From 6885c10083cc05494d4bfad43968355666c233a5 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 29 Jan 2020 17:53:05 +0000 Subject: [PATCH 01/86] Implement GET endpoints for account_data in clientapi (#861) * Implement GET endpoints for account_data in clientapi * Fix accountDB parameter * Remove fmt.Println --- clientapi/routing/account_data.go | 32 +++++++++++++++++++++++++++++++ clientapi/routing/routing.go | 20 +++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/clientapi/routing/account_data.go b/clientapi/routing/account_data.go index d57a6d370..bbc8c258e 100644 --- a/clientapi/routing/account_data.go +++ b/clientapi/routing/account_data.go @@ -28,6 +28,38 @@ import ( "github.com/matrix-org/util" ) +// GetAccountData implements GET /user/{userId}/[rooms/{roomid}/]account_data/{type} +func GetAccountData( + req *http.Request, accountDB *accounts.Database, device *authtypes.Device, + userID string, roomID string, dataType string, +) util.JSONResponse { + if userID != device.UserID { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("userID does not match the current user"), + } + } + + localpart, _, err := gomatrixserverlib.SplitID('@', userID) + if err != nil { + return httputil.LogThenError(req, err) + } + + if data, err := accountDB.GetAccountDataByType( + req.Context(), localpart, roomID, dataType, + ); err == nil { + return util.JSONResponse{ + Code: http.StatusOK, + JSON: data, + } + } + + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.Forbidden("data not found"), + } +} + // SaveAccountData implements PUT /user/{userId}/[rooms/{roomId}/]account_data/{type} func SaveAccountData( req *http.Request, accountDB *accounts.Database, device *authtypes.Device, diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 4a36661db..5c98cd0d8 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -430,6 +430,26 @@ func Setup( }), ).Methods(http.MethodPut, http.MethodOptions) + r0mux.Handle("/user/{userID}/account_data/{type}", + common.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return GetAccountData(req, accountDB, device, vars["userID"], "", vars["type"]) + }), + ).Methods(http.MethodGet) + + r0mux.Handle("/user/{userID}/rooms/{roomID}/account_data/{type}", + common.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return GetAccountData(req, accountDB, device, vars["userID"], vars["roomID"], vars["type"]) + }), + ).Methods(http.MethodGet) + r0mux.Handle("/rooms/{roomID}/members", common.MakeAuthAPI("rooms_members", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { vars, err := common.URLDecodeMapValues(mux.Vars(req)) From 68ee11d96c8d81a398d81a02c5ba32eb9d3022ba Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 30 Jan 2020 11:20:44 +0000 Subject: [PATCH 02/86] Add empty push rules into account data on account creation (#862) --- clientapi/auth/storage/accounts/storage.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/clientapi/auth/storage/accounts/storage.go b/clientapi/auth/storage/accounts/storage.go index 020a38376..7cfc63c01 100644 --- a/clientapi/auth/storage/accounts/storage.go +++ b/clientapi/auth/storage/accounts/storage.go @@ -140,6 +140,17 @@ func (d *Database) CreateAccount( } return nil, err } + if err := d.SaveAccountData(ctx, localpart, "", "m.push_rules", `{ + "global": { + "content": [], + "override": [], + "room": [], + "sender": [], + "underride": [] + } + }`); err != nil { + return nil, err + } return d.accounts.insertAccount(ctx, localpart, hash, appserviceID) } From 4113599f61690fd1fe660ad246f4e1c4c65b4865 Mon Sep 17 00:00:00 2001 From: Prateek Sachan <42961174+prateek2211@users.noreply.github.com> Date: Thu, 30 Jan 2020 16:55:57 +0530 Subject: [PATCH 03/86] Handle kind=guest query parameter on /register (#860) * Handle kind=guest query parameter on /register * Reorganized imports * Pass device_id as nil * Added tests to systest-whitelist --- clientapi/routing/register.go | 57 +++++++++++++++++++++++++++++++++++ go.mod | 2 ++ sytest-whitelist | 17 +++++++++++ 3 files changed, 76 insertions(+) diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index d0f36a6fd..4e50e57b9 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -43,6 +43,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/userutil" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrixserverlib/tokens" "github.com/matrix-org/util" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" @@ -449,6 +450,9 @@ func Register( if resErr != nil { return *resErr } + if req.URL.Query().Get("kind") == "guest" { + return handleGuestRegistration(req, r, cfg, accountDB, deviceDB) + } // Retrieve or generate the sessionID sessionID := r.Auth.Session @@ -505,6 +509,59 @@ func Register( return handleRegistrationFlow(req, r, sessionID, cfg, accountDB, deviceDB) } +func handleGuestRegistration( + req *http.Request, + r registerRequest, + cfg *config.Dendrite, + accountDB *accounts.Database, + deviceDB *devices.Database, +) util.JSONResponse { + + //Generate numeric local part for guest user + id, err := accountDB.GetNewNumericLocalpart(req.Context()) + if err != nil { + return httputil.LogThenError(req, err) + } + + localpart := strconv.FormatInt(id, 10) + acc, err := accountDB.CreateAccount(req.Context(), localpart, "", "") + if err != nil { + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: jsonerror.Unknown("failed to create account: " + err.Error()), + } + } + token, err := tokens.GenerateLoginToken(tokens.TokenOptions{ + ServerPrivateKey: cfg.Matrix.PrivateKey.Seed(), + ServerName: string(acc.ServerName), + UserID: acc.UserID, + }) + + if err != nil { + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: jsonerror.Unknown("Failed to generate access token"), + } + } + //we don't allow guests to specify their own device_id + dev, err := deviceDB.CreateDevice(req.Context(), acc.Localpart, nil, token, r.InitialDisplayName) + if err != nil { + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: jsonerror.Unknown("failed to create device: " + err.Error()), + } + } + return util.JSONResponse{ + Code: http.StatusOK, + JSON: registerResponse{ + UserID: dev.UserID, + AccessToken: dev.AccessToken, + HomeServer: acc.ServerName, + DeviceID: dev.ID, + }, + } +} + // handleRegistrationFlow will direct and complete registration flow stages // that the client has requested. // nolint: gocyclo diff --git a/go.mod b/go.mod index 990b839eb..4c894ad99 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,8 @@ require ( github.com/matrix-org/naffka v0.0.0-20171115094957-662bfd0841d0 github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 github.com/miekg/dns v1.1.12 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 github.com/opentracing/opentracing-go v1.0.2 github.com/pierrec/lz4 v0.0.0-20161206202305-5c9560bfa9ac // indirect diff --git a/sytest-whitelist b/sytest-whitelist index 4c333d3cb..7b704f62b 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -210,3 +210,20 @@ Message history can be paginated Getting messages going forward is limited for a departed room (SPEC-216) m.room.history_visibility == "world_readable" allows/forbids appropriately for Real users Backfill works correctly with history visibility set to joined +Guest user cannot call /events globally +Guest users can join guest_access rooms +Guest user can set display names +Guest user cannot upgrade other users +m.room.history_visibility == "world_readable" allows/forbids appropriately for Guest users +Guest non-joined user cannot call /events on shared room +Guest non-joined user cannot call /events on invited room +Guest non-joined user cannot call /events on joined room +Guest non-joined user cannot call /events on default room +Guest non-joined users can get state for world_readable rooms +Guest non-joined users can get individual state for world_readable rooms +Guest non-joined users cannot room initalSync for non-world_readable rooms +Guest non-joined users can get individual state for world_readable rooms after leaving +Guest non-joined users cannot send messages to guest_access rooms if not joined +Guest users can sync from world_readable guest_access rooms if joined +Guest users can sync from default guest_access rooms if joined +Real non-joined users cannot room initalSync for non-world_readable rooms From ed457049fb0cce8287e0e08ffb5e7bbf7dc4fd65 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 30 Jan 2020 11:29:52 +0000 Subject: [PATCH 04/86] Update sytest-whitelist --- sytest-whitelist | 1 + 1 file changed, 1 insertion(+) diff --git a/sytest-whitelist b/sytest-whitelist index 7b704f62b..b6741ff2e 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -227,3 +227,4 @@ Guest non-joined users cannot send messages to guest_access rooms if not joined Guest users can sync from world_readable guest_access rooms if joined Guest users can sync from default guest_access rooms if joined Real non-joined users cannot room initalSync for non-world_readable rooms +Push rules come down in an initial /sync From 4da26309048e48ed6c466155421ec17ac0935c5f Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Sat, 1 Feb 2020 23:19:20 +0000 Subject: [PATCH 05/86] Blacklist 'displayname updates affect room member events' (#859) --- sytest-blacklist | 3 +++ sytest-whitelist | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/sytest-blacklist b/sytest-blacklist index dd5e2cd5c..4c36e4778 100644 --- a/sytest-blacklist +++ b/sytest-blacklist @@ -7,6 +7,9 @@ POST /login can log in as a user with just the local part of the id # Blacklisted due to flakiness avatar_url updates affect room member events +# Blacklisted due to flakiness +displayname updates affect room member events + # Blacklisted due to flakiness Room members can override their displayname on a room-specific basis diff --git a/sytest-whitelist b/sytest-whitelist index b6741ff2e..0600e2e5a 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -67,7 +67,6 @@ Can get rooms/{roomId}/members for a departed room (SPEC-216) 3pid invite join valid signature but revoked keys are rejected 3pid invite join valid signature but unreachable ID server are rejected Room members can join a room with an overridden displayname -displayname updates affect room member events Real non-joined user cannot call /events on shared room Real non-joined user cannot call /events on invited room Real non-joined user cannot call /events on joined room From 880d8ae0246c8b123fdc827d2c03b4cb1b6920bc Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 5 Feb 2020 16:25:58 +0000 Subject: [PATCH 06/86] Room version abstractions (#865) * Rough first pass at adding room version abstractions * Define newer room versions * Update room version metadata * Fix roomserver/versions * Try to fix whitespace in roomsSchema --- roomserver/input/events.go | 12 +- roomserver/input/latest_events.go | 11 +- roomserver/query/query.go | 40 +- roomserver/state/database/database.go | 48 + roomserver/state/state.go | 1005 +------------------- roomserver/state/v1/state.go | 927 ++++++++++++++++++ roomserver/state/{ => v1}/state_test.go | 2 +- roomserver/storage/postgres/rooms_table.go | 19 +- roomserver/storage/postgres/storage.go | 8 + roomserver/storage/storage.go | 2 + roomserver/version/version.go | 94 ++ 11 files changed, 1201 insertions(+), 967 deletions(-) create mode 100644 roomserver/state/database/database.go create mode 100644 roomserver/state/v1/state.go rename roomserver/state/{ => v1}/state_test.go (99%) create mode 100644 roomserver/version/version.go diff --git a/roomserver/input/events.go b/roomserver/input/events.go index b30c39928..10ccb6485 100644 --- a/roomserver/input/events.go +++ b/roomserver/input/events.go @@ -21,13 +21,14 @@ import ( "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/state" + "github.com/matrix-org/dendrite/roomserver/state/database" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" ) // A RoomEventDatabase has the storage APIs needed to store a room event. type RoomEventDatabase interface { - state.RoomStateDatabase + database.RoomStateDatabase // Stores a matrix room event in the database StoreEvent( ctx context.Context, @@ -149,7 +150,12 @@ func calculateAndSetState( stateAtEvent *types.StateAtEvent, event gomatrixserverlib.Event, ) error { - var err error + // TODO: get the correct room version + state, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, db) + if err != nil { + return err + } + if input.HasState { // We've been told what the state at the event is so we don't need to calculate it. // Check that those state events are in the database and store the state. @@ -163,7 +169,7 @@ func calculateAndSetState( } } else { // We haven't been told what the state at the event is so we need to calculate it from the prev_events - if stateAtEvent.BeforeStateSnapshotNID, err = state.CalculateAndStoreStateBeforeEvent(ctx, db, event, roomNID); err != nil { + if stateAtEvent.BeforeStateSnapshotNID, err = state.CalculateAndStoreStateBeforeEvent(ctx, event, roomNID); err != nil { return err } } diff --git a/roomserver/input/latest_events.go b/roomserver/input/latest_events.go index c2f06393f..760677db2 100644 --- a/roomserver/input/latest_events.go +++ b/roomserver/input/latest_events.go @@ -171,27 +171,32 @@ func (u *latestEventsUpdater) doUpdateLatestEvents() error { func (u *latestEventsUpdater) latestState() error { var err error + // TODO: get the correct room version + state, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, u.db) + if err != nil { + return err + } latestStateAtEvents := make([]types.StateAtEvent, len(u.latest)) for i := range u.latest { latestStateAtEvents[i] = u.latest[i].StateAtEvent } u.newStateNID, err = state.CalculateAndStoreStateAfterEvents( - u.ctx, u.db, u.roomNID, latestStateAtEvents, + u.ctx, u.roomNID, latestStateAtEvents, ) if err != nil { return err } u.removed, u.added, err = state.DifferenceBetweeenStateSnapshots( - u.ctx, u.db, u.oldStateNID, u.newStateNID, + u.ctx, u.oldStateNID, u.newStateNID, ) if err != nil { return err } u.stateBeforeEventRemoves, u.stateBeforeEventAdds, err = state.DifferenceBetweeenStateSnapshots( - u.ctx, u.db, u.newStateNID, u.stateAtEvent.BeforeStateSnapshotNID, + u.ctx, u.newStateNID, u.stateAtEvent.BeforeStateSnapshotNID, ) return err } diff --git a/roomserver/query/query.go b/roomserver/query/query.go index da8fe23e5..d318fc001 100644 --- a/roomserver/query/query.go +++ b/roomserver/query/query.go @@ -23,6 +23,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/auth" "github.com/matrix-org/dendrite/roomserver/state" + "github.com/matrix-org/dendrite/roomserver/state/database" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -39,7 +40,7 @@ type RoomserverQueryAPIEventDB interface { // RoomserverQueryAPIDatabase has the storage APIs needed to implement the query API. type RoomserverQueryAPIDatabase interface { - state.RoomStateDatabase + database.RoomStateDatabase RoomserverQueryAPIEventDB // Look up the numeric ID for the room. // Returns 0 if the room doesn't exists. @@ -98,6 +99,11 @@ func (r *RoomserverQueryAPI) QueryLatestEventsAndState( request *api.QueryLatestEventsAndStateRequest, response *api.QueryLatestEventsAndStateResponse, ) error { + // TODO: get the correct room version + state, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, r.DB) + if err != nil { + return err + } response.QueryLatestEventsAndStateRequest = *request roomNID, err := r.DB.RoomNID(ctx, request.RoomID) if err != nil { @@ -116,7 +122,7 @@ func (r *RoomserverQueryAPI) QueryLatestEventsAndState( // Look up the currrent state for the requested tuples. stateEntries, err := state.LoadStateAtSnapshotForStringTuples( - ctx, r.DB, currentStateSnapshotNID, request.StateToFetch, + ctx, currentStateSnapshotNID, request.StateToFetch, ) if err != nil { return err @@ -137,6 +143,11 @@ func (r *RoomserverQueryAPI) QueryStateAfterEvents( request *api.QueryStateAfterEventsRequest, response *api.QueryStateAfterEventsResponse, ) error { + // TODO: get the correct room version + state, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, r.DB) + if err != nil { + return err + } response.QueryStateAfterEventsRequest = *request roomNID, err := r.DB.RoomNID(ctx, request.RoomID) if err != nil { @@ -160,7 +171,7 @@ func (r *RoomserverQueryAPI) QueryStateAfterEvents( // Look up the currrent state for the requested tuples. stateEntries, err := state.LoadStateAfterEventsForStringTuples( - ctx, r.DB, prevStates, request.StateToFetch, + ctx, prevStates, request.StateToFetch, ) if err != nil { return err @@ -315,6 +326,11 @@ func (r *RoomserverQueryAPI) QueryMembershipsForRoom( func (r *RoomserverQueryAPI) getMembershipsBeforeEventNID( ctx context.Context, eventNID types.EventNID, joinedOnly bool, ) ([]types.Event, error) { + // TODO: get the correct room version + state, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, r.DB) + if err != nil { + return []types.Event{}, err + } events := []types.Event{} // Lookup the event NID eIDs, err := r.DB.EventIDs(ctx, []types.EventNID{eventNID}) @@ -329,7 +345,7 @@ func (r *RoomserverQueryAPI) getMembershipsBeforeEventNID( } // Fetch the state as it was when this event was fired - stateEntries, err := state.LoadCombinedStateAfterEvents(ctx, r.DB, prevState) + stateEntries, err := state.LoadCombinedStateAfterEvents(ctx, prevState) if err != nil { return nil, err } @@ -416,7 +432,13 @@ func (r *RoomserverQueryAPI) QueryServerAllowedToSeeEvent( func (r *RoomserverQueryAPI) checkServerAllowedToSeeEvent( ctx context.Context, eventID string, serverName gomatrixserverlib.ServerName, ) (bool, error) { - stateEntries, err := state.LoadStateAtEvent(ctx, r.DB, eventID) + // TODO: get the correct room version + state, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, r.DB) + if err != nil { + return false, err + } + + stateEntries, err := state.LoadStateAtEvent(ctx, eventID) if err != nil { return false, err } @@ -570,6 +592,12 @@ func (r *RoomserverQueryAPI) QueryStateAndAuthChain( request *api.QueryStateAndAuthChainRequest, response *api.QueryStateAndAuthChainResponse, ) error { + // TODO: get the correct room version + state, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, r.DB) + if err != nil { + return err + } + response.QueryStateAndAuthChainRequest = *request roomNID, err := r.DB.RoomNID(ctx, request.RoomID) if err != nil { @@ -593,7 +621,7 @@ func (r *RoomserverQueryAPI) QueryStateAndAuthChain( // Look up the currrent state for the requested tuples. stateEntries, err := state.LoadCombinedStateAfterEvents( - ctx, r.DB, prevStates, + ctx, prevStates, ) if err != nil { return err diff --git a/roomserver/state/database/database.go b/roomserver/state/database/database.go new file mode 100644 index 000000000..546f06e87 --- /dev/null +++ b/roomserver/state/database/database.go @@ -0,0 +1,48 @@ +package database + +import ( + "context" + + "github.com/matrix-org/dendrite/roomserver/types" +) + +// A RoomStateDatabase has the storage APIs needed to load state from the database +type RoomStateDatabase interface { + // Store the room state at an event in the database + AddState( + ctx context.Context, + roomNID types.RoomNID, + stateBlockNIDs []types.StateBlockNID, + state []types.StateEntry, + ) (types.StateSnapshotNID, error) + // Look up the state of a room at each event for a list of string event IDs. + // Returns an error if there is an error talking to the database + // Returns a types.MissingEventError if the room state for the event IDs aren't in the database + StateAtEventIDs(ctx context.Context, eventIDs []string) ([]types.StateAtEvent, error) + // Look up the numeric IDs for a list of string event types. + // Returns a map from string event type to numeric ID for the event type. + EventTypeNIDs(ctx context.Context, eventTypes []string) (map[string]types.EventTypeNID, error) + // Look up the numeric IDs for a list of string event state keys. + // Returns a map from string state key to numeric ID for the state key. + EventStateKeyNIDs(ctx context.Context, eventStateKeys []string) (map[string]types.EventStateKeyNID, error) + // Look up the numeric state data IDs for each numeric state snapshot ID + // The returned slice is sorted by numeric state snapshot ID. + StateBlockNIDs(ctx context.Context, stateNIDs []types.StateSnapshotNID) ([]types.StateBlockNIDList, error) + // Look up the state data for each numeric state data ID + // The returned slice is sorted by numeric state data ID. + StateEntries(ctx context.Context, stateBlockNIDs []types.StateBlockNID) ([]types.StateEntryList, error) + // Look up the state data for the state key tuples for each numeric state block ID + // This is used to fetch a subset of the room state at a snapshot. + // If a block doesn't contain any of the requested tuples then it can be discarded from the result. + // The returned slice is sorted by numeric state block ID. + StateEntriesForTuples( + ctx context.Context, + stateBlockNIDs []types.StateBlockNID, + stateKeyTuples []types.StateKeyTuple, + ) ([]types.StateEntryList, error) + // Look up the Events for a list of numeric event IDs. + // Returns a sorted list of events. + Events(ctx context.Context, eventNIDs []types.EventNID) ([]types.Event, error) + // Look up snapshot NID for an event ID string + SnapshotNIDFromEventID(ctx context.Context, eventID string) (types.StateSnapshotNID, error) +} diff --git a/roomserver/state/state.go b/roomserver/state/state.go index 2a0b7f574..bbc27ad81 100644 --- a/roomserver/state/state.go +++ b/roomserver/state/state.go @@ -1,966 +1,65 @@ -// Copyright 2017 Vector Creations Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package state provides functions for reading state from the database. -// The functions for writing state to the database are the input package. package state import ( "context" - "fmt" - "sort" - "time" + "errors" + + "github.com/matrix-org/dendrite/roomserver/state/database" + v1 "github.com/matrix-org/dendrite/roomserver/state/v1" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" - "github.com/prometheus/client_golang/prometheus" ) -// A RoomStateDatabase has the storage APIs needed to load state from the database -type RoomStateDatabase interface { - // Store the room state at an event in the database - AddState( +type StateResolutionVersion int + +const ( + StateResolutionAlgorithmV1 StateResolutionVersion = iota + 1 + StateResolutionAlgorithmV2 +) + +func GetStateResolutionAlgorithm( + version StateResolutionVersion, db database.RoomStateDatabase, +) (StateResolutionImpl, error) { + switch version { + case StateResolutionAlgorithmV1: + return v1.Prepare(db), nil + default: + return nil, errors.New("unsupported room version") + } +} + +type StateResolutionImpl interface { + LoadStateAtSnapshot( + ctx context.Context, stateNID types.StateSnapshotNID, + ) ([]types.StateEntry, error) + LoadStateAtEvent( + ctx context.Context, eventID string, + ) ([]types.StateEntry, error) + LoadCombinedStateAfterEvents( + ctx context.Context, prevStates []types.StateAtEvent, + ) ([]types.StateEntry, error) + DifferenceBetweeenStateSnapshots( + ctx context.Context, oldStateNID, newStateNID types.StateSnapshotNID, + ) (removed, added []types.StateEntry, err error) + LoadStateAtSnapshotForStringTuples( + ctx context.Context, + stateNID types.StateSnapshotNID, + stateKeyTuples []gomatrixserverlib.StateKeyTuple, + ) ([]types.StateEntry, error) + LoadStateAfterEventsForStringTuples( + ctx context.Context, + prevStates []types.StateAtEvent, + stateKeyTuples []gomatrixserverlib.StateKeyTuple, + ) ([]types.StateEntry, error) + CalculateAndStoreStateBeforeEvent( + ctx context.Context, + event gomatrixserverlib.Event, + roomNID types.RoomNID, + ) (types.StateSnapshotNID, error) + CalculateAndStoreStateAfterEvents( ctx context.Context, roomNID types.RoomNID, - stateBlockNIDs []types.StateBlockNID, - state []types.StateEntry, + prevStates []types.StateAtEvent, ) (types.StateSnapshotNID, error) - // Look up the state of a room at each event for a list of string event IDs. - // Returns an error if there is an error talking to the database - // Returns a types.MissingEventError if the room state for the event IDs aren't in the database - StateAtEventIDs(ctx context.Context, eventIDs []string) ([]types.StateAtEvent, error) - // Look up the numeric IDs for a list of string event types. - // Returns a map from string event type to numeric ID for the event type. - EventTypeNIDs(ctx context.Context, eventTypes []string) (map[string]types.EventTypeNID, error) - // Look up the numeric IDs for a list of string event state keys. - // Returns a map from string state key to numeric ID for the state key. - EventStateKeyNIDs(ctx context.Context, eventStateKeys []string) (map[string]types.EventStateKeyNID, error) - // Look up the numeric state data IDs for each numeric state snapshot ID - // The returned slice is sorted by numeric state snapshot ID. - StateBlockNIDs(ctx context.Context, stateNIDs []types.StateSnapshotNID) ([]types.StateBlockNIDList, error) - // Look up the state data for each numeric state data ID - // The returned slice is sorted by numeric state data ID. - StateEntries(ctx context.Context, stateBlockNIDs []types.StateBlockNID) ([]types.StateEntryList, error) - // Look up the state data for the state key tuples for each numeric state block ID - // This is used to fetch a subset of the room state at a snapshot. - // If a block doesn't contain any of the requested tuples then it can be discarded from the result. - // The returned slice is sorted by numeric state block ID. - StateEntriesForTuples( - ctx context.Context, - stateBlockNIDs []types.StateBlockNID, - stateKeyTuples []types.StateKeyTuple, - ) ([]types.StateEntryList, error) - // Look up the Events for a list of numeric event IDs. - // Returns a sorted list of events. - Events(ctx context.Context, eventNIDs []types.EventNID) ([]types.Event, error) - // Look up snapshot NID for an event ID string - SnapshotNIDFromEventID(ctx context.Context, eventID string) (types.StateSnapshotNID, error) -} - -// LoadStateAtSnapshot loads the full state of a room at a particular snapshot. -// This is typically the state before an event or the current state of a room. -// Returns a sorted list of state entries or an error if there was a problem talking to the database. -func LoadStateAtSnapshot( - ctx context.Context, db RoomStateDatabase, stateNID types.StateSnapshotNID, -) ([]types.StateEntry, error) { - stateBlockNIDLists, err := db.StateBlockNIDs(ctx, []types.StateSnapshotNID{stateNID}) - if err != nil { - return nil, err - } - // We've asked for exactly one snapshot from the db so we should have exactly one entry in the result. - stateBlockNIDList := stateBlockNIDLists[0] - - stateEntryLists, err := db.StateEntries(ctx, stateBlockNIDList.StateBlockNIDs) - if err != nil { - return nil, err - } - stateEntriesMap := stateEntryListMap(stateEntryLists) - - // Combine all the state entries for this snapshot. - // The order of state block NIDs in the list tells us the order to combine them in. - var fullState []types.StateEntry - for _, stateBlockNID := range stateBlockNIDList.StateBlockNIDs { - entries, ok := stateEntriesMap.lookup(stateBlockNID) - if !ok { - // This should only get hit if the database is corrupt. - // It should be impossible for an event to reference a NID that doesn't exist - panic(fmt.Errorf("Corrupt DB: Missing state block numeric ID %d", stateBlockNID)) - } - fullState = append(fullState, entries...) - } - - // Stable sort so that the most recent entry for each state key stays - // remains later in the list than the older entries for the same state key. - sort.Stable(stateEntryByStateKeySorter(fullState)) - // Unique returns the last entry and hence the most recent entry for each state key. - fullState = fullState[:util.Unique(stateEntryByStateKeySorter(fullState))] - return fullState, nil -} - -// LoadStateAtEvent loads the full state of a room at a particular event. -func LoadStateAtEvent( - ctx context.Context, db RoomStateDatabase, eventID string, -) ([]types.StateEntry, error) { - snapshotNID, err := db.SnapshotNIDFromEventID(ctx, eventID) - if err != nil { - return nil, err - } - - stateEntries, err := LoadStateAtSnapshot(ctx, db, snapshotNID) - if err != nil { - return nil, err - } - - return stateEntries, nil -} - -// LoadCombinedStateAfterEvents loads a snapshot of the state after each of the events -// and combines those snapshots together into a single list. -func LoadCombinedStateAfterEvents( - ctx context.Context, db RoomStateDatabase, prevStates []types.StateAtEvent, -) ([]types.StateEntry, error) { - stateNIDs := make([]types.StateSnapshotNID, len(prevStates)) - for i, state := range prevStates { - stateNIDs[i] = state.BeforeStateSnapshotNID - } - // Fetch the state snapshots for the state before the each prev event from the database. - // Deduplicate the IDs before passing them to the database. - // There could be duplicates because the events could be state events where - // the snapshot of the room state before them was the same. - stateBlockNIDLists, err := db.StateBlockNIDs(ctx, uniqueStateSnapshotNIDs(stateNIDs)) - if err != nil { - return nil, err - } - - var stateBlockNIDs []types.StateBlockNID - for _, list := range stateBlockNIDLists { - stateBlockNIDs = append(stateBlockNIDs, list.StateBlockNIDs...) - } - // Fetch the state entries that will be combined to create the snapshots. - // Deduplicate the IDs before passing them to the database. - // There could be duplicates because a block of state entries could be reused by - // multiple snapshots. - stateEntryLists, err := db.StateEntries(ctx, uniqueStateBlockNIDs(stateBlockNIDs)) - if err != nil { - return nil, err - } - stateBlockNIDsMap := stateBlockNIDListMap(stateBlockNIDLists) - stateEntriesMap := stateEntryListMap(stateEntryLists) - - // Combine the entries from all the snapshots of state after each prev event into a single list. - var combined []types.StateEntry - for _, prevState := range prevStates { - // Grab the list of state data NIDs for this snapshot. - stateBlockNIDs, ok := stateBlockNIDsMap.lookup(prevState.BeforeStateSnapshotNID) - if !ok { - // This should only get hit if the database is corrupt. - // It should be impossible for an event to reference a NID that doesn't exist - panic(fmt.Errorf("Corrupt DB: Missing state snapshot numeric ID %d", prevState.BeforeStateSnapshotNID)) - } - - // Combine all the state entries for this snapshot. - // The order of state block NIDs in the list tells us the order to combine them in. - var fullState []types.StateEntry - for _, stateBlockNID := range stateBlockNIDs { - entries, ok := stateEntriesMap.lookup(stateBlockNID) - if !ok { - // This should only get hit if the database is corrupt. - // It should be impossible for an event to reference a NID that doesn't exist - panic(fmt.Errorf("Corrupt DB: Missing state block numeric ID %d", stateBlockNID)) - } - fullState = append(fullState, entries...) - } - if prevState.IsStateEvent() { - // If the prev event was a state event then add an entry for the event itself - // so that we get the state after the event rather than the state before. - fullState = append(fullState, prevState.StateEntry) - } - - // Stable sort so that the most recent entry for each state key stays - // remains later in the list than the older entries for the same state key. - sort.Stable(stateEntryByStateKeySorter(fullState)) - // Unique returns the last entry and hence the most recent entry for each state key. - fullState = fullState[:util.Unique(stateEntryByStateKeySorter(fullState))] - // Add the full state for this StateSnapshotNID. - combined = append(combined, fullState...) - } - return combined, nil -} - -// DifferenceBetweeenStateSnapshots works out which state entries have been added and removed between two snapshots. -func DifferenceBetweeenStateSnapshots( - ctx context.Context, db RoomStateDatabase, oldStateNID, newStateNID types.StateSnapshotNID, -) (removed, added []types.StateEntry, err error) { - if oldStateNID == newStateNID { - // If the snapshot NIDs are the same then nothing has changed - return nil, nil, nil - } - - var oldEntries []types.StateEntry - var newEntries []types.StateEntry - if oldStateNID != 0 { - oldEntries, err = LoadStateAtSnapshot(ctx, db, oldStateNID) - if err != nil { - return nil, nil, err - } - } - if newStateNID != 0 { - newEntries, err = LoadStateAtSnapshot(ctx, db, newStateNID) - if err != nil { - return nil, nil, err - } - } - - var oldI int - var newI int - for { - switch { - case oldI == len(oldEntries): - // We've reached the end of the old entries. - // The rest of the new list must have been newly added. - added = append(added, newEntries[newI:]...) - return - case newI == len(newEntries): - // We've reached the end of the new entries. - // The rest of the old list must be have been removed. - removed = append(removed, oldEntries[oldI:]...) - return - case oldEntries[oldI] == newEntries[newI]: - // The entry is in both lists so skip over it. - oldI++ - newI++ - case oldEntries[oldI].LessThan(newEntries[newI]): - // The lists are sorted so the old entry being less than the new entry means that it only appears in the old list. - removed = append(removed, oldEntries[oldI]) - oldI++ - default: - // Reaching the default case implies that the new entry is less than the old entry. - // Since the lists are sorted this means that it only appears in the new list. - added = append(added, newEntries[newI]) - newI++ - } - } -} - -// LoadStateAtSnapshotForStringTuples loads the state for a list of event type and state key pairs at a snapshot. -// This is used when we only want to load a subset of the room state at a snapshot. -// If there is no entry for a given event type and state key pair then it will be discarded. -// This is typically the state before an event or the current state of a room. -// Returns a sorted list of state entries or an error if there was a problem talking to the database. -func LoadStateAtSnapshotForStringTuples( - ctx context.Context, - db RoomStateDatabase, - stateNID types.StateSnapshotNID, - stateKeyTuples []gomatrixserverlib.StateKeyTuple, -) ([]types.StateEntry, error) { - numericTuples, err := stringTuplesToNumericTuples(ctx, db, stateKeyTuples) - if err != nil { - return nil, err - } - return loadStateAtSnapshotForNumericTuples(ctx, db, stateNID, numericTuples) -} - -// stringTuplesToNumericTuples converts the string state key tuples into numeric IDs -// If there isn't a numeric ID for either the event type or the event state key then the tuple is discarded. -// Returns an error if there was a problem talking to the database. -func stringTuplesToNumericTuples( - ctx context.Context, - db RoomStateDatabase, - stringTuples []gomatrixserverlib.StateKeyTuple, -) ([]types.StateKeyTuple, error) { - eventTypes := make([]string, len(stringTuples)) - stateKeys := make([]string, len(stringTuples)) - for i := range stringTuples { - eventTypes[i] = stringTuples[i].EventType - stateKeys[i] = stringTuples[i].StateKey - } - eventTypes = util.UniqueStrings(eventTypes) - eventTypeMap, err := db.EventTypeNIDs(ctx, eventTypes) - if err != nil { - return nil, err - } - stateKeys = util.UniqueStrings(stateKeys) - stateKeyMap, err := db.EventStateKeyNIDs(ctx, stateKeys) - if err != nil { - return nil, err - } - - var result []types.StateKeyTuple - for _, stringTuple := range stringTuples { - var numericTuple types.StateKeyTuple - var ok1, ok2 bool - numericTuple.EventTypeNID, ok1 = eventTypeMap[stringTuple.EventType] - numericTuple.EventStateKeyNID, ok2 = stateKeyMap[stringTuple.StateKey] - // Discard the tuple if there wasn't a numeric ID for either the event type or the state key. - if ok1 && ok2 { - result = append(result, numericTuple) - } - } - - return result, nil -} - -// loadStateAtSnapshotForNumericTuples loads the state for a list of event type and state key pairs at a snapshot. -// This is used when we only want to load a subset of the room state at a snapshot. -// If there is no entry for a given event type and state key pair then it will be discarded. -// This is typically the state before an event or the current state of a room. -// Returns a sorted list of state entries or an error if there was a problem talking to the database. -func loadStateAtSnapshotForNumericTuples( - ctx context.Context, - db RoomStateDatabase, - stateNID types.StateSnapshotNID, - stateKeyTuples []types.StateKeyTuple, -) ([]types.StateEntry, error) { - stateBlockNIDLists, err := db.StateBlockNIDs(ctx, []types.StateSnapshotNID{stateNID}) - if err != nil { - return nil, err - } - // We've asked for exactly one snapshot from the db so we should have exactly one entry in the result. - stateBlockNIDList := stateBlockNIDLists[0] - - stateEntryLists, err := db.StateEntriesForTuples( - ctx, stateBlockNIDList.StateBlockNIDs, stateKeyTuples, - ) - if err != nil { - return nil, err - } - stateEntriesMap := stateEntryListMap(stateEntryLists) - - // Combine all the state entries for this snapshot. - // The order of state block NIDs in the list tells us the order to combine them in. - var fullState []types.StateEntry - for _, stateBlockNID := range stateBlockNIDList.StateBlockNIDs { - entries, ok := stateEntriesMap.lookup(stateBlockNID) - if !ok { - // If the block is missing from the map it means that none of its entries matched a requested tuple. - // This can happen if the block doesn't contain an update for one of the requested tuples. - // If none of the requested tuples are in the block then it can be safely skipped. - continue - } - fullState = append(fullState, entries...) - } - - // Stable sort so that the most recent entry for each state key stays - // remains later in the list than the older entries for the same state key. - sort.Stable(stateEntryByStateKeySorter(fullState)) - // Unique returns the last entry and hence the most recent entry for each state key. - fullState = fullState[:util.Unique(stateEntryByStateKeySorter(fullState))] - return fullState, nil -} - -// LoadStateAfterEventsForStringTuples loads the state for a list of event type -// and state key pairs after list of events. -// This is used when we only want to load a subset of the room state after a list of events. -// If there is no entry for a given event type and state key pair then it will be discarded. -// This is typically the state before an event. -// Returns a sorted list of state entries or an error if there was a problem talking to the database. -func LoadStateAfterEventsForStringTuples( - ctx context.Context, - db RoomStateDatabase, - prevStates []types.StateAtEvent, - stateKeyTuples []gomatrixserverlib.StateKeyTuple, -) ([]types.StateEntry, error) { - numericTuples, err := stringTuplesToNumericTuples(ctx, db, stateKeyTuples) - if err != nil { - return nil, err - } - return loadStateAfterEventsForNumericTuples(ctx, db, prevStates, numericTuples) -} - -func loadStateAfterEventsForNumericTuples( - ctx context.Context, - db RoomStateDatabase, - prevStates []types.StateAtEvent, - stateKeyTuples []types.StateKeyTuple, -) ([]types.StateEntry, error) { - if len(prevStates) == 1 { - // Fast path for a single event. - prevState := prevStates[0] - result, err := loadStateAtSnapshotForNumericTuples( - ctx, db, prevState.BeforeStateSnapshotNID, stateKeyTuples, - ) - if err != nil { - return nil, err - } - if prevState.IsStateEvent() { - // The result is current the state before the requested event. - // We want the state after the requested event. - // If the requested event was a state event then we need to - // update that key in the result. - // If the requested event wasn't a state event then the state after - // it is the same as the state before it. - for i := range result { - if result[i].StateKeyTuple == prevState.StateKeyTuple { - result[i] = prevState.StateEntry - } - } - } - return result, nil - } - - // Slow path for more that one event. - // Load the entire state so that we can do conflict resolution if we need to. - // TODO: The are some optimistations we could do here: - // 1) We only need to do conflict resolution if there is a conflict in the - // requested tuples so we might try loading just those tuples and then - // checking for conflicts. - // 2) When there is a conflict we still only need to load the state - // needed to do conflict resolution which would save us having to load - // the full state. - - // TODO: Add metrics for this as it could take a long time for big rooms - // with large conflicts. - fullState, _, _, err := calculateStateAfterManyEvents(ctx, db, prevStates) - if err != nil { - return nil, err - } - - // Sort the full state so we can use it as a map. - sort.Sort(stateEntrySorter(fullState)) - - // Filter the full state down to the required tuples. - var result []types.StateEntry - for _, tuple := range stateKeyTuples { - eventNID, ok := stateEntryMap(fullState).lookup(tuple) - if ok { - result = append(result, types.StateEntry{ - StateKeyTuple: tuple, - EventNID: eventNID, - }) - } - } - sort.Sort(stateEntrySorter(result)) - return result, nil -} - -var calculateStateDurations = prometheus.NewSummaryVec( - prometheus.SummaryOpts{ - Namespace: "dendrite", - Subsystem: "roomserver", - Name: "calculate_state_duration_microseconds", - Help: "How long it takes to calculate the state after a list of events", - }, - // Takes two labels: - // algorithm: - // The algorithm used to calculate the state or the step it failed on if it failed. - // Labels starting with "_" are used to indicate when the algorithm fails halfway. - // outcome: - // Whether the state was successfully calculated. - // - // The possible values for algorithm are: - // empty_state -> The list of events was empty so the state is empty. - // no_change -> The state hasn't changed. - // single_delta -> There was a single event added to the state in a way that can be encoded as a single delta - // full_state_no_conflicts -> We created a new copy of the full room state, but didn't enounter any conflicts - // while doing so. - // full_state_with_conflicts -> We created a new copy of the full room state and had to resolve conflicts to do so. - // _load_state_block_nids -> Failed loading the state block nids for a single previous state. - // _load_combined_state -> Failed to load the combined state. - // _resolve_conflicts -> Failed to resolve conflicts. - []string{"algorithm", "outcome"}, -) - -var calculateStatePrevEventLength = prometheus.NewSummaryVec( - prometheus.SummaryOpts{ - Namespace: "dendrite", - Subsystem: "roomserver", - Name: "calculate_state_prev_event_length", - Help: "The length of the list of events to calculate the state after", - }, - []string{"algorithm", "outcome"}, -) - -var calculateStateFullStateLength = prometheus.NewSummaryVec( - prometheus.SummaryOpts{ - Namespace: "dendrite", - Subsystem: "roomserver", - Name: "calculate_state_full_state_length", - Help: "The length of the full room state.", - }, - []string{"algorithm", "outcome"}, -) - -var calculateStateConflictLength = prometheus.NewSummaryVec( - prometheus.SummaryOpts{ - Namespace: "dendrite", - Subsystem: "roomserver", - Name: "calculate_state_conflict_state_length", - Help: "The length of the conflicted room state.", - }, - []string{"algorithm", "outcome"}, -) - -type calculateStateMetrics struct { - algorithm string - startTime time.Time - prevEventLength int - fullStateLength int - conflictLength int -} - -func (c *calculateStateMetrics) stop(stateNID types.StateSnapshotNID, err error) (types.StateSnapshotNID, error) { - var outcome string - if err == nil { - outcome = "success" - } else { - outcome = "failure" - } - endTime := time.Now() - calculateStateDurations.WithLabelValues(c.algorithm, outcome).Observe( - float64(endTime.Sub(c.startTime).Nanoseconds()) / 1000., - ) - calculateStatePrevEventLength.WithLabelValues(c.algorithm, outcome).Observe( - float64(c.prevEventLength), - ) - calculateStateFullStateLength.WithLabelValues(c.algorithm, outcome).Observe( - float64(c.fullStateLength), - ) - calculateStateConflictLength.WithLabelValues(c.algorithm, outcome).Observe( - float64(c.conflictLength), - ) - return stateNID, err -} - -func init() { - prometheus.MustRegister( - calculateStateDurations, calculateStatePrevEventLength, - calculateStateFullStateLength, calculateStateConflictLength, - ) -} - -// CalculateAndStoreStateBeforeEvent calculates a snapshot of the state of a room before an event. -// Stores the snapshot of the state in the database. -// Returns a numeric ID for the snapshot of the state before the event. -func CalculateAndStoreStateBeforeEvent( - ctx context.Context, - db RoomStateDatabase, - event gomatrixserverlib.Event, - roomNID types.RoomNID, -) (types.StateSnapshotNID, error) { - // Load the state at the prev events. - prevEventRefs := event.PrevEvents() - prevEventIDs := make([]string, len(prevEventRefs)) - for i := range prevEventRefs { - prevEventIDs[i] = prevEventRefs[i].EventID - } - - prevStates, err := db.StateAtEventIDs(ctx, prevEventIDs) - if err != nil { - return 0, err - } - - // The state before this event will be the state after the events that came before it. - return CalculateAndStoreStateAfterEvents(ctx, db, roomNID, prevStates) -} - -// CalculateAndStoreStateAfterEvents finds the room state after the given events. -// Stores the resulting state in the database and returns a numeric ID for that snapshot. -func CalculateAndStoreStateAfterEvents( - ctx context.Context, - db RoomStateDatabase, - roomNID types.RoomNID, - prevStates []types.StateAtEvent, -) (types.StateSnapshotNID, error) { - metrics := calculateStateMetrics{startTime: time.Now(), prevEventLength: len(prevStates)} - - if len(prevStates) == 0 { - // 2) There weren't any prev_events for this event so the state is - // empty. - metrics.algorithm = "empty_state" - return metrics.stop(db.AddState(ctx, roomNID, nil, nil)) - } - - if len(prevStates) == 1 { - prevState := prevStates[0] - if prevState.EventStateKeyNID == 0 { - // 3) None of the previous events were state events and they all - // have the same state, so this event has exactly the same state - // as the previous events. - // This should be the common case. - metrics.algorithm = "no_change" - return metrics.stop(prevState.BeforeStateSnapshotNID, nil) - } - // The previous event was a state event so we need to store a copy - // of the previous state updated with that event. - stateBlockNIDLists, err := db.StateBlockNIDs( - ctx, []types.StateSnapshotNID{prevState.BeforeStateSnapshotNID}, - ) - if err != nil { - metrics.algorithm = "_load_state_blocks" - return metrics.stop(0, err) - } - stateBlockNIDs := stateBlockNIDLists[0].StateBlockNIDs - if len(stateBlockNIDs) < maxStateBlockNIDs { - // 4) The number of state data blocks is small enough that we can just - // add the state event as a block of size one to the end of the blocks. - metrics.algorithm = "single_delta" - return metrics.stop(db.AddState( - ctx, roomNID, stateBlockNIDs, []types.StateEntry{prevState.StateEntry}, - )) - } - // If there are too many deltas then we need to calculate the full state - // So fall through to calculateAndStoreStateAfterManyEvents - } - - return calculateAndStoreStateAfterManyEvents(ctx, db, roomNID, prevStates, metrics) -} - -// maxStateBlockNIDs is the maximum number of state data blocks to use to encode a snapshot of room state. -// Increasing this number means that we can encode more of the state changes as simple deltas which means that -// we need fewer entries in the state data table. However making this number bigger will increase the size of -// the rows in the state table itself and will require more index lookups when retrieving a snapshot. -// TODO: Tune this to get the right balance between size and lookup performance. -const maxStateBlockNIDs = 64 - -// calculateAndStoreStateAfterManyEvents finds the room state after the given events. -// This handles the slow path of calculateAndStoreStateAfterEvents for when there is more than one event. -// Stores the resulting state and returns a numeric ID for the snapshot. -func calculateAndStoreStateAfterManyEvents( - ctx context.Context, - db RoomStateDatabase, - roomNID types.RoomNID, - prevStates []types.StateAtEvent, - metrics calculateStateMetrics, -) (types.StateSnapshotNID, error) { - - state, algorithm, conflictLength, err := - calculateStateAfterManyEvents(ctx, db, prevStates) - metrics.algorithm = algorithm - if err != nil { - return metrics.stop(0, err) - } - - // TODO: Check if we can encode the new state as a delta against the - // previous state. - metrics.conflictLength = conflictLength - metrics.fullStateLength = len(state) - return metrics.stop(db.AddState(ctx, roomNID, nil, state)) -} - -func calculateStateAfterManyEvents( - ctx context.Context, db RoomStateDatabase, prevStates []types.StateAtEvent, -) (state []types.StateEntry, algorithm string, conflictLength int, err error) { - var combined []types.StateEntry - // Conflict resolution. - // First stage: load the state after each of the prev events. - combined, err = LoadCombinedStateAfterEvents(ctx, db, prevStates) - if err != nil { - algorithm = "_load_combined_state" - return - } - - // Collect all the entries with the same type and key together. - // We don't care about the order here because the conflict resolution - // algorithm doesn't depend on the order of the prev events. - // Remove duplicate entires. - combined = combined[:util.SortAndUnique(stateEntrySorter(combined))] - - // Find the conflicts - conflicts := findDuplicateStateKeys(combined) - - if len(conflicts) > 0 { - conflictLength = len(conflicts) - - // 5) There are conflicting state events, for each conflict workout - // what the appropriate state event is. - - // Work out which entries aren't conflicted. - var notConflicted []types.StateEntry - for _, entry := range combined { - if _, ok := stateEntryMap(conflicts).lookup(entry.StateKeyTuple); !ok { - notConflicted = append(notConflicted, entry) - } - } - - var resolved []types.StateEntry - resolved, err = resolveConflicts(ctx, db, notConflicted, conflicts) - if err != nil { - algorithm = "_resolve_conflicts" - return - } - algorithm = "full_state_with_conflicts" - state = resolved - } else { - algorithm = "full_state_no_conflicts" - // 6) There weren't any conflicts - state = combined - } - return -} - -// resolveConflicts resolves a list of conflicted state entries. It takes two lists. -// The first is a list of all state entries that are not conflicted. -// The second is a list of all state entries that are conflicted -// A state entry is conflicted when there is more than one numeric event ID for the same state key tuple. -// Returns a list that combines the entries without conflicts with the result of state resolution for the entries with conflicts. -// The returned list is sorted by state key tuple. -// Returns an error if there was a problem talking to the database. -func resolveConflicts( - ctx context.Context, - db RoomStateDatabase, - notConflicted, conflicted []types.StateEntry, -) ([]types.StateEntry, error) { - - // Load the conflicted events - conflictedEvents, eventIDMap, err := loadStateEvents(ctx, db, conflicted) - if err != nil { - return nil, err - } - - // Work out which auth events we need to load. - needed := gomatrixserverlib.StateNeededForAuth(conflictedEvents) - - // Find the numeric IDs for the necessary state keys. - var neededStateKeys []string - neededStateKeys = append(neededStateKeys, needed.Member...) - neededStateKeys = append(neededStateKeys, needed.ThirdPartyInvite...) - stateKeyNIDMap, err := db.EventStateKeyNIDs(ctx, neededStateKeys) - if err != nil { - return nil, err - } - - // Load the necessary auth events. - tuplesNeeded := stateKeyTuplesNeeded(stateKeyNIDMap, needed) - var authEntries []types.StateEntry - for _, tuple := range tuplesNeeded { - if eventNID, ok := stateEntryMap(notConflicted).lookup(tuple); ok { - authEntries = append(authEntries, types.StateEntry{ - StateKeyTuple: tuple, - EventNID: eventNID, - }) - } - } - authEvents, _, err := loadStateEvents(ctx, db, authEntries) - if err != nil { - return nil, err - } - - // Resolve the conflicts. - resolvedEvents := gomatrixserverlib.ResolveStateConflicts(conflictedEvents, authEvents) - - // Map from the full events back to numeric state entries. - for _, resolvedEvent := range resolvedEvents { - entry, ok := eventIDMap[resolvedEvent.EventID()] - if !ok { - panic(fmt.Errorf("Missing state entry for event ID %q", resolvedEvent.EventID())) - } - notConflicted = append(notConflicted, entry) - } - - // Sort the result so it can be searched. - sort.Sort(stateEntrySorter(notConflicted)) - return notConflicted, nil -} - -// stateKeyTuplesNeeded works out which numeric state key tuples we need to authenticate some events. -func stateKeyTuplesNeeded(stateKeyNIDMap map[string]types.EventStateKeyNID, stateNeeded gomatrixserverlib.StateNeeded) []types.StateKeyTuple { - var keyTuples []types.StateKeyTuple - if stateNeeded.Create { - keyTuples = append(keyTuples, types.StateKeyTuple{ - EventTypeNID: types.MRoomCreateNID, - EventStateKeyNID: types.EmptyStateKeyNID, - }) - } - if stateNeeded.PowerLevels { - keyTuples = append(keyTuples, types.StateKeyTuple{ - EventTypeNID: types.MRoomPowerLevelsNID, - EventStateKeyNID: types.EmptyStateKeyNID, - }) - } - if stateNeeded.JoinRules { - keyTuples = append(keyTuples, types.StateKeyTuple{ - EventTypeNID: types.MRoomJoinRulesNID, - EventStateKeyNID: types.EmptyStateKeyNID, - }) - } - for _, member := range stateNeeded.Member { - stateKeyNID, ok := stateKeyNIDMap[member] - if ok { - keyTuples = append(keyTuples, types.StateKeyTuple{ - EventTypeNID: types.MRoomMemberNID, - EventStateKeyNID: stateKeyNID, - }) - } - } - for _, token := range stateNeeded.ThirdPartyInvite { - stateKeyNID, ok := stateKeyNIDMap[token] - if ok { - keyTuples = append(keyTuples, types.StateKeyTuple{ - EventTypeNID: types.MRoomThirdPartyInviteNID, - EventStateKeyNID: stateKeyNID, - }) - } - } - return keyTuples -} - -// loadStateEvents loads the matrix events for a list of state entries. -// Returns a list of state events in no particular order and a map from string event ID back to state entry. -// The map can be used to recover which numeric state entry a given event is for. -// Returns an error if there was a problem talking to the database. -func loadStateEvents( - ctx context.Context, db RoomStateDatabase, entries []types.StateEntry, -) ([]gomatrixserverlib.Event, map[string]types.StateEntry, error) { - eventNIDs := make([]types.EventNID, len(entries)) - for i := range entries { - eventNIDs[i] = entries[i].EventNID - } - events, err := db.Events(ctx, eventNIDs) - if err != nil { - return nil, nil, err - } - eventIDMap := map[string]types.StateEntry{} - result := make([]gomatrixserverlib.Event, len(entries)) - for i := range entries { - event, ok := eventMap(events).lookup(entries[i].EventNID) - if !ok { - panic(fmt.Errorf("Corrupt DB: Missing event numeric ID %d", entries[i].EventNID)) - } - result[i] = event.Event - eventIDMap[event.Event.EventID()] = entries[i] - } - return result, eventIDMap, nil -} - -// findDuplicateStateKeys finds the state entries where the state key tuple appears more than once in a sorted list. -// Returns a sorted list of those state entries. -func findDuplicateStateKeys(a []types.StateEntry) []types.StateEntry { - var result []types.StateEntry - // j is the starting index of a block of entries with the same state key tuple. - j := 0 - for i := 1; i < len(a); i++ { - // Check if the state key tuple matches the start of the block - if a[j].StateKeyTuple != a[i].StateKeyTuple { - // If the state key tuple is different then we've reached the end of a block of duplicates. - // Check if the size of the block is bigger than one. - // If the size is one then there was only a single entry with that state key tuple so we don't add it to the result - if j+1 != i { - // Add the block to the result. - result = append(result, a[j:i]...) - } - // Start a new block for the next state key tuple. - j = i - } - } - // Check if the last block with the same state key tuple had more than one event in it. - if j+1 != len(a) { - result = append(result, a[j:]...) - } - return result -} - -type stateEntrySorter []types.StateEntry - -func (s stateEntrySorter) Len() int { return len(s) } -func (s stateEntrySorter) Less(i, j int) bool { return s[i].LessThan(s[j]) } -func (s stateEntrySorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } - -type stateBlockNIDListMap []types.StateBlockNIDList - -func (m stateBlockNIDListMap) lookup(stateNID types.StateSnapshotNID) (stateBlockNIDs []types.StateBlockNID, ok bool) { - list := []types.StateBlockNIDList(m) - i := sort.Search(len(list), func(i int) bool { - return list[i].StateSnapshotNID >= stateNID - }) - if i < len(list) && list[i].StateSnapshotNID == stateNID { - ok = true - stateBlockNIDs = list[i].StateBlockNIDs - } - return -} - -type stateEntryListMap []types.StateEntryList - -func (m stateEntryListMap) lookup(stateBlockNID types.StateBlockNID) (stateEntries []types.StateEntry, ok bool) { - list := []types.StateEntryList(m) - i := sort.Search(len(list), func(i int) bool { - return list[i].StateBlockNID >= stateBlockNID - }) - if i < len(list) && list[i].StateBlockNID == stateBlockNID { - ok = true - stateEntries = list[i].StateEntries - } - return -} - -type stateEntryByStateKeySorter []types.StateEntry - -func (s stateEntryByStateKeySorter) Len() int { return len(s) } -func (s stateEntryByStateKeySorter) Less(i, j int) bool { - return s[i].StateKeyTuple.LessThan(s[j].StateKeyTuple) -} -func (s stateEntryByStateKeySorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } - -type stateNIDSorter []types.StateSnapshotNID - -func (s stateNIDSorter) Len() int { return len(s) } -func (s stateNIDSorter) Less(i, j int) bool { return s[i] < s[j] } -func (s stateNIDSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } - -func uniqueStateSnapshotNIDs(nids []types.StateSnapshotNID) []types.StateSnapshotNID { - return nids[:util.SortAndUnique(stateNIDSorter(nids))] -} - -type stateBlockNIDSorter []types.StateBlockNID - -func (s stateBlockNIDSorter) Len() int { return len(s) } -func (s stateBlockNIDSorter) Less(i, j int) bool { return s[i] < s[j] } -func (s stateBlockNIDSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } - -func uniqueStateBlockNIDs(nids []types.StateBlockNID) []types.StateBlockNID { - return nids[:util.SortAndUnique(stateBlockNIDSorter(nids))] -} - -// Map from event type, state key tuple to numeric event ID. -// Implemented using binary search on a sorted array. -type stateEntryMap []types.StateEntry - -// lookup an entry in the event map. -func (m stateEntryMap) lookup(stateKey types.StateKeyTuple) (eventNID types.EventNID, ok bool) { - // Since the list is sorted we can implement this using binary search. - // This is faster than using a hash map. - // We don't have to worry about pathological cases because the keys are fixed - // size and are controlled by us. - list := []types.StateEntry(m) - i := sort.Search(len(list), func(i int) bool { - return !list[i].StateKeyTuple.LessThan(stateKey) - }) - if i < len(list) && list[i].StateKeyTuple == stateKey { - ok = true - eventNID = list[i].EventNID - } - return -} - -// Map from numeric event ID to event. -// Implemented using binary search on a sorted array. -type eventMap []types.Event - -// lookup an entry in the event map. -func (m eventMap) lookup(eventNID types.EventNID) (event *types.Event, ok bool) { - // Since the list is sorted we can implement this using binary search. - // This is faster than using a hash map. - // We don't have to worry about pathological cases because the keys are fixed - // size are controlled by us. - list := []types.Event(m) - i := sort.Search(len(list), func(i int) bool { - return list[i].EventNID >= eventNID - }) - if i < len(list) && list[i].EventNID == eventNID { - ok = true - event = &list[i] - } - return } diff --git a/roomserver/state/v1/state.go b/roomserver/state/v1/state.go new file mode 100644 index 000000000..5683745bf --- /dev/null +++ b/roomserver/state/v1/state.go @@ -0,0 +1,927 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package state provides functions for reading state from the database. +// The functions for writing state to the database are the input package. +package v1 + +import ( + "context" + "fmt" + "sort" + "time" + + "github.com/matrix-org/dendrite/roomserver/state/database" + "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" + "github.com/prometheus/client_golang/prometheus" +) + +type StateResolutionV1 struct { + db database.RoomStateDatabase +} + +func Prepare(db database.RoomStateDatabase) StateResolutionV1 { + return StateResolutionV1{ + db: db, + } +} + +// LoadStateAtSnapshot loads the full state of a room at a particular snapshot. +// This is typically the state before an event or the current state of a room. +// Returns a sorted list of state entries or an error if there was a problem talking to the database. +func (v StateResolutionV1) LoadStateAtSnapshot( + ctx context.Context, stateNID types.StateSnapshotNID, +) ([]types.StateEntry, error) { + stateBlockNIDLists, err := v.db.StateBlockNIDs(ctx, []types.StateSnapshotNID{stateNID}) + if err != nil { + return nil, err + } + // We've asked for exactly one snapshot from the db so we should have exactly one entry in the result. + stateBlockNIDList := stateBlockNIDLists[0] + + stateEntryLists, err := v.db.StateEntries(ctx, stateBlockNIDList.StateBlockNIDs) + if err != nil { + return nil, err + } + stateEntriesMap := stateEntryListMap(stateEntryLists) + + // Combine all the state entries for this snapshot. + // The order of state block NIDs in the list tells us the order to combine them in. + var fullState []types.StateEntry + for _, stateBlockNID := range stateBlockNIDList.StateBlockNIDs { + entries, ok := stateEntriesMap.lookup(stateBlockNID) + if !ok { + // This should only get hit if the database is corrupt. + // It should be impossible for an event to reference a NID that doesn't exist + panic(fmt.Errorf("Corrupt DB: Missing state block numeric ID %d", stateBlockNID)) + } + fullState = append(fullState, entries...) + } + + // Stable sort so that the most recent entry for each state key stays + // remains later in the list than the older entries for the same state key. + sort.Stable(stateEntryByStateKeySorter(fullState)) + // Unique returns the last entry and hence the most recent entry for each state key. + fullState = fullState[:util.Unique(stateEntryByStateKeySorter(fullState))] + return fullState, nil +} + +// LoadStateAtEvent loads the full state of a room at a particular event. +func (v StateResolutionV1) LoadStateAtEvent( + ctx context.Context, eventID string, +) ([]types.StateEntry, error) { + snapshotNID, err := v.db.SnapshotNIDFromEventID(ctx, eventID) + if err != nil { + return nil, err + } + + stateEntries, err := v.LoadStateAtSnapshot(ctx, snapshotNID) + if err != nil { + return nil, err + } + + return stateEntries, nil +} + +// LoadCombinedStateAfterEvents loads a snapshot of the state after each of the events +// and combines those snapshots together into a single list. +func (v StateResolutionV1) LoadCombinedStateAfterEvents( + ctx context.Context, prevStates []types.StateAtEvent, +) ([]types.StateEntry, error) { + stateNIDs := make([]types.StateSnapshotNID, len(prevStates)) + for i, state := range prevStates { + stateNIDs[i] = state.BeforeStateSnapshotNID + } + // Fetch the state snapshots for the state before the each prev event from the database. + // Deduplicate the IDs before passing them to the database. + // There could be duplicates because the events could be state events where + // the snapshot of the room state before them was the same. + stateBlockNIDLists, err := v.db.StateBlockNIDs(ctx, uniqueStateSnapshotNIDs(stateNIDs)) + if err != nil { + return nil, err + } + + var stateBlockNIDs []types.StateBlockNID + for _, list := range stateBlockNIDLists { + stateBlockNIDs = append(stateBlockNIDs, list.StateBlockNIDs...) + } + // Fetch the state entries that will be combined to create the snapshots. + // Deduplicate the IDs before passing them to the database. + // There could be duplicates because a block of state entries could be reused by + // multiple snapshots. + stateEntryLists, err := v.db.StateEntries(ctx, uniqueStateBlockNIDs(stateBlockNIDs)) + if err != nil { + return nil, err + } + stateBlockNIDsMap := stateBlockNIDListMap(stateBlockNIDLists) + stateEntriesMap := stateEntryListMap(stateEntryLists) + + // Combine the entries from all the snapshots of state after each prev event into a single list. + var combined []types.StateEntry + for _, prevState := range prevStates { + // Grab the list of state data NIDs for this snapshot. + stateBlockNIDs, ok := stateBlockNIDsMap.lookup(prevState.BeforeStateSnapshotNID) + if !ok { + // This should only get hit if the database is corrupt. + // It should be impossible for an event to reference a NID that doesn't exist + panic(fmt.Errorf("Corrupt DB: Missing state snapshot numeric ID %d", prevState.BeforeStateSnapshotNID)) + } + + // Combine all the state entries for this snapshot. + // The order of state block NIDs in the list tells us the order to combine them in. + var fullState []types.StateEntry + for _, stateBlockNID := range stateBlockNIDs { + entries, ok := stateEntriesMap.lookup(stateBlockNID) + if !ok { + // This should only get hit if the database is corrupt. + // It should be impossible for an event to reference a NID that doesn't exist + panic(fmt.Errorf("Corrupt DB: Missing state block numeric ID %d", stateBlockNID)) + } + fullState = append(fullState, entries...) + } + if prevState.IsStateEvent() { + // If the prev event was a state event then add an entry for the event itself + // so that we get the state after the event rather than the state before. + fullState = append(fullState, prevState.StateEntry) + } + + // Stable sort so that the most recent entry for each state key stays + // remains later in the list than the older entries for the same state key. + sort.Stable(stateEntryByStateKeySorter(fullState)) + // Unique returns the last entry and hence the most recent entry for each state key. + fullState = fullState[:util.Unique(stateEntryByStateKeySorter(fullState))] + // Add the full state for this StateSnapshotNID. + combined = append(combined, fullState...) + } + return combined, nil +} + +// DifferenceBetweeenStateSnapshots works out which state entries have been added and removed between two snapshots. +func (v StateResolutionV1) DifferenceBetweeenStateSnapshots( + ctx context.Context, oldStateNID, newStateNID types.StateSnapshotNID, +) (removed, added []types.StateEntry, err error) { + if oldStateNID == newStateNID { + // If the snapshot NIDs are the same then nothing has changed + return nil, nil, nil + } + + var oldEntries []types.StateEntry + var newEntries []types.StateEntry + if oldStateNID != 0 { + oldEntries, err = v.LoadStateAtSnapshot(ctx, oldStateNID) + if err != nil { + return nil, nil, err + } + } + if newStateNID != 0 { + newEntries, err = v.LoadStateAtSnapshot(ctx, newStateNID) + if err != nil { + return nil, nil, err + } + } + + var oldI int + var newI int + for { + switch { + case oldI == len(oldEntries): + // We've reached the end of the old entries. + // The rest of the new list must have been newly added. + added = append(added, newEntries[newI:]...) + return + case newI == len(newEntries): + // We've reached the end of the new entries. + // The rest of the old list must be have been removed. + removed = append(removed, oldEntries[oldI:]...) + return + case oldEntries[oldI] == newEntries[newI]: + // The entry is in both lists so skip over it. + oldI++ + newI++ + case oldEntries[oldI].LessThan(newEntries[newI]): + // The lists are sorted so the old entry being less than the new entry means that it only appears in the old list. + removed = append(removed, oldEntries[oldI]) + oldI++ + default: + // Reaching the default case implies that the new entry is less than the old entry. + // Since the lists are sorted this means that it only appears in the new list. + added = append(added, newEntries[newI]) + newI++ + } + } +} + +// LoadStateAtSnapshotForStringTuples loads the state for a list of event type and state key pairs at a snapshot. +// This is used when we only want to load a subset of the room state at a snapshot. +// If there is no entry for a given event type and state key pair then it will be discarded. +// This is typically the state before an event or the current state of a room. +// Returns a sorted list of state entries or an error if there was a problem talking to the database. +func (v StateResolutionV1) LoadStateAtSnapshotForStringTuples( + ctx context.Context, + stateNID types.StateSnapshotNID, + stateKeyTuples []gomatrixserverlib.StateKeyTuple, +) ([]types.StateEntry, error) { + numericTuples, err := v.stringTuplesToNumericTuples(ctx, stateKeyTuples) + if err != nil { + return nil, err + } + return v.loadStateAtSnapshotForNumericTuples(ctx, stateNID, numericTuples) +} + +// stringTuplesToNumericTuples converts the string state key tuples into numeric IDs +// If there isn't a numeric ID for either the event type or the event state key then the tuple is discarded. +// Returns an error if there was a problem talking to the database. +func (v StateResolutionV1) stringTuplesToNumericTuples( + ctx context.Context, + stringTuples []gomatrixserverlib.StateKeyTuple, +) ([]types.StateKeyTuple, error) { + eventTypes := make([]string, len(stringTuples)) + stateKeys := make([]string, len(stringTuples)) + for i := range stringTuples { + eventTypes[i] = stringTuples[i].EventType + stateKeys[i] = stringTuples[i].StateKey + } + eventTypes = util.UniqueStrings(eventTypes) + eventTypeMap, err := v.db.EventTypeNIDs(ctx, eventTypes) + if err != nil { + return nil, err + } + stateKeys = util.UniqueStrings(stateKeys) + stateKeyMap, err := v.db.EventStateKeyNIDs(ctx, stateKeys) + if err != nil { + return nil, err + } + + var result []types.StateKeyTuple + for _, stringTuple := range stringTuples { + var numericTuple types.StateKeyTuple + var ok1, ok2 bool + numericTuple.EventTypeNID, ok1 = eventTypeMap[stringTuple.EventType] + numericTuple.EventStateKeyNID, ok2 = stateKeyMap[stringTuple.StateKey] + // Discard the tuple if there wasn't a numeric ID for either the event type or the state key. + if ok1 && ok2 { + result = append(result, numericTuple) + } + } + + return result, nil +} + +// loadStateAtSnapshotForNumericTuples loads the state for a list of event type and state key pairs at a snapshot. +// This is used when we only want to load a subset of the room state at a snapshot. +// If there is no entry for a given event type and state key pair then it will be discarded. +// This is typically the state before an event or the current state of a room. +// Returns a sorted list of state entries or an error if there was a problem talking to the database. +func (v StateResolutionV1) loadStateAtSnapshotForNumericTuples( + ctx context.Context, + stateNID types.StateSnapshotNID, + stateKeyTuples []types.StateKeyTuple, +) ([]types.StateEntry, error) { + stateBlockNIDLists, err := v.db.StateBlockNIDs(ctx, []types.StateSnapshotNID{stateNID}) + if err != nil { + return nil, err + } + // We've asked for exactly one snapshot from the db so we should have exactly one entry in the result. + stateBlockNIDList := stateBlockNIDLists[0] + + stateEntryLists, err := v.db.StateEntriesForTuples( + ctx, stateBlockNIDList.StateBlockNIDs, stateKeyTuples, + ) + if err != nil { + return nil, err + } + stateEntriesMap := stateEntryListMap(stateEntryLists) + + // Combine all the state entries for this snapshot. + // The order of state block NIDs in the list tells us the order to combine them in. + var fullState []types.StateEntry + for _, stateBlockNID := range stateBlockNIDList.StateBlockNIDs { + entries, ok := stateEntriesMap.lookup(stateBlockNID) + if !ok { + // If the block is missing from the map it means that none of its entries matched a requested tuple. + // This can happen if the block doesn't contain an update for one of the requested tuples. + // If none of the requested tuples are in the block then it can be safely skipped. + continue + } + fullState = append(fullState, entries...) + } + + // Stable sort so that the most recent entry for each state key stays + // remains later in the list than the older entries for the same state key. + sort.Stable(stateEntryByStateKeySorter(fullState)) + // Unique returns the last entry and hence the most recent entry for each state key. + fullState = fullState[:util.Unique(stateEntryByStateKeySorter(fullState))] + return fullState, nil +} + +// LoadStateAfterEventsForStringTuples loads the state for a list of event type +// and state key pairs after list of events. +// This is used when we only want to load a subset of the room state after a list of events. +// If there is no entry for a given event type and state key pair then it will be discarded. +// This is typically the state before an event. +// Returns a sorted list of state entries or an error if there was a problem talking to the database. +func (v StateResolutionV1) LoadStateAfterEventsForStringTuples( + ctx context.Context, + prevStates []types.StateAtEvent, + stateKeyTuples []gomatrixserverlib.StateKeyTuple, +) ([]types.StateEntry, error) { + numericTuples, err := v.stringTuplesToNumericTuples(ctx, stateKeyTuples) + if err != nil { + return nil, err + } + return v.loadStateAfterEventsForNumericTuples(ctx, prevStates, numericTuples) +} + +func (v StateResolutionV1) loadStateAfterEventsForNumericTuples( + ctx context.Context, + prevStates []types.StateAtEvent, + stateKeyTuples []types.StateKeyTuple, +) ([]types.StateEntry, error) { + if len(prevStates) == 1 { + // Fast path for a single event. + prevState := prevStates[0] + result, err := v.loadStateAtSnapshotForNumericTuples( + ctx, prevState.BeforeStateSnapshotNID, stateKeyTuples, + ) + if err != nil { + return nil, err + } + if prevState.IsStateEvent() { + // The result is current the state before the requested event. + // We want the state after the requested event. + // If the requested event was a state event then we need to + // update that key in the result. + // If the requested event wasn't a state event then the state after + // it is the same as the state before it. + for i := range result { + if result[i].StateKeyTuple == prevState.StateKeyTuple { + result[i] = prevState.StateEntry + } + } + } + return result, nil + } + + // Slow path for more that one event. + // Load the entire state so that we can do conflict resolution if we need to. + // TODO: The are some optimistations we could do here: + // 1) We only need to do conflict resolution if there is a conflict in the + // requested tuples so we might try loading just those tuples and then + // checking for conflicts. + // 2) When there is a conflict we still only need to load the state + // needed to do conflict resolution which would save us having to load + // the full state. + + // TODO: Add metrics for this as it could take a long time for big rooms + // with large conflicts. + fullState, _, _, err := v.calculateStateAfterManyEvents(ctx, prevStates) + if err != nil { + return nil, err + } + + // Sort the full state so we can use it as a map. + sort.Sort(stateEntrySorter(fullState)) + + // Filter the full state down to the required tuples. + var result []types.StateEntry + for _, tuple := range stateKeyTuples { + eventNID, ok := stateEntryMap(fullState).lookup(tuple) + if ok { + result = append(result, types.StateEntry{ + StateKeyTuple: tuple, + EventNID: eventNID, + }) + } + } + sort.Sort(stateEntrySorter(result)) + return result, nil +} + +var calculateStateDurations = prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Namespace: "dendrite", + Subsystem: "roomserver", + Name: "calculate_state_duration_microseconds", + Help: "How long it takes to calculate the state after a list of events", + }, + // Takes two labels: + // algorithm: + // The algorithm used to calculate the state or the step it failed on if it failed. + // Labels starting with "_" are used to indicate when the algorithm fails halfway. + // outcome: + // Whether the state was successfully calculated. + // + // The possible values for algorithm are: + // empty_state -> The list of events was empty so the state is empty. + // no_change -> The state hasn't changed. + // single_delta -> There was a single event added to the state in a way that can be encoded as a single delta + // full_state_no_conflicts -> We created a new copy of the full room state, but didn't enounter any conflicts + // while doing so. + // full_state_with_conflicts -> We created a new copy of the full room state and had to resolve conflicts to do so. + // _load_state_block_nids -> Failed loading the state block nids for a single previous state. + // _load_combined_state -> Failed to load the combined state. + // _resolve_conflicts -> Failed to resolve conflicts. + []string{"algorithm", "outcome"}, +) + +var calculateStatePrevEventLength = prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Namespace: "dendrite", + Subsystem: "roomserver", + Name: "calculate_state_prev_event_length", + Help: "The length of the list of events to calculate the state after", + }, + []string{"algorithm", "outcome"}, +) + +var calculateStateFullStateLength = prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Namespace: "dendrite", + Subsystem: "roomserver", + Name: "calculate_state_full_state_length", + Help: "The length of the full room state.", + }, + []string{"algorithm", "outcome"}, +) + +var calculateStateConflictLength = prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Namespace: "dendrite", + Subsystem: "roomserver", + Name: "calculate_state_conflict_state_length", + Help: "The length of the conflicted room state.", + }, + []string{"algorithm", "outcome"}, +) + +type calculateStateMetrics struct { + algorithm string + startTime time.Time + prevEventLength int + fullStateLength int + conflictLength int +} + +func (c *calculateStateMetrics) stop(stateNID types.StateSnapshotNID, err error) (types.StateSnapshotNID, error) { + var outcome string + if err == nil { + outcome = "success" + } else { + outcome = "failure" + } + endTime := time.Now() + calculateStateDurations.WithLabelValues(c.algorithm, outcome).Observe( + float64(endTime.Sub(c.startTime).Nanoseconds()) / 1000., + ) + calculateStatePrevEventLength.WithLabelValues(c.algorithm, outcome).Observe( + float64(c.prevEventLength), + ) + calculateStateFullStateLength.WithLabelValues(c.algorithm, outcome).Observe( + float64(c.fullStateLength), + ) + calculateStateConflictLength.WithLabelValues(c.algorithm, outcome).Observe( + float64(c.conflictLength), + ) + return stateNID, err +} + +func init() { + prometheus.MustRegister( + calculateStateDurations, calculateStatePrevEventLength, + calculateStateFullStateLength, calculateStateConflictLength, + ) +} + +// CalculateAndStoreStateBeforeEvent calculates a snapshot of the state of a room before an event. +// Stores the snapshot of the state in the database. +// Returns a numeric ID for the snapshot of the state before the event. +func (v StateResolutionV1) CalculateAndStoreStateBeforeEvent( + ctx context.Context, + event gomatrixserverlib.Event, + roomNID types.RoomNID, +) (types.StateSnapshotNID, error) { + // Load the state at the prev events. + prevEventRefs := event.PrevEvents() + prevEventIDs := make([]string, len(prevEventRefs)) + for i := range prevEventRefs { + prevEventIDs[i] = prevEventRefs[i].EventID + } + + prevStates, err := v.db.StateAtEventIDs(ctx, prevEventIDs) + if err != nil { + return 0, err + } + + // The state before this event will be the state after the events that came before it. + return v.CalculateAndStoreStateAfterEvents(ctx, roomNID, prevStates) +} + +// CalculateAndStoreStateAfterEvents finds the room state after the given events. +// Stores the resulting state in the database and returns a numeric ID for that snapshot. +func (v StateResolutionV1) CalculateAndStoreStateAfterEvents( + ctx context.Context, + roomNID types.RoomNID, + prevStates []types.StateAtEvent, +) (types.StateSnapshotNID, error) { + metrics := calculateStateMetrics{startTime: time.Now(), prevEventLength: len(prevStates)} + + if len(prevStates) == 0 { + // 2) There weren't any prev_events for this event so the state is + // empty. + metrics.algorithm = "empty_state" + return metrics.stop(v.db.AddState(ctx, roomNID, nil, nil)) + } + + if len(prevStates) == 1 { + prevState := prevStates[0] + if prevState.EventStateKeyNID == 0 { + // 3) None of the previous events were state events and they all + // have the same state, so this event has exactly the same state + // as the previous events. + // This should be the common case. + metrics.algorithm = "no_change" + return metrics.stop(prevState.BeforeStateSnapshotNID, nil) + } + // The previous event was a state event so we need to store a copy + // of the previous state updated with that event. + stateBlockNIDLists, err := v.db.StateBlockNIDs( + ctx, []types.StateSnapshotNID{prevState.BeforeStateSnapshotNID}, + ) + if err != nil { + metrics.algorithm = "_load_state_blocks" + return metrics.stop(0, err) + } + stateBlockNIDs := stateBlockNIDLists[0].StateBlockNIDs + if len(stateBlockNIDs) < maxStateBlockNIDs { + // 4) The number of state data blocks is small enough that we can just + // add the state event as a block of size one to the end of the blocks. + metrics.algorithm = "single_delta" + return metrics.stop(v.db.AddState( + ctx, roomNID, stateBlockNIDs, []types.StateEntry{prevState.StateEntry}, + )) + } + // If there are too many deltas then we need to calculate the full state + // So fall through to calculateAndStoreStateAfterManyEvents + } + + return v.calculateAndStoreStateAfterManyEvents(ctx, roomNID, prevStates, metrics) +} + +// maxStateBlockNIDs is the maximum number of state data blocks to use to encode a snapshot of room state. +// Increasing this number means that we can encode more of the state changes as simple deltas which means that +// we need fewer entries in the state data table. However making this number bigger will increase the size of +// the rows in the state table itself and will require more index lookups when retrieving a snapshot. +// TODO: Tune this to get the right balance between size and lookup performance. +const maxStateBlockNIDs = 64 + +// calculateAndStoreStateAfterManyEvents finds the room state after the given events. +// This handles the slow path of calculateAndStoreStateAfterEvents for when there is more than one event. +// Stores the resulting state and returns a numeric ID for the snapshot. +func (v StateResolutionV1) calculateAndStoreStateAfterManyEvents( + ctx context.Context, + roomNID types.RoomNID, + prevStates []types.StateAtEvent, + metrics calculateStateMetrics, +) (types.StateSnapshotNID, error) { + + state, algorithm, conflictLength, err := + v.calculateStateAfterManyEvents(ctx, prevStates) + metrics.algorithm = algorithm + if err != nil { + return metrics.stop(0, err) + } + + // TODO: Check if we can encode the new state as a delta against the + // previous state. + metrics.conflictLength = conflictLength + metrics.fullStateLength = len(state) + return metrics.stop(v.db.AddState(ctx, roomNID, nil, state)) +} + +func (v StateResolutionV1) calculateStateAfterManyEvents( + ctx context.Context, prevStates []types.StateAtEvent, +) (state []types.StateEntry, algorithm string, conflictLength int, err error) { + var combined []types.StateEntry + // Conflict resolution. + // First stage: load the state after each of the prev events. + combined, err = v.LoadCombinedStateAfterEvents(ctx, prevStates) + if err != nil { + algorithm = "_load_combined_state" + return + } + + // Collect all the entries with the same type and key together. + // We don't care about the order here because the conflict resolution + // algorithm doesn't depend on the order of the prev events. + // Remove duplicate entires. + combined = combined[:util.SortAndUnique(stateEntrySorter(combined))] + + // Find the conflicts + conflicts := findDuplicateStateKeys(combined) + + if len(conflicts) > 0 { + conflictLength = len(conflicts) + + // 5) There are conflicting state events, for each conflict workout + // what the appropriate state event is. + + // Work out which entries aren't conflicted. + var notConflicted []types.StateEntry + for _, entry := range combined { + if _, ok := stateEntryMap(conflicts).lookup(entry.StateKeyTuple); !ok { + notConflicted = append(notConflicted, entry) + } + } + + var resolved []types.StateEntry + resolved, err = v.resolveConflicts(ctx, notConflicted, conflicts) + if err != nil { + algorithm = "_resolve_conflicts" + return + } + algorithm = "full_state_with_conflicts" + state = resolved + } else { + algorithm = "full_state_no_conflicts" + // 6) There weren't any conflicts + state = combined + } + return +} + +// resolveConflicts resolves a list of conflicted state entries. It takes two lists. +// The first is a list of all state entries that are not conflicted. +// The second is a list of all state entries that are conflicted +// A state entry is conflicted when there is more than one numeric event ID for the same state key tuple. +// Returns a list that combines the entries without conflicts with the result of state resolution for the entries with conflicts. +// The returned list is sorted by state key tuple. +// Returns an error if there was a problem talking to the database. +func (v StateResolutionV1) resolveConflicts( + ctx context.Context, + notConflicted, conflicted []types.StateEntry, +) ([]types.StateEntry, error) { + + // Load the conflicted events + conflictedEvents, eventIDMap, err := v.loadStateEvents(ctx, conflicted) + if err != nil { + return nil, err + } + + // Work out which auth events we need to load. + needed := gomatrixserverlib.StateNeededForAuth(conflictedEvents) + + // Find the numeric IDs for the necessary state keys. + var neededStateKeys []string + neededStateKeys = append(neededStateKeys, needed.Member...) + neededStateKeys = append(neededStateKeys, needed.ThirdPartyInvite...) + stateKeyNIDMap, err := v.db.EventStateKeyNIDs(ctx, neededStateKeys) + if err != nil { + return nil, err + } + + // Load the necessary auth events. + tuplesNeeded := v.stateKeyTuplesNeeded(stateKeyNIDMap, needed) + var authEntries []types.StateEntry + for _, tuple := range tuplesNeeded { + if eventNID, ok := stateEntryMap(notConflicted).lookup(tuple); ok { + authEntries = append(authEntries, types.StateEntry{ + StateKeyTuple: tuple, + EventNID: eventNID, + }) + } + } + authEvents, _, err := v.loadStateEvents(ctx, authEntries) + if err != nil { + return nil, err + } + + // Resolve the conflicts. + resolvedEvents := gomatrixserverlib.ResolveStateConflicts(conflictedEvents, authEvents) + + // Map from the full events back to numeric state entries. + for _, resolvedEvent := range resolvedEvents { + entry, ok := eventIDMap[resolvedEvent.EventID()] + if !ok { + panic(fmt.Errorf("Missing state entry for event ID %q", resolvedEvent.EventID())) + } + notConflicted = append(notConflicted, entry) + } + + // Sort the result so it can be searched. + sort.Sort(stateEntrySorter(notConflicted)) + return notConflicted, nil +} + +// stateKeyTuplesNeeded works out which numeric state key tuples we need to authenticate some events. +func (v StateResolutionV1) stateKeyTuplesNeeded(stateKeyNIDMap map[string]types.EventStateKeyNID, stateNeeded gomatrixserverlib.StateNeeded) []types.StateKeyTuple { + var keyTuples []types.StateKeyTuple + if stateNeeded.Create { + keyTuples = append(keyTuples, types.StateKeyTuple{ + EventTypeNID: types.MRoomCreateNID, + EventStateKeyNID: types.EmptyStateKeyNID, + }) + } + if stateNeeded.PowerLevels { + keyTuples = append(keyTuples, types.StateKeyTuple{ + EventTypeNID: types.MRoomPowerLevelsNID, + EventStateKeyNID: types.EmptyStateKeyNID, + }) + } + if stateNeeded.JoinRules { + keyTuples = append(keyTuples, types.StateKeyTuple{ + EventTypeNID: types.MRoomJoinRulesNID, + EventStateKeyNID: types.EmptyStateKeyNID, + }) + } + for _, member := range stateNeeded.Member { + stateKeyNID, ok := stateKeyNIDMap[member] + if ok { + keyTuples = append(keyTuples, types.StateKeyTuple{ + EventTypeNID: types.MRoomMemberNID, + EventStateKeyNID: stateKeyNID, + }) + } + } + for _, token := range stateNeeded.ThirdPartyInvite { + stateKeyNID, ok := stateKeyNIDMap[token] + if ok { + keyTuples = append(keyTuples, types.StateKeyTuple{ + EventTypeNID: types.MRoomThirdPartyInviteNID, + EventStateKeyNID: stateKeyNID, + }) + } + } + return keyTuples +} + +// loadStateEvents loads the matrix events for a list of state entries. +// Returns a list of state events in no particular order and a map from string event ID back to state entry. +// The map can be used to recover which numeric state entry a given event is for. +// Returns an error if there was a problem talking to the database. +func (v StateResolutionV1) loadStateEvents( + ctx context.Context, entries []types.StateEntry, +) ([]gomatrixserverlib.Event, map[string]types.StateEntry, error) { + eventNIDs := make([]types.EventNID, len(entries)) + for i := range entries { + eventNIDs[i] = entries[i].EventNID + } + events, err := v.db.Events(ctx, eventNIDs) + if err != nil { + return nil, nil, err + } + eventIDMap := map[string]types.StateEntry{} + result := make([]gomatrixserverlib.Event, len(entries)) + for i := range entries { + event, ok := eventMap(events).lookup(entries[i].EventNID) + if !ok { + panic(fmt.Errorf("Corrupt DB: Missing event numeric ID %d", entries[i].EventNID)) + } + result[i] = event.Event + eventIDMap[event.Event.EventID()] = entries[i] + } + return result, eventIDMap, nil +} + +// findDuplicateStateKeys finds the state entries where the state key tuple appears more than once in a sorted list. +// Returns a sorted list of those state entries. +func findDuplicateStateKeys(a []types.StateEntry) []types.StateEntry { + var result []types.StateEntry + // j is the starting index of a block of entries with the same state key tuple. + j := 0 + for i := 1; i < len(a); i++ { + // Check if the state key tuple matches the start of the block + if a[j].StateKeyTuple != a[i].StateKeyTuple { + // If the state key tuple is different then we've reached the end of a block of duplicates. + // Check if the size of the block is bigger than one. + // If the size is one then there was only a single entry with that state key tuple so we don't add it to the result + if j+1 != i { + // Add the block to the result. + result = append(result, a[j:i]...) + } + // Start a new block for the next state key tuple. + j = i + } + } + // Check if the last block with the same state key tuple had more than one event in it. + if j+1 != len(a) { + result = append(result, a[j:]...) + } + return result +} + +type stateEntrySorter []types.StateEntry + +func (s stateEntrySorter) Len() int { return len(s) } +func (s stateEntrySorter) Less(i, j int) bool { return s[i].LessThan(s[j]) } +func (s stateEntrySorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +type stateBlockNIDListMap []types.StateBlockNIDList + +func (m stateBlockNIDListMap) lookup(stateNID types.StateSnapshotNID) (stateBlockNIDs []types.StateBlockNID, ok bool) { + list := []types.StateBlockNIDList(m) + i := sort.Search(len(list), func(i int) bool { + return list[i].StateSnapshotNID >= stateNID + }) + if i < len(list) && list[i].StateSnapshotNID == stateNID { + ok = true + stateBlockNIDs = list[i].StateBlockNIDs + } + return +} + +type stateEntryListMap []types.StateEntryList + +func (m stateEntryListMap) lookup(stateBlockNID types.StateBlockNID) (stateEntries []types.StateEntry, ok bool) { + list := []types.StateEntryList(m) + i := sort.Search(len(list), func(i int) bool { + return list[i].StateBlockNID >= stateBlockNID + }) + if i < len(list) && list[i].StateBlockNID == stateBlockNID { + ok = true + stateEntries = list[i].StateEntries + } + return +} + +type stateEntryByStateKeySorter []types.StateEntry + +func (s stateEntryByStateKeySorter) Len() int { return len(s) } +func (s stateEntryByStateKeySorter) Less(i, j int) bool { + return s[i].StateKeyTuple.LessThan(s[j].StateKeyTuple) +} +func (s stateEntryByStateKeySorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +type stateNIDSorter []types.StateSnapshotNID + +func (s stateNIDSorter) Len() int { return len(s) } +func (s stateNIDSorter) Less(i, j int) bool { return s[i] < s[j] } +func (s stateNIDSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +func uniqueStateSnapshotNIDs(nids []types.StateSnapshotNID) []types.StateSnapshotNID { + return nids[:util.SortAndUnique(stateNIDSorter(nids))] +} + +type stateBlockNIDSorter []types.StateBlockNID + +func (s stateBlockNIDSorter) Len() int { return len(s) } +func (s stateBlockNIDSorter) Less(i, j int) bool { return s[i] < s[j] } +func (s stateBlockNIDSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +func uniqueStateBlockNIDs(nids []types.StateBlockNID) []types.StateBlockNID { + return nids[:util.SortAndUnique(stateBlockNIDSorter(nids))] +} + +// Map from event type, state key tuple to numeric event ID. +// Implemented using binary search on a sorted array. +type stateEntryMap []types.StateEntry + +// lookup an entry in the event map. +func (m stateEntryMap) lookup(stateKey types.StateKeyTuple) (eventNID types.EventNID, ok bool) { + // Since the list is sorted we can implement this using binary search. + // This is faster than using a hash map. + // We don't have to worry about pathological cases because the keys are fixed + // size and are controlled by us. + list := []types.StateEntry(m) + i := sort.Search(len(list), func(i int) bool { + return !list[i].StateKeyTuple.LessThan(stateKey) + }) + if i < len(list) && list[i].StateKeyTuple == stateKey { + ok = true + eventNID = list[i].EventNID + } + return +} + +// Map from numeric event ID to event. +// Implemented using binary search on a sorted array. +type eventMap []types.Event + +// lookup an entry in the event map. +func (m eventMap) lookup(eventNID types.EventNID) (event *types.Event, ok bool) { + // Since the list is sorted we can implement this using binary search. + // This is faster than using a hash map. + // We don't have to worry about pathological cases because the keys are fixed + // size are controlled by us. + list := []types.Event(m) + i := sort.Search(len(list), func(i int) bool { + return list[i].EventNID >= eventNID + }) + if i < len(list) && list[i].EventNID == eventNID { + ok = true + event = &list[i] + } + return +} diff --git a/roomserver/state/state_test.go b/roomserver/state/v1/state_test.go similarity index 99% rename from roomserver/state/state_test.go rename to roomserver/state/v1/state_test.go index 67af18671..414ca2a11 100644 --- a/roomserver/state/state_test.go +++ b/roomserver/state/v1/state_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package state +package v1 import ( "testing" diff --git a/roomserver/storage/postgres/rooms_table.go b/roomserver/storage/postgres/rooms_table.go index ccc201b18..edd15a338 100644 --- a/roomserver/storage/postgres/rooms_table.go +++ b/roomserver/storage/postgres/rooms_table.go @@ -39,7 +39,10 @@ CREATE TABLE IF NOT EXISTS roomserver_rooms ( last_event_sent_nid BIGINT NOT NULL DEFAULT 0, -- The state of the room after the current set of latest events. -- This will be 0 if there are no latest events in the room. - state_snapshot_nid BIGINT NOT NULL DEFAULT 0 + state_snapshot_nid BIGINT NOT NULL DEFAULT 0, + -- The version of the room, which will assist in determining the state resolution + -- algorithm, event ID format, etc. + room_version BIGINT NOT NULL DEFAULT 1 ); ` @@ -61,12 +64,16 @@ const selectLatestEventNIDsForUpdateSQL = "" + const updateLatestEventNIDsSQL = "" + "UPDATE roomserver_rooms SET latest_event_nids = $2, last_event_sent_nid = $3, state_snapshot_nid = $4 WHERE room_nid = $1" +const selectRoomVersionForRoomNIDSQL = "" + + "SELECT room_version FROM roomserver_rooms WHERE room_nid = $1" + type roomStatements struct { insertRoomNIDStmt *sql.Stmt selectRoomNIDStmt *sql.Stmt selectLatestEventNIDsStmt *sql.Stmt selectLatestEventNIDsForUpdateStmt *sql.Stmt updateLatestEventNIDsStmt *sql.Stmt + selectRoomVersionForRoomNIDStmt *sql.Stmt } func (s *roomStatements) prepare(db *sql.DB) (err error) { @@ -80,6 +87,7 @@ func (s *roomStatements) prepare(db *sql.DB) (err error) { {&s.selectLatestEventNIDsStmt, selectLatestEventNIDsSQL}, {&s.selectLatestEventNIDsForUpdateStmt, selectLatestEventNIDsForUpdateSQL}, {&s.updateLatestEventNIDsStmt, updateLatestEventNIDsSQL}, + {&s.selectRoomVersionForRoomNIDStmt, selectRoomVersionForRoomNIDSQL}, }.prepare(db) } @@ -154,3 +162,12 @@ func (s *roomStatements) updateLatestEventNIDs( ) return err } + +func (s *roomStatements) selectRoomVersionForRoomNID( + ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, +) (int64, error) { + var roomVersion int64 + stmt := common.TxStmt(txn, s.selectRoomVersionForRoomNIDStmt) + err := stmt.QueryRowContext(ctx, roomNID).Scan(&roomVersion) + return roomVersion, err +} diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go index 93450e5a5..77a792d68 100644 --- a/roomserver/storage/postgres/storage.go +++ b/roomserver/storage/postgres/storage.go @@ -697,6 +697,14 @@ func (d *Database) EventsFromIDs(ctx context.Context, eventIDs []string) ([]type return d.Events(ctx, nids) } +func (d *Database) GetRoomVersionForRoom( + ctx context.Context, roomNID types.RoomNID, +) (int64, error) { + return d.statements.selectRoomVersionForRoomNID( + ctx, nil, roomNID, + ) +} + type transaction struct { ctx context.Context txn *sql.Tx diff --git a/roomserver/storage/storage.go b/roomserver/storage/storage.go index df08c124b..67efe656a 100644 --- a/roomserver/storage/storage.go +++ b/roomserver/storage/storage.go @@ -54,6 +54,8 @@ type Database interface { GetMembership(ctx context.Context, roomNID types.RoomNID, requestSenderUserID string) (membershipEventNID types.EventNID, stillInRoom bool, err error) GetMembershipEventNIDsForRoom(ctx context.Context, roomNID types.RoomNID, joinOnly bool) ([]types.EventNID, error) EventsFromIDs(ctx context.Context, eventIDs []string) ([]types.Event, error) + GetRoomVersionForRoom(ctx context.Context, roomNID types.RoomNID) (int64, error) + //GetRoomVersionForEvent(ctx context.Context, eventNID types.EventNID) int64 } // NewPublicRoomsServerDatabase opens a database connection. diff --git a/roomserver/version/version.go b/roomserver/version/version.go new file mode 100644 index 000000000..74d04d9bb --- /dev/null +++ b/roomserver/version/version.go @@ -0,0 +1,94 @@ +package version + +import ( + "errors" + + "github.com/matrix-org/dendrite/roomserver/state" +) + +type RoomVersionID int +type EventFormatID int + +const ( + RoomVersionV1 RoomVersionID = iota + 1 + RoomVersionV2 + RoomVersionV3 + RoomVersionV4 + RoomVersionV5 +) + +const ( + EventFormatV1 EventFormatID = iota + 1 // original event ID formatting + EventFormatV2 // event ID is event hash + EventFormatV3 // event ID is URL-safe base64 event hash +) + +type RoomVersionDescription struct { + Supported bool + Stable bool + StateResolution state.StateResolutionVersion + EventFormat EventFormatID + EnforceSigningKeyValidity bool +} + +var roomVersions = map[RoomVersionID]RoomVersionDescription{ + RoomVersionV1: RoomVersionDescription{ + Supported: true, + Stable: true, + StateResolution: state.StateResolutionAlgorithmV1, + EventFormat: EventFormatV1, + EnforceSigningKeyValidity: false, + }, + RoomVersionV2: RoomVersionDescription{ + Supported: false, + Stable: true, + StateResolution: state.StateResolutionAlgorithmV2, + EventFormat: EventFormatV1, + EnforceSigningKeyValidity: false, + }, + RoomVersionV3: RoomVersionDescription{ + Supported: false, + Stable: true, + StateResolution: state.StateResolutionAlgorithmV2, + EventFormat: EventFormatV2, + EnforceSigningKeyValidity: false, + }, + RoomVersionV4: RoomVersionDescription{ + Supported: false, + Stable: true, + StateResolution: state.StateResolutionAlgorithmV2, + EventFormat: EventFormatV3, + EnforceSigningKeyValidity: false, + }, + RoomVersionV5: RoomVersionDescription{ + Supported: false, + Stable: true, + StateResolution: state.StateResolutionAlgorithmV2, + EventFormat: EventFormatV3, + EnforceSigningKeyValidity: true, + }, +} + +func GetRoomVersions() map[RoomVersionID]RoomVersionDescription { + return roomVersions +} + +func GetSupportedRoomVersions() map[RoomVersionID]RoomVersionDescription { + versions := make(map[RoomVersionID]RoomVersionDescription) + for id, version := range GetRoomVersions() { + if version.Supported { + versions[id] = version + } + } + return versions +} + +func GetSupportedRoomVersion(version RoomVersionID) (desc RoomVersionDescription, err error) { + if version, ok := roomVersions[version]; ok { + desc = version + } + if !desc.Supported { + err = errors.New("unsupported room version") + } + return +} From c20109a57357fbb2cd0857485e2cca4a58c37d1b Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 5 Feb 2020 18:06:39 +0000 Subject: [PATCH 07/86] Implement room version capabilities in CS API (#866) * Add wiring for querying the roomserver for the default room version * Try to implement /capabilities for room versions * Update copyright notices * Update sytests, add /capabilities endpoint into CS API * Update sytest-whitelist * Add GetDefaultRoomVersion * Fix cases where state package was shadowed * Fix version formatting * Update Dockerfile to Go 1.13.6 * oh yes types I remember * And fix the default too --- clientapi/routing/capabilities.go | 51 ++++++++++++++++++++++++ clientapi/routing/routing.go | 6 +++ docker/Dockerfile | 2 +- roomserver/api/query.go | 34 ++++++++++++++++ roomserver/input/events.go | 6 ++- roomserver/input/latest_events.go | 10 +++-- roomserver/query/query.go | 57 ++++++++++++++++++++++----- roomserver/state/database/database.go | 16 ++++++++ roomserver/state/state.go | 16 ++++++++ roomserver/state/v1/state_test.go | 2 + roomserver/storage/storage.go | 1 - roomserver/version/version.go | 18 +++++++++ sytest-blacklist | 3 ++ sytest-whitelist | 5 ++- 14 files changed, 208 insertions(+), 19 deletions(-) create mode 100644 clientapi/routing/capabilities.go diff --git a/clientapi/routing/capabilities.go b/clientapi/routing/capabilities.go new file mode 100644 index 000000000..c8743386f --- /dev/null +++ b/clientapi/routing/capabilities.go @@ -0,0 +1,51 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routing + +import ( + "net/http" + + "github.com/matrix-org/dendrite/clientapi/httputil" + roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + + "github.com/matrix-org/util" +) + +// SendMembership implements PUT /rooms/{roomID}/(join|kick|ban|unban|leave|invite) +// by building a m.room.member event then sending it to the room server +func GetCapabilities( + req *http.Request, queryAPI roomserverAPI.RoomserverQueryAPI, +) util.JSONResponse { + roomVersionsQueryReq := roomserverAPI.QueryRoomVersionCapabilitiesRequest{} + var roomVersionsQueryRes roomserverAPI.QueryRoomVersionCapabilitiesResponse + if err := queryAPI.QueryRoomVersionCapabilities( + req.Context(), + &roomVersionsQueryReq, + &roomVersionsQueryRes, + ); err != nil { + return httputil.LogThenError(req, err) + } + + response := map[string]interface{}{ + "capabilities": map[string]interface{}{ + "m.room_versions": roomVersionsQueryRes, + }, + } + + return util.JSONResponse{ + Code: http.StatusOK, + JSON: response, + } +} diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 5c98cd0d8..bd326900a 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -551,4 +551,10 @@ func Setup( return DeleteTag(req, accountDB, device, vars["userId"], vars["roomId"], vars["tag"], syncProducer) }), ).Methods(http.MethodDelete, http.MethodOptions) + + r0mux.Handle("/capabilities", + common.MakeAuthAPI("capabilities", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + return GetCapabilities(req, queryAPI) + }), + ).Methods(http.MethodGet) } diff --git a/docker/Dockerfile b/docker/Dockerfile index d8f9de422..c88b77617 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM docker.io/golang:1.12.5-alpine3.9 +FROM docker.io/golang:1.13.6-alpine RUN mkdir /build diff --git a/roomserver/api/query.go b/roomserver/api/query.go index b3fa01840..e1850e723 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -1,4 +1,6 @@ // Copyright 2017 Vector Creations Ltd +// Copyright 2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -244,6 +246,15 @@ type QueryServersInRoomAtEventResponse struct { Servers []gomatrixserverlib.ServerName `json:"servers"` } +// QueryRoomVersionCapabilities asks for the default room version +type QueryRoomVersionCapabilitiesRequest struct{} + +// QueryRoomVersionCapabilitiesResponse is a response to QueryServersInRoomAtEventResponse +type QueryRoomVersionCapabilitiesResponse struct { + DefaultRoomVersion string `json:"default"` + AvailableRoomVersions map[string]string `json:"available"` +} + // RoomserverQueryAPI is used to query information from the room server. type RoomserverQueryAPI interface { // Query the latest events and state for a room from the room server. @@ -323,6 +334,13 @@ type RoomserverQueryAPI interface { request *QueryServersInRoomAtEventRequest, response *QueryServersInRoomAtEventResponse, ) error + + // Asks for the default room version as preferred by the server. + QueryRoomVersionCapabilities( + ctx context.Context, + request *QueryRoomVersionCapabilitiesRequest, + response *QueryRoomVersionCapabilitiesResponse, + ) error } // RoomserverQueryLatestEventsAndStatePath is the HTTP path for the QueryLatestEventsAndState API. @@ -358,6 +376,9 @@ const RoomserverQueryBackfillPath = "/api/roomserver/queryBackfill" // RoomserverQueryServersInRoomAtEventPath is the HTTP path for the QueryServersInRoomAtEvent API const RoomserverQueryServersInRoomAtEventPath = "/api/roomserver/queryServersInRoomAtEvents" +// RoomserverQueryRoomVersionCapabilitiesPath is the HTTP path for the QueryRoomVersionCapabilities API +const RoomserverQueryRoomVersionCapabilitiesPath = "/api/roomserver/queryRoomVersionCapabilities" + // NewRoomserverQueryAPIHTTP creates a RoomserverQueryAPI implemented by talking to a HTTP POST API. // If httpClient is nil then it uses the http.DefaultClient func NewRoomserverQueryAPIHTTP(roomserverURL string, httpClient *http.Client) RoomserverQueryAPI { @@ -514,3 +535,16 @@ func (h *httpRoomserverQueryAPI) QueryServersInRoomAtEvent( apiURL := h.roomserverURL + RoomserverQueryServersInRoomAtEventPath return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } + +// QueryServersInRoomAtEvent implements RoomServerQueryAPI +func (h *httpRoomserverQueryAPI) QueryRoomVersionCapabilities( + ctx context.Context, + request *QueryRoomVersionCapabilitiesRequest, + response *QueryRoomVersionCapabilitiesResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "QueryRoomVersionCapabilities") + defer span.Finish() + + apiURL := h.roomserverURL + RoomserverQueryRoomVersionCapabilitiesPath + return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} diff --git a/roomserver/input/events.go b/roomserver/input/events.go index 10ccb6485..03023a4af 100644 --- a/roomserver/input/events.go +++ b/roomserver/input/events.go @@ -1,4 +1,6 @@ // Copyright 2017 Vector Creations Ltd +// Copyright 2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -151,7 +153,7 @@ func calculateAndSetState( event gomatrixserverlib.Event, ) error { // TODO: get the correct room version - state, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, db) + roomState, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, db) if err != nil { return err } @@ -169,7 +171,7 @@ func calculateAndSetState( } } else { // We haven't been told what the state at the event is so we need to calculate it from the prev_events - if stateAtEvent.BeforeStateSnapshotNID, err = state.CalculateAndStoreStateBeforeEvent(ctx, event, roomNID); err != nil { + if stateAtEvent.BeforeStateSnapshotNID, err = roomState.CalculateAndStoreStateBeforeEvent(ctx, event, roomNID); err != nil { return err } } diff --git a/roomserver/input/latest_events.go b/roomserver/input/latest_events.go index 760677db2..7e03d544a 100644 --- a/roomserver/input/latest_events.go +++ b/roomserver/input/latest_events.go @@ -1,4 +1,6 @@ // Copyright 2017 Vector Creations Ltd +// Copyright 2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -172,7 +174,7 @@ func (u *latestEventsUpdater) doUpdateLatestEvents() error { func (u *latestEventsUpdater) latestState() error { var err error // TODO: get the correct room version - state, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, u.db) + roomState, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, u.db) if err != nil { return err } @@ -181,21 +183,21 @@ func (u *latestEventsUpdater) latestState() error { for i := range u.latest { latestStateAtEvents[i] = u.latest[i].StateAtEvent } - u.newStateNID, err = state.CalculateAndStoreStateAfterEvents( + u.newStateNID, err = roomState.CalculateAndStoreStateAfterEvents( u.ctx, u.roomNID, latestStateAtEvents, ) if err != nil { return err } - u.removed, u.added, err = state.DifferenceBetweeenStateSnapshots( + u.removed, u.added, err = roomState.DifferenceBetweeenStateSnapshots( u.ctx, u.oldStateNID, u.newStateNID, ) if err != nil { return err } - u.stateBeforeEventRemoves, u.stateBeforeEventAdds, err = state.DifferenceBetweeenStateSnapshots( + u.stateBeforeEventRemoves, u.stateBeforeEventAdds, err = roomState.DifferenceBetweeenStateSnapshots( u.ctx, u.newStateNID, u.stateAtEvent.BeforeStateSnapshotNID, ) return err diff --git a/roomserver/query/query.go b/roomserver/query/query.go index d318fc001..f138686b5 100644 --- a/roomserver/query/query.go +++ b/roomserver/query/query.go @@ -1,4 +1,6 @@ // Copyright 2017 Vector Creations Ltd +// Copyright 2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +20,7 @@ import ( "context" "encoding/json" "net/http" + "strconv" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/api" @@ -25,6 +28,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/state" "github.com/matrix-org/dendrite/roomserver/state/database" "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/dendrite/roomserver/version" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -100,7 +104,7 @@ func (r *RoomserverQueryAPI) QueryLatestEventsAndState( response *api.QueryLatestEventsAndStateResponse, ) error { // TODO: get the correct room version - state, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, r.DB) + roomState, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, r.DB) if err != nil { return err } @@ -121,7 +125,7 @@ func (r *RoomserverQueryAPI) QueryLatestEventsAndState( } // Look up the currrent state for the requested tuples. - stateEntries, err := state.LoadStateAtSnapshotForStringTuples( + stateEntries, err := roomState.LoadStateAtSnapshotForStringTuples( ctx, currentStateSnapshotNID, request.StateToFetch, ) if err != nil { @@ -144,7 +148,7 @@ func (r *RoomserverQueryAPI) QueryStateAfterEvents( response *api.QueryStateAfterEventsResponse, ) error { // TODO: get the correct room version - state, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, r.DB) + roomState, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, r.DB) if err != nil { return err } @@ -170,7 +174,7 @@ func (r *RoomserverQueryAPI) QueryStateAfterEvents( response.PrevEventsExist = true // Look up the currrent state for the requested tuples. - stateEntries, err := state.LoadStateAfterEventsForStringTuples( + stateEntries, err := roomState.LoadStateAfterEventsForStringTuples( ctx, prevStates, request.StateToFetch, ) if err != nil { @@ -327,7 +331,7 @@ func (r *RoomserverQueryAPI) getMembershipsBeforeEventNID( ctx context.Context, eventNID types.EventNID, joinedOnly bool, ) ([]types.Event, error) { // TODO: get the correct room version - state, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, r.DB) + roomState, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, r.DB) if err != nil { return []types.Event{}, err } @@ -345,7 +349,7 @@ func (r *RoomserverQueryAPI) getMembershipsBeforeEventNID( } // Fetch the state as it was when this event was fired - stateEntries, err := state.LoadCombinedStateAfterEvents(ctx, prevState) + stateEntries, err := roomState.LoadCombinedStateAfterEvents(ctx, prevState) if err != nil { return nil, err } @@ -433,12 +437,12 @@ func (r *RoomserverQueryAPI) checkServerAllowedToSeeEvent( ctx context.Context, eventID string, serverName gomatrixserverlib.ServerName, ) (bool, error) { // TODO: get the correct room version - state, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, r.DB) + roomState, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, r.DB) if err != nil { return false, err } - stateEntries, err := state.LoadStateAtEvent(ctx, eventID) + stateEntries, err := roomState.LoadStateAtEvent(ctx, eventID) if err != nil { return false, err } @@ -593,7 +597,7 @@ func (r *RoomserverQueryAPI) QueryStateAndAuthChain( response *api.QueryStateAndAuthChainResponse, ) error { // TODO: get the correct room version - state, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, r.DB) + roomState, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, r.DB) if err != nil { return err } @@ -620,7 +624,7 @@ func (r *RoomserverQueryAPI) QueryStateAndAuthChain( response.PrevEventsExist = true // Look up the currrent state for the requested tuples. - stateEntries, err := state.LoadCombinedStateAfterEvents( + stateEntries, err := roomState.LoadCombinedStateAfterEvents( ctx, prevStates, ) if err != nil { @@ -723,6 +727,25 @@ func (r *RoomserverQueryAPI) QueryServersInRoomAtEvent( return nil } +// QueryRoomVersionCapabilities implements api.RoomserverQueryAPI +func (r *RoomserverQueryAPI) QueryRoomVersionCapabilities( + ctx context.Context, + request *api.QueryRoomVersionCapabilitiesRequest, + response *api.QueryRoomVersionCapabilitiesResponse, +) error { + response.DefaultRoomVersion = strconv.Itoa(int(version.GetDefaultRoomVersion())) + response.AvailableRoomVersions = make(map[string]string) + for v, desc := range version.GetSupportedRoomVersions() { + sv := strconv.Itoa(int(v)) + if desc.Stable { + response.AvailableRoomVersions[sv] = "stable" + } else { + response.AvailableRoomVersions[sv] = "unstable" + } + } + return nil +} + // SetupHTTP adds the RoomserverQueryAPI handlers to the http.ServeMux. // nolint: gocyclo func (r *RoomserverQueryAPI) SetupHTTP(servMux *http.ServeMux) { @@ -880,4 +903,18 @@ func (r *RoomserverQueryAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) + servMux.Handle( + api.RoomserverQueryRoomVersionCapabilitiesPath, + common.MakeInternalAPI("QueryRoomVersionCapabilities", func(req *http.Request) util.JSONResponse { + var request api.QueryRoomVersionCapabilitiesRequest + var response api.QueryRoomVersionCapabilitiesResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryRoomVersionCapabilities(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) } diff --git a/roomserver/state/database/database.go b/roomserver/state/database/database.go index 546f06e87..ede6c5ec3 100644 --- a/roomserver/state/database/database.go +++ b/roomserver/state/database/database.go @@ -1,3 +1,19 @@ +// Copyright 2017 Vector Creations Ltd +// Copyright 2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package database import ( diff --git a/roomserver/state/state.go b/roomserver/state/state.go index bbc27ad81..687a120e3 100644 --- a/roomserver/state/state.go +++ b/roomserver/state/state.go @@ -1,3 +1,19 @@ +// Copyright 2017 Vector Creations Ltd +// Copyright 2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package state import ( diff --git a/roomserver/state/v1/state_test.go b/roomserver/state/v1/state_test.go index 414ca2a11..4dc7e52ec 100644 --- a/roomserver/state/v1/state_test.go +++ b/roomserver/state/v1/state_test.go @@ -1,4 +1,6 @@ // Copyright 2017 Vector Creations Ltd +// Copyright 2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/roomserver/storage/storage.go b/roomserver/storage/storage.go index 67efe656a..908411681 100644 --- a/roomserver/storage/storage.go +++ b/roomserver/storage/storage.go @@ -55,7 +55,6 @@ type Database interface { GetMembershipEventNIDsForRoom(ctx context.Context, roomNID types.RoomNID, joinOnly bool) ([]types.EventNID, error) EventsFromIDs(ctx context.Context, eventIDs []string) ([]types.Event, error) GetRoomVersionForRoom(ctx context.Context, roomNID types.RoomNID) (int64, error) - //GetRoomVersionForEvent(ctx context.Context, eventNID types.EventNID) int64 } // NewPublicRoomsServerDatabase opens a database connection. diff --git a/roomserver/version/version.go b/roomserver/version/version.go index 74d04d9bb..0943e3843 100644 --- a/roomserver/version/version.go +++ b/roomserver/version/version.go @@ -1,3 +1,17 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package version import ( @@ -69,6 +83,10 @@ var roomVersions = map[RoomVersionID]RoomVersionDescription{ }, } +func GetDefaultRoomVersion() RoomVersionID { + return RoomVersionV1 +} + func GetRoomVersions() map[RoomVersionID]RoomVersionDescription { return roomVersions } diff --git a/sytest-blacklist b/sytest-blacklist index 4c36e4778..5ddc18f67 100644 --- a/sytest-blacklist +++ b/sytest-blacklist @@ -19,3 +19,6 @@ Alias creators can delete alias with no ops # Blacklisted because matrix-org/dendrite#847 might have broken it but we're not # really sure and we need it pretty badly anyway Real non-joined users can get individual state for world_readable rooms after leaving + +# Blacklisted until matrix-org/dendrite#862 is reverted due to Riot bug +Latest account data appears in v2 /sync diff --git a/sytest-whitelist b/sytest-whitelist index 0600e2e5a..efc97b5e6 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -112,7 +112,7 @@ User can invite local user to room with version 4 Should reject keys claiming to belong to a different user Can add account data Can add account data to room -Latest account data appears in v2 /sync +#Latest account data appears in v2 /sync New account data appears in incremental v2 /sync Checking local federation server Inbound federation can query profile data @@ -227,3 +227,6 @@ Guest users can sync from world_readable guest_access rooms if joined Guest users can sync from default guest_access rooms if joined Real non-joined users cannot room initalSync for non-world_readable rooms Push rules come down in an initial /sync +Regular users can add and delete aliases in the default room configuration +Regular users can add and delete aliases when m.room.aliases is restricted +GET /r0/capabilities is not public From b72d7eb0cfdb6cead864c6e7cc0ccec77efa5805 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 6 Feb 2020 11:54:26 +0000 Subject: [PATCH 08/86] Update documentation for Go 1.13 (#867) --- CONTRIBUTING.md | 54 +++++++++++++++++++++++++++++-------------------- INSTALL.md | 8 ++++---- README.md | 25 +++++++++++++---------- 3 files changed, 50 insertions(+), 37 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4d413a29c..0bcd2bb1e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,34 +20,40 @@ should pick up any unit test and run it). There are also [scripts](scripts) for [linting](scripts/find-lint.sh) and doing a [build/test/lint run](scripts/build-test-lint.sh). +As of February 2020, we are deprecating support for Go 1.11 and Go 1.12 and are +now targeting Go 1.13 or later. Please ensure that you are using at least Go +1.13 when developing for Dendrite - our CI will lint and run tests against this +version. + ## Continuous Integration When a Pull Request is submitted, continuous integration jobs are run -automatically to ensure the code builds and is relatively well-written. The -jobs are run on [Buildkite](https://buildkite.com/matrix-dot-org/dendrite/), -and the Buildkite pipeline configuration can be found in Matrix.org's -[pipelines repository](https://github.com/matrix-org/pipelines). +automatically to ensure the code builds and is relatively well-written. The jobs +are run on [Buildkite](https://buildkite.com/matrix-dot-org/dendrite/), and the +Buildkite pipeline configuration can be found in Matrix.org's [pipelines +repository](https://github.com/matrix-org/pipelines). If a job fails, click the "details" button and you should be taken to the job's logs. -![Click the details button on the failing build step](docs/images/details-button-location.jpg) +![Click the details button on the failing build +step](docs/images/details-button-location.jpg) -Scroll down to the failing step and you should see some log output. Scan -the logs until you find what it's complaining about, fix it, submit a new -commit, then rinse and repeat until CI passes. +Scroll down to the failing step and you should see some log output. Scan the +logs until you find what it's complaining about, fix it, submit a new commit, +then rinse and repeat until CI passes. ### Running CI Tests Locally To save waiting for CI to finish after every commit, it is ideal to run the -checks locally before pushing, fixing errors first. This also saves other -people time as only so many PRs can be tested at a given time. +checks locally before pushing, fixing errors first. This also saves other people +time as only so many PRs can be tested at a given time. -To execute what Buildkite tests, first run `./scripts/build-test-lint.sh`; -this script will build the code, lint it, and run `go test ./...` with race -condition checking enabled. If something needs to be changed, fix it and then -run the script again until it no longer complains. Be warned that the linting -can take a significant amount of CPU and RAM. +To execute what Buildkite tests, first run `./scripts/build-test-lint.sh`; this +script will build the code, lint it, and run `go test ./...` with race condition +checking enabled. If something needs to be changed, fix it and then run the +script again until it no longer complains. Be warned that the linting can take a +significant amount of CPU and RAM. Once the code builds, run [Sytest](https://github.com/matrix-org/sytest) according to the guide in @@ -61,16 +67,18 @@ tests. ## Picking Things To Do -If you're new then feel free to pick up an issue labelled [good first issue](https://github.com/matrix-org/dendrite/labels/good%20first%20issue). +If you're new then feel free to pick up an issue labelled [good first +issue](https://github.com/matrix-org/dendrite/labels/good%20first%20issue). These should be well-contained, small pieces of work that can be picked up to help you get familiar with the code base. Once you're comfortable with hacking on Dendrite there are issues lablled as -[help wanted](https://github.com/matrix-org/dendrite/labels/help%20wanted), these -are often slightly larger or more complicated pieces of work but are hopefully -nonetheless fairly well-contained. +[help wanted](https://github.com/matrix-org/dendrite/labels/help%20wanted), +these are often slightly larger or more complicated pieces of work but are +hopefully nonetheless fairly well-contained. -We ask people who are familiar with Dendrite to leave the [good first issue](https://github.com/matrix-org/dendrite/labels/good%20first%20issue) +We ask people who are familiar with Dendrite to leave the [good first +issue](https://github.com/matrix-org/dendrite/labels/good%20first%20issue) issues so that there is always a way for new people to come and get involved. ## Getting Help @@ -79,9 +87,11 @@ For questions related to developing on Dendrite we have a dedicated room on Matrix [#dendrite-dev:matrix.org](https://matrix.to/#/#dendrite-dev:matrix.org) where we're happy to help. -For more general questions please use [#dendrite:matrix.org](https://matrix.to/#/#dendrite:matrix.org). +For more general questions please use +[#dendrite:matrix.org](https://matrix.to/#/#dendrite:matrix.org). ## Sign off We ask that everyone who contributes to the project signs off their -contributions, in accordance with the [DCO](https://github.com/matrix-org/matrix-doc/blob/master/CONTRIBUTING.rst#sign-off). +contributions, in accordance with the +[DCO](https://github.com/matrix-org/matrix-doc/blob/master/CONTRIBUTING.rst#sign-off). diff --git a/INSTALL.md b/INSTALL.md index 0fb0c08e5..4173e705e 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -12,7 +12,7 @@ Dendrite can be run in one of two configurations: ## Requirements - - Go 1.11+ + - Go 1.13+ - Postgres 9.5+ - For Kafka (optional if using the monolith server): - Unix-based system (https://kafka.apache.org/documentation/#os) @@ -22,7 +22,7 @@ Dendrite can be run in one of two configurations: ## Setting up a development environment -Assumes Go 1.10+ and JDK 1.8+ are already installed and are on PATH. +Assumes Go 1.13+ and JDK 1.8+ are already installed and are on PATH. ```bash # Get the code @@ -101,7 +101,7 @@ Create config file, based on `dendrite-config.yaml`. Call it `dendrite.yaml`. Th It is possible to use 'naffka' as an in-process replacement to Kafka when using the monolith server. To do this, set `use_naffka: true` in `dendrite.yaml` and uncomment -the necessary line related to naffka in the `database` section. Be sure to update the +the necessary line related to naffka in the `database` section. Be sure to update the database username and password if needed. The monolith server can be started as shown below. By default it listens for @@ -255,7 +255,7 @@ you want to support federation. ./bin/dendrite-federation-sender-server --config dendrite.yaml ``` -### Run an appservice server +### Run an appservice server This sends events from the network to [application services](https://matrix.org/docs/spec/application_service/unstable.html) diff --git a/README.md b/README.md index 2dadb1f4f..801d0e3ca 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,30 @@ # Dendrite [![Build Status](https://badge.buildkite.com/4be40938ab19f2bbc4a6c6724517353ee3ec1422e279faf374.svg?branch=master)](https://buildkite.com/matrix-dot-org/dendrite) [![Dendrite Dev on Matrix](https://img.shields.io/matrix/dendrite-dev:matrix.org.svg?label=%23dendrite-dev%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite-dev:matrix.org) [![Dendrite on Matrix](https://img.shields.io/matrix/dendrite:matrix.org.svg?label=%23dendrite%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite:matrix.org) -Dendrite will be a matrix homeserver written in go. +Dendrite will be a second-generation Matrix homeserver written in Go. -It's still very much a work in progress, but installation instructions can -be found in [INSTALL.md](INSTALL.md) +It's still very much a work in progress, but installation instructions can be +found in [INSTALL.md](INSTALL.md). It is not recommended to use Dendrite as a +production homeserver at this time. -An overview of the design can be found in [DESIGN.md](DESIGN.md) +An overview of the design can be found in [DESIGN.md](DESIGN.md). # Contributing -Everyone is welcome to help out and contribute! See [CONTRIBUTING.md](CONTRIBUTING.md) -to get started! +Everyone is welcome to help out and contribute! See +[CONTRIBUTING.md](CONTRIBUTING.md) to get started! -We aim to try and make it as easy as possible to jump in. +Please note that, as of February 2020, Dendrite now only targets Go 1.13 or +later. Please ensure that you are using at least Go 1.13 when developing for +Dendrite. # Discussion For questions about Dendrite we have a dedicated room on Matrix -[#dendrite:matrix.org](https://matrix.to/#/#dendrite:matrix.org). -Development discussion should happen in +[#dendrite:matrix.org](https://matrix.to/#/#dendrite:matrix.org). Development +discussion should happen in [#dendrite-dev:matrix.org](https://matrix.to/#/#dendrite-dev:matrix.org). # Progress -There's plenty still to do to make Dendrite usable! We're tracking progress in -a [project board](https://github.com/matrix-org/dendrite/projects/2). +There's plenty still to do to make Dendrite usable! We're tracking progress in a +[project board](https://github.com/matrix-org/dendrite/projects/2). From 9937c05beae7d76e351eae35ad6f441b1c177303 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Behouba=20Manass=C3=A9?= Date: Tue, 11 Feb 2020 14:18:12 +0300 Subject: [PATCH 09/86] Pass cfg by reference around the codebase (#819) * Pass cfg by reference around the codebase * Merge branch 'master' into pass-cfg-by-ref Co-authored-by: Neil Alexander --- appservice/appservice.go | 2 +- appservice/routing/routing.go | 2 +- clientapi/clientapi.go | 2 +- clientapi/routing/auth_fallback.go | 8 ++++---- clientapi/routing/createroom.go | 6 +++--- clientapi/routing/getevent.go | 4 ++-- clientapi/routing/joinroom.go | 4 ++-- clientapi/routing/login.go | 2 +- clientapi/routing/membership.go | 8 ++++---- clientapi/routing/memberships.go | 2 +- clientapi/routing/profile.go | 2 +- clientapi/routing/register.go | 2 +- clientapi/routing/routing.go | 20 ++++++++++---------- clientapi/routing/sendevent.go | 4 ++-- clientapi/routing/threepid.go | 4 ++-- clientapi/routing/voip.go | 2 +- clientapi/threepid/invites.go | 8 ++++---- clientapi/threepid/threepid.go | 8 ++++---- common/events.go | 2 +- federationapi/federationapi.go | 6 +++--- federationapi/routing/backfill.go | 2 +- federationapi/routing/invite.go | 2 +- federationapi/routing/join.go | 4 ++-- federationapi/routing/keys.go | 4 ++-- federationapi/routing/leave.go | 4 ++-- federationapi/routing/profile.go | 2 +- federationapi/routing/query.go | 2 +- federationapi/routing/routing.go | 2 +- federationapi/routing/send.go | 2 +- federationapi/routing/threepid.go | 10 +++++----- 30 files changed, 66 insertions(+), 66 deletions(-) diff --git a/appservice/appservice.go b/appservice/appservice.go index 8703959f8..f2cbcce24 100644 --- a/appservice/appservice.go +++ b/appservice/appservice.go @@ -100,7 +100,7 @@ func SetupAppServiceAPIComponent( // Set up HTTP Endpoints routing.Setup( - base.APIMux, *base.Cfg, roomserverQueryAPI, roomserverAliasAPI, + base.APIMux, base.Cfg, roomserverQueryAPI, roomserverAliasAPI, accountsDB, federation, transactionsCache, ) diff --git a/appservice/routing/routing.go b/appservice/routing/routing.go index 0e4bd6bab..8a24caad0 100644 --- a/appservice/routing/routing.go +++ b/appservice/routing/routing.go @@ -36,7 +36,7 @@ const pathPrefixApp = "/_matrix/app/v1" // applied: // nolint: gocyclo func Setup( - apiMux *mux.Router, cfg config.Dendrite, // nolint: unparam + apiMux *mux.Router, cfg *config.Dendrite, // nolint: unparam queryAPI api.RoomserverQueryAPI, aliasAPI api.RoomserverAliasAPI, // nolint: unparam accountDB *accounts.Database, // nolint: unparam federation *gomatrixserverlib.FederationClient, // nolint: unparam diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go index f3f3e08cf..c911fecce 100644 --- a/clientapi/clientapi.go +++ b/clientapi/clientapi.go @@ -67,7 +67,7 @@ func SetupClientAPIComponent( } routing.Setup( - base.APIMux, *base.Cfg, roomserverProducer, queryAPI, aliasAPI, asAPI, + base.APIMux, base.Cfg, roomserverProducer, queryAPI, aliasAPI, asAPI, accountsDB, deviceDB, federation, *keyRing, userUpdateProducer, syncProducer, typingProducer, transactionsCache, fedSenderAPI, ) diff --git a/clientapi/routing/auth_fallback.go b/clientapi/routing/auth_fallback.go index cd4530d1b..5332226c4 100644 --- a/clientapi/routing/auth_fallback.go +++ b/clientapi/routing/auth_fallback.go @@ -102,7 +102,7 @@ func serveTemplate(w http.ResponseWriter, templateHTML string, data map[string]s // AuthFallback implements GET and POST /auth/{authType}/fallback/web?session={sessionID} func AuthFallback( w http.ResponseWriter, req *http.Request, authType string, - cfg config.Dendrite, + cfg *config.Dendrite, ) *util.JSONResponse { sessionID := req.URL.Query().Get("session") @@ -130,7 +130,7 @@ func AuthFallback( if req.Method == http.MethodGet { // Handle Recaptcha if authType == authtypes.LoginTypeRecaptcha { - if err := checkRecaptchaEnabled(&cfg, w, req); err != nil { + if err := checkRecaptchaEnabled(cfg, w, req); err != nil { return err } @@ -144,7 +144,7 @@ func AuthFallback( } else if req.Method == http.MethodPost { // Handle Recaptcha if authType == authtypes.LoginTypeRecaptcha { - if err := checkRecaptchaEnabled(&cfg, w, req); err != nil { + if err := checkRecaptchaEnabled(cfg, w, req); err != nil { return err } @@ -156,7 +156,7 @@ func AuthFallback( } response := req.Form.Get("g-recaptcha-response") - if err := validateRecaptcha(&cfg, response, clientIP); err != nil { + if err := validateRecaptcha(cfg, response, clientIP); err != nil { util.GetLogger(req.Context()).Error(err) return err } diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index 620246d28..f6f06421e 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -134,7 +134,7 @@ type fledglingEvent struct { // CreateRoom implements /createRoom func CreateRoom( req *http.Request, device *authtypes.Device, - cfg config.Dendrite, producer *producers.RoomserverProducer, + cfg *config.Dendrite, producer *producers.RoomserverProducer, accountDB *accounts.Database, aliasAPI roomserverAPI.RoomserverAliasAPI, asAPI appserviceAPI.AppServiceQueryAPI, ) util.JSONResponse { @@ -148,7 +148,7 @@ func CreateRoom( // nolint: gocyclo func createRoom( req *http.Request, device *authtypes.Device, - cfg config.Dendrite, roomID string, producer *producers.RoomserverProducer, + cfg *config.Dendrite, roomID string, producer *producers.RoomserverProducer, accountDB *accounts.Database, aliasAPI roomserverAPI.RoomserverAliasAPI, asAPI appserviceAPI.AppServiceQueryAPI, ) util.JSONResponse { @@ -344,7 +344,7 @@ func createRoom( func buildEvent( builder *gomatrixserverlib.EventBuilder, provider gomatrixserverlib.AuthEventProvider, - cfg config.Dendrite, + cfg *config.Dendrite, evTime time.Time, ) (*gomatrixserverlib.Event, error) { eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(builder) diff --git a/clientapi/routing/getevent.go b/clientapi/routing/getevent.go index 7071d16f0..115286bd6 100644 --- a/clientapi/routing/getevent.go +++ b/clientapi/routing/getevent.go @@ -31,7 +31,7 @@ type getEventRequest struct { device *authtypes.Device roomID string eventID string - cfg config.Dendrite + cfg *config.Dendrite federation *gomatrixserverlib.FederationClient keyRing gomatrixserverlib.KeyRing requestedEvent gomatrixserverlib.Event @@ -44,7 +44,7 @@ func GetEvent( device *authtypes.Device, roomID string, eventID string, - cfg config.Dendrite, + cfg *config.Dendrite, queryAPI api.RoomserverQueryAPI, federation *gomatrixserverlib.FederationClient, keyRing gomatrixserverlib.KeyRing, diff --git a/clientapi/routing/joinroom.go b/clientapi/routing/joinroom.go index 06bd7207b..8b3f3740b 100644 --- a/clientapi/routing/joinroom.go +++ b/clientapi/routing/joinroom.go @@ -39,7 +39,7 @@ func JoinRoomByIDOrAlias( req *http.Request, device *authtypes.Device, roomIDOrAlias string, - cfg config.Dendrite, + cfg *config.Dendrite, federation *gomatrixserverlib.FederationClient, producer *producers.RoomserverProducer, queryAPI roomserverAPI.RoomserverQueryAPI, @@ -98,7 +98,7 @@ type joinRoomReq struct { evTime time.Time content map[string]interface{} userID string - cfg config.Dendrite + cfg *config.Dendrite federation *gomatrixserverlib.FederationClient producer *producers.RoomserverProducer queryAPI roomserverAPI.RoomserverQueryAPI diff --git a/clientapi/routing/login.go b/clientapi/routing/login.go index 939472ff5..2f4fb83c1 100644 --- a/clientapi/routing/login.go +++ b/clientapi/routing/login.go @@ -71,7 +71,7 @@ func passwordLogin() loginFlows { // Login implements GET and POST /login func Login( req *http.Request, accountDB *accounts.Database, deviceDB *devices.Database, - cfg config.Dendrite, + cfg *config.Dendrite, ) util.JSONResponse { if req.Method == http.MethodGet { // TODO: support other forms of login other than password, depending on config options return util.JSONResponse{ diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index c71ac2de2..8b8b3a0f9 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -41,7 +41,7 @@ var errMissingUserID = errors.New("'user_id' must be supplied") // by building a m.room.member event then sending it to the room server func SendMembership( req *http.Request, accountDB *accounts.Database, device *authtypes.Device, - roomID string, membership string, cfg config.Dendrite, + roomID string, membership string, cfg *config.Dendrite, queryAPI roomserverAPI.RoomserverQueryAPI, asAPI appserviceAPI.AppServiceQueryAPI, producer *producers.RoomserverProducer, ) util.JSONResponse { @@ -119,7 +119,7 @@ func buildMembershipEvent( body threepid.MembershipRequest, accountDB *accounts.Database, device *authtypes.Device, membership, roomID string, - cfg config.Dendrite, evTime time.Time, + cfg *config.Dendrite, evTime time.Time, queryAPI roomserverAPI.RoomserverQueryAPI, asAPI appserviceAPI.AppServiceQueryAPI, ) (*gomatrixserverlib.Event, error) { stateKey, reason, err := getMembershipStateKey(body, device, membership) @@ -165,7 +165,7 @@ func buildMembershipEvent( func loadProfile( ctx context.Context, userID string, - cfg config.Dendrite, + cfg *config.Dendrite, accountDB *accounts.Database, asAPI appserviceAPI.AppServiceQueryAPI, ) (*authtypes.Profile, error) { @@ -214,7 +214,7 @@ func checkAndProcessThreepid( req *http.Request, device *authtypes.Device, body *threepid.MembershipRequest, - cfg config.Dendrite, + cfg *config.Dendrite, queryAPI roomserverAPI.RoomserverQueryAPI, accountDB *accounts.Database, producer *producers.RoomserverProducer, diff --git a/clientapi/routing/memberships.go b/clientapi/routing/memberships.go index 5b8903287..e6fca505f 100644 --- a/clientapi/routing/memberships.go +++ b/clientapi/routing/memberships.go @@ -33,7 +33,7 @@ type response struct { // GetMemberships implements GET /rooms/{roomId}/members func GetMemberships( req *http.Request, device *authtypes.Device, roomID string, joinedOnly bool, - _ config.Dendrite, + _ *config.Dendrite, queryAPI api.RoomserverQueryAPI, ) util.JSONResponse { queryReq := api.QueryMembershipsForRoomRequest{ diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index a87c6f743..4688b19e9 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -343,7 +343,7 @@ func buildMembershipEvents( return nil, err } - event, err := common.BuildEvent(ctx, &builder, *cfg, evTime, queryAPI, nil) + event, err := common.BuildEvent(ctx, &builder, cfg, evTime, queryAPI, nil) if err != nil { return nil, err } diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index 4e50e57b9..4375faaf2 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -991,7 +991,7 @@ type availableResponse struct { // RegisterAvailable checks if the username is already taken or invalid. func RegisterAvailable( req *http.Request, - cfg config.Dendrite, + cfg *config.Dendrite, accountDB *accounts.Database, ) util.JSONResponse { username := req.URL.Query().Get("username") diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index bd326900a..f3cf351e6 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -47,7 +47,7 @@ const pathPrefixUnstable = "/_matrix/client/unstable" // applied: // nolint: gocyclo func Setup( - apiMux *mux.Router, cfg config.Dendrite, + apiMux *mux.Router, cfg *config.Dendrite, producer *producers.RoomserverProducer, queryAPI roomserverAPI.RoomserverQueryAPI, aliasAPI roomserverAPI.RoomserverAliasAPI, @@ -170,11 +170,11 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/register", common.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse { - return Register(req, accountDB, deviceDB, &cfg) + return Register(req, accountDB, deviceDB, cfg) })).Methods(http.MethodPost, http.MethodOptions) v1mux.Handle("/register", common.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse { - return LegacyRegister(req, accountDB, deviceDB, &cfg) + return LegacyRegister(req, accountDB, deviceDB, cfg) })).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/register/available", common.MakeExternalAPI("registerAvailable", func(req *http.Request) util.JSONResponse { @@ -187,7 +187,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return DirectoryRoom(req, vars["roomAlias"], federation, &cfg, aliasAPI, federationSender) + return DirectoryRoom(req, vars["roomAlias"], federation, cfg, aliasAPI, federationSender) }), ).Methods(http.MethodGet, http.MethodOptions) @@ -197,7 +197,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SetLocalAlias(req, device, vars["roomAlias"], &cfg, aliasAPI) + return SetLocalAlias(req, device, vars["roomAlias"], cfg, aliasAPI) }), ).Methods(http.MethodPut, http.MethodOptions) @@ -301,7 +301,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return GetProfile(req, accountDB, &cfg, vars["userID"], asAPI, federation) + return GetProfile(req, accountDB, cfg, vars["userID"], asAPI, federation) }), ).Methods(http.MethodGet, http.MethodOptions) @@ -311,7 +311,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return GetAvatarURL(req, accountDB, &cfg, vars["userID"], asAPI, federation) + return GetAvatarURL(req, accountDB, cfg, vars["userID"], asAPI, federation) }), ).Methods(http.MethodGet, http.MethodOptions) @@ -321,7 +321,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SetAvatarURL(req, accountDB, device, vars["userID"], userUpdateProducer, &cfg, producer, queryAPI) + return SetAvatarURL(req, accountDB, device, vars["userID"], userUpdateProducer, cfg, producer, queryAPI) }), ).Methods(http.MethodPut, http.MethodOptions) // Browsers use the OPTIONS HTTP method to check if the CORS policy allows @@ -333,7 +333,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return GetDisplayName(req, accountDB, &cfg, vars["userID"], asAPI, federation) + return GetDisplayName(req, accountDB, cfg, vars["userID"], asAPI, federation) }), ).Methods(http.MethodGet, http.MethodOptions) @@ -343,7 +343,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SetDisplayName(req, accountDB, device, vars["userID"], userUpdateProducer, &cfg, producer, queryAPI) + return SetDisplayName(req, accountDB, device, vars["userID"], userUpdateProducer, cfg, producer, queryAPI) }), ).Methods(http.MethodPut, http.MethodOptions) // Browsers use the OPTIONS HTTP method to check if the CORS policy allows diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go index 76e36cd46..e6de187f2 100644 --- a/clientapi/routing/sendevent.go +++ b/clientapi/routing/sendevent.go @@ -43,7 +43,7 @@ func SendEvent( req *http.Request, device *authtypes.Device, roomID, eventType string, txnID, stateKey *string, - cfg config.Dendrite, + cfg *config.Dendrite, queryAPI api.RoomserverQueryAPI, producer *producers.RoomserverProducer, txnCache *transactions.Cache, @@ -93,7 +93,7 @@ func generateSendEvent( req *http.Request, device *authtypes.Device, roomID, eventType string, stateKey *string, - cfg config.Dendrite, + cfg *config.Dendrite, queryAPI api.RoomserverQueryAPI, ) (*gomatrixserverlib.Event, *util.JSONResponse) { // parse the incoming http request diff --git a/clientapi/routing/threepid.go b/clientapi/routing/threepid.go index 897d13b63..88b02fe46 100644 --- a/clientapi/routing/threepid.go +++ b/clientapi/routing/threepid.go @@ -39,7 +39,7 @@ type threePIDsResponse struct { // RequestEmailToken implements: // POST /account/3pid/email/requestToken // POST /register/email/requestToken -func RequestEmailToken(req *http.Request, accountDB *accounts.Database, cfg config.Dendrite) util.JSONResponse { +func RequestEmailToken(req *http.Request, accountDB *accounts.Database, cfg *config.Dendrite) util.JSONResponse { var body threepid.EmailAssociationRequest if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil { return *reqErr @@ -83,7 +83,7 @@ func RequestEmailToken(req *http.Request, accountDB *accounts.Database, cfg conf // CheckAndSave3PIDAssociation implements POST /account/3pid func CheckAndSave3PIDAssociation( req *http.Request, accountDB *accounts.Database, device *authtypes.Device, - cfg config.Dendrite, + cfg *config.Dendrite, ) util.JSONResponse { var body threepid.EmailAssociationCheckRequest if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil { diff --git a/clientapi/routing/voip.go b/clientapi/routing/voip.go index b9121633f..872e64473 100644 --- a/clientapi/routing/voip.go +++ b/clientapi/routing/voip.go @@ -31,7 +31,7 @@ import ( // RequestTurnServer implements: // GET /voip/turnServer -func RequestTurnServer(req *http.Request, device *authtypes.Device, cfg config.Dendrite) util.JSONResponse { +func RequestTurnServer(req *http.Request, device *authtypes.Device, cfg *config.Dendrite) util.JSONResponse { turnConfig := cfg.TURN // TODO Guest Support diff --git a/clientapi/threepid/invites.go b/clientapi/threepid/invites.go index bfe5060a8..2cf88d6e3 100644 --- a/clientapi/threepid/invites.go +++ b/clientapi/threepid/invites.go @@ -86,7 +86,7 @@ var ( // can be emitted. func CheckAndProcessInvite( ctx context.Context, - device *authtypes.Device, body *MembershipRequest, cfg config.Dendrite, + device *authtypes.Device, body *MembershipRequest, cfg *config.Dendrite, queryAPI api.RoomserverQueryAPI, db *accounts.Database, producer *producers.RoomserverProducer, membership string, roomID string, evTime time.Time, @@ -137,7 +137,7 @@ func CheckAndProcessInvite( // Returns an error if a check or a request failed. func queryIDServer( ctx context.Context, - db *accounts.Database, cfg config.Dendrite, device *authtypes.Device, + db *accounts.Database, cfg *config.Dendrite, device *authtypes.Device, body *MembershipRequest, roomID string, ) (lookupRes *idServerLookupResponse, storeInviteRes *idServerStoreInviteResponse, err error) { if err = isTrusted(body.IDServer, cfg); err != nil { @@ -206,7 +206,7 @@ func queryIDServerLookup(ctx context.Context, body *MembershipRequest) (*idServe // Returns an error if the request failed to send or if the response couldn't be parsed. func queryIDServerStoreInvite( ctx context.Context, - db *accounts.Database, cfg config.Dendrite, device *authtypes.Device, + db *accounts.Database, cfg *config.Dendrite, device *authtypes.Device, body *MembershipRequest, roomID string, ) (*idServerStoreInviteResponse, error) { // Retrieve the sender's profile to get their display name @@ -330,7 +330,7 @@ func checkIDServerSignatures( func emit3PIDInviteEvent( ctx context.Context, body *MembershipRequest, res *idServerStoreInviteResponse, - device *authtypes.Device, roomID string, cfg config.Dendrite, + device *authtypes.Device, roomID string, cfg *config.Dendrite, queryAPI api.RoomserverQueryAPI, producer *producers.RoomserverProducer, evTime time.Time, ) error { diff --git a/clientapi/threepid/threepid.go b/clientapi/threepid/threepid.go index e5b3305e3..a7f26c295 100644 --- a/clientapi/threepid/threepid.go +++ b/clientapi/threepid/threepid.go @@ -53,7 +53,7 @@ type Credentials struct { // Returns an error if there was a problem sending the request or decoding the // response, or if the identity server responded with a non-OK status. func CreateSession( - ctx context.Context, req EmailAssociationRequest, cfg config.Dendrite, + ctx context.Context, req EmailAssociationRequest, cfg *config.Dendrite, ) (string, error) { if err := isTrusted(req.IDServer, cfg); err != nil { return "", err @@ -101,7 +101,7 @@ func CreateSession( // Returns an error if there was a problem sending the request or decoding the // response, or if the identity server responded with a non-OK status. func CheckAssociation( - ctx context.Context, creds Credentials, cfg config.Dendrite, + ctx context.Context, creds Credentials, cfg *config.Dendrite, ) (bool, string, string, error) { if err := isTrusted(creds.IDServer, cfg); err != nil { return false, "", "", err @@ -142,7 +142,7 @@ func CheckAssociation( // identifier and a Matrix ID. // Returns an error if there was a problem sending the request or decoding the // response, or if the identity server responded with a non-OK status. -func PublishAssociation(creds Credentials, userID string, cfg config.Dendrite) error { +func PublishAssociation(creds Credentials, userID string, cfg *config.Dendrite) error { if err := isTrusted(creds.IDServer, cfg); err != nil { return err } @@ -177,7 +177,7 @@ func PublishAssociation(creds Credentials, userID string, cfg config.Dendrite) e // isTrusted checks if a given identity server is part of the list of trusted // identity servers in the configuration file. // Returns an error if the server isn't trusted. -func isTrusted(idServer string, cfg config.Dendrite) error { +func isTrusted(idServer string, cfg *config.Dendrite) error { for _, server := range cfg.Matrix.TrustedIDServers { if idServer == server { return nil diff --git a/common/events.go b/common/events.go index 5c87c0e56..3c060ee65 100644 --- a/common/events.go +++ b/common/events.go @@ -39,7 +39,7 @@ var ErrRoomNoExists = errors.New("Room does not exist") // Returns an error if something else went wrong func BuildEvent( ctx context.Context, - builder *gomatrixserverlib.EventBuilder, cfg config.Dendrite, evTime time.Time, + builder *gomatrixserverlib.EventBuilder, cfg *config.Dendrite, evTime time.Time, queryAPI api.RoomserverQueryAPI, queryRes *api.QueryLatestEventsAndStateResponse, ) (*gomatrixserverlib.Event, error) { err := AddPrevEventsToEvent(ctx, builder, queryAPI, queryRes) diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go index c2aef06c8..53851bc51 100644 --- a/federationapi/federationapi.go +++ b/federationapi/federationapi.go @@ -45,8 +45,8 @@ func SetupFederationAPIComponent( roomserverProducer := producers.NewRoomserverProducer(inputAPI) routing.Setup( - base.APIMux, *base.Cfg, queryAPI, aliasAPI, asAPI, - roomserverProducer, federationSenderAPI, *keyRing, federation, accountsDB, - deviceDB, + base.APIMux, base.Cfg, queryAPI, aliasAPI, asAPI, + roomserverProducer, federationSenderAPI, *keyRing, + federation, accountsDB, deviceDB, ) } diff --git a/federationapi/routing/backfill.go b/federationapi/routing/backfill.go index 5c6b0087f..cb388f50d 100644 --- a/federationapi/routing/backfill.go +++ b/federationapi/routing/backfill.go @@ -34,7 +34,7 @@ func Backfill( request *gomatrixserverlib.FederationRequest, query api.RoomserverQueryAPI, roomID string, - cfg config.Dendrite, + cfg *config.Dendrite, ) util.JSONResponse { var res api.QueryBackfillResponse var eIDs []string diff --git a/federationapi/routing/invite.go b/federationapi/routing/invite.go index 01a1bed23..9a04a0880 100644 --- a/federationapi/routing/invite.go +++ b/federationapi/routing/invite.go @@ -32,7 +32,7 @@ func Invite( request *gomatrixserverlib.FederationRequest, roomID string, eventID string, - cfg config.Dendrite, + cfg *config.Dendrite, producer *producers.RoomserverProducer, keys gomatrixserverlib.KeyRing, ) util.JSONResponse { diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go index e2885dd99..325b99374 100644 --- a/federationapi/routing/join.go +++ b/federationapi/routing/join.go @@ -33,7 +33,7 @@ import ( func MakeJoin( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, - cfg config.Dendrite, + cfg *config.Dendrite, query api.RoomserverQueryAPI, roomID, userID string, ) util.JSONResponse { @@ -97,7 +97,7 @@ func MakeJoin( func SendJoin( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, - cfg config.Dendrite, + cfg *config.Dendrite, query api.RoomserverQueryAPI, producer *producers.RoomserverProducer, keys gomatrixserverlib.KeyRing, diff --git a/federationapi/routing/keys.go b/federationapi/routing/keys.go index 9c53d177e..3eb88567d 100644 --- a/federationapi/routing/keys.go +++ b/federationapi/routing/keys.go @@ -27,7 +27,7 @@ import ( // LocalKeys returns the local keys for the server. // See https://matrix.org/docs/spec/server_server/unstable.html#publishing-keys -func LocalKeys(cfg config.Dendrite) util.JSONResponse { +func LocalKeys(cfg *config.Dendrite) util.JSONResponse { keys, err := localKeys(cfg, time.Now().Add(cfg.Matrix.KeyValidityPeriod)) if err != nil { return util.ErrorResponse(err) @@ -35,7 +35,7 @@ func LocalKeys(cfg config.Dendrite) util.JSONResponse { return util.JSONResponse{Code: http.StatusOK, JSON: keys} } -func localKeys(cfg config.Dendrite, validUntil time.Time) (*gomatrixserverlib.ServerKeys, error) { +func localKeys(cfg *config.Dendrite, validUntil time.Time) (*gomatrixserverlib.ServerKeys, error) { var keys gomatrixserverlib.ServerKeys keys.ServerName = cfg.Matrix.ServerName diff --git a/federationapi/routing/leave.go b/federationapi/routing/leave.go index a982b87f8..958158084 100644 --- a/federationapi/routing/leave.go +++ b/federationapi/routing/leave.go @@ -31,7 +31,7 @@ import ( func MakeLeave( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, - cfg config.Dendrite, + cfg *config.Dendrite, query api.RoomserverQueryAPI, roomID, userID string, ) util.JSONResponse { @@ -95,7 +95,7 @@ func MakeLeave( func SendLeave( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, - cfg config.Dendrite, + cfg *config.Dendrite, producer *producers.RoomserverProducer, keys gomatrixserverlib.KeyRing, roomID, eventID string, diff --git a/federationapi/routing/profile.go b/federationapi/routing/profile.go index 2b478cfbf..3be729c29 100644 --- a/federationapi/routing/profile.go +++ b/federationapi/routing/profile.go @@ -31,7 +31,7 @@ import ( func GetProfile( httpReq *http.Request, accountDB *accounts.Database, - cfg config.Dendrite, + cfg *config.Dendrite, asAPI appserviceAPI.AppServiceQueryAPI, ) util.JSONResponse { userID, field := httpReq.FormValue("user_id"), httpReq.FormValue("field") diff --git a/federationapi/routing/query.go b/federationapi/routing/query.go index ed2d8b741..5277f0acd 100644 --- a/federationapi/routing/query.go +++ b/federationapi/routing/query.go @@ -32,7 +32,7 @@ import ( func RoomAliasToID( httpReq *http.Request, federation *gomatrixserverlib.FederationClient, - cfg config.Dendrite, + cfg *config.Dendrite, aliasAPI roomserverAPI.RoomserverAliasAPI, senderAPI federationSenderAPI.FederationSenderQueryAPI, ) util.JSONResponse { diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index 302f7ed0b..13ed24f3a 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -43,7 +43,7 @@ const ( // nolint: gocyclo func Setup( apiMux *mux.Router, - cfg config.Dendrite, + cfg *config.Dendrite, query roomserverAPI.RoomserverQueryAPI, aliasAPI roomserverAPI.RoomserverAliasAPI, asAPI appserviceAPI.AppServiceQueryAPI, diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index eab248745..5513a088f 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -34,7 +34,7 @@ func Send( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, txnID gomatrixserverlib.TransactionID, - cfg config.Dendrite, + cfg *config.Dendrite, query api.RoomserverQueryAPI, producer *producers.RoomserverProducer, keys gomatrixserverlib.KeyRing, diff --git a/federationapi/routing/threepid.go b/federationapi/routing/threepid.go index 7fa02be91..06e00eeaf 100644 --- a/federationapi/routing/threepid.go +++ b/federationapi/routing/threepid.go @@ -59,7 +59,7 @@ var ( // CreateInvitesFrom3PIDInvites implements POST /_matrix/federation/v1/3pid/onbind func CreateInvitesFrom3PIDInvites( req *http.Request, queryAPI roomserverAPI.RoomserverQueryAPI, - asAPI appserviceAPI.AppServiceQueryAPI, cfg config.Dendrite, + asAPI appserviceAPI.AppServiceQueryAPI, cfg *config.Dendrite, producer *producers.RoomserverProducer, federation *gomatrixserverlib.FederationClient, accountDB *accounts.Database, ) util.JSONResponse { @@ -98,7 +98,7 @@ func ExchangeThirdPartyInvite( request *gomatrixserverlib.FederationRequest, roomID string, queryAPI roomserverAPI.RoomserverQueryAPI, - cfg config.Dendrite, + cfg *config.Dendrite, federation *gomatrixserverlib.FederationClient, producer *producers.RoomserverProducer, ) util.JSONResponse { @@ -172,7 +172,7 @@ func ExchangeThirdPartyInvite( // necessary data to do so. func createInviteFrom3PIDInvite( ctx context.Context, queryAPI roomserverAPI.RoomserverQueryAPI, - asAPI appserviceAPI.AppServiceQueryAPI, cfg config.Dendrite, + asAPI appserviceAPI.AppServiceQueryAPI, cfg *config.Dendrite, inv invite, federation *gomatrixserverlib.FederationClient, accountDB *accounts.Database, ) (*gomatrixserverlib.Event, error) { @@ -230,7 +230,7 @@ func createInviteFrom3PIDInvite( func buildMembershipEvent( ctx context.Context, builder *gomatrixserverlib.EventBuilder, queryAPI roomserverAPI.RoomserverQueryAPI, - cfg config.Dendrite, + cfg *config.Dendrite, ) (*gomatrixserverlib.Event, error) { eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(builder) if err != nil { @@ -290,7 +290,7 @@ func buildMembershipEvent( // them responded with an error. func sendToRemoteServer( ctx context.Context, inv invite, - federation *gomatrixserverlib.FederationClient, _ config.Dendrite, + federation *gomatrixserverlib.FederationClient, _ *config.Dendrite, builder gomatrixserverlib.EventBuilder, ) (err error) { remoteServers := make([]gomatrixserverlib.ServerName, 2) From 3dfafd4824a2fc58701e12a54ac67ceff44330a0 Mon Sep 17 00:00:00 2001 From: S7evinK Date: Tue, 11 Feb 2020 13:13:38 +0100 Subject: [PATCH 10/86] Implement missing device management features (#835) * Implement missing device management features Signed-off-by: Till Faelligen * Add a little more documentation * Undo changes * Use non-anonymous struct to decode devices list * Update sytest-whitelist * Update sytest-whitelist * Update sytest-blacklist Co-authored-by: Neil Alexander --- .../auth/storage/devices/devices_table.go | 24 +++++++- clientapi/auth/storage/devices/storage.go | 15 +++++ clientapi/routing/device.go | 55 +++++++++++++++++++ clientapi/routing/routing.go | 16 ++++++ sytest-blacklist | 6 ++ 5 files changed, 114 insertions(+), 2 deletions(-) diff --git a/clientapi/auth/storage/devices/devices_table.go b/clientapi/auth/storage/devices/devices_table.go index d011d25c9..c5773ce39 100644 --- a/clientapi/auth/storage/devices/devices_table.go +++ b/clientapi/auth/storage/devices/devices_table.go @@ -19,10 +19,10 @@ import ( "database/sql" "time" - "github.com/matrix-org/dendrite/common" - + "github.com/lib/pq" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/userutil" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/gomatrixserverlib" ) @@ -80,6 +80,9 @@ const deleteDeviceSQL = "" + const deleteDevicesByLocalpartSQL = "" + "DELETE FROM device_devices WHERE localpart = $1" +const deleteDevicesSQL = "" + + "DELETE FROM device_devices WHERE localpart = $1 AND device_id = ANY($2)" + type devicesStatements struct { insertDeviceStmt *sql.Stmt selectDeviceByTokenStmt *sql.Stmt @@ -88,6 +91,7 @@ type devicesStatements struct { updateDeviceNameStmt *sql.Stmt deleteDeviceStmt *sql.Stmt deleteDevicesByLocalpartStmt *sql.Stmt + deleteDevicesStmt *sql.Stmt serverName gomatrixserverlib.ServerName } @@ -117,6 +121,9 @@ func (s *devicesStatements) prepare(db *sql.DB, server gomatrixserverlib.ServerN if s.deleteDevicesByLocalpartStmt, err = db.Prepare(deleteDevicesByLocalpartSQL); err != nil { return } + if s.deleteDevicesStmt, err = db.Prepare(deleteDevicesSQL); err != nil { + return + } s.serverName = server return } @@ -142,6 +149,7 @@ func (s *devicesStatements) insertDevice( }, nil } +// deleteDevice removes a single device by id and user localpart. func (s *devicesStatements) deleteDevice( ctx context.Context, txn *sql.Tx, id, localpart string, ) error { @@ -150,6 +158,18 @@ func (s *devicesStatements) deleteDevice( return err } +// deleteDevices removes a single or multiple devices by ids and user localpart. +// Returns an error if the execution failed. +func (s *devicesStatements) deleteDevices( + ctx context.Context, txn *sql.Tx, localpart string, devices []string, +) error { + stmt := common.TxStmt(txn, s.deleteDevicesStmt) + _, err := stmt.ExecContext(ctx, localpart, pq.Array(devices)) + return err +} + +// deleteDevicesByLocalpart removes all devices for the +// given user localpart. func (s *devicesStatements) deleteDevicesByLocalpart( ctx context.Context, txn *sql.Tx, localpart string, ) error { diff --git a/clientapi/auth/storage/devices/storage.go b/clientapi/auth/storage/devices/storage.go index 82c8e97a2..150180c1e 100644 --- a/clientapi/auth/storage/devices/storage.go +++ b/clientapi/auth/storage/devices/storage.go @@ -152,6 +152,21 @@ func (d *Database) RemoveDevice( }) } +// RemoveDevices revokes one or more devices by deleting the entry in the database +// matching with the given device IDs and user ID localpart. +// If the devices don't exist, it will not return an error +// If something went wrong during the deletion, it will return the SQL error. +func (d *Database) RemoveDevices( + ctx context.Context, localpart string, devices []string, +) error { + return common.WithTransaction(d.db, func(txn *sql.Tx) error { + if err := d.devices.deleteDevices(ctx, txn, localpart, devices); err != sql.ErrNoRows { + return err + } + return nil + }) +} + // RemoveAllDevices revokes devices by deleting the entry in the // database matching the given user ID localpart. // If something went wrong during the deletion, it will return the SQL error. diff --git a/clientapi/routing/device.go b/clientapi/routing/device.go index c858e88aa..eb7cd0b0c 100644 --- a/clientapi/routing/device.go +++ b/clientapi/routing/device.go @@ -40,6 +40,10 @@ type deviceUpdateJSON struct { DisplayName *string `json:"display_name"` } +type devicesDeleteJSON struct { + Devices []string `json:"devices"` +} + // GetDeviceByID handles /devices/{deviceID} func GetDeviceByID( req *http.Request, deviceDB *devices.Database, device *authtypes.Device, @@ -146,3 +150,54 @@ func UpdateDeviceByID( JSON: struct{}{}, } } + +// DeleteDeviceById handles DELETE requests to /devices/{deviceId} +func DeleteDeviceById( + req *http.Request, deviceDB *devices.Database, device *authtypes.Device, + deviceID string, +) util.JSONResponse { + localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) + if err != nil { + return httputil.LogThenError(req, err) + } + ctx := req.Context() + + defer req.Body.Close() // nolint: errcheck + + if err := deviceDB.RemoveDevice(ctx, deviceID, localpart); err != nil { + return httputil.LogThenError(req, err) + } + + return util.JSONResponse{ + Code: http.StatusOK, + JSON: struct{}{}, + } +} + +// DeleteDevices handles POST requests to /delete_devices +func DeleteDevices( + req *http.Request, deviceDB *devices.Database, device *authtypes.Device, +) util.JSONResponse { + localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) + if err != nil { + return httputil.LogThenError(req, err) + } + + ctx := req.Context() + payload := devicesDeleteJSON{} + + if err := json.NewDecoder(req.Body).Decode(&payload); err != nil { + return httputil.LogThenError(req, err) + } + + defer req.Body.Close() // nolint: errcheck + + if err := deviceDB.RemoveDevices(ctx, localpart, payload.Devices); err != nil { + return httputil.LogThenError(req, err) + } + + return util.JSONResponse{ + Code: http.StatusOK, + JSON: struct{}{}, + } +} diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index f3cf351e6..f519523ac 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -503,6 +503,22 @@ func Setup( }), ).Methods(http.MethodPut, http.MethodOptions) + r0mux.Handle("/devices/{deviceID}", + common.MakeAuthAPI("delete_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return DeleteDeviceById(req, deviceDB, device, vars["deviceID"]) + }), + ).Methods(http.MethodDelete, http.MethodOptions) + + r0mux.Handle("/delete_devices", + common.MakeAuthAPI("delete_devices", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + return DeleteDevices(req, deviceDB, device) + }), + ).Methods(http.MethodPost, http.MethodOptions) + // Stub implementations for sytest r0mux.Handle("/events", common.MakeExternalAPI("events", func(req *http.Request) util.JSONResponse { diff --git a/sytest-blacklist b/sytest-blacklist index 5ddc18f67..2df2b3a86 100644 --- a/sytest-blacklist +++ b/sytest-blacklist @@ -22,3 +22,9 @@ Real non-joined users can get individual state for world_readable rooms after le # Blacklisted until matrix-org/dendrite#862 is reverted due to Riot bug Latest account data appears in v2 /sync + +# Blacklisted due to flakiness +Outbound federation can backfill events + +# Blacklisted due to alias work on Synapse +Alias creators can delete canonical alias with no ops From 880545bfa8248185c7664d2adbead7f91b1b87d5 Mon Sep 17 00:00:00 2001 From: Thibaut CHARLES Date: Tue, 11 Feb 2020 14:53:00 +0100 Subject: [PATCH 11/86] Adding sslmode: disable to sytest server config (#813) Co-authored-by: Neil Alexander --- docs/sytest.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/sytest.md b/docs/sytest.md index 6d03270bb..9385ebff3 100644 --- a/docs/sytest.md +++ b/docs/sytest.md @@ -44,6 +44,7 @@ args: user: dendrite database: dendrite host: 127.0.0.1 + sslmode: disable type: pg EOF ``` From d45f869cdd35b07ed9b44445732b27935ca1910d Mon Sep 17 00:00:00 2001 From: aditsachde <23707194+aditsachde@users.noreply.github.com> Date: Tue, 11 Feb 2020 08:53:54 -0500 Subject: [PATCH 12/86] Fix AppService bind addrs in test (#805) Co-authored-by: Neil Alexander --- common/test/config.go | 1 + 1 file changed, 1 insertion(+) diff --git a/common/test/config.go b/common/test/config.go index 693555619..0fed252ae 100644 --- a/common/test/config.go +++ b/common/test/config.go @@ -111,6 +111,7 @@ func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*con // Bind to the same address as the listen address // All microservices are run on the same host in testing cfg.Bind.ClientAPI = cfg.Listen.ClientAPI + cfg.Bind.AppServiceAPI = cfg.Listen.AppServiceAPI cfg.Bind.FederationAPI = cfg.Listen.FederationAPI cfg.Bind.MediaAPI = cfg.Listen.MediaAPI cfg.Bind.RoomServer = cfg.Listen.RoomServer From d5dbe546e461261056b5fda1a2ac9fc6d36c69e1 Mon Sep 17 00:00:00 2001 From: Kiril Vladimiroff Date: Tue, 11 Feb 2020 16:12:21 +0200 Subject: [PATCH 13/86] Always defer *sql.Rows.Close and consult with Err (#844) * Always defer *sql.Rows.Close and consult with Err database/sql.Rows.Next() makes sure to call Close only after exhausting result rows which would NOT happen when returning early from a bad Scan. Close being idempotent makes it a great candidate to get always deferred regardless of what happens later on the result set. This change also makes sure call Err() after exhausting Next() and propagate non-nil results from it as the documentation advises. Closes #764 Signed-off-by: Kiril Vladimiroff * Override named result parameters in last returns Signed-off-by: Kiril Vladimiroff * Do the same over new changes that got merged Signed-off-by: Kiril Vladimiroff Co-authored-by: Neil Alexander --- .../storage/accounts/account_data_table.go | 4 ++-- .../auth/storage/accounts/membership_table.go | 7 +++---- .../auth/storage/accounts/threepid_table.go | 4 ++-- .../auth/storage/devices/devices_table.go | 3 ++- common/keydb/postgres/server_key_table.go | 2 +- common/partition_offset_table.go | 2 +- .../storage/postgres/joined_hosts_table.go | 2 +- mediaapi/storage/postgres/thumbnail_table.go | 3 ++- .../storage/postgres/public_rooms_table.go | 3 ++- .../storage/postgres/event_json_table.go | 2 +- .../postgres/event_state_keys_table.go | 4 ++-- .../storage/postgres/event_types_table.go | 2 +- roomserver/storage/postgres/events_table.go | 21 ++++++++++++++++--- roomserver/storage/postgres/invite_table.go | 12 ++++++----- .../storage/postgres/membership_table.go | 7 +++++-- .../storage/postgres/room_aliases_table.go | 12 +++++------ .../storage/postgres/state_block_table.go | 9 +++++--- .../storage/postgres/state_snapshot_table.go | 5 ++++- .../storage/postgres/account_data_table.go | 4 ++-- .../postgres/backward_extremities_table.go | 3 ++- .../postgres/current_room_state_table.go | 6 +++--- syncapi/storage/postgres/invites_table.go | 2 +- .../postgres/output_room_events_table.go | 5 +++-- .../output_room_events_topology_table.go | 6 ++++-- 24 files changed, 81 insertions(+), 49 deletions(-) diff --git a/clientapi/auth/storage/accounts/account_data_table.go b/clientapi/auth/storage/accounts/account_data_table.go index 080ca3f38..1b7484d81 100644 --- a/clientapi/auth/storage/accounts/account_data_table.go +++ b/clientapi/auth/storage/accounts/account_data_table.go @@ -90,6 +90,7 @@ func (s *accountDataStatements) selectAccountData( if err != nil { return } + defer rows.Close() // nolint: errcheck global = []gomatrixserverlib.ClientEvent{} rooms = make(map[string][]gomatrixserverlib.ClientEvent) @@ -114,8 +115,7 @@ func (s *accountDataStatements) selectAccountData( global = append(global, ac) } } - - return + return global, rooms, rows.Err() } func (s *accountDataStatements) selectAccountDataByType( diff --git a/clientapi/auth/storage/accounts/membership_table.go b/clientapi/auth/storage/accounts/membership_table.go index 6185065c6..7b7c50ac6 100644 --- a/clientapi/auth/storage/accounts/membership_table.go +++ b/clientapi/auth/storage/accounts/membership_table.go @@ -122,11 +122,10 @@ func (s *membershipStatements) selectMembershipsByLocalpart( for rows.Next() { var m authtypes.Membership m.Localpart = localpart - if err := rows.Scan(&m.RoomID, &m.EventID); err != nil { - return nil, err + if err = rows.Scan(&m.RoomID, &m.EventID); err != nil { + return } memberships = append(memberships, m) } - - return + return memberships, rows.Err() } diff --git a/clientapi/auth/storage/accounts/threepid_table.go b/clientapi/auth/storage/accounts/threepid_table.go index 5900260a2..a03aa4f89 100644 --- a/clientapi/auth/storage/accounts/threepid_table.go +++ b/clientapi/auth/storage/accounts/threepid_table.go @@ -97,6 +97,7 @@ func (s *threepidStatements) selectThreePIDsForLocalpart( if err != nil { return } + defer rows.Close() // nolint: errcheck threepids = []authtypes.ThreePID{} for rows.Next() { @@ -110,8 +111,7 @@ func (s *threepidStatements) selectThreePIDsForLocalpart( Medium: medium, }) } - - return + return threepids, rows.Err() } func (s *threepidStatements) insertThreePID( diff --git a/clientapi/auth/storage/devices/devices_table.go b/clientapi/auth/storage/devices/devices_table.go index c5773ce39..997412470 100644 --- a/clientapi/auth/storage/devices/devices_table.go +++ b/clientapi/auth/storage/devices/devices_table.go @@ -226,6 +226,7 @@ func (s *devicesStatements) selectDevicesByLocalpart( if err != nil { return devices, err } + defer rows.Close() // nolint: errcheck for rows.Next() { var dev authtypes.Device @@ -237,5 +238,5 @@ func (s *devicesStatements) selectDevicesByLocalpart( devices = append(devices, dev) } - return devices, nil + return devices, rows.Err() } diff --git a/common/keydb/postgres/server_key_table.go b/common/keydb/postgres/server_key_table.go index 8fb9a0ee9..6b13cc3c2 100644 --- a/common/keydb/postgres/server_key_table.go +++ b/common/keydb/postgres/server_key_table.go @@ -117,7 +117,7 @@ func (s *serverKeyStatements) bulkSelectServerKeys( ExpiredTS: gomatrixserverlib.Timestamp(expiredTS), } } - return results, nil + return results, rows.Err() } func (s *serverKeyStatements) upsertServerKeys( diff --git a/common/partition_offset_table.go b/common/partition_offset_table.go index bf37e2ed5..6955ac365 100644 --- a/common/partition_offset_table.go +++ b/common/partition_offset_table.go @@ -99,7 +99,7 @@ func (s *PartitionOffsetStatements) selectPartitionOffsets( } results = append(results, offset) } - return results, nil + return results, rows.Err() } // UpsertPartitionOffset updates or inserts the partition offset for the given topic. diff --git a/federationsender/storage/postgres/joined_hosts_table.go b/federationsender/storage/postgres/joined_hosts_table.go index bd580e3b5..e5c30a010 100644 --- a/federationsender/storage/postgres/joined_hosts_table.go +++ b/federationsender/storage/postgres/joined_hosts_table.go @@ -132,5 +132,5 @@ func joinedHostsFromStmt( }) } - return result, nil + return result, rows.Err() } diff --git a/mediaapi/storage/postgres/thumbnail_table.go b/mediaapi/storage/postgres/thumbnail_table.go index 167e37957..127b86bb9 100644 --- a/mediaapi/storage/postgres/thumbnail_table.go +++ b/mediaapi/storage/postgres/thumbnail_table.go @@ -144,6 +144,7 @@ func (s *thumbnailStatements) selectThumbnails( if err != nil { return nil, err } + defer rows.Close() // nolint: errcheck var thumbnails []*types.ThumbnailMetadata for rows.Next() { @@ -167,5 +168,5 @@ func (s *thumbnailStatements) selectThumbnails( thumbnails = append(thumbnails, &thumbnailMetadata) } - return thumbnails, err + return thumbnails, rows.Err() } diff --git a/publicroomsapi/storage/postgres/public_rooms_table.go b/publicroomsapi/storage/postgres/public_rooms_table.go index 852afe770..edf9ad2ab 100644 --- a/publicroomsapi/storage/postgres/public_rooms_table.go +++ b/publicroomsapi/storage/postgres/public_rooms_table.go @@ -203,6 +203,7 @@ func (s *publicRoomsStatements) selectPublicRooms( if err != nil { return []types.PublicRoom{}, nil } + defer rows.Close() // nolint: errcheck rooms := []types.PublicRoom{} for rows.Next() { @@ -222,7 +223,7 @@ func (s *publicRoomsStatements) selectPublicRooms( rooms = append(rooms, r) } - return rooms, nil + return rooms, rows.Err() } func (s *publicRoomsStatements) selectRoomVisibility( diff --git a/roomserver/storage/postgres/event_json_table.go b/roomserver/storage/postgres/event_json_table.go index 415fb84eb..0b7ef6aa7 100644 --- a/roomserver/storage/postgres/event_json_table.go +++ b/roomserver/storage/postgres/event_json_table.go @@ -102,5 +102,5 @@ func (s *eventJSONStatements) bulkSelectEventJSON( } result.EventNID = types.EventNID(eventNID) } - return results[:i], nil + return results[:i], rows.Err() } diff --git a/roomserver/storage/postgres/event_state_keys_table.go b/roomserver/storage/postgres/event_state_keys_table.go index c3aaa498e..cbc29a69d 100644 --- a/roomserver/storage/postgres/event_state_keys_table.go +++ b/roomserver/storage/postgres/event_state_keys_table.go @@ -125,7 +125,7 @@ func (s *eventStateKeyStatements) bulkSelectEventStateKeyNID( } result[stateKey] = types.EventStateKeyNID(stateKeyNID) } - return result, nil + return result, rows.Err() } func (s *eventStateKeyStatements) bulkSelectEventStateKey( @@ -150,5 +150,5 @@ func (s *eventStateKeyStatements) bulkSelectEventStateKey( } result[types.EventStateKeyNID(stateKeyNID)] = stateKey } - return result, nil + return result, rows.Err() } diff --git a/roomserver/storage/postgres/event_types_table.go b/roomserver/storage/postgres/event_types_table.go index 1ec2e7cde..faa887545 100644 --- a/roomserver/storage/postgres/event_types_table.go +++ b/roomserver/storage/postgres/event_types_table.go @@ -143,5 +143,5 @@ func (s *eventTypeStatements) bulkSelectEventTypeNID( } result[eventType] = types.EventTypeNID(eventTypeNID) } - return result, nil + return result, rows.Err() } diff --git a/roomserver/storage/postgres/events_table.go b/roomserver/storage/postgres/events_table.go index 1e8a5665b..d9b269bc8 100644 --- a/roomserver/storage/postgres/events_table.go +++ b/roomserver/storage/postgres/events_table.go @@ -209,6 +209,9 @@ func (s *eventStatements) bulkSelectStateEventByID( return nil, err } } + if err = rows.Err(); err != nil { + return nil, err + } if i != len(eventIDs) { // If there are fewer rows returned than IDs then we were asked to lookup event IDs we don't have. // We don't know which ones were missing because we don't return the string IDs in the query. @@ -219,7 +222,7 @@ func (s *eventStatements) bulkSelectStateEventByID( fmt.Sprintf("storage: state event IDs missing from the database (%d != %d)", i, len(eventIDs)), ) } - return results, err + return results, nil } // bulkSelectStateAtEventByID lookups the state at a list of events by event ID. @@ -251,12 +254,15 @@ func (s *eventStatements) bulkSelectStateAtEventByID( ) } } + if err = rows.Err(); err != nil { + return nil, err + } if i != len(eventIDs) { return nil, types.MissingEventError( fmt.Sprintf("storage: event IDs missing from the database (%d != %d)", i, len(eventIDs)), ) } - return results, err + return results, nil } func (s *eventStatements) updateEventState( @@ -321,6 +327,9 @@ func (s *eventStatements) bulkSelectStateAtEventAndReference( result.EventID = eventID result.EventSHA256 = eventSHA256 } + if err = rows.Err(); err != nil { + return nil, err + } if i != len(eventNIDs) { return nil, fmt.Errorf("storage: event NIDs missing from the database (%d != %d)", i, len(eventNIDs)) } @@ -343,6 +352,9 @@ func (s *eventStatements) bulkSelectEventReference( return nil, err } } + if err = rows.Err(); err != nil { + return nil, err + } if i != len(eventNIDs) { return nil, fmt.Errorf("storage: event NIDs missing from the database (%d != %d)", i, len(eventNIDs)) } @@ -366,6 +378,9 @@ func (s *eventStatements) bulkSelectEventID(ctx context.Context, eventNIDs []typ } results[types.EventNID(eventNID)] = eventID } + if err = rows.Err(); err != nil { + return nil, err + } if i != len(eventNIDs) { return nil, fmt.Errorf("storage: event NIDs missing from the database (%d != %d)", i, len(eventNIDs)) } @@ -389,7 +404,7 @@ func (s *eventStatements) bulkSelectEventNID(ctx context.Context, eventIDs []str } results[eventID] = types.EventNID(eventNID) } - return results, nil + return results, rows.Err() } func (s *eventStatements) selectMaxEventDepth(ctx context.Context, eventNIDs []types.EventNID) (int64, error) { diff --git a/roomserver/storage/postgres/invite_table.go b/roomserver/storage/postgres/invite_table.go index 43cd5ba09..603fed31b 100644 --- a/roomserver/storage/postgres/invite_table.go +++ b/roomserver/storage/postgres/invite_table.go @@ -114,21 +114,23 @@ func (s *inviteStatements) insertInviteEvent( func (s *inviteStatements) updateInviteRetired( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, -) (eventIDs []string, err error) { +) ([]string, error) { stmt := common.TxStmt(txn, s.updateInviteRetiredStmt) rows, err := stmt.QueryContext(ctx, roomNID, targetUserNID) if err != nil { return nil, err } - defer (func() { err = rows.Close() })() + defer rows.Close() // nolint: errcheck + + var eventIDs []string for rows.Next() { var inviteEventID string - if err := rows.Scan(&inviteEventID); err != nil { + if err = rows.Scan(&inviteEventID); err != nil { return nil, err } eventIDs = append(eventIDs, inviteEventID) } - return + return eventIDs, rows.Err() } // selectInviteActiveForUserInRoom returns a list of sender state key NIDs @@ -151,5 +153,5 @@ func (s *inviteStatements) selectInviteActiveForUserInRoom( } result = append(result, types.EventStateKeyNID(senderUserNID)) } - return result, nil + return result, rows.Err() } diff --git a/roomserver/storage/postgres/membership_table.go b/roomserver/storage/postgres/membership_table.go index 9f41fd67b..70032fd1e 100644 --- a/roomserver/storage/postgres/membership_table.go +++ b/roomserver/storage/postgres/membership_table.go @@ -151,6 +151,7 @@ func (s *membershipStatements) selectMembershipsFromRoom( if err != nil { return } + defer rows.Close() // nolint: errcheck for rows.Next() { var eNID types.EventNID @@ -159,8 +160,9 @@ func (s *membershipStatements) selectMembershipsFromRoom( } eventNIDs = append(eventNIDs, eNID) } - return + return eventNIDs, rows.Err() } + func (s *membershipStatements) selectMembershipsFromRoomAndMembership( ctx context.Context, roomNID types.RoomNID, membership membershipState, @@ -170,6 +172,7 @@ func (s *membershipStatements) selectMembershipsFromRoomAndMembership( if err != nil { return } + defer rows.Close() // nolint: errcheck for rows.Next() { var eNID types.EventNID @@ -178,7 +181,7 @@ func (s *membershipStatements) selectMembershipsFromRoomAndMembership( } eventNIDs = append(eventNIDs, eNID) } - return + return eventNIDs, rows.Err() } func (s *membershipStatements) updateMembership( diff --git a/roomserver/storage/postgres/room_aliases_table.go b/roomserver/storage/postgres/room_aliases_table.go index ad1b560c2..6de898c41 100644 --- a/roomserver/storage/postgres/room_aliases_table.go +++ b/roomserver/storage/postgres/room_aliases_table.go @@ -90,23 +90,23 @@ func (s *roomAliasesStatements) selectRoomIDFromAlias( func (s *roomAliasesStatements) selectAliasesFromRoomID( ctx context.Context, roomID string, -) (aliases []string, err error) { - aliases = []string{} +) ([]string, error) { rows, err := s.selectAliasesFromRoomIDStmt.QueryContext(ctx, roomID) if err != nil { - return + return nil, err } + defer rows.Close() // nolint: errcheck + var aliases []string for rows.Next() { var alias string if err = rows.Scan(&alias); err != nil { - return + return nil, err } aliases = append(aliases, alias) } - - return + return aliases, rows.Err() } func (s *roomAliasesStatements) selectCreatorIDFromAlias( diff --git a/roomserver/storage/postgres/state_block_table.go b/roomserver/storage/postgres/state_block_table.go index 15e69cc98..e6f4f7fe9 100644 --- a/roomserver/storage/postgres/state_block_table.go +++ b/roomserver/storage/postgres/state_block_table.go @@ -152,7 +152,7 @@ func (s *stateBlockStatements) bulkSelectStateBlockEntries( eventNID int64 entry types.StateEntry ) - if err := rows.Scan( + if err = rows.Scan( &stateBlockNID, &eventTypeNID, &eventStateKeyNID, &eventNID, ); err != nil { return nil, err @@ -169,10 +169,13 @@ func (s *stateBlockStatements) bulkSelectStateBlockEntries( } current.StateEntries = append(current.StateEntries, entry) } + if err = rows.Err(); err != nil { + return nil, err + } if i != len(stateBlockNIDs) { return nil, fmt.Errorf("storage: state data NIDs missing from the database (%d != %d)", i, len(stateBlockNIDs)) } - return results, nil + return results, err } func (s *stateBlockStatements) bulkSelectFilteredStateBlockEntries( @@ -237,7 +240,7 @@ func (s *stateBlockStatements) bulkSelectFilteredStateBlockEntries( if current.StateEntries != nil { results = append(results, current) } - return results, nil + return results, rows.Err() } func stateBlockNIDsAsArray(stateBlockNIDs []types.StateBlockNID) pq.Int64Array { diff --git a/roomserver/storage/postgres/state_snapshot_table.go b/roomserver/storage/postgres/state_snapshot_table.go index 76f1d2b66..a1f26e228 100644 --- a/roomserver/storage/postgres/state_snapshot_table.go +++ b/roomserver/storage/postgres/state_snapshot_table.go @@ -104,7 +104,7 @@ func (s *stateSnapshotStatements) bulkSelectStateBlockNIDs( for ; rows.Next(); i++ { result := &results[i] var stateBlockNIDs pq.Int64Array - if err := rows.Scan(&result.StateSnapshotNID, &stateBlockNIDs); err != nil { + if err = rows.Scan(&result.StateSnapshotNID, &stateBlockNIDs); err != nil { return nil, err } result.StateBlockNIDs = make([]types.StateBlockNID, len(stateBlockNIDs)) @@ -112,6 +112,9 @@ func (s *stateSnapshotStatements) bulkSelectStateBlockNIDs( result.StateBlockNIDs[k] = types.StateBlockNID(stateBlockNIDs[k]) } } + if err = rows.Err(); err != nil { + return nil, err + } if i != len(stateNIDs) { return nil, fmt.Errorf("storage: state NIDs missing from the database (%d != %d)", i, len(stateNIDs)) } diff --git a/syncapi/storage/postgres/account_data_table.go b/syncapi/storage/postgres/account_data_table.go index 94e6ac41c..3d75ad999 100644 --- a/syncapi/storage/postgres/account_data_table.go +++ b/syncapi/storage/postgres/account_data_table.go @@ -118,6 +118,7 @@ func (s *accountDataStatements) selectAccountDataInRange( if err != nil { return } + defer rows.Close() // nolint: errcheck for rows.Next() { var dataType string @@ -133,8 +134,7 @@ func (s *accountDataStatements) selectAccountDataInRange( data[roomID] = []string{dataType} } } - - return + return data, rows.Err() } func (s *accountDataStatements) selectMaxAccountDataID( diff --git a/syncapi/storage/postgres/backward_extremities_table.go b/syncapi/storage/postgres/backward_extremities_table.go index 1489f7f91..d63c546e3 100644 --- a/syncapi/storage/postgres/backward_extremities_table.go +++ b/syncapi/storage/postgres/backward_extremities_table.go @@ -91,6 +91,7 @@ func (s *backwardExtremitiesStatements) selectBackwardExtremitiesForRoom( if err != nil { return } + defer rows.Close() // nolint: errcheck for rows.Next() { var eID string @@ -101,7 +102,7 @@ func (s *backwardExtremitiesStatements) selectBackwardExtremitiesForRoom( eventIDs = append(eventIDs, eID) } - return + return eventIDs, rows.Err() } func (s *backwardExtremitiesStatements) isBackwardExtremity( diff --git a/syncapi/storage/postgres/current_room_state_table.go b/syncapi/storage/postgres/current_room_state_table.go index 816cbb44a..117bd7750 100644 --- a/syncapi/storage/postgres/current_room_state_table.go +++ b/syncapi/storage/postgres/current_room_state_table.go @@ -154,7 +154,7 @@ func (s *currentRoomStateStatements) selectJoinedUsers( users = append(users, userID) result[roomID] = users } - return result, nil + return result, rows.Err() } // SelectRoomIDsWithMembership returns the list of room IDs which have the given user in the given membership state. @@ -179,7 +179,7 @@ func (s *currentRoomStateStatements) selectRoomIDsWithMembership( } result = append(result, roomID) } - return result, nil + return result, rows.Err() } // CurrentState returns all the current state events for the given room. @@ -267,7 +267,7 @@ func rowsToEvents(rows *sql.Rows) ([]gomatrixserverlib.Event, error) { } result = append(result, ev) } - return result, nil + return result, rows.Err() } func (s *currentRoomStateStatements) selectStateEvent( diff --git a/syncapi/storage/postgres/invites_table.go b/syncapi/storage/postgres/invites_table.go index ca4bbeb5c..2cb8fb199 100644 --- a/syncapi/storage/postgres/invites_table.go +++ b/syncapi/storage/postgres/invites_table.go @@ -133,7 +133,7 @@ func (s *inviteEventsStatements) selectInviteEventsInRange( result[roomID] = event } - return result, nil + return result, rows.Err() } func (s *inviteEventsStatements) selectMaxInviteID( diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go index 6d213a57e..610ff23eb 100644 --- a/syncapi/storage/postgres/output_room_events_table.go +++ b/syncapi/storage/postgres/output_room_events_table.go @@ -170,6 +170,7 @@ func (s *outputRoomEventsStatements) selectStateInRange( if err != nil { return nil, nil, err } + defer rows.Close() // nolint: errcheck // Fetch all the state change events for all rooms between the two positions then loop each event and: // - Keep a cache of the event by ID (99% of state change events are for the event itself) // - For each room ID, build up an array of event IDs which represents cumulative adds/removes @@ -226,7 +227,7 @@ func (s *outputRoomEventsStatements) selectStateInRange( } } - return stateNeeded, eventIDToEvent, nil + return stateNeeded, eventIDToEvent, rows.Err() } // MaxID returns the ID of the last inserted event in this table. 'txn' is optional. If it is not supplied, @@ -392,5 +393,5 @@ func rowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) { ExcludeFromSync: excludeFromSync, }) } - return result, nil + return result, rows.Err() } diff --git a/syncapi/storage/postgres/output_room_events_topology_table.go b/syncapi/storage/postgres/output_room_events_topology_table.go index 793d1e236..78a381da9 100644 --- a/syncapi/storage/postgres/output_room_events_topology_table.go +++ b/syncapi/storage/postgres/output_room_events_topology_table.go @@ -134,6 +134,7 @@ func (s *outputRoomEventsTopologyStatements) selectEventIDsInRange( } else if err != nil { return } + defer rows.Close() // nolint: errcheck // Return the IDs. var eventID string @@ -144,7 +145,7 @@ func (s *outputRoomEventsTopologyStatements) selectEventIDsInRange( eventIDs = append(eventIDs, eventID) } - return + return eventIDs, rows.Err() } // selectPositionInTopology returns the position of a given event in the @@ -176,6 +177,7 @@ func (s *outputRoomEventsTopologyStatements) selectEventIDsFromPosition( } else if err != nil { return } + defer rows.Close() // nolint: errcheck // Return the IDs. var eventID string for rows.Next() { @@ -184,5 +186,5 @@ func (s *outputRoomEventsTopologyStatements) selectEventIDsFromPosition( } eventIDs = append(eventIDs, eventID) } - return + return eventIDs, rows.Err() } From 6942ee1de0250235164cf0ce45570b7fc919669d Mon Sep 17 00:00:00 2001 From: Thibaut CHARLES Date: Tue, 11 Feb 2020 16:46:51 +0100 Subject: [PATCH 14/86] Upgrade gomatrixserverlib dependency (#808) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Upgrade gomatrixserverlib dependency Signed-off-by: Thibaut CHARLES cromfr@gmail.com * Added missing passing sytest Signed-off-by: Thibaut CHARLES cromfr@gmail.com * Fix login using identifier key Not a full fix, it only really supports logging in with the localpart of an mxid. Signed-off-by: Serra Allgood * Replace deprecated prometheus.InstrumentHandler and unsafe time.Ticker * goimports * re-add temporarily missing deps? * Refactor InstrumentHandlerCounter definition * URL decode args * Return server names (#833) * Remove unnecessary map->array processing * Return server names in room federation directory query * Knock off a TODO * Fix /send_join and /send_leave (#821) Fix the /send_join and /send_leave endpoints, so that they use the v2 endpoints as mandated by MSC1802. Also comment out the SyTest tests that are failing because of lack of support for the v1 endpoints. * Refuse /send_join without m.room.create (#824) Signed-off-by: Abhishek Kumar * AS should use the v1 endpoint, rather than r0 (#827) * docker: Passthrough parameters to dendrite-monolith-server * Fix copy & paste error (#812) * Use gomatrixserverlib.Transaction instead of local type (#590) (#811) * Move files back if linting fails (#810) * replaced gometalinter description with golangci-lint (#837) * Amend syncapi SQL queries to return missing columns (#840) * This commit updates a couple of the syncapi SQL queries to return additional columns that are required/expected by rowsToStreamEvents in output_room_events_table.go. It's not exactly clear to me yet what transaction_id and session_id do, but these being added n #367 results in state events breaking the /sync endpoint. This is a temporary fix. We need to come up with a better solution. * gomatrix to gomatrixserverlib on some weird line change * Tweaks from @babolivier review comments * Implement storage interfaces (#841) * Implement interfaces for federationsender storage * Implement interfaces for mediaapi storage * Implement interfaces for publicroomsapi storage * Implement interfaces for roomserver storage * Implement interfaces for syncapi storage * Implement interfaces for keydb storage * common.PartitionStorer in publicroomsapi interface * Update copyright notices * make cmd directory path absolute in build.sh (#830) * Resync testfile with current sytest pass/fail (#832) * Resync testfile with current sytest pass/fail * Add displayname test * Fall back to postgres when database connection string parsing fails (#842) * Fall back to postgres when parsing the database connection string for a URI schema fails * Fix behaviour so that it really tries postgres when URL parsing fails and it complains about unknown schema if it succeeds * Fix #842 * Fix #842 - again... * Federation fixes (#845) * Update gomatrixserverlib to p2p commit 92c0338, other tweaks * Update gomatrixserverlib to p2p commit e5dcc65 * Rewrite getAuthChain * Update gomatrixserverlib in go.mod/go.sum * Correct a couple of package refs for updated gmsl/gomatrix * Update gomatrixserverlib ref in go.mod/go.sum * Update getAuthChain comments following @babolivier review * Add a Sytest blacklist file (#849) * Add more passing tests to the testfile, add test blacklist file (#848) * CS API: Support for /messages, fixes for /sync (#847) * Merge forward * Tidy up a bit * TODO: What to do with NextBatch here? * Replace SyncPosition with PaginationToken throughout syncapi * Fix PaginationTokens * Fix lint errors * Add a couple of missing functions into the syncapi external storage interface * Some updates based on review comments from @babolivier * Some updates based on review comments from @babolivier * argh whitespacing * Fix opentracing span * Remove dead code * Don't overshadow err (fix lint issue) * Handle extremities after inserting event into topology * Try insert event topology as ON CONFLICT DO NOTHING * Prevent OOB error in addRoomDeltaToResponse * Thwarted by gocyclo again * Fix NewPaginationTokenFromString, define unit test for it * Update pagination token test * Update sytest-whitelist * Hopefully fix some of the sync batch tokens * Remove extraneous sync position func * Revert to topology tokens in addRoomDeltaToResponse etc * Fix typo * Remove prevPDUPos as dead now that backwardTopologyPos is used instead * Fix selectEventsWithEventIDsSQL * Update sytest-blacklist * Update sytest-whitelist * Some fixes for #847 (#850) * Fix a couple of cases where backfilling events we already had causes panics, hopefully fix ordering of events, update GMSL dependency for backfill URL fixes * Remove commented out lines from output_room_events_table schema * Wire up publicroomsapi for roomserver events (#851) * Wire up publicroomsapi to roomserver events * Remove parameter that was incorrectly brought over from p2p work * nolint containsBackwardExtremity for now * Store our own keys in the keydb (#853) * Store our own keys in the keydb The DirectKeyFetcher makes the assumption that you can always reach the key/v2/server endpoint of any server, including our own. We previously haven't bothered to store our own keys in the keydb so this would mean we end up making key requests to ourselves. In the libp2p world as an example, self-dialling is not possible, therefore this would render it impossible to get our own keys. This commit adds our own keys into the keydb so that we don't create unnecessarily (and maybe impossible) requests. * Use golang.org/x/crypto/ed25519 instead of crypto/ed25519 for pre-Go 1.13 * More sync fixes (#854) * Further sync tweaks * Remove unnecessary blank line * getBackwardTopologyPos always returns a usable value * Revert order fixing * Implement GET endpoints for account_data in clientapi (#861) * Implement GET endpoints for account_data in clientapi * Fix accountDB parameter * Remove fmt.Println * Add empty push rules into account data on account creation (#862) * Handle kind=guest query parameter on /register (#860) * Handle kind=guest query parameter on /register * Reorganized imports * Pass device_id as nil * Added tests to systest-whitelist * Update sytest-whitelist * Blacklist 'displayname updates affect room member events' (#859) * Room version abstractions (#865) * Rough first pass at adding room version abstractions * Define newer room versions * Update room version metadata * Fix roomserver/versions * Try to fix whitespace in roomsSchema * Implement room version capabilities in CS API (#866) * Add wiring for querying the roomserver for the default room version * Try to implement /capabilities for room versions * Update copyright notices * Update sytests, add /capabilities endpoint into CS API * Update sytest-whitelist * Add GetDefaultRoomVersion * Fix cases where state package was shadowed * Fix version formatting * Update Dockerfile to Go 1.13.6 * oh yes types I remember * And fix the default too * Update documentation for Go 1.13 (#867) * Pass cfg by reference around the codebase (#819) * Pass cfg by reference around the codebase * Merge branch 'master' into pass-cfg-by-ref Co-authored-by: Neil Alexander * Implement missing device management features (#835) * Implement missing device management features Signed-off-by: Till Faelligen * Add a little more documentation * Undo changes * Use non-anonymous struct to decode devices list * Update sytest-whitelist * Update sytest-whitelist * Update sytest-blacklist Co-authored-by: Neil Alexander * Adding sslmode: disable to sytest server config (#813) Co-authored-by: Neil Alexander * Fix AppService bind addrs in test (#805) Co-authored-by: Neil Alexander * Always defer *sql.Rows.Close and consult with Err (#844) * Always defer *sql.Rows.Close and consult with Err database/sql.Rows.Next() makes sure to call Close only after exhausting result rows which would NOT happen when returning early from a bad Scan. Close being idempotent makes it a great candidate to get always deferred regardless of what happens later on the result set. This change also makes sure call Err() after exhausting Next() and propagate non-nil results from it as the documentation advises. Closes #764 Signed-off-by: Kiril Vladimiroff * Override named result parameters in last returns Signed-off-by: Kiril Vladimiroff * Do the same over new changes that got merged Signed-off-by: Kiril Vladimiroff Co-authored-by: Neil Alexander * Clean up Co-authored-by: Serra Allgood Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Co-authored-by: Neil Alexander Co-authored-by: Brendan Abolivier Co-authored-by: Abhishek Kumar <31231064+abhishekkumar2718@users.noreply.github.com> Co-authored-by: Will Hunt Co-authored-by: S7evinK Co-authored-by: Arshpreet <30545756+arsh-7@users.noreply.github.com> Co-authored-by: Prateek Sachan <42961174+prateek2211@users.noreply.github.com> Co-authored-by: Behouba Manassé Co-authored-by: aditsachde <23707194+aditsachde@users.noreply.github.com> Co-authored-by: Kiril Vladimiroff --- go.mod | 2 - go.sum | 1 + syncapi/api/query.go | 123 ++++++++++++++++++ syncapi/routing/state.go | 7 +- .../storage/postgres/account_data_table.go | 10 +- .../postgres/current_room_state_table.go | 15 +-- .../postgres/output_room_events_table.go | 15 +-- syncapi/storage/postgres/syncserver.go | 33 +++-- syncapi/storage/storage.go | 5 +- syncapi/sync/requestpool.go | 9 +- sytest-whitelist | 1 + 11 files changed, 169 insertions(+), 52 deletions(-) create mode 100644 syncapi/api/query.go diff --git a/go.mod b/go.mod index 4c894ad99..990b839eb 100644 --- a/go.mod +++ b/go.mod @@ -18,8 +18,6 @@ require ( github.com/matrix-org/naffka v0.0.0-20171115094957-662bfd0841d0 github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 github.com/miekg/dns v1.1.12 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.1 // indirect github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 github.com/opentracing/opentracing-go v1.0.2 github.com/pierrec/lz4 v0.0.0-20161206202305-5c9560bfa9ac // indirect diff --git a/go.sum b/go.sum index d71838164..42a145d6c 100644 --- a/go.sum +++ b/go.sum @@ -183,6 +183,7 @@ gopkg.in/h2non/bimg.v1 v1.0.18 h1:qn6/RpBHt+7WQqoBcK+aF2puc6nC78eZj5LexxoalT4= gopkg.in/h2non/bimg.v1 v1.0.18/go.mod h1:PgsZL7dLwUbsGm1NYps320GxGgvQNTnecMCZqxV11So= gopkg.in/h2non/gock.v1 v1.0.14 h1:fTeu9fcUvSnLNacYvYI54h+1/XEteDyHvrVCZEEEYNM= gopkg.in/h2non/gock.v1 v1.0.14/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE= +gopkg.in/macaroon.v2 v2.1.0 h1:HZcsjBCzq9t0eBPMKqTN/uSN6JOm78ZJ2INbqcBQOUI= gopkg.in/macaroon.v2 v2.1.0/go.mod h1:OUb+TQP/OP0WOerC2Jp/3CwhIKyIa9kQjuc7H24e6/o= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= diff --git a/syncapi/api/query.go b/syncapi/api/query.go new file mode 100644 index 000000000..2993829e0 --- /dev/null +++ b/syncapi/api/query.go @@ -0,0 +1,123 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package api + +import ( + "context" + "net/http" + + commonHTTP "github.com/matrix-org/dendrite/common/http" + "github.com/matrix-org/util" + opentracing "github.com/opentracing/opentracing-go" +) + +const ( + SyncAPIQuerySyncPath = "/api/syncapi/querySync" + SyncAPIQueryStatePath = "/api/syncapi/queryState" + SyncAPIQueryStateTypePath = "/api/syncapi/queryStateType" + SyncAPIQueryMessagesPath = "/api/syncapi/queryMessages" +) + +func NewSyncQueryAPIHTTP(syncapiURL string, httpClient *http.Client) SyncQueryAPI { + if httpClient == nil { + httpClient = http.DefaultClient + } + return &httpSyncQueryAPI{syncapiURL, httpClient} +} + +type httpSyncQueryAPI struct { + syncapiURL string + httpClient *http.Client +} + +type SyncQueryAPI interface { + QuerySync(ctx context.Context, request *QuerySyncRequest, response *QuerySyncResponse) error + QueryState(ctx context.Context, request *QueryStateRequest, response *QueryStateResponse) error + QueryStateType(ctx context.Context, request *QueryStateTypeRequest, response *QueryStateTypeResponse) error + QueryMessages(ctx context.Context, request *QueryMessagesRequest, response *QueryMessagesResponse) error +} + +type QuerySyncRequest struct{} + +type QueryStateRequest struct { + RoomID string +} + +type QueryStateTypeRequest struct { + RoomID string + EventType string + StateKey string +} + +type QueryMessagesRequest struct { + RoomID string +} + +type QuerySyncResponse util.JSONResponse +type QueryStateResponse util.JSONResponse +type QueryStateTypeResponse util.JSONResponse +type QueryMessagesResponse util.JSONResponse + +// QueryLatestEventsAndState implements SyncQueryAPI +func (h *httpSyncQueryAPI) QuerySync( + ctx context.Context, + request *QuerySyncRequest, + response *QuerySyncResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "QuerySync") + defer span.Finish() + + apiURL := h.syncapiURL + SyncAPIQuerySyncPath + return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +// QueryStateAfterEvents implements SyncQueryAPI +func (h *httpSyncQueryAPI) QueryState( + ctx context.Context, + request *QueryStateRequest, + response *QueryStateResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "QueryState") + defer span.Finish() + + apiURL := h.syncapiURL + SyncAPIQueryStatePath + return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +// QueryEventsByID implements SyncQueryAPI +func (h *httpSyncQueryAPI) QueryStateType( + ctx context.Context, + request *QueryStateTypeRequest, + response *QueryStateTypeResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "QueryStateType") + defer span.Finish() + + apiURL := h.syncapiURL + SyncAPIQueryStateTypePath + return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +// QueryMembershipForUser implements SyncQueryAPI +func (h *httpSyncQueryAPI) QueryMessages( + ctx context.Context, + request *QueryMessagesRequest, + response *QueryMessagesResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "QueryMessages") + defer span.Finish() + + apiURL := h.syncapiURL + SyncAPIQueryMessagesPath + return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} diff --git a/syncapi/routing/state.go b/syncapi/routing/state.go index dbee267d6..cf67f7522 100644 --- a/syncapi/routing/state.go +++ b/syncapi/routing/state.go @@ -22,7 +22,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/types" - "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" log "github.com/sirupsen/logrus" @@ -45,10 +44,10 @@ func OnIncomingStateRequest(req *http.Request, db storage.Database, roomID strin // TODO(#287): Auth request and handle the case where the user has left (where // we should return the state at the poin they left) - stateFilterPart := gomatrix.DefaultFilterPart() - // TODO: stateFilterPart should not limit the number of state events (or only limits abusive number of events) + stateFilter := gomatrixserverlib.DefaultStateFilter() + // TODO: stateFilter should not limit the number of state events (or only limits abusive number of events) - stateEvents, err := db.GetStateEventsForRoom(req.Context(), roomID, &stateFilterPart) + stateEvents, err := db.GetStateEventsForRoom(req.Context(), roomID, &stateFilter) if err != nil { return httputil.LogThenError(req, err) } diff --git a/syncapi/storage/postgres/account_data_table.go b/syncapi/storage/postgres/account_data_table.go index 3d75ad999..d1811aa66 100644 --- a/syncapi/storage/postgres/account_data_table.go +++ b/syncapi/storage/postgres/account_data_table.go @@ -22,7 +22,7 @@ import ( "github.com/lib/pq" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/syncapi/types" - "github.com/matrix-org/gomatrix" + "github.com/matrix-org/gomatrixserverlib" ) const accountDataSchema = ` @@ -99,7 +99,7 @@ func (s *accountDataStatements) selectAccountDataInRange( ctx context.Context, userID string, oldPos, newPos types.StreamPosition, - accountDataFilterPart *gomatrix.FilterPart, + accountDataEventFilter *gomatrixserverlib.EventFilter, ) (data map[string][]string, err error) { data = make(map[string][]string) @@ -111,9 +111,9 @@ func (s *accountDataStatements) selectAccountDataInRange( } rows, err := s.selectAccountDataInRangeStmt.QueryContext(ctx, userID, oldPos, newPos, - pq.StringArray(filterConvertTypeWildcardToSQL(accountDataFilterPart.Types)), - pq.StringArray(filterConvertTypeWildcardToSQL(accountDataFilterPart.NotTypes)), - accountDataFilterPart.Limit, + pq.StringArray(filterConvertTypeWildcardToSQL(accountDataEventFilter.Types)), + pq.StringArray(filterConvertTypeWildcardToSQL(accountDataEventFilter.NotTypes)), + accountDataEventFilter.Limit, ) if err != nil { return diff --git a/syncapi/storage/postgres/current_room_state_table.go b/syncapi/storage/postgres/current_room_state_table.go index 117bd7750..6f5c1e803 100644 --- a/syncapi/storage/postgres/current_room_state_table.go +++ b/syncapi/storage/postgres/current_room_state_table.go @@ -23,7 +23,6 @@ import ( "github.com/lib/pq" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/syncapi/types" - "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" ) @@ -185,16 +184,16 @@ func (s *currentRoomStateStatements) selectRoomIDsWithMembership( // CurrentState returns all the current state events for the given room. func (s *currentRoomStateStatements) selectCurrentState( ctx context.Context, txn *sql.Tx, roomID string, - stateFilterPart *gomatrix.FilterPart, + stateFilter *gomatrixserverlib.StateFilter, ) ([]gomatrixserverlib.Event, error) { stmt := common.TxStmt(txn, s.selectCurrentStateStmt) rows, err := stmt.QueryContext(ctx, roomID, - pq.StringArray(stateFilterPart.Senders), - pq.StringArray(stateFilterPart.NotSenders), - pq.StringArray(filterConvertTypeWildcardToSQL(stateFilterPart.Types)), - pq.StringArray(filterConvertTypeWildcardToSQL(stateFilterPart.NotTypes)), - stateFilterPart.ContainsURL, - stateFilterPart.Limit, + pq.StringArray(stateFilter.Senders), + pq.StringArray(stateFilter.NotSenders), + pq.StringArray(filterConvertTypeWildcardToSQL(stateFilter.Types)), + pq.StringArray(filterConvertTypeWildcardToSQL(stateFilter.NotTypes)), + stateFilter.ContainsURL, + stateFilter.Limit, ) if err != nil { return nil, err diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go index 610ff23eb..2db46c5db 100644 --- a/syncapi/storage/postgres/output_room_events_table.go +++ b/syncapi/storage/postgres/output_room_events_table.go @@ -23,7 +23,6 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/types" - "github.com/matrix-org/gomatrix" "github.com/lib/pq" "github.com/matrix-org/dendrite/common" @@ -154,18 +153,18 @@ func (s *outputRoomEventsStatements) prepare(db *sql.DB) (err error) { // two positions, only the most recent state is returned. func (s *outputRoomEventsStatements) selectStateInRange( ctx context.Context, txn *sql.Tx, oldPos, newPos types.StreamPosition, - stateFilterPart *gomatrix.FilterPart, + stateFilter *gomatrixserverlib.StateFilter, ) (map[string]map[string]bool, map[string]types.StreamEvent, error) { stmt := common.TxStmt(txn, s.selectStateInRangeStmt) rows, err := stmt.QueryContext( ctx, oldPos, newPos, - pq.StringArray(stateFilterPart.Senders), - pq.StringArray(stateFilterPart.NotSenders), - pq.StringArray(filterConvertTypeWildcardToSQL(stateFilterPart.Types)), - pq.StringArray(filterConvertTypeWildcardToSQL(stateFilterPart.NotTypes)), - stateFilterPart.ContainsURL, - stateFilterPart.Limit, + pq.StringArray(stateFilter.Senders), + pq.StringArray(stateFilter.NotSenders), + pq.StringArray(filterConvertTypeWildcardToSQL(stateFilter.Types)), + pq.StringArray(filterConvertTypeWildcardToSQL(stateFilter.NotTypes)), + stateFilter.ContainsURL, + stateFilter.Limit, ) if err != nil { return nil, nil, err diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go index f391c5784..aec37185d 100644 --- a/syncapi/storage/postgres/syncserver.go +++ b/syncapi/storage/postgres/syncserver.go @@ -26,7 +26,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/gomatrix" // Import the postgres database driver. _ "github.com/lib/pq" @@ -237,10 +236,10 @@ func (d *SyncServerDatasource) GetStateEvent( // Returns an empty slice if no state events could be found for this room. // Returns an error if there was an issue with the retrieval. func (d *SyncServerDatasource) GetStateEventsForRoom( - ctx context.Context, roomID string, stateFilterPart *gomatrix.FilterPart, + ctx context.Context, roomID string, stateFilter *gomatrixserverlib.StateFilter, ) (stateEvents []gomatrixserverlib.Event, err error) { err = common.WithTransaction(d.db, func(txn *sql.Tx) error { - stateEvents, err = d.roomstate.selectCurrentState(ctx, txn, roomID, stateFilterPart) + stateEvents, err = d.roomstate.selectCurrentState(ctx, txn, roomID, stateFilter) return err }) return @@ -422,7 +421,7 @@ func (d *SyncServerDatasource) addPDUDeltaToResponse( var succeeded bool defer common.EndTransaction(txn, &succeeded) - stateFilterPart := gomatrix.DefaultFilterPart() // TODO: use filter provided in request + stateFilter := gomatrixserverlib.DefaultStateFilter() // TODO: use filter provided in request // Work out which rooms to return in the response. This is done by getting not only the currently // joined rooms, but also which rooms have membership transitions for this user between the 2 PDU stream positions. @@ -432,11 +431,11 @@ func (d *SyncServerDatasource) addPDUDeltaToResponse( var joinedRoomIDs []string if !wantFullState { deltas, joinedRoomIDs, err = d.getStateDeltas( - ctx, &device, txn, fromPos, toPos, device.UserID, &stateFilterPart, + ctx, &device, txn, fromPos, toPos, device.UserID, &stateFilter, ) } else { deltas, joinedRoomIDs, err = d.getStateDeltasForFullStateSync( - ctx, &device, txn, fromPos, toPos, device.UserID, &stateFilterPart, + ctx, &device, txn, fromPos, toPos, device.UserID, &stateFilter, ) } if err != nil { @@ -587,12 +586,12 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( return } - stateFilterPart := gomatrix.DefaultFilterPart() // TODO: use filter provided in request + stateFilter := gomatrixserverlib.DefaultStateFilter() // TODO: use filter provided in request // Build up a /sync response. Add joined rooms. for _, roomID := range joinedRoomIDs { var stateEvents []gomatrixserverlib.Event - stateEvents, err = d.roomstate.selectCurrentState(ctx, txn, roomID, &stateFilterPart) + stateEvents, err = d.roomstate.selectCurrentState(ctx, txn, roomID, &stateFilter) if err != nil { return } @@ -681,7 +680,7 @@ var txReadOnlySnapshot = sql.TxOptions{ // If there was an issue with the retrieval, returns an error func (d *SyncServerDatasource) GetAccountDataInRange( ctx context.Context, userID string, oldPos, newPos types.StreamPosition, - accountDataFilterPart *gomatrix.FilterPart, + accountDataFilterPart *gomatrixserverlib.EventFilter, ) (map[string][]string, error) { return d.accountData.selectAccountDataInRange(ctx, userID, oldPos, newPos, accountDataFilterPart) } @@ -931,7 +930,7 @@ func (d *SyncServerDatasource) fetchMissingStateEvents( func (d *SyncServerDatasource) getStateDeltas( ctx context.Context, device *authtypes.Device, txn *sql.Tx, fromPos, toPos types.StreamPosition, userID string, - stateFilterPart *gomatrix.FilterPart, + stateFilter *gomatrixserverlib.StateFilter, ) ([]stateDelta, []string, error) { // Implement membership change algorithm: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L821 // - Get membership list changes for this user in this sync response @@ -944,7 +943,7 @@ func (d *SyncServerDatasource) getStateDeltas( var deltas []stateDelta // get all the state events ever between these two positions - stateNeeded, eventMap, err := d.events.selectStateInRange(ctx, txn, fromPos, toPos, stateFilterPart) + stateNeeded, eventMap, err := d.events.selectStateInRange(ctx, txn, fromPos, toPos, stateFilter) if err != nil { return nil, nil, err } @@ -964,7 +963,7 @@ func (d *SyncServerDatasource) getStateDeltas( if membership == gomatrixserverlib.Join { // send full room state down instead of a delta var s []types.StreamEvent - s, err = d.currentStateStreamEventsForRoom(ctx, txn, roomID, stateFilterPart) + s, err = d.currentStateStreamEventsForRoom(ctx, txn, roomID, stateFilter) if err != nil { return nil, nil, err } @@ -1006,7 +1005,7 @@ func (d *SyncServerDatasource) getStateDeltas( func (d *SyncServerDatasource) getStateDeltasForFullStateSync( ctx context.Context, device *authtypes.Device, txn *sql.Tx, fromPos, toPos types.StreamPosition, userID string, - stateFilterPart *gomatrix.FilterPart, + stateFilter *gomatrixserverlib.StateFilter, ) ([]stateDelta, []string, error) { joinedRoomIDs, err := d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, gomatrixserverlib.Join) if err != nil { @@ -1018,7 +1017,7 @@ func (d *SyncServerDatasource) getStateDeltasForFullStateSync( // Add full states for all joined rooms for _, joinedRoomID := range joinedRoomIDs { - s, stateErr := d.currentStateStreamEventsForRoom(ctx, txn, joinedRoomID, stateFilterPart) + s, stateErr := d.currentStateStreamEventsForRoom(ctx, txn, joinedRoomID, stateFilter) if stateErr != nil { return nil, nil, stateErr } @@ -1030,7 +1029,7 @@ func (d *SyncServerDatasource) getStateDeltasForFullStateSync( } // Get all the state events ever between these two positions - stateNeeded, eventMap, err := d.events.selectStateInRange(ctx, txn, fromPos, toPos, stateFilterPart) + stateNeeded, eventMap, err := d.events.selectStateInRange(ctx, txn, fromPos, toPos, stateFilter) if err != nil { return nil, nil, err } @@ -1061,9 +1060,9 @@ func (d *SyncServerDatasource) getStateDeltasForFullStateSync( func (d *SyncServerDatasource) currentStateStreamEventsForRoom( ctx context.Context, txn *sql.Tx, roomID string, - stateFilterPart *gomatrix.FilterPart, + stateFilter *gomatrixserverlib.StateFilter, ) ([]types.StreamEvent, error) { - allState, err := d.roomstate.selectCurrentState(ctx, txn, roomID, stateFilterPart) + allState, err := d.roomstate.selectCurrentState(ctx, txn, roomID, stateFilter) if err != nil { return nil, err } diff --git a/syncapi/storage/storage.go b/syncapi/storage/storage.go index 4e8a2c837..e63928440 100644 --- a/syncapi/storage/storage.go +++ b/syncapi/storage/storage.go @@ -25,7 +25,6 @@ import ( "github.com/matrix-org/dendrite/syncapi/storage/postgres" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/dendrite/typingserver/cache" - "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" ) @@ -35,11 +34,11 @@ type Database interface { Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.Event, error) WriteEvent(context.Context, *gomatrixserverlib.Event, []gomatrixserverlib.Event, []string, []string, *api.TransactionID, bool) (types.StreamPosition, error) GetStateEvent(ctx context.Context, roomID, evType, stateKey string) (*gomatrixserverlib.Event, error) - GetStateEventsForRoom(ctx context.Context, roomID string, stateFilterPart *gomatrix.FilterPart) (stateEvents []gomatrixserverlib.Event, err error) + GetStateEventsForRoom(ctx context.Context, roomID string, stateFilterPart *gomatrixserverlib.StateFilter) (stateEvents []gomatrixserverlib.Event, err error) SyncPosition(ctx context.Context) (types.PaginationToken, error) IncrementalSync(ctx context.Context, device authtypes.Device, fromPos, toPos types.PaginationToken, numRecentEventsPerRoom int, wantFullState bool) (*types.Response, error) CompleteSync(ctx context.Context, userID string, numRecentEventsPerRoom int) (*types.Response, error) - GetAccountDataInRange(ctx context.Context, userID string, oldPos, newPos types.StreamPosition, accountDataFilterPart *gomatrix.FilterPart) (map[string][]string, error) + GetAccountDataInRange(ctx context.Context, userID string, oldPos, newPos types.StreamPosition, accountDataFilterPart *gomatrixserverlib.EventFilter) (map[string][]string, error) UpsertAccountData(ctx context.Context, userID, roomID, dataType string) (types.StreamPosition, error) AddInviteEvent(ctx context.Context, inviteEvent gomatrixserverlib.Event) (types.StreamPosition, error) RetireInviteEvent(ctx context.Context, inviteEventID string) error diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index 5a3ae8807..06a8d6d8a 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -24,7 +24,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/types" - "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" log "github.com/sirupsen/logrus" @@ -142,14 +141,14 @@ func (rp *RequestPool) currentSyncForUser(req syncRequest, latestPos types.Pagin return } - accountDataFilter := gomatrix.DefaultFilterPart() // TODO: use filter provided in req instead - res, err = rp.appendAccountData(res, req.device.UserID, req, int64(latestPos.PDUPosition), &accountDataFilter) + accountDataFilter := gomatrixserverlib.DefaultEventFilter() // TODO: use filter provided in req instead + res, err = rp.appendAccountData(res, req.device.UserID, req, latestPos.PDUPosition, &accountDataFilter) return } func (rp *RequestPool) appendAccountData( - data *types.Response, userID string, req syncRequest, currentPos int64, - accountDataFilter *gomatrix.FilterPart, + data *types.Response, userID string, req syncRequest, currentPos types.StreamPosition, + accountDataFilter *gomatrixserverlib.EventFilter, ) (*types.Response, error) { // TODO: Account data doesn't have a sync position of its own, meaning that // account data might be sent multiple time to the client if multiple account diff --git a/sytest-whitelist b/sytest-whitelist index efc97b5e6..47fd58286 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -81,6 +81,7 @@ Can't forget room you're still in Can get rooms/{roomId}/members Can create filter Can download filter +Lazy loading parameters in the filter are strictly boolean Can sync Can sync a joined room Newly joined room is included in an incremental sync From b6ea1bc67ab51667b9e139dd05e0778aca025501 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Thu, 13 Feb 2020 17:27:33 +0000 Subject: [PATCH 15/86] Support sqlite in addition to postgres (#869) * Move current work into single branch * Initial massaging of clientapi etc (not working yet) * Interfaces for accounts/devices databases * Duplicate postgres package for sqlite3 (no changes made to it yet) * Some keydb, accountdb, devicedb, common partition fixes, some more syncapi tweaking * Fix accounts DB, device DB * Update naffka dependency for SQLite * Naffka SQLite * Update naffka to latest master * SQLite support for federationsender * Mostly not-bad support for SQLite in syncapi (although there are problems where lots of events get classed incorrectly as backward extremities, probably because of IN/ANY clauses that are badly supported) * Update Dockerfile -> Go 1.13.7, add build-base (as gcc and friends are needed for SQLite) * Implement GET endpoints for account_data in clientapi * Nuke filtering for now... * Revert "Implement GET endpoints for account_data in clientapi" This reverts commit 4d80dff4583d278620d9b3ed437e9fcd8d4674ee. * Implement GET endpoints for account_data in clientapi (#861) * Implement GET endpoints for account_data in clientapi * Fix accountDB parameter * Remove fmt.Println * Fix insertAccountData SQLite query * Fix accountDB storage interfaces * Add empty push rules into account data on account creation (#862) * Put SaveAccountData into the right function this time * Not sure if roomserver is better or worse now * sqlite work * Allow empty last sent ID for the first event * sqlite: room creation works * Support sending messages * Nuke fmt.println * Move QueryVariadic etc into common, other device fixes * Fix some linter issues * Fix bugs * Fix some linting errors * Fix errcheck lint errors * Make naffka use postgres as fallback, fix couple of compile errors * What on earth happened to the /rooms/{roomID}/send/{eventType} routing Co-authored-by: Neil Alexander --- appservice/api/query.go | 2 +- appservice/appservice.go | 8 +- appservice/consumers/roomserver.go | 4 +- appservice/routing/routing.go | 2 +- .../{ => postgres}/account_data_table.go | 2 +- .../accounts/{ => postgres}/accounts_table.go | 2 +- .../accounts/{ => postgres}/filter_table.go | 2 +- .../{ => postgres}/membership_table.go | 2 +- .../accounts/{ => postgres}/profile_table.go | 2 +- .../auth/storage/accounts/postgres/storage.go | 392 ++++++ .../accounts/postgres/threepid_table.go | 129 ++ .../accounts/sqlite3/account_data_table.go | 141 ++ .../accounts/sqlite3/accounts_table.go | 151 +++ .../storage/accounts/sqlite3/filter_table.go | 139 ++ .../accounts/sqlite3/membership_table.go | 131 ++ .../storage/accounts/sqlite3/profile_table.go | 107 ++ .../auth/storage/accounts/sqlite3/storage.go | 392 ++++++ .../accounts/{ => sqlite3}/threepid_table.go | 2 +- clientapi/auth/storage/accounts/storage.go | 406 +----- .../devices/{ => postgres}/devices_table.go | 2 +- .../auth/storage/devices/postgres/storage.go | 182 +++ .../storage/devices/sqlite3/devices_table.go | 243 ++++ .../auth/storage/devices/sqlite3/storage.go | 184 +++ clientapi/auth/storage/devices/storage.go | 191 +-- clientapi/clientapi.go | 4 +- clientapi/consumers/roomserver.go | 4 +- clientapi/routing/account_data.go | 4 +- clientapi/routing/createroom.go | 4 +- clientapi/routing/device.go | 10 +- clientapi/routing/filter.go | 4 +- clientapi/routing/joinroom.go | 2 +- clientapi/routing/login.go | 4 +- clientapi/routing/logout.go | 4 +- clientapi/routing/membership.go | 8 +- clientapi/routing/profile.go | 12 +- clientapi/routing/register.go | 30 +- clientapi/routing/room_tagging.go | 10 +- clientapi/routing/routing.go | 4 +- clientapi/routing/sendtyping.go | 2 +- clientapi/routing/threepid.go | 8 +- clientapi/threepid/invites.go | 6 +- cmd/kafka-producer/main.go | 2 +- common/basecomponent/base.go | 76 +- common/keydb/keydb.go | 3 + common/keydb/sqlite3/keydb.go | 115 ++ common/keydb/sqlite3/server_key_table.go | 142 ++ common/partition_offset_table.go | 4 +- common/sql.go | 33 +- docker/Dockerfile | 6 +- docker/docker-compose.yml | 10 +- federationapi/federationapi.go | 4 +- federationapi/routing/devices.go | 2 +- federationapi/routing/profile.go | 2 +- federationapi/routing/routing.go | 4 +- federationapi/routing/threepid.go | 4 +- federationsender/storage/postgres/storage.go | 2 +- .../storage/sqlite3/joined_hosts_table.go | 139 ++ .../storage/sqlite3/room_table.go | 101 ++ federationsender/storage/sqlite3/storage.go | 124 ++ federationsender/storage/storage.go | 3 + go.mod | 20 +- go.sum | 42 +- mediaapi/mediaapi.go | 2 +- mediaapi/routing/routing.go | 2 +- publicroomsapi/publicroomsapi.go | 2 +- publicroomsapi/routing/routing.go | 2 +- publicroomsapi/storage/sqlite3/prepare.go | 36 + .../storage/sqlite3/public_rooms_table.go | 277 ++++ publicroomsapi/storage/sqlite3/storage.go | 256 ++++ roomserver/input/events.go | 7 +- roomserver/input/latest_events.go | 7 +- .../storage/sqlite3/event_json_table.go | 108 ++ .../storage/sqlite3/event_state_keys_table.go | 156 +++ .../storage/sqlite3/event_types_table.go | 153 +++ roomserver/storage/sqlite3/events_table.go | 479 +++++++ roomserver/storage/sqlite3/invite_table.go | 142 ++ roomserver/storage/sqlite3/list.go | 18 + .../storage/sqlite3/membership_table.go | 180 +++ roomserver/storage/sqlite3/prepare.go | 36 + .../storage/sqlite3/previous_events_table.go | 92 ++ .../storage/sqlite3/room_aliases_table.go | 135 ++ roomserver/storage/sqlite3/rooms_table.go | 165 +++ roomserver/storage/sqlite3/sql.go | 60 + .../storage/sqlite3/state_block_table.go | 292 ++++ .../storage/sqlite3/state_block_table_test.go | 86 ++ .../storage/sqlite3/state_snapshot_table.go | 120 ++ roomserver/storage/sqlite3/storage.go | 864 ++++++++++++ .../storage/sqlite3/transactions_table.go | 86 ++ roomserver/storage/storage.go | 14 +- syncapi/routing/routing.go | 2 +- syncapi/storage/postgres/syncserver.go | 17 +- syncapi/storage/sqlite3/account_data_table.go | 143 ++ .../sqlite3/backward_extremities_table.go | 124 ++ .../sqlite3/current_room_state_table.go | 276 ++++ syncapi/storage/sqlite3/filtering.go | 36 + syncapi/storage/sqlite3/invites_table.go | 157 +++ .../sqlite3/output_room_events_table.go | 411 ++++++ .../output_room_events_topology_table.go | 192 +++ syncapi/storage/sqlite3/stream_id_table.go | 58 + syncapi/storage/sqlite3/syncserver.go | 1197 +++++++++++++++++ syncapi/storage/storage.go | 3 + syncapi/sync/requestpool.go | 4 +- syncapi/syncapi.go | 4 +- 103 files changed, 9467 insertions(+), 710 deletions(-) rename clientapi/auth/storage/accounts/{ => postgres}/account_data_table.go (99%) rename clientapi/auth/storage/accounts/{ => postgres}/accounts_table.go (99%) rename clientapi/auth/storage/accounts/{ => postgres}/filter_table.go (99%) rename clientapi/auth/storage/accounts/{ => postgres}/membership_table.go (99%) rename clientapi/auth/storage/accounts/{ => postgres}/profile_table.go (99%) create mode 100644 clientapi/auth/storage/accounts/postgres/storage.go create mode 100644 clientapi/auth/storage/accounts/postgres/threepid_table.go create mode 100644 clientapi/auth/storage/accounts/sqlite3/account_data_table.go create mode 100644 clientapi/auth/storage/accounts/sqlite3/accounts_table.go create mode 100644 clientapi/auth/storage/accounts/sqlite3/filter_table.go create mode 100644 clientapi/auth/storage/accounts/sqlite3/membership_table.go create mode 100644 clientapi/auth/storage/accounts/sqlite3/profile_table.go create mode 100644 clientapi/auth/storage/accounts/sqlite3/storage.go rename clientapi/auth/storage/accounts/{ => sqlite3}/threepid_table.go (99%) rename clientapi/auth/storage/devices/{ => postgres}/devices_table.go (99%) create mode 100644 clientapi/auth/storage/devices/postgres/storage.go create mode 100644 clientapi/auth/storage/devices/sqlite3/devices_table.go create mode 100644 clientapi/auth/storage/devices/sqlite3/storage.go create mode 100644 common/keydb/sqlite3/keydb.go create mode 100644 common/keydb/sqlite3/server_key_table.go create mode 100644 federationsender/storage/sqlite3/joined_hosts_table.go create mode 100644 federationsender/storage/sqlite3/room_table.go create mode 100644 federationsender/storage/sqlite3/storage.go create mode 100644 publicroomsapi/storage/sqlite3/prepare.go create mode 100644 publicroomsapi/storage/sqlite3/public_rooms_table.go create mode 100644 publicroomsapi/storage/sqlite3/storage.go create mode 100644 roomserver/storage/sqlite3/event_json_table.go create mode 100644 roomserver/storage/sqlite3/event_state_keys_table.go create mode 100644 roomserver/storage/sqlite3/event_types_table.go create mode 100644 roomserver/storage/sqlite3/events_table.go create mode 100644 roomserver/storage/sqlite3/invite_table.go create mode 100644 roomserver/storage/sqlite3/list.go create mode 100644 roomserver/storage/sqlite3/membership_table.go create mode 100644 roomserver/storage/sqlite3/prepare.go create mode 100644 roomserver/storage/sqlite3/previous_events_table.go create mode 100644 roomserver/storage/sqlite3/room_aliases_table.go create mode 100644 roomserver/storage/sqlite3/rooms_table.go create mode 100644 roomserver/storage/sqlite3/sql.go create mode 100644 roomserver/storage/sqlite3/state_block_table.go create mode 100644 roomserver/storage/sqlite3/state_block_table_test.go create mode 100644 roomserver/storage/sqlite3/state_snapshot_table.go create mode 100644 roomserver/storage/sqlite3/storage.go create mode 100644 roomserver/storage/sqlite3/transactions_table.go create mode 100644 syncapi/storage/sqlite3/account_data_table.go create mode 100644 syncapi/storage/sqlite3/backward_extremities_table.go create mode 100644 syncapi/storage/sqlite3/current_room_state_table.go create mode 100644 syncapi/storage/sqlite3/filtering.go create mode 100644 syncapi/storage/sqlite3/invites_table.go create mode 100644 syncapi/storage/sqlite3/output_room_events_table.go create mode 100644 syncapi/storage/sqlite3/output_room_events_topology_table.go create mode 100644 syncapi/storage/sqlite3/stream_id_table.go create mode 100644 syncapi/storage/sqlite3/syncserver.go diff --git a/appservice/api/query.go b/appservice/api/query.go index 9542df565..7e61d6233 100644 --- a/appservice/api/query.go +++ b/appservice/api/query.go @@ -140,7 +140,7 @@ func RetrieveUserProfile( ctx context.Context, userID string, asAPI AppServiceQueryAPI, - accountDB *accounts.Database, + accountDB accounts.Database, ) (*authtypes.Profile, error) { localpart, _, err := gomatrixserverlib.SplitID('@', userID) if err != nil { diff --git a/appservice/appservice.go b/appservice/appservice.go index f2cbcce24..181799879 100644 --- a/appservice/appservice.go +++ b/appservice/appservice.go @@ -41,8 +41,8 @@ import ( // component. func SetupAppServiceAPIComponent( base *basecomponent.BaseDendrite, - accountsDB *accounts.Database, - deviceDB *devices.Database, + accountsDB accounts.Database, + deviceDB devices.Database, federation *gomatrixserverlib.FederationClient, roomserverAliasAPI roomserverAPI.RoomserverAliasAPI, roomserverQueryAPI roomserverAPI.RoomserverQueryAPI, @@ -111,8 +111,8 @@ func SetupAppServiceAPIComponent( // `sender_localpart` field of each application service if it doesn't // exist already func generateAppServiceAccount( - accountsDB *accounts.Database, - deviceDB *devices.Database, + accountsDB accounts.Database, + deviceDB devices.Database, as config.ApplicationService, ) error { ctx := context.Background() diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go index dbdae5320..b9a567954 100644 --- a/appservice/consumers/roomserver.go +++ b/appservice/consumers/roomserver.go @@ -33,7 +33,7 @@ import ( // OutputRoomEventConsumer consumes events that originated in the room server. type OutputRoomEventConsumer struct { roomServerConsumer *common.ContinualConsumer - db *accounts.Database + db accounts.Database asDB *storage.Database query api.RoomserverQueryAPI alias api.RoomserverAliasAPI @@ -46,7 +46,7 @@ type OutputRoomEventConsumer struct { func NewOutputRoomEventConsumer( cfg *config.Dendrite, kafkaConsumer sarama.Consumer, - store *accounts.Database, + store accounts.Database, appserviceDB *storage.Database, queryAPI api.RoomserverQueryAPI, aliasAPI api.RoomserverAliasAPI, diff --git a/appservice/routing/routing.go b/appservice/routing/routing.go index 8a24caad0..42fa80520 100644 --- a/appservice/routing/routing.go +++ b/appservice/routing/routing.go @@ -38,7 +38,7 @@ const pathPrefixApp = "/_matrix/app/v1" func Setup( apiMux *mux.Router, cfg *config.Dendrite, // nolint: unparam queryAPI api.RoomserverQueryAPI, aliasAPI api.RoomserverAliasAPI, // nolint: unparam - accountDB *accounts.Database, // nolint: unparam + accountDB accounts.Database, // nolint: unparam federation *gomatrixserverlib.FederationClient, // nolint: unparam transactionsCache *transactions.Cache, // nolint: unparam ) { diff --git a/clientapi/auth/storage/accounts/account_data_table.go b/clientapi/auth/storage/accounts/postgres/account_data_table.go similarity index 99% rename from clientapi/auth/storage/accounts/account_data_table.go rename to clientapi/auth/storage/accounts/postgres/account_data_table.go index 1b7484d81..d0cfcc0cf 100644 --- a/clientapi/auth/storage/accounts/account_data_table.go +++ b/clientapi/auth/storage/accounts/postgres/account_data_table.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package accounts +package postgres import ( "context" diff --git a/clientapi/auth/storage/accounts/accounts_table.go b/clientapi/auth/storage/accounts/postgres/accounts_table.go similarity index 99% rename from clientapi/auth/storage/accounts/accounts_table.go rename to clientapi/auth/storage/accounts/postgres/accounts_table.go index e86654eca..6b8ed3728 100644 --- a/clientapi/auth/storage/accounts/accounts_table.go +++ b/clientapi/auth/storage/accounts/postgres/accounts_table.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package accounts +package postgres import ( "context" diff --git a/clientapi/auth/storage/accounts/filter_table.go b/clientapi/auth/storage/accounts/postgres/filter_table.go similarity index 99% rename from clientapi/auth/storage/accounts/filter_table.go rename to clientapi/auth/storage/accounts/postgres/filter_table.go index 2b07ef17e..c54e4bc42 100644 --- a/clientapi/auth/storage/accounts/filter_table.go +++ b/clientapi/auth/storage/accounts/postgres/filter_table.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package accounts +package postgres import ( "context" diff --git a/clientapi/auth/storage/accounts/membership_table.go b/clientapi/auth/storage/accounts/postgres/membership_table.go similarity index 99% rename from clientapi/auth/storage/accounts/membership_table.go rename to clientapi/auth/storage/accounts/postgres/membership_table.go index 7b7c50ac6..426c2d6ac 100644 --- a/clientapi/auth/storage/accounts/membership_table.go +++ b/clientapi/auth/storage/accounts/postgres/membership_table.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package accounts +package postgres import ( "context" diff --git a/clientapi/auth/storage/accounts/profile_table.go b/clientapi/auth/storage/accounts/postgres/profile_table.go similarity index 99% rename from clientapi/auth/storage/accounts/profile_table.go rename to clientapi/auth/storage/accounts/postgres/profile_table.go index 157bb99b0..38c76c40f 100644 --- a/clientapi/auth/storage/accounts/profile_table.go +++ b/clientapi/auth/storage/accounts/postgres/profile_table.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package accounts +package postgres import ( "context" diff --git a/clientapi/auth/storage/accounts/postgres/storage.go b/clientapi/auth/storage/accounts/postgres/storage.go new file mode 100644 index 000000000..cb74d1315 --- /dev/null +++ b/clientapi/auth/storage/accounts/postgres/storage.go @@ -0,0 +1,392 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package postgres + +import ( + "context" + "database/sql" + "errors" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/gomatrixserverlib" + "golang.org/x/crypto/bcrypt" + + // Import the postgres database driver. + _ "github.com/lib/pq" +) + +// Database represents an account database +type Database struct { + db *sql.DB + common.PartitionOffsetStatements + accounts accountsStatements + profiles profilesStatements + memberships membershipStatements + accountDatas accountDataStatements + threepids threepidStatements + filter filterStatements + serverName gomatrixserverlib.ServerName +} + +// NewDatabase creates a new accounts and profiles database +func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) { + var db *sql.DB + var err error + if db, err = sql.Open("postgres", dataSourceName); err != nil { + return nil, err + } + partitions := common.PartitionOffsetStatements{} + if err = partitions.Prepare(db, "account"); err != nil { + return nil, err + } + a := accountsStatements{} + if err = a.prepare(db, serverName); err != nil { + return nil, err + } + p := profilesStatements{} + if err = p.prepare(db); err != nil { + return nil, err + } + m := membershipStatements{} + if err = m.prepare(db); err != nil { + return nil, err + } + ac := accountDataStatements{} + if err = ac.prepare(db); err != nil { + return nil, err + } + t := threepidStatements{} + if err = t.prepare(db); err != nil { + return nil, err + } + f := filterStatements{} + if err = f.prepare(db); err != nil { + return nil, err + } + return &Database{db, partitions, a, p, m, ac, t, f, serverName}, nil +} + +// GetAccountByPassword returns the account associated with the given localpart and password. +// Returns sql.ErrNoRows if no account exists which matches the given localpart. +func (d *Database) GetAccountByPassword( + ctx context.Context, localpart, plaintextPassword string, +) (*authtypes.Account, error) { + hash, err := d.accounts.selectPasswordHash(ctx, localpart) + if err != nil { + return nil, err + } + if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(plaintextPassword)); err != nil { + return nil, err + } + return d.accounts.selectAccountByLocalpart(ctx, localpart) +} + +// GetProfileByLocalpart returns the profile associated with the given localpart. +// Returns sql.ErrNoRows if no profile exists which matches the given localpart. +func (d *Database) GetProfileByLocalpart( + ctx context.Context, localpart string, +) (*authtypes.Profile, error) { + return d.profiles.selectProfileByLocalpart(ctx, localpart) +} + +// SetAvatarURL updates the avatar URL of the profile associated with the given +// localpart. Returns an error if something went wrong with the SQL query +func (d *Database) SetAvatarURL( + ctx context.Context, localpart string, avatarURL string, +) error { + return d.profiles.setAvatarURL(ctx, localpart, avatarURL) +} + +// SetDisplayName updates the display name of the profile associated with the given +// localpart. Returns an error if something went wrong with the SQL query +func (d *Database) SetDisplayName( + ctx context.Context, localpart string, displayName string, +) error { + return d.profiles.setDisplayName(ctx, localpart, displayName) +} + +// CreateAccount makes a new account with the given login name and password, and creates an empty profile +// for this account. If no password is supplied, the account will be a passwordless account. If the +// account already exists, it will return nil, nil. +func (d *Database) CreateAccount( + ctx context.Context, localpart, plaintextPassword, appserviceID string, +) (*authtypes.Account, error) { + var err error + + // Generate a password hash if this is not a password-less user + hash := "" + if plaintextPassword != "" { + hash, err = hashPassword(plaintextPassword) + if err != nil { + return nil, err + } + } + if err := d.profiles.insertProfile(ctx, localpart); err != nil { + if common.IsUniqueConstraintViolationErr(err) { + return nil, nil + } + return nil, err + } + if err := d.SaveAccountData(ctx, localpart, "", "m.push_rules", `{ + "global": { + "content": [], + "override": [], + "room": [], + "sender": [], + "underride": [] + } + }`); err != nil { + return nil, err + } + return d.accounts.insertAccount(ctx, localpart, hash, appserviceID) +} + +// SaveMembership saves the user matching a given localpart as a member of a given +// room. It also stores the ID of the membership event. +// If a membership already exists between the user and the room, or if the +// insert fails, returns the SQL error +func (d *Database) saveMembership( + ctx context.Context, txn *sql.Tx, localpart, roomID, eventID string, +) error { + return d.memberships.insertMembership(ctx, txn, localpart, roomID, eventID) +} + +// removeMembershipsByEventIDs removes the memberships corresponding to the +// `join` membership events IDs in the eventIDs slice. +// If the removal fails, or if there is no membership to remove, returns an error +func (d *Database) removeMembershipsByEventIDs( + ctx context.Context, txn *sql.Tx, eventIDs []string, +) error { + return d.memberships.deleteMembershipsByEventIDs(ctx, txn, eventIDs) +} + +// UpdateMemberships adds the "join" membership events included in a given state +// events array, and removes those which ID is included in a given array of events +// IDs. All of the process is run in a transaction, which commits only once/if every +// insertion and deletion has been successfully processed. +// Returns a SQL error if there was an issue with any part of the process +func (d *Database) UpdateMemberships( + ctx context.Context, eventsToAdd []gomatrixserverlib.Event, idsToRemove []string, +) error { + return common.WithTransaction(d.db, func(txn *sql.Tx) error { + if err := d.removeMembershipsByEventIDs(ctx, txn, idsToRemove); err != nil { + return err + } + + for _, event := range eventsToAdd { + if err := d.newMembership(ctx, txn, event); err != nil { + return err + } + } + + return nil + }) +} + +// GetMembershipInRoomByLocalpart returns the membership for an user +// matching the given localpart if he is a member of the room matching roomID, +// if not sql.ErrNoRows is returned. +// If there was an issue during the retrieval, returns the SQL error +func (d *Database) GetMembershipInRoomByLocalpart( + ctx context.Context, localpart, roomID string, +) (authtypes.Membership, error) { + return d.memberships.selectMembershipInRoomByLocalpart(ctx, localpart, roomID) +} + +// GetMembershipsByLocalpart returns an array containing the memberships for all +// the rooms a user matching a given localpart is a member of +// If no membership match the given localpart, returns an empty array +// If there was an issue during the retrieval, returns the SQL error +func (d *Database) GetMembershipsByLocalpart( + ctx context.Context, localpart string, +) (memberships []authtypes.Membership, err error) { + return d.memberships.selectMembershipsByLocalpart(ctx, localpart) +} + +// newMembership saves a new membership in the database. +// If the event isn't a valid m.room.member event with type `join`, does nothing. +// If an error occurred, returns the SQL error +func (d *Database) newMembership( + ctx context.Context, txn *sql.Tx, ev gomatrixserverlib.Event, +) error { + if ev.Type() == "m.room.member" && ev.StateKey() != nil { + localpart, serverName, err := gomatrixserverlib.SplitID('@', *ev.StateKey()) + if err != nil { + return err + } + + // We only want state events from local users + if string(serverName) != string(d.serverName) { + return nil + } + + eventID := ev.EventID() + roomID := ev.RoomID() + membership, err := ev.Membership() + if err != nil { + return err + } + + // Only "join" membership events can be considered as new memberships + if membership == gomatrixserverlib.Join { + if err := d.saveMembership(ctx, txn, localpart, roomID, eventID); err != nil { + return err + } + } + } + return nil +} + +// SaveAccountData saves new account data for a given user and a given room. +// If the account data is not specific to a room, the room ID should be an empty string +// If an account data already exists for a given set (user, room, data type), it will +// update the corresponding row with the new content +// Returns a SQL error if there was an issue with the insertion/update +func (d *Database) SaveAccountData( + ctx context.Context, localpart, roomID, dataType, content string, +) error { + return d.accountDatas.insertAccountData(ctx, localpart, roomID, dataType, content) +} + +// GetAccountData returns account data related to a given localpart +// If no account data could be found, returns an empty arrays +// Returns an error if there was an issue with the retrieval +func (d *Database) GetAccountData(ctx context.Context, localpart string) ( + global []gomatrixserverlib.ClientEvent, + rooms map[string][]gomatrixserverlib.ClientEvent, + err error, +) { + return d.accountDatas.selectAccountData(ctx, localpart) +} + +// GetAccountDataByType returns account data matching a given +// localpart, room ID and type. +// If no account data could be found, returns nil +// Returns an error if there was an issue with the retrieval +func (d *Database) GetAccountDataByType( + ctx context.Context, localpart, roomID, dataType string, +) (data *gomatrixserverlib.ClientEvent, err error) { + return d.accountDatas.selectAccountDataByType( + ctx, localpart, roomID, dataType, + ) +} + +// GetNewNumericLocalpart generates and returns a new unused numeric localpart +func (d *Database) GetNewNumericLocalpart( + ctx context.Context, +) (int64, error) { + return d.accounts.selectNewNumericLocalpart(ctx) +} + +func hashPassword(plaintext string) (hash string, err error) { + hashBytes, err := bcrypt.GenerateFromPassword([]byte(plaintext), bcrypt.DefaultCost) + return string(hashBytes), err +} + +// Err3PIDInUse is the error returned when trying to save an association involving +// a third-party identifier which is already associated to a local user. +var Err3PIDInUse = errors.New("This third-party identifier is already in use") + +// SaveThreePIDAssociation saves the association between a third party identifier +// and a local Matrix user (identified by the user's ID's local part). +// If the third-party identifier is already part of an association, returns Err3PIDInUse. +// Returns an error if there was a problem talking to the database. +func (d *Database) SaveThreePIDAssociation( + ctx context.Context, threepid, localpart, medium string, +) (err error) { + return common.WithTransaction(d.db, func(txn *sql.Tx) error { + user, err := d.threepids.selectLocalpartForThreePID( + ctx, txn, threepid, medium, + ) + if err != nil { + return err + } + + if len(user) > 0 { + return Err3PIDInUse + } + + return d.threepids.insertThreePID(ctx, txn, threepid, medium, localpart) + }) +} + +// RemoveThreePIDAssociation removes the association involving a given third-party +// identifier. +// If no association exists involving this third-party identifier, returns nothing. +// If there was a problem talking to the database, returns an error. +func (d *Database) RemoveThreePIDAssociation( + ctx context.Context, threepid string, medium string, +) (err error) { + return d.threepids.deleteThreePID(ctx, threepid, medium) +} + +// GetLocalpartForThreePID looks up the localpart associated with a given third-party +// identifier. +// If no association involves the given third-party idenfitier, returns an empty +// string. +// Returns an error if there was a problem talking to the database. +func (d *Database) GetLocalpartForThreePID( + ctx context.Context, threepid string, medium string, +) (localpart string, err error) { + return d.threepids.selectLocalpartForThreePID(ctx, nil, threepid, medium) +} + +// GetThreePIDsForLocalpart looks up the third-party identifiers associated with +// a given local user. +// If no association is known for this user, returns an empty slice. +// Returns an error if there was an issue talking to the database. +func (d *Database) GetThreePIDsForLocalpart( + ctx context.Context, localpart string, +) (threepids []authtypes.ThreePID, err error) { + return d.threepids.selectThreePIDsForLocalpart(ctx, localpart) +} + +// GetFilter looks up the filter associated with a given local user and filter ID. +// Returns a filter structure. Otherwise returns an error if no such filter exists +// or if there was an error talking to the database. +func (d *Database) GetFilter( + ctx context.Context, localpart string, filterID string, +) (*gomatrixserverlib.Filter, error) { + return d.filter.selectFilter(ctx, localpart, filterID) +} + +// PutFilter puts the passed filter into the database. +// Returns the filterID as a string. Otherwise returns an error if something +// goes wrong. +func (d *Database) PutFilter( + ctx context.Context, localpart string, filter *gomatrixserverlib.Filter, +) (string, error) { + return d.filter.insertFilter(ctx, filter, localpart) +} + +// CheckAccountAvailability checks if the username/localpart is already present +// in the database. +// If the DB returns sql.ErrNoRows the Localpart isn't taken. +func (d *Database) CheckAccountAvailability(ctx context.Context, localpart string) (bool, error) { + _, err := d.accounts.selectAccountByLocalpart(ctx, localpart) + if err == sql.ErrNoRows { + return true, nil + } + return false, err +} + +// GetAccountByLocalpart returns the account associated with the given localpart. +// This function assumes the request is authenticated or the account data is used only internally. +// Returns sql.ErrNoRows if no account exists which matches the given localpart. +func (d *Database) GetAccountByLocalpart(ctx context.Context, localpart string, +) (*authtypes.Account, error) { + return d.accounts.selectAccountByLocalpart(ctx, localpart) +} diff --git a/clientapi/auth/storage/accounts/postgres/threepid_table.go b/clientapi/auth/storage/accounts/postgres/threepid_table.go new file mode 100644 index 000000000..851b4a90b --- /dev/null +++ b/clientapi/auth/storage/accounts/postgres/threepid_table.go @@ -0,0 +1,129 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package postgres + +import ( + "context" + "database/sql" + + "github.com/matrix-org/dendrite/common" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" +) + +const threepidSchema = ` +-- Stores data about third party identifiers +CREATE TABLE IF NOT EXISTS account_threepid ( + -- The third party identifier + threepid TEXT NOT NULL, + -- The 3PID medium + medium TEXT NOT NULL DEFAULT 'email', + -- The localpart of the Matrix user ID associated to this 3PID + localpart TEXT NOT NULL, + + PRIMARY KEY(threepid, medium) +); + +CREATE INDEX IF NOT EXISTS account_threepid_localpart ON account_threepid(localpart); +` + +const selectLocalpartForThreePIDSQL = "" + + "SELECT localpart FROM account_threepid WHERE threepid = $1 AND medium = $2" + +const selectThreePIDsForLocalpartSQL = "" + + "SELECT threepid, medium FROM account_threepid WHERE localpart = $1" + +const insertThreePIDSQL = "" + + "INSERT INTO account_threepid (threepid, medium, localpart) VALUES ($1, $2, $3)" + +const deleteThreePIDSQL = "" + + "DELETE FROM account_threepid WHERE threepid = $1 AND medium = $2" + +type threepidStatements struct { + selectLocalpartForThreePIDStmt *sql.Stmt + selectThreePIDsForLocalpartStmt *sql.Stmt + insertThreePIDStmt *sql.Stmt + deleteThreePIDStmt *sql.Stmt +} + +func (s *threepidStatements) prepare(db *sql.DB) (err error) { + _, err = db.Exec(threepidSchema) + if err != nil { + return + } + if s.selectLocalpartForThreePIDStmt, err = db.Prepare(selectLocalpartForThreePIDSQL); err != nil { + return + } + if s.selectThreePIDsForLocalpartStmt, err = db.Prepare(selectThreePIDsForLocalpartSQL); err != nil { + return + } + if s.insertThreePIDStmt, err = db.Prepare(insertThreePIDSQL); err != nil { + return + } + if s.deleteThreePIDStmt, err = db.Prepare(deleteThreePIDSQL); err != nil { + return + } + + return +} + +func (s *threepidStatements) selectLocalpartForThreePID( + ctx context.Context, txn *sql.Tx, threepid string, medium string, +) (localpart string, err error) { + stmt := common.TxStmt(txn, s.selectLocalpartForThreePIDStmt) + err = stmt.QueryRowContext(ctx, threepid, medium).Scan(&localpart) + if err == sql.ErrNoRows { + return "", nil + } + return +} + +func (s *threepidStatements) selectThreePIDsForLocalpart( + ctx context.Context, localpart string, +) (threepids []authtypes.ThreePID, err error) { + rows, err := s.selectThreePIDsForLocalpartStmt.QueryContext(ctx, localpart) + if err != nil { + return + } + + threepids = []authtypes.ThreePID{} + for rows.Next() { + var threepid string + var medium string + if err = rows.Scan(&threepid, &medium); err != nil { + return + } + threepids = append(threepids, authtypes.ThreePID{ + Address: threepid, + Medium: medium, + }) + } + + return +} + +func (s *threepidStatements) insertThreePID( + ctx context.Context, txn *sql.Tx, threepid, medium, localpart string, +) (err error) { + stmt := common.TxStmt(txn, s.insertThreePIDStmt) + _, err = stmt.ExecContext(ctx, threepid, medium, localpart) + return +} + +func (s *threepidStatements) deleteThreePID( + ctx context.Context, threepid string, medium string) (err error) { + _, err = s.deleteThreePIDStmt.ExecContext(ctx, threepid, medium) + return +} diff --git a/clientapi/auth/storage/accounts/sqlite3/account_data_table.go b/clientapi/auth/storage/accounts/sqlite3/account_data_table.go new file mode 100644 index 000000000..c2143881b --- /dev/null +++ b/clientapi/auth/storage/accounts/sqlite3/account_data_table.go @@ -0,0 +1,141 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + + "github.com/matrix-org/gomatrixserverlib" +) + +const accountDataSchema = ` +-- Stores data about accounts data. +CREATE TABLE IF NOT EXISTS account_data ( + -- The Matrix user ID localpart for this account + localpart TEXT NOT NULL, + -- The room ID for this data (empty string if not specific to a room) + room_id TEXT, + -- The account data type + type TEXT NOT NULL, + -- The account data content + content TEXT NOT NULL, + + PRIMARY KEY(localpart, room_id, type) +); +` + +const insertAccountDataSQL = ` + INSERT INTO account_data(localpart, room_id, type, content) VALUES($1, $2, $3, $4) + ON CONFLICT (localpart, room_id, type) DO UPDATE SET content = $4 +` + +const selectAccountDataSQL = "" + + "SELECT room_id, type, content FROM account_data WHERE localpart = $1" + +const selectAccountDataByTypeSQL = "" + + "SELECT content FROM account_data WHERE localpart = $1 AND room_id = $2 AND type = $3" + +type accountDataStatements struct { + insertAccountDataStmt *sql.Stmt + selectAccountDataStmt *sql.Stmt + selectAccountDataByTypeStmt *sql.Stmt +} + +func (s *accountDataStatements) prepare(db *sql.DB) (err error) { + _, err = db.Exec(accountDataSchema) + if err != nil { + return + } + if s.insertAccountDataStmt, err = db.Prepare(insertAccountDataSQL); err != nil { + return + } + if s.selectAccountDataStmt, err = db.Prepare(selectAccountDataSQL); err != nil { + return + } + if s.selectAccountDataByTypeStmt, err = db.Prepare(selectAccountDataByTypeSQL); err != nil { + return + } + return +} + +func (s *accountDataStatements) insertAccountData( + ctx context.Context, localpart, roomID, dataType, content string, +) (err error) { + stmt := s.insertAccountDataStmt + _, err = stmt.ExecContext(ctx, localpart, roomID, dataType, content) + return +} + +func (s *accountDataStatements) selectAccountData( + ctx context.Context, localpart string, +) ( + global []gomatrixserverlib.ClientEvent, + rooms map[string][]gomatrixserverlib.ClientEvent, + err error, +) { + rows, err := s.selectAccountDataStmt.QueryContext(ctx, localpart) + if err != nil { + return + } + + global = []gomatrixserverlib.ClientEvent{} + rooms = make(map[string][]gomatrixserverlib.ClientEvent) + + for rows.Next() { + var roomID string + var dataType string + var content []byte + + if err = rows.Scan(&roomID, &dataType, &content); err != nil { + return + } + + ac := gomatrixserverlib.ClientEvent{ + Type: dataType, + Content: content, + } + + if len(roomID) > 0 { + rooms[roomID] = append(rooms[roomID], ac) + } else { + global = append(global, ac) + } + } + + return +} + +func (s *accountDataStatements) selectAccountDataByType( + ctx context.Context, localpart, roomID, dataType string, +) (data *gomatrixserverlib.ClientEvent, err error) { + stmt := s.selectAccountDataByTypeStmt + var content []byte + + if err = stmt.QueryRowContext(ctx, localpart, roomID, dataType).Scan(&content); err != nil { + if err == sql.ErrNoRows { + return nil, nil + } + + return + } + + data = &gomatrixserverlib.ClientEvent{ + Type: dataType, + Content: content, + } + + return +} diff --git a/clientapi/auth/storage/accounts/sqlite3/accounts_table.go b/clientapi/auth/storage/accounts/sqlite3/accounts_table.go new file mode 100644 index 000000000..b029951f1 --- /dev/null +++ b/clientapi/auth/storage/accounts/sqlite3/accounts_table.go @@ -0,0 +1,151 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + "time" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/clientapi/userutil" + "github.com/matrix-org/gomatrixserverlib" + + log "github.com/sirupsen/logrus" +) + +const accountsSchema = ` +-- Stores data about accounts. +CREATE TABLE IF NOT EXISTS account_accounts ( + -- The Matrix user ID localpart for this account + localpart TEXT NOT NULL PRIMARY KEY, + -- When this account was first created, as a unix timestamp (ms resolution). + created_ts BIGINT NOT NULL, + -- The password hash for this account. Can be NULL if this is a passwordless account. + password_hash TEXT, + -- Identifies which application service this account belongs to, if any. + appservice_id TEXT + -- TODO: + -- is_guest, is_admin, upgraded_ts, devices, any email reset stuff? +); +` + +const insertAccountSQL = "" + + "INSERT INTO account_accounts(localpart, created_ts, password_hash, appservice_id) VALUES ($1, $2, $3, $4)" + +const selectAccountByLocalpartSQL = "" + + "SELECT localpart, appservice_id FROM account_accounts WHERE localpart = $1" + +const selectPasswordHashSQL = "" + + "SELECT password_hash FROM account_accounts WHERE localpart = $1" + +const selectNewNumericLocalpartSQL = "" + + "SELECT COUNT(localpart) FROM account_accounts" + +// TODO: Update password + +type accountsStatements struct { + insertAccountStmt *sql.Stmt + selectAccountByLocalpartStmt *sql.Stmt + selectPasswordHashStmt *sql.Stmt + selectNewNumericLocalpartStmt *sql.Stmt + serverName gomatrixserverlib.ServerName +} + +func (s *accountsStatements) prepare(db *sql.DB, server gomatrixserverlib.ServerName) (err error) { + _, err = db.Exec(accountsSchema) + if err != nil { + return + } + if s.insertAccountStmt, err = db.Prepare(insertAccountSQL); err != nil { + return + } + if s.selectAccountByLocalpartStmt, err = db.Prepare(selectAccountByLocalpartSQL); err != nil { + return + } + if s.selectPasswordHashStmt, err = db.Prepare(selectPasswordHashSQL); err != nil { + return + } + if s.selectNewNumericLocalpartStmt, err = db.Prepare(selectNewNumericLocalpartSQL); err != nil { + return + } + s.serverName = server + return +} + +// insertAccount creates a new account. 'hash' should be the password hash for this account. If it is missing, +// this account will be passwordless. Returns an error if this account already exists. Returns the account +// on success. +func (s *accountsStatements) insertAccount( + ctx context.Context, localpart, hash, appserviceID string, +) (*authtypes.Account, error) { + createdTimeMS := time.Now().UnixNano() / 1000000 + stmt := s.insertAccountStmt + + var err error + if appserviceID == "" { + _, err = stmt.ExecContext(ctx, localpart, createdTimeMS, hash, nil) + } else { + _, err = stmt.ExecContext(ctx, localpart, createdTimeMS, hash, appserviceID) + } + if err != nil { + return nil, err + } + + return &authtypes.Account{ + Localpart: localpart, + UserID: userutil.MakeUserID(localpart, s.serverName), + ServerName: s.serverName, + AppServiceID: appserviceID, + }, nil +} + +func (s *accountsStatements) selectPasswordHash( + ctx context.Context, localpart string, +) (hash string, err error) { + err = s.selectPasswordHashStmt.QueryRowContext(ctx, localpart).Scan(&hash) + return +} + +func (s *accountsStatements) selectAccountByLocalpart( + ctx context.Context, localpart string, +) (*authtypes.Account, error) { + var appserviceIDPtr sql.NullString + var acc authtypes.Account + + stmt := s.selectAccountByLocalpartStmt + err := stmt.QueryRowContext(ctx, localpart).Scan(&acc.Localpart, &appserviceIDPtr) + if err != nil { + if err != sql.ErrNoRows { + log.WithError(err).Error("Unable to retrieve user from the db") + } + return nil, err + } + if appserviceIDPtr.Valid { + acc.AppServiceID = appserviceIDPtr.String + } + + acc.UserID = userutil.MakeUserID(localpart, s.serverName) + acc.ServerName = s.serverName + + return &acc, nil +} + +func (s *accountsStatements) selectNewNumericLocalpart( + ctx context.Context, +) (id int64, err error) { + err = s.selectNewNumericLocalpartStmt.QueryRowContext(ctx).Scan(&id) + return +} diff --git a/clientapi/auth/storage/accounts/sqlite3/filter_table.go b/clientapi/auth/storage/accounts/sqlite3/filter_table.go new file mode 100644 index 000000000..691ead775 --- /dev/null +++ b/clientapi/auth/storage/accounts/sqlite3/filter_table.go @@ -0,0 +1,139 @@ +// Copyright 2017 Jan Christian Grünhage +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + "encoding/json" + + "github.com/matrix-org/gomatrixserverlib" +) + +const filterSchema = ` +-- Stores data about filters +CREATE TABLE IF NOT EXISTS account_filter ( + -- The filter + filter TEXT NOT NULL, + -- The ID + id INTEGER PRIMARY KEY AUTOINCREMENT, + -- The localpart of the Matrix user ID associated to this filter + localpart TEXT NOT NULL, + + UNIQUE (id, localpart) +); + +CREATE INDEX IF NOT EXISTS account_filter_localpart ON account_filter(localpart); +` + +const selectFilterSQL = "" + + "SELECT filter FROM account_filter WHERE localpart = $1 AND id = $2" + +const selectFilterIDByContentSQL = "" + + "SELECT id FROM account_filter WHERE localpart = $1 AND filter = $2" + +const insertFilterSQL = "" + + "INSERT INTO account_filter (filter, localpart) VALUES ($1, $2)" + +const selectLastInsertedFilterIDSQL = "" + + "SELECT id FROM account_filter WHERE rowid = last_insert_rowid()" + +type filterStatements struct { + selectFilterStmt *sql.Stmt + selectLastInsertedFilterIDStmt *sql.Stmt + selectFilterIDByContentStmt *sql.Stmt + insertFilterStmt *sql.Stmt +} + +func (s *filterStatements) prepare(db *sql.DB) (err error) { + _, err = db.Exec(filterSchema) + if err != nil { + return + } + if s.selectFilterStmt, err = db.Prepare(selectFilterSQL); err != nil { + return + } + if s.selectLastInsertedFilterIDStmt, err = db.Prepare(selectLastInsertedFilterIDSQL); err != nil { + return + } + if s.selectFilterIDByContentStmt, err = db.Prepare(selectFilterIDByContentSQL); err != nil { + return + } + if s.insertFilterStmt, err = db.Prepare(insertFilterSQL); err != nil { + return + } + return +} + +func (s *filterStatements) selectFilter( + ctx context.Context, localpart string, filterID string, +) (*gomatrixserverlib.Filter, error) { + // Retrieve filter from database (stored as canonical JSON) + var filterData []byte + err := s.selectFilterStmt.QueryRowContext(ctx, localpart, filterID).Scan(&filterData) + if err != nil { + return nil, err + } + + // Unmarshal JSON into Filter struct + var filter gomatrixserverlib.Filter + if err = json.Unmarshal(filterData, &filter); err != nil { + return nil, err + } + return &filter, nil +} + +func (s *filterStatements) insertFilter( + ctx context.Context, filter *gomatrixserverlib.Filter, localpart string, +) (filterID string, err error) { + var existingFilterID string + + // Serialise json + filterJSON, err := json.Marshal(filter) + if err != nil { + return "", err + } + // Remove whitespaces and sort JSON data + // needed to prevent from inserting the same filter multiple times + filterJSON, err = gomatrixserverlib.CanonicalJSON(filterJSON) + if err != nil { + return "", err + } + + // Check if filter already exists in the database using its localpart and content + // + // This can result in a race condition when two clients try to insert the + // same filter and localpart at the same time, however this is not a + // problem as both calls will result in the same filterID + err = s.selectFilterIDByContentStmt.QueryRowContext(ctx, + localpart, filterJSON).Scan(&existingFilterID) + if err != nil && err != sql.ErrNoRows { + return "", err + } + // If it does, return the existing ID + if existingFilterID != "" { + return existingFilterID, err + } + + // Otherwise insert the filter and return the new ID + if _, err = s.insertFilterStmt.ExecContext(ctx, filterJSON, localpart); err != nil { + return "", err + } + row := s.selectLastInsertedFilterIDStmt.QueryRowContext(ctx) + if err := row.Scan(&filterID); err != nil { + return "", err + } + return +} diff --git a/clientapi/auth/storage/accounts/sqlite3/membership_table.go b/clientapi/auth/storage/accounts/sqlite3/membership_table.go new file mode 100644 index 000000000..8e5e69bad --- /dev/null +++ b/clientapi/auth/storage/accounts/sqlite3/membership_table.go @@ -0,0 +1,131 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + + "github.com/lib/pq" + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" +) + +const membershipSchema = ` +-- Stores data about users memberships to rooms. +CREATE TABLE IF NOT EXISTS account_memberships ( + -- The Matrix user ID localpart for the member + localpart TEXT NOT NULL, + -- The room this user is a member of + room_id TEXT NOT NULL, + -- The ID of the join membership event + event_id TEXT NOT NULL, + + -- A user can only be member of a room once + PRIMARY KEY (localpart, room_id), + + UNIQUE (event_id) +); +` + +const insertMembershipSQL = ` + INSERT INTO account_memberships(localpart, room_id, event_id) VALUES ($1, $2, $3) + ON CONFLICT (localpart, room_id) DO UPDATE SET event_id = EXCLUDED.event_id +` + +const selectMembershipsByLocalpartSQL = "" + + "SELECT room_id, event_id FROM account_memberships WHERE localpart = $1" + +const selectMembershipInRoomByLocalpartSQL = "" + + "SELECT event_id FROM account_memberships WHERE localpart = $1 AND room_id = $2" + +const deleteMembershipsByEventIDsSQL = "" + + "DELETE FROM account_memberships WHERE event_id IN ($1)" + +type membershipStatements struct { + deleteMembershipsByEventIDsStmt *sql.Stmt + insertMembershipStmt *sql.Stmt + selectMembershipInRoomByLocalpartStmt *sql.Stmt + selectMembershipsByLocalpartStmt *sql.Stmt +} + +func (s *membershipStatements) prepare(db *sql.DB) (err error) { + _, err = db.Exec(membershipSchema) + if err != nil { + return + } + if s.deleteMembershipsByEventIDsStmt, err = db.Prepare(deleteMembershipsByEventIDsSQL); err != nil { + return + } + if s.insertMembershipStmt, err = db.Prepare(insertMembershipSQL); err != nil { + return + } + if s.selectMembershipInRoomByLocalpartStmt, err = db.Prepare(selectMembershipInRoomByLocalpartSQL); err != nil { + return + } + if s.selectMembershipsByLocalpartStmt, err = db.Prepare(selectMembershipsByLocalpartSQL); err != nil { + return + } + return +} + +func (s *membershipStatements) insertMembership( + ctx context.Context, txn *sql.Tx, localpart, roomID, eventID string, +) (err error) { + stmt := txn.Stmt(s.insertMembershipStmt) + _, err = stmt.ExecContext(ctx, localpart, roomID, eventID) + return +} + +func (s *membershipStatements) deleteMembershipsByEventIDs( + ctx context.Context, txn *sql.Tx, eventIDs []string, +) (err error) { + stmt := txn.Stmt(s.deleteMembershipsByEventIDsStmt) + _, err = stmt.ExecContext(ctx, pq.StringArray(eventIDs)) + return +} + +func (s *membershipStatements) selectMembershipInRoomByLocalpart( + ctx context.Context, localpart, roomID string, +) (authtypes.Membership, error) { + membership := authtypes.Membership{Localpart: localpart, RoomID: roomID} + stmt := s.selectMembershipInRoomByLocalpartStmt + err := stmt.QueryRowContext(ctx, localpart, roomID).Scan(&membership.EventID) + + return membership, err +} + +func (s *membershipStatements) selectMembershipsByLocalpart( + ctx context.Context, localpart string, +) (memberships []authtypes.Membership, err error) { + stmt := s.selectMembershipsByLocalpartStmt + rows, err := stmt.QueryContext(ctx, localpart) + if err != nil { + return + } + + memberships = []authtypes.Membership{} + + defer rows.Close() // nolint: errcheck + for rows.Next() { + var m authtypes.Membership + m.Localpart = localpart + if err := rows.Scan(&m.RoomID, &m.EventID); err != nil { + return nil, err + } + memberships = append(memberships, m) + } + + return +} diff --git a/clientapi/auth/storage/accounts/sqlite3/profile_table.go b/clientapi/auth/storage/accounts/sqlite3/profile_table.go new file mode 100644 index 000000000..7af8307e1 --- /dev/null +++ b/clientapi/auth/storage/accounts/sqlite3/profile_table.go @@ -0,0 +1,107 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" +) + +const profilesSchema = ` +-- Stores data about accounts profiles. +CREATE TABLE IF NOT EXISTS account_profiles ( + -- The Matrix user ID localpart for this account + localpart TEXT NOT NULL PRIMARY KEY, + -- The display name for this account + display_name TEXT, + -- The URL of the avatar for this account + avatar_url TEXT +); +` + +const insertProfileSQL = "" + + "INSERT INTO account_profiles(localpart, display_name, avatar_url) VALUES ($1, $2, $3)" + +const selectProfileByLocalpartSQL = "" + + "SELECT localpart, display_name, avatar_url FROM account_profiles WHERE localpart = $1" + +const setAvatarURLSQL = "" + + "UPDATE account_profiles SET avatar_url = $1 WHERE localpart = $2" + +const setDisplayNameSQL = "" + + "UPDATE account_profiles SET display_name = $1 WHERE localpart = $2" + +type profilesStatements struct { + insertProfileStmt *sql.Stmt + selectProfileByLocalpartStmt *sql.Stmt + setAvatarURLStmt *sql.Stmt + setDisplayNameStmt *sql.Stmt +} + +func (s *profilesStatements) prepare(db *sql.DB) (err error) { + _, err = db.Exec(profilesSchema) + if err != nil { + return + } + if s.insertProfileStmt, err = db.Prepare(insertProfileSQL); err != nil { + return + } + if s.selectProfileByLocalpartStmt, err = db.Prepare(selectProfileByLocalpartSQL); err != nil { + return + } + if s.setAvatarURLStmt, err = db.Prepare(setAvatarURLSQL); err != nil { + return + } + if s.setDisplayNameStmt, err = db.Prepare(setDisplayNameSQL); err != nil { + return + } + return +} + +func (s *profilesStatements) insertProfile( + ctx context.Context, localpart string, +) (err error) { + _, err = s.insertProfileStmt.ExecContext(ctx, localpart, "", "") + return +} + +func (s *profilesStatements) selectProfileByLocalpart( + ctx context.Context, localpart string, +) (*authtypes.Profile, error) { + var profile authtypes.Profile + err := s.selectProfileByLocalpartStmt.QueryRowContext(ctx, localpart).Scan( + &profile.Localpart, &profile.DisplayName, &profile.AvatarURL, + ) + if err != nil { + return nil, err + } + return &profile, nil +} + +func (s *profilesStatements) setAvatarURL( + ctx context.Context, localpart string, avatarURL string, +) (err error) { + _, err = s.setAvatarURLStmt.ExecContext(ctx, avatarURL, localpart) + return +} + +func (s *profilesStatements) setDisplayName( + ctx context.Context, localpart string, displayName string, +) (err error) { + _, err = s.setDisplayNameStmt.ExecContext(ctx, displayName, localpart) + return +} diff --git a/clientapi/auth/storage/accounts/sqlite3/storage.go b/clientapi/auth/storage/accounts/sqlite3/storage.go new file mode 100644 index 000000000..199c4606e --- /dev/null +++ b/clientapi/auth/storage/accounts/sqlite3/storage.go @@ -0,0 +1,392 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + "errors" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/gomatrixserverlib" + "golang.org/x/crypto/bcrypt" + + // Import the postgres database driver. + _ "github.com/mattn/go-sqlite3" +) + +// Database represents an account database +type Database struct { + db *sql.DB + common.PartitionOffsetStatements + accounts accountsStatements + profiles profilesStatements + memberships membershipStatements + accountDatas accountDataStatements + threepids threepidStatements + filter filterStatements + serverName gomatrixserverlib.ServerName +} + +// NewDatabase creates a new accounts and profiles database +func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) { + var db *sql.DB + var err error + if db, err = sql.Open("sqlite3", dataSourceName); err != nil { + return nil, err + } + partitions := common.PartitionOffsetStatements{} + if err = partitions.Prepare(db, "account"); err != nil { + return nil, err + } + a := accountsStatements{} + if err = a.prepare(db, serverName); err != nil { + return nil, err + } + p := profilesStatements{} + if err = p.prepare(db); err != nil { + return nil, err + } + m := membershipStatements{} + if err = m.prepare(db); err != nil { + return nil, err + } + ac := accountDataStatements{} + if err = ac.prepare(db); err != nil { + return nil, err + } + t := threepidStatements{} + if err = t.prepare(db); err != nil { + return nil, err + } + f := filterStatements{} + if err = f.prepare(db); err != nil { + return nil, err + } + return &Database{db, partitions, a, p, m, ac, t, f, serverName}, nil +} + +// GetAccountByPassword returns the account associated with the given localpart and password. +// Returns sql.ErrNoRows if no account exists which matches the given localpart. +func (d *Database) GetAccountByPassword( + ctx context.Context, localpart, plaintextPassword string, +) (*authtypes.Account, error) { + hash, err := d.accounts.selectPasswordHash(ctx, localpart) + if err != nil { + return nil, err + } + if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(plaintextPassword)); err != nil { + return nil, err + } + return d.accounts.selectAccountByLocalpart(ctx, localpart) +} + +// GetProfileByLocalpart returns the profile associated with the given localpart. +// Returns sql.ErrNoRows if no profile exists which matches the given localpart. +func (d *Database) GetProfileByLocalpart( + ctx context.Context, localpart string, +) (*authtypes.Profile, error) { + return d.profiles.selectProfileByLocalpart(ctx, localpart) +} + +// SetAvatarURL updates the avatar URL of the profile associated with the given +// localpart. Returns an error if something went wrong with the SQL query +func (d *Database) SetAvatarURL( + ctx context.Context, localpart string, avatarURL string, +) error { + return d.profiles.setAvatarURL(ctx, localpart, avatarURL) +} + +// SetDisplayName updates the display name of the profile associated with the given +// localpart. Returns an error if something went wrong with the SQL query +func (d *Database) SetDisplayName( + ctx context.Context, localpart string, displayName string, +) error { + return d.profiles.setDisplayName(ctx, localpart, displayName) +} + +// CreateAccount makes a new account with the given login name and password, and creates an empty profile +// for this account. If no password is supplied, the account will be a passwordless account. If the +// account already exists, it will return nil, nil. +func (d *Database) CreateAccount( + ctx context.Context, localpart, plaintextPassword, appserviceID string, +) (*authtypes.Account, error) { + var err error + + // Generate a password hash if this is not a password-less user + hash := "" + if plaintextPassword != "" { + hash, err = hashPassword(plaintextPassword) + if err != nil { + return nil, err + } + } + if err := d.profiles.insertProfile(ctx, localpart); err != nil { + if common.IsUniqueConstraintViolationErr(err) { + return nil, nil + } + return nil, err + } + if err := d.SaveAccountData(ctx, localpart, "", "m.push_rules", `{ + "global": { + "content": [], + "override": [], + "room": [], + "sender": [], + "underride": [] + } + }`); err != nil { + return nil, err + } + return d.accounts.insertAccount(ctx, localpart, hash, appserviceID) +} + +// SaveMembership saves the user matching a given localpart as a member of a given +// room. It also stores the ID of the membership event. +// If a membership already exists between the user and the room, or if the +// insert fails, returns the SQL error +func (d *Database) saveMembership( + ctx context.Context, txn *sql.Tx, localpart, roomID, eventID string, +) error { + return d.memberships.insertMembership(ctx, txn, localpart, roomID, eventID) +} + +// removeMembershipsByEventIDs removes the memberships corresponding to the +// `join` membership events IDs in the eventIDs slice. +// If the removal fails, or if there is no membership to remove, returns an error +func (d *Database) removeMembershipsByEventIDs( + ctx context.Context, txn *sql.Tx, eventIDs []string, +) error { + return d.memberships.deleteMembershipsByEventIDs(ctx, txn, eventIDs) +} + +// UpdateMemberships adds the "join" membership events included in a given state +// events array, and removes those which ID is included in a given array of events +// IDs. All of the process is run in a transaction, which commits only once/if every +// insertion and deletion has been successfully processed. +// Returns a SQL error if there was an issue with any part of the process +func (d *Database) UpdateMemberships( + ctx context.Context, eventsToAdd []gomatrixserverlib.Event, idsToRemove []string, +) error { + return common.WithTransaction(d.db, func(txn *sql.Tx) error { + if err := d.removeMembershipsByEventIDs(ctx, txn, idsToRemove); err != nil { + return err + } + + for _, event := range eventsToAdd { + if err := d.newMembership(ctx, txn, event); err != nil { + return err + } + } + + return nil + }) +} + +// GetMembershipInRoomByLocalpart returns the membership for an user +// matching the given localpart if he is a member of the room matching roomID, +// if not sql.ErrNoRows is returned. +// If there was an issue during the retrieval, returns the SQL error +func (d *Database) GetMembershipInRoomByLocalpart( + ctx context.Context, localpart, roomID string, +) (authtypes.Membership, error) { + return d.memberships.selectMembershipInRoomByLocalpart(ctx, localpart, roomID) +} + +// GetMembershipsByLocalpart returns an array containing the memberships for all +// the rooms a user matching a given localpart is a member of +// If no membership match the given localpart, returns an empty array +// If there was an issue during the retrieval, returns the SQL error +func (d *Database) GetMembershipsByLocalpart( + ctx context.Context, localpart string, +) (memberships []authtypes.Membership, err error) { + return d.memberships.selectMembershipsByLocalpart(ctx, localpart) +} + +// newMembership saves a new membership in the database. +// If the event isn't a valid m.room.member event with type `join`, does nothing. +// If an error occurred, returns the SQL error +func (d *Database) newMembership( + ctx context.Context, txn *sql.Tx, ev gomatrixserverlib.Event, +) error { + if ev.Type() == "m.room.member" && ev.StateKey() != nil { + localpart, serverName, err := gomatrixserverlib.SplitID('@', *ev.StateKey()) + if err != nil { + return err + } + + // We only want state events from local users + if string(serverName) != string(d.serverName) { + return nil + } + + eventID := ev.EventID() + roomID := ev.RoomID() + membership, err := ev.Membership() + if err != nil { + return err + } + + // Only "join" membership events can be considered as new memberships + if membership == gomatrixserverlib.Join { + if err := d.saveMembership(ctx, txn, localpart, roomID, eventID); err != nil { + return err + } + } + } + return nil +} + +// SaveAccountData saves new account data for a given user and a given room. +// If the account data is not specific to a room, the room ID should be an empty string +// If an account data already exists for a given set (user, room, data type), it will +// update the corresponding row with the new content +// Returns a SQL error if there was an issue with the insertion/update +func (d *Database) SaveAccountData( + ctx context.Context, localpart, roomID, dataType, content string, +) error { + return d.accountDatas.insertAccountData(ctx, localpart, roomID, dataType, content) +} + +// GetAccountData returns account data related to a given localpart +// If no account data could be found, returns an empty arrays +// Returns an error if there was an issue with the retrieval +func (d *Database) GetAccountData(ctx context.Context, localpart string) ( + global []gomatrixserverlib.ClientEvent, + rooms map[string][]gomatrixserverlib.ClientEvent, + err error, +) { + return d.accountDatas.selectAccountData(ctx, localpart) +} + +// GetAccountDataByType returns account data matching a given +// localpart, room ID and type. +// If no account data could be found, returns nil +// Returns an error if there was an issue with the retrieval +func (d *Database) GetAccountDataByType( + ctx context.Context, localpart, roomID, dataType string, +) (data *gomatrixserverlib.ClientEvent, err error) { + return d.accountDatas.selectAccountDataByType( + ctx, localpart, roomID, dataType, + ) +} + +// GetNewNumericLocalpart generates and returns a new unused numeric localpart +func (d *Database) GetNewNumericLocalpart( + ctx context.Context, +) (int64, error) { + return d.accounts.selectNewNumericLocalpart(ctx) +} + +func hashPassword(plaintext string) (hash string, err error) { + hashBytes, err := bcrypt.GenerateFromPassword([]byte(plaintext), bcrypt.DefaultCost) + return string(hashBytes), err +} + +// Err3PIDInUse is the error returned when trying to save an association involving +// a third-party identifier which is already associated to a local user. +var Err3PIDInUse = errors.New("This third-party identifier is already in use") + +// SaveThreePIDAssociation saves the association between a third party identifier +// and a local Matrix user (identified by the user's ID's local part). +// If the third-party identifier is already part of an association, returns Err3PIDInUse. +// Returns an error if there was a problem talking to the database. +func (d *Database) SaveThreePIDAssociation( + ctx context.Context, threepid, localpart, medium string, +) (err error) { + return common.WithTransaction(d.db, func(txn *sql.Tx) error { + user, err := d.threepids.selectLocalpartForThreePID( + ctx, txn, threepid, medium, + ) + if err != nil { + return err + } + + if len(user) > 0 { + return Err3PIDInUse + } + + return d.threepids.insertThreePID(ctx, txn, threepid, medium, localpart) + }) +} + +// RemoveThreePIDAssociation removes the association involving a given third-party +// identifier. +// If no association exists involving this third-party identifier, returns nothing. +// If there was a problem talking to the database, returns an error. +func (d *Database) RemoveThreePIDAssociation( + ctx context.Context, threepid string, medium string, +) (err error) { + return d.threepids.deleteThreePID(ctx, threepid, medium) +} + +// GetLocalpartForThreePID looks up the localpart associated with a given third-party +// identifier. +// If no association involves the given third-party idenfitier, returns an empty +// string. +// Returns an error if there was a problem talking to the database. +func (d *Database) GetLocalpartForThreePID( + ctx context.Context, threepid string, medium string, +) (localpart string, err error) { + return d.threepids.selectLocalpartForThreePID(ctx, nil, threepid, medium) +} + +// GetThreePIDsForLocalpart looks up the third-party identifiers associated with +// a given local user. +// If no association is known for this user, returns an empty slice. +// Returns an error if there was an issue talking to the database. +func (d *Database) GetThreePIDsForLocalpart( + ctx context.Context, localpart string, +) (threepids []authtypes.ThreePID, err error) { + return d.threepids.selectThreePIDsForLocalpart(ctx, localpart) +} + +// GetFilter looks up the filter associated with a given local user and filter ID. +// Returns a filter structure. Otherwise returns an error if no such filter exists +// or if there was an error talking to the database. +func (d *Database) GetFilter( + ctx context.Context, localpart string, filterID string, +) (*gomatrixserverlib.Filter, error) { + return d.filter.selectFilter(ctx, localpart, filterID) +} + +// PutFilter puts the passed filter into the database. +// Returns the filterID as a string. Otherwise returns an error if something +// goes wrong. +func (d *Database) PutFilter( + ctx context.Context, localpart string, filter *gomatrixserverlib.Filter, +) (string, error) { + return d.filter.insertFilter(ctx, filter, localpart) +} + +// CheckAccountAvailability checks if the username/localpart is already present +// in the database. +// If the DB returns sql.ErrNoRows the Localpart isn't taken. +func (d *Database) CheckAccountAvailability(ctx context.Context, localpart string) (bool, error) { + _, err := d.accounts.selectAccountByLocalpart(ctx, localpart) + if err == sql.ErrNoRows { + return true, nil + } + return false, err +} + +// GetAccountByLocalpart returns the account associated with the given localpart. +// This function assumes the request is authenticated or the account data is used only internally. +// Returns sql.ErrNoRows if no account exists which matches the given localpart. +func (d *Database) GetAccountByLocalpart(ctx context.Context, localpart string, +) (*authtypes.Account, error) { + return d.accounts.selectAccountByLocalpart(ctx, localpart) +} diff --git a/clientapi/auth/storage/accounts/threepid_table.go b/clientapi/auth/storage/accounts/sqlite3/threepid_table.go similarity index 99% rename from clientapi/auth/storage/accounts/threepid_table.go rename to clientapi/auth/storage/accounts/sqlite3/threepid_table.go index a03aa4f89..53f6408d1 100644 --- a/clientapi/auth/storage/accounts/threepid_table.go +++ b/clientapi/auth/storage/accounts/sqlite3/threepid_table.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package accounts +package sqlite3 import ( "context" diff --git a/clientapi/auth/storage/accounts/storage.go b/clientapi/auth/storage/accounts/storage.go index 7cfc63c01..1dfd5f1f4 100644 --- a/clientapi/auth/storage/accounts/storage.go +++ b/clientapi/auth/storage/accounts/storage.go @@ -1,392 +1,56 @@ -// Copyright 2017 Vector Creations Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package accounts import ( "context" - "database/sql" "errors" + "net/url" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/postgres" + "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/sqlite3" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/gomatrixserverlib" - "golang.org/x/crypto/bcrypt" - - // Import the postgres database driver. - _ "github.com/lib/pq" ) -// Database represents an account database -type Database struct { - db *sql.DB - common.PartitionOffsetStatements - accounts accountsStatements - profiles profilesStatements - memberships membershipStatements - accountDatas accountDataStatements - threepids threepidStatements - filter filterStatements - serverName gomatrixserverlib.ServerName +type Database interface { + common.PartitionStorer + GetAccountByPassword(ctx context.Context, localpart, plaintextPassword string) (*authtypes.Account, error) + GetProfileByLocalpart(ctx context.Context, localpart string) (*authtypes.Profile, error) + SetAvatarURL(ctx context.Context, localpart string, avatarURL string) error + SetDisplayName(ctx context.Context, localpart string, displayName string) error + CreateAccount(ctx context.Context, localpart, plaintextPassword, appserviceID string) (*authtypes.Account, error) + UpdateMemberships(ctx context.Context, eventsToAdd []gomatrixserverlib.Event, idsToRemove []string) error + GetMembershipInRoomByLocalpart(ctx context.Context, localpart, roomID string) (authtypes.Membership, error) + GetMembershipsByLocalpart(ctx context.Context, localpart string) (memberships []authtypes.Membership, err error) + SaveAccountData(ctx context.Context, localpart, roomID, dataType, content string) error + GetAccountData(ctx context.Context, localpart string) (global []gomatrixserverlib.ClientEvent, rooms map[string][]gomatrixserverlib.ClientEvent, err error) + GetAccountDataByType(ctx context.Context, localpart, roomID, dataType string) (data *gomatrixserverlib.ClientEvent, err error) + GetNewNumericLocalpart(ctx context.Context) (int64, error) + SaveThreePIDAssociation(ctx context.Context, threepid, localpart, medium string) (err error) + RemoveThreePIDAssociation(ctx context.Context, threepid string, medium string) (err error) + GetLocalpartForThreePID(ctx context.Context, threepid string, medium string) (localpart string, err error) + GetThreePIDsForLocalpart(ctx context.Context, localpart string) (threepids []authtypes.ThreePID, err error) + GetFilter(ctx context.Context, localpart string, filterID string) (*gomatrixserverlib.Filter, error) + PutFilter(ctx context.Context, localpart string, filter *gomatrixserverlib.Filter) (string, error) + CheckAccountAvailability(ctx context.Context, localpart string) (bool, error) + GetAccountByLocalpart(ctx context.Context, localpart string) (*authtypes.Account, error) } -// NewDatabase creates a new accounts and profiles database -func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) { - var db *sql.DB - var err error - if db, err = sql.Open("postgres", dataSourceName); err != nil { - return nil, err - } - partitions := common.PartitionOffsetStatements{} - if err = partitions.Prepare(db, "account"); err != nil { - return nil, err - } - a := accountsStatements{} - if err = a.prepare(db, serverName); err != nil { - return nil, err - } - p := profilesStatements{} - if err = p.prepare(db); err != nil { - return nil, err - } - m := membershipStatements{} - if err = m.prepare(db); err != nil { - return nil, err - } - ac := accountDataStatements{} - if err = ac.prepare(db); err != nil { - return nil, err - } - t := threepidStatements{} - if err = t.prepare(db); err != nil { - return nil, err - } - f := filterStatements{} - if err = f.prepare(db); err != nil { - return nil, err - } - return &Database{db, partitions, a, p, m, ac, t, f, serverName}, nil -} - -// GetAccountByPassword returns the account associated with the given localpart and password. -// Returns sql.ErrNoRows if no account exists which matches the given localpart. -func (d *Database) GetAccountByPassword( - ctx context.Context, localpart, plaintextPassword string, -) (*authtypes.Account, error) { - hash, err := d.accounts.selectPasswordHash(ctx, localpart) +func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (Database, error) { + uri, err := url.Parse(dataSourceName) if err != nil { - return nil, err + return postgres.NewDatabase(dataSourceName, serverName) } - if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(plaintextPassword)); err != nil { - return nil, err + switch uri.Scheme { + case "postgres": + return postgres.NewDatabase(dataSourceName, serverName) + case "file": + return sqlite3.NewDatabase(dataSourceName, serverName) + default: + return postgres.NewDatabase(dataSourceName, serverName) } - return d.accounts.selectAccountByLocalpart(ctx, localpart) -} - -// GetProfileByLocalpart returns the profile associated with the given localpart. -// Returns sql.ErrNoRows if no profile exists which matches the given localpart. -func (d *Database) GetProfileByLocalpart( - ctx context.Context, localpart string, -) (*authtypes.Profile, error) { - return d.profiles.selectProfileByLocalpart(ctx, localpart) -} - -// SetAvatarURL updates the avatar URL of the profile associated with the given -// localpart. Returns an error if something went wrong with the SQL query -func (d *Database) SetAvatarURL( - ctx context.Context, localpart string, avatarURL string, -) error { - return d.profiles.setAvatarURL(ctx, localpart, avatarURL) -} - -// SetDisplayName updates the display name of the profile associated with the given -// localpart. Returns an error if something went wrong with the SQL query -func (d *Database) SetDisplayName( - ctx context.Context, localpart string, displayName string, -) error { - return d.profiles.setDisplayName(ctx, localpart, displayName) -} - -// CreateAccount makes a new account with the given login name and password, and creates an empty profile -// for this account. If no password is supplied, the account will be a passwordless account. If the -// account already exists, it will return nil, nil. -func (d *Database) CreateAccount( - ctx context.Context, localpart, plaintextPassword, appserviceID string, -) (*authtypes.Account, error) { - var err error - - // Generate a password hash if this is not a password-less user - hash := "" - if plaintextPassword != "" { - hash, err = hashPassword(plaintextPassword) - if err != nil { - return nil, err - } - } - if err := d.profiles.insertProfile(ctx, localpart); err != nil { - if common.IsUniqueConstraintViolationErr(err) { - return nil, nil - } - return nil, err - } - if err := d.SaveAccountData(ctx, localpart, "", "m.push_rules", `{ - "global": { - "content": [], - "override": [], - "room": [], - "sender": [], - "underride": [] - } - }`); err != nil { - return nil, err - } - return d.accounts.insertAccount(ctx, localpart, hash, appserviceID) -} - -// SaveMembership saves the user matching a given localpart as a member of a given -// room. It also stores the ID of the membership event. -// If a membership already exists between the user and the room, or if the -// insert fails, returns the SQL error -func (d *Database) saveMembership( - ctx context.Context, txn *sql.Tx, localpart, roomID, eventID string, -) error { - return d.memberships.insertMembership(ctx, txn, localpart, roomID, eventID) -} - -// removeMembershipsByEventIDs removes the memberships corresponding to the -// `join` membership events IDs in the eventIDs slice. -// If the removal fails, or if there is no membership to remove, returns an error -func (d *Database) removeMembershipsByEventIDs( - ctx context.Context, txn *sql.Tx, eventIDs []string, -) error { - return d.memberships.deleteMembershipsByEventIDs(ctx, txn, eventIDs) -} - -// UpdateMemberships adds the "join" membership events included in a given state -// events array, and removes those which ID is included in a given array of events -// IDs. All of the process is run in a transaction, which commits only once/if every -// insertion and deletion has been successfully processed. -// Returns a SQL error if there was an issue with any part of the process -func (d *Database) UpdateMemberships( - ctx context.Context, eventsToAdd []gomatrixserverlib.Event, idsToRemove []string, -) error { - return common.WithTransaction(d.db, func(txn *sql.Tx) error { - if err := d.removeMembershipsByEventIDs(ctx, txn, idsToRemove); err != nil { - return err - } - - for _, event := range eventsToAdd { - if err := d.newMembership(ctx, txn, event); err != nil { - return err - } - } - - return nil - }) -} - -// GetMembershipInRoomByLocalpart returns the membership for an user -// matching the given localpart if he is a member of the room matching roomID, -// if not sql.ErrNoRows is returned. -// If there was an issue during the retrieval, returns the SQL error -func (d *Database) GetMembershipInRoomByLocalpart( - ctx context.Context, localpart, roomID string, -) (authtypes.Membership, error) { - return d.memberships.selectMembershipInRoomByLocalpart(ctx, localpart, roomID) -} - -// GetMembershipsByLocalpart returns an array containing the memberships for all -// the rooms a user matching a given localpart is a member of -// If no membership match the given localpart, returns an empty array -// If there was an issue during the retrieval, returns the SQL error -func (d *Database) GetMembershipsByLocalpart( - ctx context.Context, localpart string, -) (memberships []authtypes.Membership, err error) { - return d.memberships.selectMembershipsByLocalpart(ctx, localpart) -} - -// newMembership saves a new membership in the database. -// If the event isn't a valid m.room.member event with type `join`, does nothing. -// If an error occurred, returns the SQL error -func (d *Database) newMembership( - ctx context.Context, txn *sql.Tx, ev gomatrixserverlib.Event, -) error { - if ev.Type() == "m.room.member" && ev.StateKey() != nil { - localpart, serverName, err := gomatrixserverlib.SplitID('@', *ev.StateKey()) - if err != nil { - return err - } - - // We only want state events from local users - if string(serverName) != string(d.serverName) { - return nil - } - - eventID := ev.EventID() - roomID := ev.RoomID() - membership, err := ev.Membership() - if err != nil { - return err - } - - // Only "join" membership events can be considered as new memberships - if membership == gomatrixserverlib.Join { - if err := d.saveMembership(ctx, txn, localpart, roomID, eventID); err != nil { - return err - } - } - } - return nil -} - -// SaveAccountData saves new account data for a given user and a given room. -// If the account data is not specific to a room, the room ID should be an empty string -// If an account data already exists for a given set (user, room, data type), it will -// update the corresponding row with the new content -// Returns a SQL error if there was an issue with the insertion/update -func (d *Database) SaveAccountData( - ctx context.Context, localpart, roomID, dataType, content string, -) error { - return d.accountDatas.insertAccountData(ctx, localpart, roomID, dataType, content) -} - -// GetAccountData returns account data related to a given localpart -// If no account data could be found, returns an empty arrays -// Returns an error if there was an issue with the retrieval -func (d *Database) GetAccountData(ctx context.Context, localpart string) ( - global []gomatrixserverlib.ClientEvent, - rooms map[string][]gomatrixserverlib.ClientEvent, - err error, -) { - return d.accountDatas.selectAccountData(ctx, localpart) -} - -// GetAccountDataByType returns account data matching a given -// localpart, room ID and type. -// If no account data could be found, returns nil -// Returns an error if there was an issue with the retrieval -func (d *Database) GetAccountDataByType( - ctx context.Context, localpart, roomID, dataType string, -) (data *gomatrixserverlib.ClientEvent, err error) { - return d.accountDatas.selectAccountDataByType( - ctx, localpart, roomID, dataType, - ) -} - -// GetNewNumericLocalpart generates and returns a new unused numeric localpart -func (d *Database) GetNewNumericLocalpart( - ctx context.Context, -) (int64, error) { - return d.accounts.selectNewNumericLocalpart(ctx) -} - -func hashPassword(plaintext string) (hash string, err error) { - hashBytes, err := bcrypt.GenerateFromPassword([]byte(plaintext), bcrypt.DefaultCost) - return string(hashBytes), err } // Err3PIDInUse is the error returned when trying to save an association involving // a third-party identifier which is already associated to a local user. var Err3PIDInUse = errors.New("This third-party identifier is already in use") - -// SaveThreePIDAssociation saves the association between a third party identifier -// and a local Matrix user (identified by the user's ID's local part). -// If the third-party identifier is already part of an association, returns Err3PIDInUse. -// Returns an error if there was a problem talking to the database. -func (d *Database) SaveThreePIDAssociation( - ctx context.Context, threepid, localpart, medium string, -) (err error) { - return common.WithTransaction(d.db, func(txn *sql.Tx) error { - user, err := d.threepids.selectLocalpartForThreePID( - ctx, txn, threepid, medium, - ) - if err != nil { - return err - } - - if len(user) > 0 { - return Err3PIDInUse - } - - return d.threepids.insertThreePID(ctx, txn, threepid, medium, localpart) - }) -} - -// RemoveThreePIDAssociation removes the association involving a given third-party -// identifier. -// If no association exists involving this third-party identifier, returns nothing. -// If there was a problem talking to the database, returns an error. -func (d *Database) RemoveThreePIDAssociation( - ctx context.Context, threepid string, medium string, -) (err error) { - return d.threepids.deleteThreePID(ctx, threepid, medium) -} - -// GetLocalpartForThreePID looks up the localpart associated with a given third-party -// identifier. -// If no association involves the given third-party idenfitier, returns an empty -// string. -// Returns an error if there was a problem talking to the database. -func (d *Database) GetLocalpartForThreePID( - ctx context.Context, threepid string, medium string, -) (localpart string, err error) { - return d.threepids.selectLocalpartForThreePID(ctx, nil, threepid, medium) -} - -// GetThreePIDsForLocalpart looks up the third-party identifiers associated with -// a given local user. -// If no association is known for this user, returns an empty slice. -// Returns an error if there was an issue talking to the database. -func (d *Database) GetThreePIDsForLocalpart( - ctx context.Context, localpart string, -) (threepids []authtypes.ThreePID, err error) { - return d.threepids.selectThreePIDsForLocalpart(ctx, localpart) -} - -// GetFilter looks up the filter associated with a given local user and filter ID. -// Returns a filter structure. Otherwise returns an error if no such filter exists -// or if there was an error talking to the database. -func (d *Database) GetFilter( - ctx context.Context, localpart string, filterID string, -) (*gomatrixserverlib.Filter, error) { - return d.filter.selectFilter(ctx, localpart, filterID) -} - -// PutFilter puts the passed filter into the database. -// Returns the filterID as a string. Otherwise returns an error if something -// goes wrong. -func (d *Database) PutFilter( - ctx context.Context, localpart string, filter *gomatrixserverlib.Filter, -) (string, error) { - return d.filter.insertFilter(ctx, filter, localpart) -} - -// CheckAccountAvailability checks if the username/localpart is already present -// in the database. -// If the DB returns sql.ErrNoRows the Localpart isn't taken. -func (d *Database) CheckAccountAvailability(ctx context.Context, localpart string) (bool, error) { - _, err := d.accounts.selectAccountByLocalpart(ctx, localpart) - if err == sql.ErrNoRows { - return true, nil - } - return false, err -} - -// GetAccountByLocalpart returns the account associated with the given localpart. -// This function assumes the request is authenticated or the account data is used only internally. -// Returns sql.ErrNoRows if no account exists which matches the given localpart. -func (d *Database) GetAccountByLocalpart(ctx context.Context, localpart string, -) (*authtypes.Account, error) { - return d.accounts.selectAccountByLocalpart(ctx, localpart) -} diff --git a/clientapi/auth/storage/devices/devices_table.go b/clientapi/auth/storage/devices/postgres/devices_table.go similarity index 99% rename from clientapi/auth/storage/devices/devices_table.go rename to clientapi/auth/storage/devices/postgres/devices_table.go index 997412470..349bf1ef7 100644 --- a/clientapi/auth/storage/devices/devices_table.go +++ b/clientapi/auth/storage/devices/postgres/devices_table.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package devices +package postgres import ( "context" diff --git a/clientapi/auth/storage/devices/postgres/storage.go b/clientapi/auth/storage/devices/postgres/storage.go new file mode 100644 index 000000000..221c3998e --- /dev/null +++ b/clientapi/auth/storage/devices/postgres/storage.go @@ -0,0 +1,182 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package postgres + +import ( + "context" + "crypto/rand" + "database/sql" + "encoding/base64" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/gomatrixserverlib" +) + +// The length of generated device IDs +var deviceIDByteLength = 6 + +// Database represents a device database. +type Database struct { + db *sql.DB + devices devicesStatements +} + +// NewDatabase creates a new device database +func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) { + var db *sql.DB + var err error + if db, err = sql.Open("postgres", dataSourceName); err != nil { + return nil, err + } + d := devicesStatements{} + if err = d.prepare(db, serverName); err != nil { + return nil, err + } + return &Database{db, d}, nil +} + +// GetDeviceByAccessToken returns the device matching the given access token. +// Returns sql.ErrNoRows if no matching device was found. +func (d *Database) GetDeviceByAccessToken( + ctx context.Context, token string, +) (*authtypes.Device, error) { + return d.devices.selectDeviceByToken(ctx, token) +} + +// GetDeviceByID returns the device matching the given ID. +// Returns sql.ErrNoRows if no matching device was found. +func (d *Database) GetDeviceByID( + ctx context.Context, localpart, deviceID string, +) (*authtypes.Device, error) { + return d.devices.selectDeviceByID(ctx, localpart, deviceID) +} + +// GetDevicesByLocalpart returns the devices matching the given localpart. +func (d *Database) GetDevicesByLocalpart( + ctx context.Context, localpart string, +) ([]authtypes.Device, error) { + return d.devices.selectDevicesByLocalpart(ctx, localpart) +} + +// CreateDevice makes a new device associated with the given user ID localpart. +// If there is already a device with the same device ID for this user, that access token will be revoked +// and replaced with the given accessToken. If the given accessToken is already in use for another device, +// an error will be returned. +// If no device ID is given one is generated. +// Returns the device on success. +func (d *Database) CreateDevice( + ctx context.Context, localpart string, deviceID *string, accessToken string, + displayName *string, +) (dev *authtypes.Device, returnErr error) { + if deviceID != nil { + returnErr = common.WithTransaction(d.db, func(txn *sql.Tx) error { + var err error + // Revoke existing tokens for this device + if err = d.devices.deleteDevice(ctx, txn, *deviceID, localpart); err != nil { + return err + } + + dev, err = d.devices.insertDevice(ctx, txn, *deviceID, localpart, accessToken, displayName) + return err + }) + } else { + // We generate device IDs in a loop in case its already taken. + // We cap this at going round 5 times to ensure we don't spin forever + var newDeviceID string + for i := 1; i <= 5; i++ { + newDeviceID, returnErr = generateDeviceID() + if returnErr != nil { + return + } + + returnErr = common.WithTransaction(d.db, func(txn *sql.Tx) error { + var err error + dev, err = d.devices.insertDevice(ctx, txn, newDeviceID, localpart, accessToken, displayName) + return err + }) + if returnErr == nil { + return + } + } + } + return +} + +// generateDeviceID creates a new device id. Returns an error if failed to generate +// random bytes. +func generateDeviceID() (string, error) { + b := make([]byte, deviceIDByteLength) + _, err := rand.Read(b) + if err != nil { + return "", err + } + // url-safe no padding + return base64.RawURLEncoding.EncodeToString(b), nil +} + +// UpdateDevice updates the given device with the display name. +// Returns SQL error if there are problems and nil on success. +func (d *Database) UpdateDevice( + ctx context.Context, localpart, deviceID string, displayName *string, +) error { + return common.WithTransaction(d.db, func(txn *sql.Tx) error { + return d.devices.updateDeviceName(ctx, txn, localpart, deviceID, displayName) + }) +} + +// RemoveDevice revokes a device by deleting the entry in the database +// matching with the given device ID and user ID localpart. +// If the device doesn't exist, it will not return an error +// If something went wrong during the deletion, it will return the SQL error. +func (d *Database) RemoveDevice( + ctx context.Context, deviceID, localpart string, +) error { + return common.WithTransaction(d.db, func(txn *sql.Tx) error { + if err := d.devices.deleteDevice(ctx, txn, deviceID, localpart); err != sql.ErrNoRows { + return err + } + return nil + }) +} + +// RemoveDevices revokes one or more devices by deleting the entry in the database +// matching with the given device IDs and user ID localpart. +// If the devices don't exist, it will not return an error +// If something went wrong during the deletion, it will return the SQL error. +func (d *Database) RemoveDevices( + ctx context.Context, localpart string, devices []string, +) error { + return common.WithTransaction(d.db, func(txn *sql.Tx) error { + if err := d.devices.deleteDevices(ctx, txn, localpart, devices); err != sql.ErrNoRows { + return err + } + return nil + }) +} + +// RemoveAllDevices revokes devices by deleting the entry in the +// database matching the given user ID localpart. +// If something went wrong during the deletion, it will return the SQL error. +func (d *Database) RemoveAllDevices( + ctx context.Context, localpart string, +) error { + return common.WithTransaction(d.db, func(txn *sql.Tx) error { + if err := d.devices.deleteDevicesByLocalpart(ctx, txn, localpart); err != sql.ErrNoRows { + return err + } + return nil + }) +} diff --git a/clientapi/auth/storage/devices/sqlite3/devices_table.go b/clientapi/auth/storage/devices/sqlite3/devices_table.go new file mode 100644 index 000000000..dc88890d3 --- /dev/null +++ b/clientapi/auth/storage/devices/sqlite3/devices_table.go @@ -0,0 +1,243 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + "strings" + "time" + + "github.com/matrix-org/dendrite/common" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/clientapi/userutil" + "github.com/matrix-org/gomatrixserverlib" +) + +const devicesSchema = ` +-- This sequence is used for automatic allocation of session_id. +-- CREATE SEQUENCE IF NOT EXISTS device_session_id_seq START 1; + +-- Stores data about devices. +CREATE TABLE IF NOT EXISTS device_devices ( + access_token TEXT PRIMARY KEY, + session_id INTEGER, + device_id TEXT , + localpart TEXT , + created_ts BIGINT, + display_name TEXT, + + UNIQUE (localpart, device_id) +); +` + +const insertDeviceSQL = "" + + "INSERT INTO device_devices (device_id, localpart, access_token, created_ts, display_name, session_id)" + + " VALUES ($1, $2, $3, $4, $5, $6)" + +const selectDevicesCountSQL = "" + + "SELECT COUNT(access_token) FROM device_devices" + +const selectDeviceByTokenSQL = "" + + "SELECT session_id, device_id, localpart FROM device_devices WHERE access_token = $1" + +const selectDeviceByIDSQL = "" + + "SELECT display_name FROM device_devices WHERE localpart = $1 and device_id = $2" + +const selectDevicesByLocalpartSQL = "" + + "SELECT device_id, display_name FROM device_devices WHERE localpart = $1" + +const updateDeviceNameSQL = "" + + "UPDATE device_devices SET display_name = $1 WHERE localpart = $2 AND device_id = $3" + +const deleteDeviceSQL = "" + + "DELETE FROM device_devices WHERE device_id = $1 AND localpart = $2" + +const deleteDevicesByLocalpartSQL = "" + + "DELETE FROM device_devices WHERE localpart = $1" + +const deleteDevicesSQL = "" + + "DELETE FROM device_devices WHERE localpart = $1 AND device_id IN ($2)" + +type devicesStatements struct { + db *sql.DB + insertDeviceStmt *sql.Stmt + selectDevicesCountStmt *sql.Stmt + selectDeviceByTokenStmt *sql.Stmt + selectDeviceByIDStmt *sql.Stmt + selectDevicesByLocalpartStmt *sql.Stmt + updateDeviceNameStmt *sql.Stmt + deleteDeviceStmt *sql.Stmt + deleteDevicesByLocalpartStmt *sql.Stmt + serverName gomatrixserverlib.ServerName +} + +func (s *devicesStatements) prepare(db *sql.DB, server gomatrixserverlib.ServerName) (err error) { + s.db = db + _, err = db.Exec(devicesSchema) + if err != nil { + return + } + if s.insertDeviceStmt, err = db.Prepare(insertDeviceSQL); err != nil { + return + } + if s.selectDevicesCountStmt, err = db.Prepare(selectDevicesCountSQL); err != nil { + return + } + if s.selectDeviceByTokenStmt, err = db.Prepare(selectDeviceByTokenSQL); err != nil { + return + } + if s.selectDeviceByIDStmt, err = db.Prepare(selectDeviceByIDSQL); err != nil { + return + } + if s.selectDevicesByLocalpartStmt, err = db.Prepare(selectDevicesByLocalpartSQL); err != nil { + return + } + if s.updateDeviceNameStmt, err = db.Prepare(updateDeviceNameSQL); err != nil { + return + } + if s.deleteDeviceStmt, err = db.Prepare(deleteDeviceSQL); err != nil { + return + } + if s.deleteDevicesByLocalpartStmt, err = db.Prepare(deleteDevicesByLocalpartSQL); err != nil { + return + } + s.serverName = server + return +} + +// insertDevice creates a new device. Returns an error if any device with the same access token already exists. +// Returns an error if the user already has a device with the given device ID. +// Returns the device on success. +func (s *devicesStatements) insertDevice( + ctx context.Context, txn *sql.Tx, id, localpart, accessToken string, + displayName *string, +) (*authtypes.Device, error) { + createdTimeMS := time.Now().UnixNano() / 1000000 + var sessionID int64 + countStmt := common.TxStmt(txn, s.selectDevicesCountStmt) + insertStmt := common.TxStmt(txn, s.insertDeviceStmt) + if err := countStmt.QueryRowContext(ctx).Scan(&sessionID); err != nil { + return nil, err + } + sessionID++ + if _, err := insertStmt.ExecContext(ctx, id, localpart, accessToken, createdTimeMS, displayName, sessionID); err != nil { + return nil, err + } + return &authtypes.Device{ + ID: id, + UserID: userutil.MakeUserID(localpart, s.serverName), + AccessToken: accessToken, + SessionID: sessionID, + }, nil +} + +func (s *devicesStatements) deleteDevice( + ctx context.Context, txn *sql.Tx, id, localpart string, +) error { + stmt := common.TxStmt(txn, s.deleteDeviceStmt) + _, err := stmt.ExecContext(ctx, id, localpart) + return err +} + +func (s *devicesStatements) deleteDevices( + ctx context.Context, txn *sql.Tx, localpart string, devices []string, +) error { + orig := strings.Replace(deleteDevicesSQL, "($1)", common.QueryVariadic(len(devices)), 1) + prep, err := s.db.Prepare(orig) + if err != nil { + return err + } + stmt := common.TxStmt(txn, prep) + params := make([]interface{}, len(devices)+1) + params[0] = localpart + for i, v := range devices { + params[i+1] = v + } + params = append(params, params...) + _, err = stmt.ExecContext(ctx, params...) + return err +} + +func (s *devicesStatements) deleteDevicesByLocalpart( + ctx context.Context, txn *sql.Tx, localpart string, +) error { + stmt := common.TxStmt(txn, s.deleteDevicesByLocalpartStmt) + _, err := stmt.ExecContext(ctx, localpart) + return err +} + +func (s *devicesStatements) updateDeviceName( + ctx context.Context, txn *sql.Tx, localpart, deviceID string, displayName *string, +) error { + stmt := common.TxStmt(txn, s.updateDeviceNameStmt) + _, err := stmt.ExecContext(ctx, displayName, localpart, deviceID) + return err +} + +func (s *devicesStatements) selectDeviceByToken( + ctx context.Context, accessToken string, +) (*authtypes.Device, error) { + var dev authtypes.Device + var localpart string + stmt := s.selectDeviceByTokenStmt + err := stmt.QueryRowContext(ctx, accessToken).Scan(&dev.SessionID, &dev.ID, &localpart) + if err == nil { + dev.UserID = userutil.MakeUserID(localpart, s.serverName) + dev.AccessToken = accessToken + } + return &dev, err +} + +// selectDeviceByID retrieves a device from the database with the given user +// localpart and deviceID +func (s *devicesStatements) selectDeviceByID( + ctx context.Context, localpart, deviceID string, +) (*authtypes.Device, error) { + var dev authtypes.Device + var created sql.NullInt64 + stmt := s.selectDeviceByIDStmt + err := stmt.QueryRowContext(ctx, localpart, deviceID).Scan(&created) + if err == nil { + dev.ID = deviceID + dev.UserID = userutil.MakeUserID(localpart, s.serverName) + } + return &dev, err +} + +func (s *devicesStatements) selectDevicesByLocalpart( + ctx context.Context, localpart string, +) ([]authtypes.Device, error) { + devices := []authtypes.Device{} + + rows, err := s.selectDevicesByLocalpartStmt.QueryContext(ctx, localpart) + + if err != nil { + return devices, err + } + + for rows.Next() { + var dev authtypes.Device + err = rows.Scan(&dev.ID) + if err != nil { + return devices, err + } + dev.UserID = userutil.MakeUserID(localpart, s.serverName) + devices = append(devices, dev) + } + + return devices, nil +} diff --git a/clientapi/auth/storage/devices/sqlite3/storage.go b/clientapi/auth/storage/devices/sqlite3/storage.go new file mode 100644 index 000000000..e1ce6f00d --- /dev/null +++ b/clientapi/auth/storage/devices/sqlite3/storage.go @@ -0,0 +1,184 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "crypto/rand" + "database/sql" + "encoding/base64" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/gomatrixserverlib" + + _ "github.com/mattn/go-sqlite3" +) + +// The length of generated device IDs +var deviceIDByteLength = 6 + +// Database represents a device database. +type Database struct { + db *sql.DB + devices devicesStatements +} + +// NewDatabase creates a new device database +func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) { + var db *sql.DB + var err error + if db, err = sql.Open("sqlite3", dataSourceName); err != nil { + return nil, err + } + d := devicesStatements{} + if err = d.prepare(db, serverName); err != nil { + return nil, err + } + return &Database{db, d}, nil +} + +// GetDeviceByAccessToken returns the device matching the given access token. +// Returns sql.ErrNoRows if no matching device was found. +func (d *Database) GetDeviceByAccessToken( + ctx context.Context, token string, +) (*authtypes.Device, error) { + return d.devices.selectDeviceByToken(ctx, token) +} + +// GetDeviceByID returns the device matching the given ID. +// Returns sql.ErrNoRows if no matching device was found. +func (d *Database) GetDeviceByID( + ctx context.Context, localpart, deviceID string, +) (*authtypes.Device, error) { + return d.devices.selectDeviceByID(ctx, localpart, deviceID) +} + +// GetDevicesByLocalpart returns the devices matching the given localpart. +func (d *Database) GetDevicesByLocalpart( + ctx context.Context, localpart string, +) ([]authtypes.Device, error) { + return d.devices.selectDevicesByLocalpart(ctx, localpart) +} + +// CreateDevice makes a new device associated with the given user ID localpart. +// If there is already a device with the same device ID for this user, that access token will be revoked +// and replaced with the given accessToken. If the given accessToken is already in use for another device, +// an error will be returned. +// If no device ID is given one is generated. +// Returns the device on success. +func (d *Database) CreateDevice( + ctx context.Context, localpart string, deviceID *string, accessToken string, + displayName *string, +) (dev *authtypes.Device, returnErr error) { + if deviceID != nil { + returnErr = common.WithTransaction(d.db, func(txn *sql.Tx) error { + var err error + // Revoke existing tokens for this device + if err = d.devices.deleteDevice(ctx, txn, *deviceID, localpart); err != nil { + return err + } + + dev, err = d.devices.insertDevice(ctx, txn, *deviceID, localpart, accessToken, displayName) + return err + }) + } else { + // We generate device IDs in a loop in case its already taken. + // We cap this at going round 5 times to ensure we don't spin forever + var newDeviceID string + for i := 1; i <= 5; i++ { + newDeviceID, returnErr = generateDeviceID() + if returnErr != nil { + return + } + + returnErr = common.WithTransaction(d.db, func(txn *sql.Tx) error { + var err error + dev, err = d.devices.insertDevice(ctx, txn, newDeviceID, localpart, accessToken, displayName) + return err + }) + if returnErr == nil { + return + } + } + } + return +} + +// generateDeviceID creates a new device id. Returns an error if failed to generate +// random bytes. +func generateDeviceID() (string, error) { + b := make([]byte, deviceIDByteLength) + _, err := rand.Read(b) + if err != nil { + return "", err + } + // url-safe no padding + return base64.RawURLEncoding.EncodeToString(b), nil +} + +// UpdateDevice updates the given device with the display name. +// Returns SQL error if there are problems and nil on success. +func (d *Database) UpdateDevice( + ctx context.Context, localpart, deviceID string, displayName *string, +) error { + return common.WithTransaction(d.db, func(txn *sql.Tx) error { + return d.devices.updateDeviceName(ctx, txn, localpart, deviceID, displayName) + }) +} + +// RemoveDevice revokes a device by deleting the entry in the database +// matching with the given device ID and user ID localpart. +// If the device doesn't exist, it will not return an error +// If something went wrong during the deletion, it will return the SQL error. +func (d *Database) RemoveDevice( + ctx context.Context, deviceID, localpart string, +) error { + return common.WithTransaction(d.db, func(txn *sql.Tx) error { + if err := d.devices.deleteDevice(ctx, txn, deviceID, localpart); err != sql.ErrNoRows { + return err + } + return nil + }) +} + +// RemoveDevices revokes one or more devices by deleting the entry in the database +// matching with the given device IDs and user ID localpart. +// If the devices don't exist, it will not return an error +// If something went wrong during the deletion, it will return the SQL error. +func (d *Database) RemoveDevices( + ctx context.Context, localpart string, devices []string, +) error { + return common.WithTransaction(d.db, func(txn *sql.Tx) error { + if err := d.devices.deleteDevices(ctx, txn, localpart, devices); err != sql.ErrNoRows { + return err + } + return nil + }) +} + +// RemoveAllDevices revokes devices by deleting the entry in the +// database matching the given user ID localpart. +// If something went wrong during the deletion, it will return the SQL error. +func (d *Database) RemoveAllDevices( + ctx context.Context, localpart string, +) error { + return common.WithTransaction(d.db, func(txn *sql.Tx) error { + if err := d.devices.deleteDevicesByLocalpart(ctx, txn, localpart); err != sql.ErrNoRows { + return err + } + return nil + }) +} diff --git a/clientapi/auth/storage/devices/storage.go b/clientapi/auth/storage/devices/storage.go index 150180c1e..82f756401 100644 --- a/clientapi/auth/storage/devices/storage.go +++ b/clientapi/auth/storage/devices/storage.go @@ -1,182 +1,37 @@ -// Copyright 2017 Vector Creations Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package devices import ( "context" - "crypto/rand" - "database/sql" - "encoding/base64" + "net/url" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/clientapi/auth/storage/devices/postgres" + "github.com/matrix-org/dendrite/clientapi/auth/storage/devices/sqlite3" "github.com/matrix-org/gomatrixserverlib" ) -// The length of generated device IDs -var deviceIDByteLength = 6 - -// Database represents a device database. -type Database struct { - db *sql.DB - devices devicesStatements +type Database interface { + GetDeviceByAccessToken(ctx context.Context, token string) (*authtypes.Device, error) + GetDeviceByID(ctx context.Context, localpart, deviceID string) (*authtypes.Device, error) + GetDevicesByLocalpart(ctx context.Context, localpart string) ([]authtypes.Device, error) + CreateDevice(ctx context.Context, localpart string, deviceID *string, accessToken string, displayName *string) (dev *authtypes.Device, returnErr error) + UpdateDevice(ctx context.Context, localpart, deviceID string, displayName *string) error + RemoveDevice(ctx context.Context, deviceID, localpart string) error + RemoveDevices(ctx context.Context, localpart string, devices []string) error + RemoveAllDevices(ctx context.Context, localpart string) error } -// NewDatabase creates a new device database -func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) { - var db *sql.DB - var err error - if db, err = sql.Open("postgres", dataSourceName); err != nil { - return nil, err - } - d := devicesStatements{} - if err = d.prepare(db, serverName); err != nil { - return nil, err - } - return &Database{db, d}, nil -} - -// GetDeviceByAccessToken returns the device matching the given access token. -// Returns sql.ErrNoRows if no matching device was found. -func (d *Database) GetDeviceByAccessToken( - ctx context.Context, token string, -) (*authtypes.Device, error) { - return d.devices.selectDeviceByToken(ctx, token) -} - -// GetDeviceByID returns the device matching the given ID. -// Returns sql.ErrNoRows if no matching device was found. -func (d *Database) GetDeviceByID( - ctx context.Context, localpart, deviceID string, -) (*authtypes.Device, error) { - return d.devices.selectDeviceByID(ctx, localpart, deviceID) -} - -// GetDevicesByLocalpart returns the devices matching the given localpart. -func (d *Database) GetDevicesByLocalpart( - ctx context.Context, localpart string, -) ([]authtypes.Device, error) { - return d.devices.selectDevicesByLocalpart(ctx, localpart) -} - -// CreateDevice makes a new device associated with the given user ID localpart. -// If there is already a device with the same device ID for this user, that access token will be revoked -// and replaced with the given accessToken. If the given accessToken is already in use for another device, -// an error will be returned. -// If no device ID is given one is generated. -// Returns the device on success. -func (d *Database) CreateDevice( - ctx context.Context, localpart string, deviceID *string, accessToken string, - displayName *string, -) (dev *authtypes.Device, returnErr error) { - if deviceID != nil { - returnErr = common.WithTransaction(d.db, func(txn *sql.Tx) error { - var err error - // Revoke existing tokens for this device - if err = d.devices.deleteDevice(ctx, txn, *deviceID, localpart); err != nil { - return err - } - - dev, err = d.devices.insertDevice(ctx, txn, *deviceID, localpart, accessToken, displayName) - return err - }) - } else { - // We generate device IDs in a loop in case its already taken. - // We cap this at going round 5 times to ensure we don't spin forever - var newDeviceID string - for i := 1; i <= 5; i++ { - newDeviceID, returnErr = generateDeviceID() - if returnErr != nil { - return - } - - returnErr = common.WithTransaction(d.db, func(txn *sql.Tx) error { - var err error - dev, err = d.devices.insertDevice(ctx, txn, newDeviceID, localpart, accessToken, displayName) - return err - }) - if returnErr == nil { - return - } - } - } - return -} - -// generateDeviceID creates a new device id. Returns an error if failed to generate -// random bytes. -func generateDeviceID() (string, error) { - b := make([]byte, deviceIDByteLength) - _, err := rand.Read(b) +func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (Database, error) { + uri, err := url.Parse(dataSourceName) if err != nil { - return "", err + return postgres.NewDatabase(dataSourceName, serverName) + } + switch uri.Scheme { + case "postgres": + return postgres.NewDatabase(dataSourceName, serverName) + case "file": + return sqlite3.NewDatabase(dataSourceName, serverName) + default: + return postgres.NewDatabase(dataSourceName, serverName) } - // url-safe no padding - return base64.RawURLEncoding.EncodeToString(b), nil -} - -// UpdateDevice updates the given device with the display name. -// Returns SQL error if there are problems and nil on success. -func (d *Database) UpdateDevice( - ctx context.Context, localpart, deviceID string, displayName *string, -) error { - return common.WithTransaction(d.db, func(txn *sql.Tx) error { - return d.devices.updateDeviceName(ctx, txn, localpart, deviceID, displayName) - }) -} - -// RemoveDevice revokes a device by deleting the entry in the database -// matching with the given device ID and user ID localpart. -// If the device doesn't exist, it will not return an error -// If something went wrong during the deletion, it will return the SQL error. -func (d *Database) RemoveDevice( - ctx context.Context, deviceID, localpart string, -) error { - return common.WithTransaction(d.db, func(txn *sql.Tx) error { - if err := d.devices.deleteDevice(ctx, txn, deviceID, localpart); err != sql.ErrNoRows { - return err - } - return nil - }) -} - -// RemoveDevices revokes one or more devices by deleting the entry in the database -// matching with the given device IDs and user ID localpart. -// If the devices don't exist, it will not return an error -// If something went wrong during the deletion, it will return the SQL error. -func (d *Database) RemoveDevices( - ctx context.Context, localpart string, devices []string, -) error { - return common.WithTransaction(d.db, func(txn *sql.Tx) error { - if err := d.devices.deleteDevices(ctx, txn, localpart, devices); err != sql.ErrNoRows { - return err - } - return nil - }) -} - -// RemoveAllDevices revokes devices by deleting the entry in the -// database matching the given user ID localpart. -// If something went wrong during the deletion, it will return the SQL error. -func (d *Database) RemoveAllDevices( - ctx context.Context, localpart string, -) error { - return common.WithTransaction(d.db, func(txn *sql.Tx) error { - if err := d.devices.deleteDevicesByLocalpart(ctx, txn, localpart); err != sql.ErrNoRows { - return err - } - return nil - }) } diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go index c911fecce..bb44e016a 100644 --- a/clientapi/clientapi.go +++ b/clientapi/clientapi.go @@ -34,8 +34,8 @@ import ( // component. func SetupClientAPIComponent( base *basecomponent.BaseDendrite, - deviceDB *devices.Database, - accountsDB *accounts.Database, + deviceDB devices.Database, + accountsDB accounts.Database, federation *gomatrixserverlib.FederationClient, keyRing *gomatrixserverlib.KeyRing, aliasAPI roomserverAPI.RoomserverAliasAPI, diff --git a/clientapi/consumers/roomserver.go b/clientapi/consumers/roomserver.go index 0ee7c6bf0..a65281514 100644 --- a/clientapi/consumers/roomserver.go +++ b/clientapi/consumers/roomserver.go @@ -31,7 +31,7 @@ import ( // OutputRoomEventConsumer consumes events that originated in the room server. type OutputRoomEventConsumer struct { roomServerConsumer *common.ContinualConsumer - db *accounts.Database + db accounts.Database query api.RoomserverQueryAPI serverName string } @@ -40,7 +40,7 @@ type OutputRoomEventConsumer struct { func NewOutputRoomEventConsumer( cfg *config.Dendrite, kafkaConsumer sarama.Consumer, - store *accounts.Database, + store accounts.Database, queryAPI api.RoomserverQueryAPI, ) *OutputRoomEventConsumer { diff --git a/clientapi/routing/account_data.go b/clientapi/routing/account_data.go index bbc8c258e..8ae9de2d5 100644 --- a/clientapi/routing/account_data.go +++ b/clientapi/routing/account_data.go @@ -30,7 +30,7 @@ import ( // GetAccountData implements GET /user/{userId}/[rooms/{roomid}/]account_data/{type} func GetAccountData( - req *http.Request, accountDB *accounts.Database, device *authtypes.Device, + req *http.Request, accountDB accounts.Database, device *authtypes.Device, userID string, roomID string, dataType string, ) util.JSONResponse { if userID != device.UserID { @@ -62,7 +62,7 @@ func GetAccountData( // SaveAccountData implements PUT /user/{userId}/[rooms/{roomId}/]account_data/{type} func SaveAccountData( - req *http.Request, accountDB *accounts.Database, device *authtypes.Device, + req *http.Request, accountDB accounts.Database, device *authtypes.Device, userID string, roomID string, dataType string, syncProducer *producers.SyncAPIProducer, ) util.JSONResponse { if userID != device.UserID { diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index f6f06421e..2b1245b9a 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -135,7 +135,7 @@ type fledglingEvent struct { func CreateRoom( req *http.Request, device *authtypes.Device, cfg *config.Dendrite, producer *producers.RoomserverProducer, - accountDB *accounts.Database, aliasAPI roomserverAPI.RoomserverAliasAPI, + accountDB accounts.Database, aliasAPI roomserverAPI.RoomserverAliasAPI, asAPI appserviceAPI.AppServiceQueryAPI, ) util.JSONResponse { // TODO (#267): Check room ID doesn't clash with an existing one, and we @@ -149,7 +149,7 @@ func CreateRoom( func createRoom( req *http.Request, device *authtypes.Device, cfg *config.Dendrite, roomID string, producer *producers.RoomserverProducer, - accountDB *accounts.Database, aliasAPI roomserverAPI.RoomserverAliasAPI, + accountDB accounts.Database, aliasAPI roomserverAPI.RoomserverAliasAPI, asAPI appserviceAPI.AppServiceQueryAPI, ) util.JSONResponse { logger := util.GetLogger(req.Context()) diff --git a/clientapi/routing/device.go b/clientapi/routing/device.go index eb7cd0b0c..9b8647cd4 100644 --- a/clientapi/routing/device.go +++ b/clientapi/routing/device.go @@ -46,7 +46,7 @@ type devicesDeleteJSON struct { // GetDeviceByID handles /devices/{deviceID} func GetDeviceByID( - req *http.Request, deviceDB *devices.Database, device *authtypes.Device, + req *http.Request, deviceDB devices.Database, device *authtypes.Device, deviceID string, ) util.JSONResponse { localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) @@ -76,7 +76,7 @@ func GetDeviceByID( // GetDevicesByLocalpart handles /devices func GetDevicesByLocalpart( - req *http.Request, deviceDB *devices.Database, device *authtypes.Device, + req *http.Request, deviceDB devices.Database, device *authtypes.Device, ) util.JSONResponse { localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) if err != nil { @@ -107,7 +107,7 @@ func GetDevicesByLocalpart( // UpdateDeviceByID handles PUT on /devices/{deviceID} func UpdateDeviceByID( - req *http.Request, deviceDB *devices.Database, device *authtypes.Device, + req *http.Request, deviceDB devices.Database, device *authtypes.Device, deviceID string, ) util.JSONResponse { localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) @@ -153,7 +153,7 @@ func UpdateDeviceByID( // DeleteDeviceById handles DELETE requests to /devices/{deviceId} func DeleteDeviceById( - req *http.Request, deviceDB *devices.Database, device *authtypes.Device, + req *http.Request, deviceDB devices.Database, device *authtypes.Device, deviceID string, ) util.JSONResponse { localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) @@ -176,7 +176,7 @@ func DeleteDeviceById( // DeleteDevices handles POST requests to /delete_devices func DeleteDevices( - req *http.Request, deviceDB *devices.Database, device *authtypes.Device, + req *http.Request, deviceDB devices.Database, device *authtypes.Device, ) util.JSONResponse { localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) if err != nil { diff --git a/clientapi/routing/filter.go b/clientapi/routing/filter.go index eec501ff7..583b23957 100644 --- a/clientapi/routing/filter.go +++ b/clientapi/routing/filter.go @@ -27,7 +27,7 @@ import ( // GetFilter implements GET /_matrix/client/r0/user/{userId}/filter/{filterId} func GetFilter( - req *http.Request, device *authtypes.Device, accountDB *accounts.Database, userID string, filterID string, + req *http.Request, device *authtypes.Device, accountDB accounts.Database, userID string, filterID string, ) util.JSONResponse { if userID != device.UserID { return util.JSONResponse{ @@ -63,7 +63,7 @@ type filterResponse struct { //PutFilter implements POST /_matrix/client/r0/user/{userId}/filter func PutFilter( - req *http.Request, device *authtypes.Device, accountDB *accounts.Database, userID string, + req *http.Request, device *authtypes.Device, accountDB accounts.Database, userID string, ) util.JSONResponse { if userID != device.UserID { return util.JSONResponse{ diff --git a/clientapi/routing/joinroom.go b/clientapi/routing/joinroom.go index 8b3f3740b..5e6f3e559 100644 --- a/clientapi/routing/joinroom.go +++ b/clientapi/routing/joinroom.go @@ -45,7 +45,7 @@ func JoinRoomByIDOrAlias( queryAPI roomserverAPI.RoomserverQueryAPI, aliasAPI roomserverAPI.RoomserverAliasAPI, keyRing gomatrixserverlib.KeyRing, - accountDB *accounts.Database, + accountDB accounts.Database, ) util.JSONResponse { var content map[string]interface{} // must be a JSON object if resErr := httputil.UnmarshalJSONRequest(req, &content); resErr != nil { diff --git a/clientapi/routing/login.go b/clientapi/routing/login.go index 2f4fb83c1..b8364ed9d 100644 --- a/clientapi/routing/login.go +++ b/clientapi/routing/login.go @@ -70,7 +70,7 @@ func passwordLogin() loginFlows { // Login implements GET and POST /login func Login( - req *http.Request, accountDB *accounts.Database, deviceDB *devices.Database, + req *http.Request, accountDB accounts.Database, deviceDB devices.Database, cfg *config.Dendrite, ) util.JSONResponse { if req.Method == http.MethodGet { // TODO: support other forms of login other than password, depending on config options @@ -153,7 +153,7 @@ func Login( func getDevice( ctx context.Context, r passwordRequest, - deviceDB *devices.Database, + deviceDB devices.Database, acc *authtypes.Account, token string, ) (dev *authtypes.Device, err error) { diff --git a/clientapi/routing/logout.go b/clientapi/routing/logout.go index 3294fbcdc..0ac9ca4a5 100644 --- a/clientapi/routing/logout.go +++ b/clientapi/routing/logout.go @@ -26,7 +26,7 @@ import ( // Logout handles POST /logout func Logout( - req *http.Request, deviceDB *devices.Database, device *authtypes.Device, + req *http.Request, deviceDB devices.Database, device *authtypes.Device, ) util.JSONResponse { localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) if err != nil { @@ -45,7 +45,7 @@ func Logout( // LogoutAll handles POST /logout/all func LogoutAll( - req *http.Request, deviceDB *devices.Database, device *authtypes.Device, + req *http.Request, deviceDB devices.Database, device *authtypes.Device, ) util.JSONResponse { localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) if err != nil { diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index 8b8b3a0f9..68c131a2b 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -40,7 +40,7 @@ var errMissingUserID = errors.New("'user_id' must be supplied") // SendMembership implements PUT /rooms/{roomID}/(join|kick|ban|unban|leave|invite) // by building a m.room.member event then sending it to the room server func SendMembership( - req *http.Request, accountDB *accounts.Database, device *authtypes.Device, + req *http.Request, accountDB accounts.Database, device *authtypes.Device, roomID string, membership string, cfg *config.Dendrite, queryAPI roomserverAPI.RoomserverQueryAPI, asAPI appserviceAPI.AppServiceQueryAPI, producer *producers.RoomserverProducer, @@ -116,7 +116,7 @@ func SendMembership( func buildMembershipEvent( ctx context.Context, - body threepid.MembershipRequest, accountDB *accounts.Database, + body threepid.MembershipRequest, accountDB accounts.Database, device *authtypes.Device, membership, roomID string, cfg *config.Dendrite, evTime time.Time, @@ -166,7 +166,7 @@ func loadProfile( ctx context.Context, userID string, cfg *config.Dendrite, - accountDB *accounts.Database, + accountDB accounts.Database, asAPI appserviceAPI.AppServiceQueryAPI, ) (*authtypes.Profile, error) { _, serverName, err := gomatrixserverlib.SplitID('@', userID) @@ -216,7 +216,7 @@ func checkAndProcessThreepid( body *threepid.MembershipRequest, cfg *config.Dendrite, queryAPI roomserverAPI.RoomserverQueryAPI, - accountDB *accounts.Database, + accountDB accounts.Database, producer *producers.RoomserverProducer, membership, roomID string, evTime time.Time, diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index 4688b19e9..9b091ddf7 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -36,7 +36,7 @@ import ( // GetProfile implements GET /profile/{userID} func GetProfile( - req *http.Request, accountDB *accounts.Database, cfg *config.Dendrite, + req *http.Request, accountDB accounts.Database, cfg *config.Dendrite, userID string, asAPI appserviceAPI.AppServiceQueryAPI, federation *gomatrixserverlib.FederationClient, @@ -64,7 +64,7 @@ func GetProfile( // GetAvatarURL implements GET /profile/{userID}/avatar_url func GetAvatarURL( - req *http.Request, accountDB *accounts.Database, cfg *config.Dendrite, + req *http.Request, accountDB accounts.Database, cfg *config.Dendrite, userID string, asAPI appserviceAPI.AppServiceQueryAPI, federation *gomatrixserverlib.FederationClient, ) util.JSONResponse { @@ -90,7 +90,7 @@ func GetAvatarURL( // SetAvatarURL implements PUT /profile/{userID}/avatar_url func SetAvatarURL( - req *http.Request, accountDB *accounts.Database, device *authtypes.Device, + req *http.Request, accountDB accounts.Database, device *authtypes.Device, userID string, producer *producers.UserUpdateProducer, cfg *config.Dendrite, rsProducer *producers.RoomserverProducer, queryAPI api.RoomserverQueryAPI, ) util.JSONResponse { @@ -170,7 +170,7 @@ func SetAvatarURL( // GetDisplayName implements GET /profile/{userID}/displayname func GetDisplayName( - req *http.Request, accountDB *accounts.Database, cfg *config.Dendrite, + req *http.Request, accountDB accounts.Database, cfg *config.Dendrite, userID string, asAPI appserviceAPI.AppServiceQueryAPI, federation *gomatrixserverlib.FederationClient, ) util.JSONResponse { @@ -196,7 +196,7 @@ func GetDisplayName( // SetDisplayName implements PUT /profile/{userID}/displayname func SetDisplayName( - req *http.Request, accountDB *accounts.Database, device *authtypes.Device, + req *http.Request, accountDB accounts.Database, device *authtypes.Device, userID string, producer *producers.UserUpdateProducer, cfg *config.Dendrite, rsProducer *producers.RoomserverProducer, queryAPI api.RoomserverQueryAPI, ) util.JSONResponse { @@ -279,7 +279,7 @@ func SetDisplayName( // Returns an error when something goes wrong or specifically // common.ErrProfileNoExists when the profile doesn't exist. func getProfile( - ctx context.Context, accountDB *accounts.Database, cfg *config.Dendrite, + ctx context.Context, accountDB accounts.Database, cfg *config.Dendrite, userID string, asAPI appserviceAPI.AppServiceQueryAPI, federation *gomatrixserverlib.FederationClient, diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index 4375faaf2..9d67d9982 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -440,8 +440,8 @@ func validateApplicationService( // http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#post-matrix-client-unstable-register func Register( req *http.Request, - accountDB *accounts.Database, - deviceDB *devices.Database, + accountDB accounts.Database, + deviceDB devices.Database, cfg *config.Dendrite, ) util.JSONResponse { @@ -513,8 +513,8 @@ func handleGuestRegistration( req *http.Request, r registerRequest, cfg *config.Dendrite, - accountDB *accounts.Database, - deviceDB *devices.Database, + accountDB accounts.Database, + deviceDB devices.Database, ) util.JSONResponse { //Generate numeric local part for guest user @@ -570,8 +570,8 @@ func handleRegistrationFlow( r registerRequest, sessionID string, cfg *config.Dendrite, - accountDB *accounts.Database, - deviceDB *devices.Database, + accountDB accounts.Database, + deviceDB devices.Database, ) util.JSONResponse { // TODO: Shared secret registration (create new user scripts) // TODO: Enable registration config flag @@ -668,8 +668,8 @@ func handleApplicationServiceRegistration( req *http.Request, r registerRequest, cfg *config.Dendrite, - accountDB *accounts.Database, - deviceDB *devices.Database, + accountDB accounts.Database, + deviceDB devices.Database, ) util.JSONResponse { // Check if we previously had issues extracting the access token from the // request. @@ -707,8 +707,8 @@ func checkAndCompleteFlow( r registerRequest, sessionID string, cfg *config.Dendrite, - accountDB *accounts.Database, - deviceDB *devices.Database, + accountDB accounts.Database, + deviceDB devices.Database, ) util.JSONResponse { if checkFlowCompleted(flow, cfg.Derived.Registration.Flows) { // This flow was completed, registration can continue @@ -730,8 +730,8 @@ func checkAndCompleteFlow( // LegacyRegister process register requests from the legacy v1 API func LegacyRegister( req *http.Request, - accountDB *accounts.Database, - deviceDB *devices.Database, + accountDB accounts.Database, + deviceDB devices.Database, cfg *config.Dendrite, ) util.JSONResponse { var r legacyRegisterRequest @@ -814,8 +814,8 @@ func parseAndValidateLegacyLogin(req *http.Request, r *legacyRegisterRequest) *u // not all func completeRegistration( ctx context.Context, - accountDB *accounts.Database, - deviceDB *devices.Database, + accountDB accounts.Database, + deviceDB devices.Database, username, password, appserviceID string, inhibitLogin common.WeakBoolean, displayName, deviceID *string, @@ -992,7 +992,7 @@ type availableResponse struct { func RegisterAvailable( req *http.Request, cfg *config.Dendrite, - accountDB *accounts.Database, + accountDB accounts.Database, ) util.JSONResponse { username := req.URL.Query().Get("username") diff --git a/clientapi/routing/room_tagging.go b/clientapi/routing/room_tagging.go index 487081c53..aa5f13c4c 100644 --- a/clientapi/routing/room_tagging.go +++ b/clientapi/routing/room_tagging.go @@ -40,7 +40,7 @@ func newTag() gomatrix.TagContent { // GetTags implements GET /_matrix/client/r0/user/{userID}/rooms/{roomID}/tags func GetTags( req *http.Request, - accountDB *accounts.Database, + accountDB accounts.Database, device *authtypes.Device, userID string, roomID string, @@ -77,7 +77,7 @@ func GetTags( // the tag to the "map" and saving the new "map" to the DB func PutTag( req *http.Request, - accountDB *accounts.Database, + accountDB accounts.Database, device *authtypes.Device, userID string, roomID string, @@ -134,7 +134,7 @@ func PutTag( // the "map" and then saving the new "map" in the DB func DeleteTag( req *http.Request, - accountDB *accounts.Database, + accountDB accounts.Database, device *authtypes.Device, userID string, roomID string, @@ -203,7 +203,7 @@ func obtainSavedTags( req *http.Request, userID string, roomID string, - accountDB *accounts.Database, + accountDB accounts.Database, ) (string, *gomatrixserverlib.ClientEvent, error) { localpart, _, err := gomatrixserverlib.SplitID('@', userID) if err != nil { @@ -222,7 +222,7 @@ func saveTagData( req *http.Request, localpart string, roomID string, - accountDB *accounts.Database, + accountDB accounts.Database, Tag gomatrix.TagContent, ) error { newTagData, err := json.Marshal(Tag) diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index f519523ac..f0841b796 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -52,8 +52,8 @@ func Setup( queryAPI roomserverAPI.RoomserverQueryAPI, aliasAPI roomserverAPI.RoomserverAliasAPI, asAPI appserviceAPI.AppServiceQueryAPI, - accountDB *accounts.Database, - deviceDB *devices.Database, + accountDB accounts.Database, + deviceDB devices.Database, federation *gomatrixserverlib.FederationClient, keyRing gomatrixserverlib.KeyRing, userUpdateProducer *producers.UserUpdateProducer, diff --git a/clientapi/routing/sendtyping.go b/clientapi/routing/sendtyping.go index 561a2d89c..db3ab28b2 100644 --- a/clientapi/routing/sendtyping.go +++ b/clientapi/routing/sendtyping.go @@ -34,7 +34,7 @@ type typingContentJSON struct { // sends the typing events to client API typingProducer func SendTyping( req *http.Request, device *authtypes.Device, roomID string, - userID string, accountDB *accounts.Database, + userID string, accountDB accounts.Database, typingProducer *producers.TypingServerProducer, ) util.JSONResponse { if device.UserID != userID { diff --git a/clientapi/routing/threepid.go b/clientapi/routing/threepid.go index 88b02fe46..69383cdf7 100644 --- a/clientapi/routing/threepid.go +++ b/clientapi/routing/threepid.go @@ -39,7 +39,7 @@ type threePIDsResponse struct { // RequestEmailToken implements: // POST /account/3pid/email/requestToken // POST /register/email/requestToken -func RequestEmailToken(req *http.Request, accountDB *accounts.Database, cfg *config.Dendrite) util.JSONResponse { +func RequestEmailToken(req *http.Request, accountDB accounts.Database, cfg *config.Dendrite) util.JSONResponse { var body threepid.EmailAssociationRequest if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil { return *reqErr @@ -82,7 +82,7 @@ func RequestEmailToken(req *http.Request, accountDB *accounts.Database, cfg *con // CheckAndSave3PIDAssociation implements POST /account/3pid func CheckAndSave3PIDAssociation( - req *http.Request, accountDB *accounts.Database, device *authtypes.Device, + req *http.Request, accountDB accounts.Database, device *authtypes.Device, cfg *config.Dendrite, ) util.JSONResponse { var body threepid.EmailAssociationCheckRequest @@ -142,7 +142,7 @@ func CheckAndSave3PIDAssociation( // GetAssociated3PIDs implements GET /account/3pid func GetAssociated3PIDs( - req *http.Request, accountDB *accounts.Database, device *authtypes.Device, + req *http.Request, accountDB accounts.Database, device *authtypes.Device, ) util.JSONResponse { localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) if err != nil { @@ -161,7 +161,7 @@ func GetAssociated3PIDs( } // Forget3PID implements POST /account/3pid/delete -func Forget3PID(req *http.Request, accountDB *accounts.Database) util.JSONResponse { +func Forget3PID(req *http.Request, accountDB accounts.Database) util.JSONResponse { var body authtypes.ThreePID if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil { return *reqErr diff --git a/clientapi/threepid/invites.go b/clientapi/threepid/invites.go index 2cf88d6e3..aa54aa9fa 100644 --- a/clientapi/threepid/invites.go +++ b/clientapi/threepid/invites.go @@ -87,7 +87,7 @@ var ( func CheckAndProcessInvite( ctx context.Context, device *authtypes.Device, body *MembershipRequest, cfg *config.Dendrite, - queryAPI api.RoomserverQueryAPI, db *accounts.Database, + queryAPI api.RoomserverQueryAPI, db accounts.Database, producer *producers.RoomserverProducer, membership string, roomID string, evTime time.Time, ) (inviteStoredOnIDServer bool, err error) { @@ -137,7 +137,7 @@ func CheckAndProcessInvite( // Returns an error if a check or a request failed. func queryIDServer( ctx context.Context, - db *accounts.Database, cfg *config.Dendrite, device *authtypes.Device, + db accounts.Database, cfg *config.Dendrite, device *authtypes.Device, body *MembershipRequest, roomID string, ) (lookupRes *idServerLookupResponse, storeInviteRes *idServerStoreInviteResponse, err error) { if err = isTrusted(body.IDServer, cfg); err != nil { @@ -206,7 +206,7 @@ func queryIDServerLookup(ctx context.Context, body *MembershipRequest) (*idServe // Returns an error if the request failed to send or if the response couldn't be parsed. func queryIDServerStoreInvite( ctx context.Context, - db *accounts.Database, cfg *config.Dendrite, device *authtypes.Device, + db accounts.Database, cfg *config.Dendrite, device *authtypes.Device, body *MembershipRequest, roomID string, ) (*idServerStoreInviteResponse, error) { // Retrieve the sender's profile to get their display name diff --git a/cmd/kafka-producer/main.go b/cmd/kafka-producer/main.go index 8a4340f21..f5f243e4e 100644 --- a/cmd/kafka-producer/main.go +++ b/cmd/kafka-producer/main.go @@ -21,7 +21,7 @@ import ( "os" "strings" - "github.com/Shopify/sarama" + sarama "gopkg.in/Shopify/sarama.v1" ) const usage = `Usage: %s diff --git a/common/basecomponent/base.go b/common/basecomponent/base.go index dc917ffe2..4274de2b6 100644 --- a/common/basecomponent/base.go +++ b/common/basecomponent/base.go @@ -18,6 +18,7 @@ import ( "database/sql" "io" "net/http" + "net/url" "golang.org/x/crypto/ed25519" @@ -68,7 +69,13 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string) *BaseDendrite { logrus.WithError(err).Panicf("failed to start opentracing") } - kafkaConsumer, kafkaProducer := setupKafka(cfg) + var kafkaConsumer sarama.Consumer + var kafkaProducer sarama.SyncProducer + if cfg.Kafka.UseNaffka { + kafkaConsumer, kafkaProducer = setupNaffka(cfg) + } else { + kafkaConsumer, kafkaProducer = setupKafka(cfg) + } return &BaseDendrite{ componentName: componentName, @@ -118,7 +125,7 @@ func (b *BaseDendrite) CreateHTTPFederationSenderAPIs() federationSenderAPI.Fede // CreateDeviceDB creates a new instance of the device database. Should only be // called once per component. -func (b *BaseDendrite) CreateDeviceDB() *devices.Database { +func (b *BaseDendrite) CreateDeviceDB() devices.Database { db, err := devices.NewDatabase(string(b.Cfg.Database.Device), b.Cfg.Matrix.ServerName) if err != nil { logrus.WithError(err).Panicf("failed to connect to devices db") @@ -129,7 +136,7 @@ func (b *BaseDendrite) CreateDeviceDB() *devices.Database { // CreateAccountsDB creates a new instance of the accounts database. Should only // be called once per component. -func (b *BaseDendrite) CreateAccountsDB() *accounts.Database { +func (b *BaseDendrite) CreateAccountsDB() accounts.Database { db, err := accounts.NewDatabase(string(b.Cfg.Database.Account), b.Cfg.Matrix.ServerName) if err != nil { logrus.WithError(err).Panicf("failed to connect to accounts db") @@ -186,28 +193,8 @@ func (b *BaseDendrite) SetupAndServeHTTP(bindaddr string, listenaddr string) { logrus.Infof("Stopped %s server on %s", b.componentName, addr) } -// setupKafka creates kafka consumer/producer pair from the config. Checks if -// should use naffka. +// setupKafka creates kafka consumer/producer pair from the config. func setupKafka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) { - if cfg.Kafka.UseNaffka { - db, err := sql.Open("postgres", string(cfg.Database.Naffka)) - if err != nil { - logrus.WithError(err).Panic("Failed to open naffka database") - } - - naffkaDB, err := naffka.NewPostgresqlDatabase(db) - if err != nil { - logrus.WithError(err).Panic("Failed to setup naffka database") - } - - naff, err := naffka.New(naffkaDB) - if err != nil { - logrus.WithError(err).Panic("Failed to setup naffka") - } - - return naff, naff - } - consumer, err := sarama.NewConsumer(cfg.Kafka.Addresses, nil) if err != nil { logrus.WithError(err).Panic("failed to start kafka consumer") @@ -220,3 +207,44 @@ func setupKafka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) { return consumer, producer } + +// setupNaffka creates kafka consumer/producer pair from the config. +func setupNaffka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) { + var err error + var db *sql.DB + var naffkaDB *naffka.DatabaseImpl + + uri, err := url.Parse(string(cfg.Database.Naffka)) + if err != nil || uri.Scheme == "file" { + db, err = sql.Open("sqlite3", string(cfg.Database.Naffka)) + if err != nil { + logrus.WithError(err).Panic("Failed to open naffka database") + } + + naffkaDB, err = naffka.NewSqliteDatabase(db) + if err != nil { + logrus.WithError(err).Panic("Failed to setup naffka database") + } + } else { + db, err = sql.Open("postgres", string(cfg.Database.Naffka)) + if err != nil { + logrus.WithError(err).Panic("Failed to open naffka database") + } + + naffkaDB, err = naffka.NewPostgresqlDatabase(db) + if err != nil { + logrus.WithError(err).Panic("Failed to setup naffka database") + } + } + + if naffkaDB == nil { + panic("naffka connection string not understood") + } + + naff, err := naffka.New(naffkaDB) + if err != nil { + logrus.WithError(err).Panic("Failed to setup naffka") + } + + return naff, naff +} diff --git a/common/keydb/keydb.go b/common/keydb/keydb.go index d1f2b7ebc..cf15c9f0c 100644 --- a/common/keydb/keydb.go +++ b/common/keydb/keydb.go @@ -21,6 +21,7 @@ import ( "golang.org/x/crypto/ed25519" "github.com/matrix-org/dendrite/common/keydb/postgres" + "github.com/matrix-org/dendrite/common/keydb/sqlite3" "github.com/matrix-org/gomatrixserverlib" ) @@ -44,6 +45,8 @@ func NewDatabase( switch uri.Scheme { case "postgres": return postgres.NewDatabase(dataSourceName, serverName, serverKey, serverKeyID) + case "file": + return sqlite3.NewDatabase(dataSourceName, serverName, serverKey, serverKeyID) default: return postgres.NewDatabase(dataSourceName, serverName, serverKey, serverKeyID) } diff --git a/common/keydb/sqlite3/keydb.go b/common/keydb/sqlite3/keydb.go new file mode 100644 index 000000000..88eb9d9fa --- /dev/null +++ b/common/keydb/sqlite3/keydb.go @@ -0,0 +1,115 @@ +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + "math" + + "golang.org/x/crypto/ed25519" + + "github.com/matrix-org/gomatrixserverlib" + + _ "github.com/mattn/go-sqlite3" +) + +// A Database implements gomatrixserverlib.KeyDatabase and is used to store +// the public keys for other matrix servers. +type Database struct { + statements serverKeyStatements +} + +// NewDatabase prepares a new key database. +// It creates the necessary tables if they don't already exist. +// It prepares all the SQL statements that it will use. +// Returns an error if there was a problem talking to the database. +func NewDatabase( + dataSourceName string, + serverName gomatrixserverlib.ServerName, + serverKey ed25519.PublicKey, + serverKeyID gomatrixserverlib.KeyID, +) (*Database, error) { + db, err := sql.Open("sqlite3", dataSourceName) + if err != nil { + return nil, err + } + d := &Database{} + err = d.statements.prepare(db) + if err != nil { + return nil, err + } + // Store our own keys so that we don't end up making HTTP requests to find our + // own keys + index := gomatrixserverlib.PublicKeyLookupRequest{ + ServerName: serverName, + KeyID: serverKeyID, + } + value := gomatrixserverlib.PublicKeyLookupResult{ + VerifyKey: gomatrixserverlib.VerifyKey{ + Key: gomatrixserverlib.Base64String(serverKey), + }, + ValidUntilTS: math.MaxUint64 >> 1, + ExpiredTS: gomatrixserverlib.PublicKeyNotExpired, + } + err = d.StoreKeys( + context.Background(), + map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult{ + index: value, + }, + ) + if err != nil { + return nil, err + } + return d, nil +} + +// FetcherName implements KeyFetcher +func (d Database) FetcherName() string { + return "KeyDatabase" +} + +// FetchKeys implements gomatrixserverlib.KeyDatabase +func (d *Database) FetchKeys( + ctx context.Context, + requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp, +) (map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, error) { + return d.statements.bulkSelectServerKeys(ctx, requests) +} + +// StoreKeys implements gomatrixserverlib.KeyDatabase +func (d *Database) StoreKeys( + ctx context.Context, + keyMap map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, +) error { + // TODO: Inserting all the keys within a single transaction may + // be more efficient since the transaction overhead can be quite + // high for a single insert statement. + var lastErr error + for request, keys := range keyMap { + if err := d.statements.upsertServerKeys(ctx, request, keys); err != nil { + // Rather than returning immediately on error we try to insert the + // remaining keys. + // Since we are inserting the keys outside of a transaction it is + // possible for some of the inserts to succeed even though some + // of the inserts have failed. + // Ensuring that we always insert all the keys we can means that + // this behaviour won't depend on the iteration order of the map. + lastErr = err + } + } + return lastErr +} diff --git a/common/keydb/sqlite3/server_key_table.go b/common/keydb/sqlite3/server_key_table.go new file mode 100644 index 000000000..6c33f30a0 --- /dev/null +++ b/common/keydb/sqlite3/server_key_table.go @@ -0,0 +1,142 @@ +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + + "github.com/lib/pq" + "github.com/matrix-org/gomatrixserverlib" +) + +const serverKeysSchema = ` +-- A cache of signing keys downloaded from remote servers. +CREATE TABLE IF NOT EXISTS keydb_server_keys ( + -- The name of the matrix server the key is for. + server_name TEXT NOT NULL, + -- The ID of the server key. + server_key_id TEXT NOT NULL, + -- Combined server name and key ID separated by the ASCII unit separator + -- to make it easier to run bulk queries. + server_name_and_key_id TEXT NOT NULL, + -- When the key is valid until as a millisecond timestamp. + -- 0 if this is an expired key (in which case expired_ts will be non-zero) + valid_until_ts BIGINT NOT NULL, + -- When the key expired as a millisecond timestamp. + -- 0 if this is an active key (in which case valid_until_ts will be non-zero) + expired_ts BIGINT NOT NULL, + -- The base64-encoded public key. + server_key TEXT NOT NULL, + UNIQUE (server_name, server_key_id) +); + +CREATE INDEX IF NOT EXISTS keydb_server_name_and_key_id ON keydb_server_keys (server_name_and_key_id); +` + +const bulkSelectServerKeysSQL = "" + + "SELECT server_name, server_key_id, valid_until_ts, expired_ts, " + + " server_key FROM keydb_server_keys" + + " WHERE server_name_and_key_id IN ($1)" + +const upsertServerKeysSQL = "" + + "INSERT INTO keydb_server_keys (server_name, server_key_id," + + " server_name_and_key_id, valid_until_ts, expired_ts, server_key)" + + " VALUES ($1, $2, $3, $4, $5, $6)" + + " ON CONFLICT (server_name, server_key_id)" + + " DO UPDATE SET valid_until_ts = $4, expired_ts = $5, server_key = $6" + +type serverKeyStatements struct { + bulkSelectServerKeysStmt *sql.Stmt + upsertServerKeysStmt *sql.Stmt +} + +func (s *serverKeyStatements) prepare(db *sql.DB) (err error) { + _, err = db.Exec(serverKeysSchema) + if err != nil { + return + } + if s.bulkSelectServerKeysStmt, err = db.Prepare(bulkSelectServerKeysSQL); err != nil { + return + } + if s.upsertServerKeysStmt, err = db.Prepare(upsertServerKeysSQL); err != nil { + return + } + return +} + +func (s *serverKeyStatements) bulkSelectServerKeys( + ctx context.Context, + requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp, +) (map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, error) { + var nameAndKeyIDs []string + for request := range requests { + nameAndKeyIDs = append(nameAndKeyIDs, nameAndKeyID(request)) + } + stmt := s.bulkSelectServerKeysStmt + rows, err := stmt.QueryContext(ctx, pq.StringArray(nameAndKeyIDs)) + if err != nil { + return nil, err + } + defer rows.Close() // nolint: errcheck + results := map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult{} + for rows.Next() { + var serverName string + var keyID string + var key string + var validUntilTS int64 + var expiredTS int64 + if err = rows.Scan(&serverName, &keyID, &validUntilTS, &expiredTS, &key); err != nil { + return nil, err + } + r := gomatrixserverlib.PublicKeyLookupRequest{ + ServerName: gomatrixserverlib.ServerName(serverName), + KeyID: gomatrixserverlib.KeyID(keyID), + } + vk := gomatrixserverlib.VerifyKey{} + err = vk.Key.Decode(key) + if err != nil { + return nil, err + } + results[r] = gomatrixserverlib.PublicKeyLookupResult{ + VerifyKey: vk, + ValidUntilTS: gomatrixserverlib.Timestamp(validUntilTS), + ExpiredTS: gomatrixserverlib.Timestamp(expiredTS), + } + } + return results, nil +} + +func (s *serverKeyStatements) upsertServerKeys( + ctx context.Context, + request gomatrixserverlib.PublicKeyLookupRequest, + key gomatrixserverlib.PublicKeyLookupResult, +) error { + _, err := s.upsertServerKeysStmt.ExecContext( + ctx, + string(request.ServerName), + string(request.KeyID), + nameAndKeyID(request), + key.ValidUntilTS, + key.ExpiredTS, + key.Key.Encode(), + ) + return err +} + +func nameAndKeyID(request gomatrixserverlib.PublicKeyLookupRequest) string { + return string(request.ServerName) + "\x1F" + string(request.KeyID) +} diff --git a/common/partition_offset_table.go b/common/partition_offset_table.go index 6955ac365..6bc066a69 100644 --- a/common/partition_offset_table.go +++ b/common/partition_offset_table.go @@ -29,7 +29,7 @@ CREATE TABLE IF NOT EXISTS ${prefix}_partition_offsets ( partition INTEGER NOT NULL, -- The 64-bit offset. partition_offset BIGINT NOT NULL, - CONSTRAINT ${prefix}_topic_partition_unique UNIQUE (topic, partition) + UNIQUE (topic, partition) ); ` @@ -38,7 +38,7 @@ const selectPartitionOffsetsSQL = "" + const upsertPartitionOffsetsSQL = "" + "INSERT INTO ${prefix}_partition_offsets (topic, partition, partition_offset) VALUES ($1, $2, $3)" + - " ON CONFLICT ON CONSTRAINT ${prefix}_topic_partition_unique" + + " ON CONFLICT (topic, partition)" + " DO UPDATE SET partition_offset = $3" // PartitionOffsetStatements represents a set of statements that can be run on a partition_offsets table. diff --git a/common/sql.go b/common/sql.go index 7ac9ac140..975930202 100644 --- a/common/sql.go +++ b/common/sql.go @@ -16,6 +16,7 @@ package common import ( "database/sql" + "fmt" "github.com/lib/pq" ) @@ -30,11 +31,13 @@ type Transaction interface { // EndTransaction ends a transaction. // If the transaction succeeded then it is committed, otherwise it is rolledback. -func EndTransaction(txn Transaction, succeeded *bool) { +// You MUST check the error returned from this function to be sure that the transaction +// was applied correctly. For example, 'database is locked' errors in sqlite will happen here. +func EndTransaction(txn Transaction, succeeded *bool) error { if *succeeded { - txn.Commit() // nolint: errcheck + return txn.Commit() // nolint: errcheck } else { - txn.Rollback() // nolint: errcheck + return txn.Rollback() // nolint: errcheck } } @@ -47,7 +50,12 @@ func WithTransaction(db *sql.DB, fn func(txn *sql.Tx) error) (err error) { return } succeeded := false - defer EndTransaction(txn, &succeeded) + defer func() { + err2 := EndTransaction(txn, &succeeded) + if err == nil && err2 != nil { // failed to commit/rollback + err = err2 + } + }() err = fn(txn) if err != nil { @@ -74,3 +82,20 @@ func IsUniqueConstraintViolationErr(err error) bool { pqErr, ok := err.(*pq.Error) return ok && pqErr.Code == "23505" } + +// Hack of the century +func QueryVariadic(count int) string { + return QueryVariadicOffset(count, 0) +} + +func QueryVariadicOffset(count, offset int) string { + str := "(" + for i := 0; i < count; i++ { + str += fmt.Sprintf("$%d", i+offset+1) + if i < (count - 1) { + str += ", " + } + } + str += ")" + return str +} diff --git a/docker/Dockerfile b/docker/Dockerfile index c88b77617..29b27dde2 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,9 +1,13 @@ +<<<<<<< HEAD +FROM docker.io/golang:1.13.7-alpine3.11 +======= FROM docker.io/golang:1.13.6-alpine +>>>>>>> master RUN mkdir /build WORKDIR /build -RUN apk --update --no-cache add openssl bash git +RUN apk --update --no-cache add openssl bash git build-base CMD ["bash", "docker/build.sh"] diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 9cf67457c..d738ed3f0 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,13 +1,21 @@ version: "3.4" services: + riot: + image: vectorim/riot-web + networks: + - internal + ports: + - "8500:80" + monolith: container_name: dendrite_monolith hostname: monolith - entrypoint: ["bash", "./docker/services/monolith.sh"] + entrypoint: ["bash", "./docker/services/monolith.sh", "--config", "/etc/dendrite/dendrite.yaml"] build: ./ volumes: - ..:/build - ./build/bin:/build/bin + - ../cfg:/etc/dendrite networks: - internal depends_on: diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go index 53851bc51..ef57da881 100644 --- a/federationapi/federationapi.go +++ b/federationapi/federationapi.go @@ -32,8 +32,8 @@ import ( // FederationAPI component. func SetupFederationAPIComponent( base *basecomponent.BaseDendrite, - accountsDB *accounts.Database, - deviceDB *devices.Database, + accountsDB accounts.Database, + deviceDB devices.Database, federation *gomatrixserverlib.FederationClient, keyRing *gomatrixserverlib.KeyRing, aliasAPI roomserverAPI.RoomserverAliasAPI, diff --git a/federationapi/routing/devices.go b/federationapi/routing/devices.go index ba8af7a9a..78021c12e 100644 --- a/federationapi/routing/devices.go +++ b/federationapi/routing/devices.go @@ -30,7 +30,7 @@ type userDevicesResponse struct { // GetUserDevices for the given user id func GetUserDevices( req *http.Request, - deviceDB *devices.Database, + deviceDB devices.Database, userID string, ) util.JSONResponse { localpart, err := userutil.ParseUsernameParam(userID, nil) diff --git a/federationapi/routing/profile.go b/federationapi/routing/profile.go index 3be729c29..31b7a343f 100644 --- a/federationapi/routing/profile.go +++ b/federationapi/routing/profile.go @@ -30,7 +30,7 @@ import ( // GetProfile implements GET /_matrix/federation/v1/query/profile func GetProfile( httpReq *http.Request, - accountDB *accounts.Database, + accountDB accounts.Database, cfg *config.Dendrite, asAPI appserviceAPI.AppServiceQueryAPI, ) util.JSONResponse { diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index 13ed24f3a..3b119301a 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -51,8 +51,8 @@ func Setup( federationSenderAPI federationSenderAPI.FederationSenderQueryAPI, keys gomatrixserverlib.KeyRing, federation *gomatrixserverlib.FederationClient, - accountDB *accounts.Database, - deviceDB *devices.Database, + accountDB accounts.Database, + deviceDB devices.Database, ) { v2keysmux := apiMux.PathPrefix(pathPrefixV2Keys).Subrouter() v1fedmux := apiMux.PathPrefix(pathPrefixV1Federation).Subrouter() diff --git a/federationapi/routing/threepid.go b/federationapi/routing/threepid.go index 06e00eeaf..a22685f25 100644 --- a/federationapi/routing/threepid.go +++ b/federationapi/routing/threepid.go @@ -61,7 +61,7 @@ func CreateInvitesFrom3PIDInvites( req *http.Request, queryAPI roomserverAPI.RoomserverQueryAPI, asAPI appserviceAPI.AppServiceQueryAPI, cfg *config.Dendrite, producer *producers.RoomserverProducer, federation *gomatrixserverlib.FederationClient, - accountDB *accounts.Database, + accountDB accounts.Database, ) util.JSONResponse { var body invites if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil { @@ -174,7 +174,7 @@ func createInviteFrom3PIDInvite( ctx context.Context, queryAPI roomserverAPI.RoomserverQueryAPI, asAPI appserviceAPI.AppServiceQueryAPI, cfg *config.Dendrite, inv invite, federation *gomatrixserverlib.FederationClient, - accountDB *accounts.Database, + accountDB accounts.Database, ) (*gomatrixserverlib.Event, error) { _, server, err := gomatrixserverlib.SplitID('@', inv.MXID) if err != nil { diff --git a/federationsender/storage/postgres/storage.go b/federationsender/storage/postgres/storage.go index c60f6dc5c..d97b5d296 100644 --- a/federationsender/storage/postgres/storage.go +++ b/federationsender/storage/postgres/storage.go @@ -87,7 +87,7 @@ func (d *Database) UpdateRoom( return nil } - if lastSentEventID != oldEventID { + if lastSentEventID != "" && lastSentEventID != oldEventID { return types.EventIDMismatchError{ DatabaseID: lastSentEventID, RoomServerID: oldEventID, } diff --git a/federationsender/storage/sqlite3/joined_hosts_table.go b/federationsender/storage/sqlite3/joined_hosts_table.go new file mode 100644 index 000000000..1437a062b --- /dev/null +++ b/federationsender/storage/sqlite3/joined_hosts_table.go @@ -0,0 +1,139 @@ +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/federationsender/types" + "github.com/matrix-org/gomatrixserverlib" +) + +const joinedHostsSchema = ` +-- The joined_hosts table stores a list of m.room.member event ids in the +-- current state for each room where the membership is "join". +-- There will be an entry for every user that is joined to the room. +CREATE TABLE IF NOT EXISTS federationsender_joined_hosts ( + -- The string ID of the room. + room_id TEXT NOT NULL, + -- The event ID of the m.room.member join event. + event_id TEXT NOT NULL, + -- The domain part of the user ID the m.room.member event is for. + server_name TEXT NOT NULL +); + +CREATE UNIQUE INDEX IF NOT EXISTS federatonsender_joined_hosts_event_id_idx + ON federationsender_joined_hosts (event_id); + +CREATE INDEX IF NOT EXISTS federatonsender_joined_hosts_room_id_idx + ON federationsender_joined_hosts (room_id) +` + +const insertJoinedHostsSQL = "" + + "INSERT INTO federationsender_joined_hosts (room_id, event_id, server_name)" + + " VALUES ($1, $2, $3)" + +const deleteJoinedHostsSQL = "" + + "DELETE FROM federationsender_joined_hosts WHERE event_id = $1" + +const selectJoinedHostsSQL = "" + + "SELECT event_id, server_name FROM federationsender_joined_hosts" + + " WHERE room_id = $1" + +type joinedHostsStatements struct { + insertJoinedHostsStmt *sql.Stmt + deleteJoinedHostsStmt *sql.Stmt + selectJoinedHostsStmt *sql.Stmt +} + +func (s *joinedHostsStatements) prepare(db *sql.DB) (err error) { + _, err = db.Exec(joinedHostsSchema) + if err != nil { + return + } + if s.insertJoinedHostsStmt, err = db.Prepare(insertJoinedHostsSQL); err != nil { + return + } + if s.deleteJoinedHostsStmt, err = db.Prepare(deleteJoinedHostsSQL); err != nil { + return + } + if s.selectJoinedHostsStmt, err = db.Prepare(selectJoinedHostsSQL); err != nil { + return + } + return +} + +func (s *joinedHostsStatements) insertJoinedHosts( + ctx context.Context, + txn *sql.Tx, + roomID, eventID string, + serverName gomatrixserverlib.ServerName, +) error { + stmt := common.TxStmt(txn, s.insertJoinedHostsStmt) + _, err := stmt.ExecContext(ctx, roomID, eventID, serverName) + return err +} + +func (s *joinedHostsStatements) deleteJoinedHosts( + ctx context.Context, txn *sql.Tx, eventIDs []string, +) error { + for _, eventID := range eventIDs { + stmt := common.TxStmt(txn, s.deleteJoinedHostsStmt) + if _, err := stmt.ExecContext(ctx, eventID); err != nil { + return err + } + } + return nil +} + +func (s *joinedHostsStatements) selectJoinedHostsWithTx( + ctx context.Context, txn *sql.Tx, roomID string, +) ([]types.JoinedHost, error) { + stmt := common.TxStmt(txn, s.selectJoinedHostsStmt) + return joinedHostsFromStmt(ctx, stmt, roomID) +} + +func (s *joinedHostsStatements) selectJoinedHosts( + ctx context.Context, roomID string, +) ([]types.JoinedHost, error) { + return joinedHostsFromStmt(ctx, s.selectJoinedHostsStmt, roomID) +} + +func joinedHostsFromStmt( + ctx context.Context, stmt *sql.Stmt, roomID string, +) ([]types.JoinedHost, error) { + rows, err := stmt.QueryContext(ctx, roomID) + if err != nil { + return nil, err + } + defer rows.Close() // nolint: errcheck + + var result []types.JoinedHost + for rows.Next() { + var eventID, serverName string + if err = rows.Scan(&eventID, &serverName); err != nil { + return nil, err + } + result = append(result, types.JoinedHost{ + MemberEventID: eventID, + ServerName: gomatrixserverlib.ServerName(serverName), + }) + } + + return result, nil +} diff --git a/federationsender/storage/sqlite3/room_table.go b/federationsender/storage/sqlite3/room_table.go new file mode 100644 index 000000000..6361400d3 --- /dev/null +++ b/federationsender/storage/sqlite3/room_table.go @@ -0,0 +1,101 @@ +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + + "github.com/matrix-org/dendrite/common" +) + +const roomSchema = ` +CREATE TABLE IF NOT EXISTS federationsender_rooms ( + -- The string ID of the room + room_id TEXT PRIMARY KEY, + -- The most recent event state by the room server. + -- We can use this to tell if our view of the room state has become + -- desynchronised. + last_event_id TEXT NOT NULL +);` + +const insertRoomSQL = "" + + "INSERT INTO federationsender_rooms (room_id, last_event_id) VALUES ($1, '')" + + " ON CONFLICT DO NOTHING" + +const selectRoomForUpdateSQL = "" + + "SELECT last_event_id FROM federationsender_rooms WHERE room_id = $1" + +const updateRoomSQL = "" + + "UPDATE federationsender_rooms SET last_event_id = $2 WHERE room_id = $1" + +type roomStatements struct { + insertRoomStmt *sql.Stmt + selectRoomForUpdateStmt *sql.Stmt + updateRoomStmt *sql.Stmt +} + +func (s *roomStatements) prepare(db *sql.DB) (err error) { + _, err = db.Exec(roomSchema) + if err != nil { + return + } + + if s.insertRoomStmt, err = db.Prepare(insertRoomSQL); err != nil { + return + } + if s.selectRoomForUpdateStmt, err = db.Prepare(selectRoomForUpdateSQL); err != nil { + return + } + if s.updateRoomStmt, err = db.Prepare(updateRoomSQL); err != nil { + return + } + return +} + +// insertRoom inserts the room if it didn't already exist. +// If the room didn't exist then last_event_id is set to the empty string. +func (s *roomStatements) insertRoom( + ctx context.Context, txn *sql.Tx, roomID string, +) error { + _, err := common.TxStmt(txn, s.insertRoomStmt).ExecContext(ctx, roomID) + return err +} + +// selectRoomForUpdate locks the row for the room and returns the last_event_id. +// The row must already exist in the table. Callers can ensure that the row +// exists by calling insertRoom first. +func (s *roomStatements) selectRoomForUpdate( + ctx context.Context, txn *sql.Tx, roomID string, +) (string, error) { + var lastEventID string + stmt := common.TxStmt(txn, s.selectRoomForUpdateStmt) + err := stmt.QueryRowContext(ctx, roomID).Scan(&lastEventID) + if err != nil { + return "", err + } + return lastEventID, nil +} + +// updateRoom updates the last_event_id for the room. selectRoomForUpdate should +// have already been called earlier within the transaction. +func (s *roomStatements) updateRoom( + ctx context.Context, txn *sql.Tx, roomID, lastEventID string, +) error { + stmt := common.TxStmt(txn, s.updateRoomStmt) + _, err := stmt.ExecContext(ctx, roomID, lastEventID) + return err +} diff --git a/federationsender/storage/sqlite3/storage.go b/federationsender/storage/sqlite3/storage.go new file mode 100644 index 000000000..f9cfaa99d --- /dev/null +++ b/federationsender/storage/sqlite3/storage.go @@ -0,0 +1,124 @@ +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + + _ "github.com/mattn/go-sqlite3" + + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/federationsender/types" +) + +// Database stores information needed by the federation sender +type Database struct { + joinedHostsStatements + roomStatements + common.PartitionOffsetStatements + db *sql.DB +} + +// NewDatabase opens a new database +func NewDatabase(dataSourceName string) (*Database, error) { + var result Database + var err error + if result.db, err = sql.Open("sqlite3", dataSourceName); err != nil { + return nil, err + } + if err = result.prepare(); err != nil { + return nil, err + } + return &result, nil +} + +func (d *Database) prepare() error { + var err error + + if err = d.joinedHostsStatements.prepare(d.db); err != nil { + return err + } + + if err = d.roomStatements.prepare(d.db); err != nil { + return err + } + + return d.PartitionOffsetStatements.Prepare(d.db, "federationsender") +} + +// UpdateRoom updates the joined hosts for a room and returns what the joined +// hosts were before the update, or nil if this was a duplicate message. +// This is called when we receive a message from kafka, so we pass in +// oldEventID and newEventID to check that we haven't missed any messages or +// this isn't a duplicate message. +func (d *Database) UpdateRoom( + ctx context.Context, + roomID, oldEventID, newEventID string, + addHosts []types.JoinedHost, + removeHosts []string, +) (joinedHosts []types.JoinedHost, err error) { + err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + err = d.insertRoom(ctx, txn, roomID) + if err != nil { + return err + } + + lastSentEventID, err := d.selectRoomForUpdate(ctx, txn, roomID) + if err != nil { + return err + } + + if lastSentEventID == newEventID { + // We've handled this message before, so let's just ignore it. + // We can only get a duplicate for the last message we processed, + // so its enough just to compare the newEventID with lastSentEventID + return nil + } + + if lastSentEventID != "" && lastSentEventID != oldEventID { + return types.EventIDMismatchError{ + DatabaseID: lastSentEventID, RoomServerID: oldEventID, + } + } + + joinedHosts, err = d.selectJoinedHostsWithTx(ctx, txn, roomID) + if err != nil { + return err + } + + for _, add := range addHosts { + err = d.insertJoinedHosts(ctx, txn, roomID, add.MemberEventID, add.ServerName) + if err != nil { + return err + } + } + if err = d.deleteJoinedHosts(ctx, txn, removeHosts); err != nil { + return err + } + return d.updateRoom(ctx, txn, roomID, newEventID) + }) + return +} + +// GetJoinedHosts returns the currently joined hosts for room, +// as known to federationserver. +// Returns an error if something goes wrong. +func (d *Database) GetJoinedHosts( + ctx context.Context, roomID string, +) ([]types.JoinedHost, error) { + return d.selectJoinedHosts(ctx, roomID) +} diff --git a/federationsender/storage/storage.go b/federationsender/storage/storage.go index 4ce151c7a..e83c1e9d2 100644 --- a/federationsender/storage/storage.go +++ b/federationsender/storage/storage.go @@ -20,6 +20,7 @@ import ( "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/federationsender/storage/postgres" + "github.com/matrix-org/dendrite/federationsender/storage/sqlite3" "github.com/matrix-org/dendrite/federationsender/types" ) @@ -36,6 +37,8 @@ func NewDatabase(dataSourceName string) (Database, error) { return postgres.NewDatabase(dataSourceName) } switch uri.Scheme { + case "file": + return sqlite3.NewDatabase(dataSourceName) case "postgres": return postgres.NewDatabase(dataSourceName) default: diff --git a/go.mod b/go.mod index 990b839eb..2d442cd0d 100644 --- a/go.mod +++ b/go.mod @@ -1,30 +1,30 @@ module github.com/matrix-org/dendrite require ( - github.com/Shopify/sarama v0.0.0-20170127151855-574d3147eee3 + github.com/DataDog/zstd v1.4.4 // indirect github.com/Shopify/toxiproxy v2.1.4+incompatible // indirect github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect - github.com/eapache/go-resiliency v0.0.0-20160104191539-b86b1ec0dd42 // indirect - github.com/eapache/go-xerial-snappy v0.0.0-20160609142408-bb955e01b934 // indirect + github.com/eapache/go-resiliency v1.2.0 // indirect + github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect github.com/eapache/queue v1.1.0 // indirect - github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db // indirect + github.com/frankban/quicktest v1.7.2 // indirect + github.com/golang/snappy v0.0.1 // indirect github.com/gorilla/mux v1.7.3 - github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6 // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect github.com/lib/pq v1.2.0 github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 - github.com/matrix-org/naffka v0.0.0-20171115094957-662bfd0841d0 + github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 + github.com/mattn/go-sqlite3 v2.0.2+incompatible github.com/miekg/dns v1.1.12 // indirect github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 github.com/opentracing/opentracing-go v1.0.2 - github.com/pierrec/lz4 v0.0.0-20161206202305-5c9560bfa9ac // indirect - github.com/pierrec/xxHash v0.0.0-20160112165351-5a004441f897 // indirect + github.com/pierrec/lz4 v2.4.1+incompatible // indirect github.com/pkg/errors v0.8.1 github.com/prometheus/client_golang v1.2.1 - github.com/rcrowley/go-metrics v0.0.0-20161128210544-1f30fe9094a5 // indirect + github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 // indirect github.com/sirupsen/logrus v1.4.2 github.com/stretchr/testify v1.4.0 // indirect github.com/uber-go/atomic v1.3.0 // indirect @@ -34,7 +34,7 @@ require ( golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 golang.org/x/net v0.0.0-20190620200207-3b0461eec859 // indirect golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 // indirect - gopkg.in/Shopify/sarama.v1 v1.11.0 + gopkg.in/Shopify/sarama.v1 v1.20.1 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect gopkg.in/h2non/bimg.v1 v1.0.18 gopkg.in/yaml.v2 v2.2.2 diff --git a/go.sum b/go.sum index 42a145d6c..7c8732f63 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -github.com/Shopify/sarama v0.0.0-20170127151855-574d3147eee3 h1:j6BAEHYn1kUyW2j7kY0mOJ/R8A0qWwXpvUAEHGemm/g= -github.com/Shopify/sarama v0.0.0-20170127151855-574d3147eee3/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= +github.com/DataDog/zstd v1.4.4 h1:+IawcoXhCBylN7ccwdwf8LOH2jKq7NavGpEPanrlTzE= +github.com/DataDog/zstd v1.4.4/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -18,13 +18,15 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/eapache/go-resiliency v0.0.0-20160104191539-b86b1ec0dd42 h1:f8ERmXYuaC+kCSv2w+y3rBK/oVu6If4DEm3jywJJ0hc= -github.com/eapache/go-resiliency v0.0.0-20160104191539-b86b1ec0dd42/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= -github.com/eapache/go-xerial-snappy v0.0.0-20160609142408-bb955e01b934 h1:oGLoaVIefp3tiOgi7+KInR/nNPvEpPM6GFo+El7fd14= -github.com/eapache/go-xerial-snappy v0.0.0-20160609142408-bb955e01b934/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= +github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q= +github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/frankban/quicktest v1.0.0/go.mod h1:R98jIehRai+d1/3Hv2//jOVCTJhW1VBavT6B6CuGq2k= +github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3BTYk= +github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= @@ -35,11 +37,13 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= @@ -48,8 +52,6 @@ github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplb github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6 h1:KAZ1BW2TCmT6PRihDPpocIy1QTtsAsrx6TneU/4+CMg= -github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -69,10 +71,12 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:kmRjpmFOenVpOaV/DRlo9p6z/IbOKlUC+hhKsAAh8Qg= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/naffka v0.0.0-20171115094957-662bfd0841d0 h1:p7WTwG+aXM86+yVrYAiCMW3ZHSmotVvuRbjtt3jC+4A= -github.com/matrix-org/naffka v0.0.0-20171115094957-662bfd0841d0/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= +github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= +github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 h1:W7l5CP4V7wPyPb4tYE11dbmeAOwtFQBTW0rf4OonOS8= github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5/go.mod h1:lePuOiXLNDott7NZfnQvJk0lAZ5HgvIuWGhel6J+RLA= +github.com/mattn/go-sqlite3 v2.0.2+incompatible h1:qzw9c2GNT8UFrgWNDhCTqRqYUSmu/Dav/9Z58LGpk7U= +github.com/mattn/go-sqlite3 v2.0.2+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.1.4 h1:rCMZsU2ScVSYcAsOXgmC6+AKOK+6pmQTOcw03nfwYV0= @@ -90,10 +94,8 @@ github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 h1:BvoENQQU+fZ9uukda/R github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/pierrec/lz4 v0.0.0-20161206202305-5c9560bfa9ac h1:tKcxwAA5OHUQjL6sWsuCIcP9OnzN+RwKfvomtIOsfy8= -github.com/pierrec/lz4 v0.0.0-20161206202305-5c9560bfa9ac/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pierrec/xxHash v0.0.0-20160112165351-5a004441f897 h1:jp3jc/PyyTrTKjJJ6rWnhTbmo7tGgBFyG9AL5FIrO1I= -github.com/pierrec/xxHash v0.0.0-20160112165351-5a004441f897/go.mod h1:w2waW5Zoa/Wc4Yqe0wgrIYAGKqRMf7czn2HNKXmuL+I= +github.com/pierrec/lz4 v2.4.1+incompatible h1:mFe7ttWaflA46Mhqh+jUfjp2qTbPYxLB2/OyBppH9dg= +github.com/pierrec/lz4 v2.4.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -114,8 +116,8 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8= github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/rcrowley/go-metrics v0.0.0-20161128210544-1f30fe9094a5 h1:gwcdIpH6NU2iF8CmcqD+CP6+1CkRBOhHaPR+iu6raBY= -github.com/rcrowley/go-metrics v0.0.0-20161128210544-1f30fe9094a5/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ= +github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -172,8 +174,8 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY= golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -gopkg.in/Shopify/sarama.v1 v1.11.0 h1:/3kaCyeYaPbr59IBjeqhIcUOB1vXlIVqXAYa5g5C5F0= -gopkg.in/Shopify/sarama.v1 v1.11.0/go.mod h1:AxnvoaevB2nBjNK17cG61A3LleFcWFwVBHBt+cot4Oc= +gopkg.in/Shopify/sarama.v1 v1.20.1 h1:Gi09A3fJXm0Jgt8kuKZ8YK+r60GfYn7MQuEmI3oq6hE= +gopkg.in/Shopify/sarama.v1 v1.20.1/go.mod h1:AxnvoaevB2nBjNK17cG61A3LleFcWFwVBHBt+cot4Oc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/mediaapi/mediaapi.go b/mediaapi/mediaapi.go index 46d1c328c..f2e614c17 100644 --- a/mediaapi/mediaapi.go +++ b/mediaapi/mediaapi.go @@ -27,7 +27,7 @@ import ( // component. func SetupMediaAPIComponent( base *basecomponent.BaseDendrite, - deviceDB *devices.Database, + deviceDB devices.Database, ) { mediaDB, err := storage.Open(string(base.Cfg.Database.MediaAPI)) if err != nil { diff --git a/mediaapi/routing/routing.go b/mediaapi/routing/routing.go index dcc6ac06c..71dad19b6 100644 --- a/mediaapi/routing/routing.go +++ b/mediaapi/routing/routing.go @@ -44,7 +44,7 @@ func Setup( apiMux *mux.Router, cfg *config.Dendrite, db storage.Database, - deviceDB *devices.Database, + deviceDB devices.Database, client *gomatrixserverlib.Client, ) { r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter() diff --git a/publicroomsapi/publicroomsapi.go b/publicroomsapi/publicroomsapi.go index 181966d3d..1e2a3f9bb 100644 --- a/publicroomsapi/publicroomsapi.go +++ b/publicroomsapi/publicroomsapi.go @@ -28,7 +28,7 @@ import ( // component. func SetupPublicRoomsAPIComponent( base *basecomponent.BaseDendrite, - deviceDB *devices.Database, + deviceDB devices.Database, rsQueryAPI roomserverAPI.RoomserverQueryAPI, ) { publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI)) diff --git a/publicroomsapi/routing/routing.go b/publicroomsapi/routing/routing.go index 3d2d2ac04..1953e04fc 100644 --- a/publicroomsapi/routing/routing.go +++ b/publicroomsapi/routing/routing.go @@ -34,7 +34,7 @@ const pathPrefixR0 = "/_matrix/client/r0" // Due to Setup being used to call many other functions, a gocyclo nolint is // applied: // nolint: gocyclo -func Setup(apiMux *mux.Router, deviceDB *devices.Database, publicRoomsDB storage.Database) { +func Setup(apiMux *mux.Router, deviceDB devices.Database, publicRoomsDB storage.Database) { r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter() authData := auth.Data{ diff --git a/publicroomsapi/storage/sqlite3/prepare.go b/publicroomsapi/storage/sqlite3/prepare.go new file mode 100644 index 000000000..482dfa2b9 --- /dev/null +++ b/publicroomsapi/storage/sqlite3/prepare.go @@ -0,0 +1,36 @@ +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "database/sql" +) + +// a statementList is a list of SQL statements to prepare and a pointer to where to store the resulting prepared statement. +type statementList []struct { + statement **sql.Stmt + sql string +} + +// prepare the SQL for each statement in the list and assign the result to the prepared statement. +func (s statementList) prepare(db *sql.DB) (err error) { + for _, statement := range s { + if *statement.statement, err = db.Prepare(statement.sql); err != nil { + return + } + } + return +} diff --git a/publicroomsapi/storage/sqlite3/public_rooms_table.go b/publicroomsapi/storage/sqlite3/public_rooms_table.go new file mode 100644 index 000000000..06c74a331 --- /dev/null +++ b/publicroomsapi/storage/sqlite3/public_rooms_table.go @@ -0,0 +1,277 @@ +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + "errors" + "fmt" + + "github.com/lib/pq" + "github.com/matrix-org/dendrite/publicroomsapi/types" +) + +var editableAttributes = []string{ + "aliases", + "canonical_alias", + "name", + "topic", + "world_readable", + "guest_can_join", + "avatar_url", + "visibility", +} + +const publicRoomsSchema = ` +-- Stores all of the rooms with data needed to create the server's room directory +CREATE TABLE IF NOT EXISTS publicroomsapi_public_rooms( + -- The room's ID + room_id TEXT NOT NULL PRIMARY KEY, + -- Number of joined members in the room + joined_members INTEGER NOT NULL DEFAULT 0, + -- Aliases of the room (empty array if none) + aliases TEXT[] NOT NULL DEFAULT '{}'::TEXT[], + -- Canonical alias of the room (empty string if none) + canonical_alias TEXT NOT NULL DEFAULT '', + -- Name of the room (empty string if none) + name TEXT NOT NULL DEFAULT '', + -- Topic of the room (empty string if none) + topic TEXT NOT NULL DEFAULT '', + -- Is the room world readable? + world_readable BOOLEAN NOT NULL DEFAULT false, + -- Can guest join the room? + guest_can_join BOOLEAN NOT NULL DEFAULT false, + -- URL of the room avatar (empty string if none) + avatar_url TEXT NOT NULL DEFAULT '', + -- Visibility of the room: true means the room is publicly visible, false + -- means the room is private + visibility BOOLEAN NOT NULL DEFAULT false +); +` + +const countPublicRoomsSQL = "" + + "SELECT COUNT(*) FROM publicroomsapi_public_rooms" + + " WHERE visibility = true" + +const selectPublicRoomsSQL = "" + + "SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" + + " FROM publicroomsapi_public_rooms WHERE visibility = true" + + " ORDER BY joined_members DESC" + + " OFFSET $1" + +const selectPublicRoomsWithLimitSQL = "" + + "SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" + + " FROM publicroomsapi_public_rooms WHERE visibility = true" + + " ORDER BY joined_members DESC" + + " OFFSET $1 LIMIT $2" + +const selectPublicRoomsWithFilterSQL = "" + + "SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" + + " FROM publicroomsapi_public_rooms" + + " WHERE visibility = true" + + " AND (LOWER(name) LIKE LOWER($1)" + + " OR LOWER(topic) LIKE LOWER($1)" + + " OR LOWER(ARRAY_TO_STRING(aliases, ',')) LIKE LOWER($1))" + + " ORDER BY joined_members DESC" + + " OFFSET $2" + +const selectPublicRoomsWithLimitAndFilterSQL = "" + + "SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" + + " FROM publicroomsapi_public_rooms" + + " WHERE visibility = true" + + " AND (LOWER(name) LIKE LOWER($1)" + + " OR LOWER(topic) LIKE LOWER($1)" + + " OR LOWER(ARRAY_TO_STRING(aliases, ',')) LIKE LOWER($1))" + + " ORDER BY joined_members DESC" + + " OFFSET $2 LIMIT $3" + +const selectRoomVisibilitySQL = "" + + "SELECT visibility FROM publicroomsapi_public_rooms" + + " WHERE room_id = $1" + +const insertNewRoomSQL = "" + + "INSERT INTO publicroomsapi_public_rooms(room_id)" + + " VALUES ($1)" + +const incrementJoinedMembersInRoomSQL = "" + + "UPDATE publicroomsapi_public_rooms" + + " SET joined_members = joined_members + 1" + + " WHERE room_id = $1" + +const decrementJoinedMembersInRoomSQL = "" + + "UPDATE publicroomsapi_public_rooms" + + " SET joined_members = joined_members - 1" + + " WHERE room_id = $1" + +const updateRoomAttributeSQL = "" + + "UPDATE publicroomsapi_public_rooms" + + " SET %s = $1" + + " WHERE room_id = $2" + +type publicRoomsStatements struct { + countPublicRoomsStmt *sql.Stmt + selectPublicRoomsStmt *sql.Stmt + selectPublicRoomsWithLimitStmt *sql.Stmt + selectPublicRoomsWithFilterStmt *sql.Stmt + selectPublicRoomsWithLimitAndFilterStmt *sql.Stmt + selectRoomVisibilityStmt *sql.Stmt + insertNewRoomStmt *sql.Stmt + incrementJoinedMembersInRoomStmt *sql.Stmt + decrementJoinedMembersInRoomStmt *sql.Stmt + updateRoomAttributeStmts map[string]*sql.Stmt +} + +func (s *publicRoomsStatements) prepare(db *sql.DB) (err error) { + _, err = db.Exec(publicRoomsSchema) + if err != nil { + return + } + + stmts := statementList{ + {&s.countPublicRoomsStmt, countPublicRoomsSQL}, + {&s.selectPublicRoomsStmt, selectPublicRoomsSQL}, + {&s.selectPublicRoomsWithLimitStmt, selectPublicRoomsWithLimitSQL}, + {&s.selectPublicRoomsWithFilterStmt, selectPublicRoomsWithFilterSQL}, + {&s.selectPublicRoomsWithLimitAndFilterStmt, selectPublicRoomsWithLimitAndFilterSQL}, + {&s.selectRoomVisibilityStmt, selectRoomVisibilitySQL}, + {&s.insertNewRoomStmt, insertNewRoomSQL}, + {&s.incrementJoinedMembersInRoomStmt, incrementJoinedMembersInRoomSQL}, + {&s.decrementJoinedMembersInRoomStmt, decrementJoinedMembersInRoomSQL}, + } + + if err = stmts.prepare(db); err != nil { + return + } + + s.updateRoomAttributeStmts = make(map[string]*sql.Stmt) + for _, editable := range editableAttributes { + stmt := fmt.Sprintf(updateRoomAttributeSQL, editable) + if s.updateRoomAttributeStmts[editable], err = db.Prepare(stmt); err != nil { + return + } + } + + return +} + +func (s *publicRoomsStatements) countPublicRooms(ctx context.Context) (nb int64, err error) { + err = s.countPublicRoomsStmt.QueryRowContext(ctx).Scan(&nb) + return +} + +func (s *publicRoomsStatements) selectPublicRooms( + ctx context.Context, offset int64, limit int16, filter string, +) ([]types.PublicRoom, error) { + var rows *sql.Rows + var err error + + if len(filter) > 0 { + pattern := "%" + filter + "%" + if limit == 0 { + rows, err = s.selectPublicRoomsWithFilterStmt.QueryContext( + ctx, pattern, offset, + ) + } else { + rows, err = s.selectPublicRoomsWithLimitAndFilterStmt.QueryContext( + ctx, pattern, offset, limit, + ) + } + } else { + if limit == 0 { + rows, err = s.selectPublicRoomsStmt.QueryContext(ctx, offset) + } else { + rows, err = s.selectPublicRoomsWithLimitStmt.QueryContext( + ctx, offset, limit, + ) + } + } + + if err != nil { + return []types.PublicRoom{}, nil + } + + rooms := []types.PublicRoom{} + for rows.Next() { + var r types.PublicRoom + var aliases pq.StringArray + + err = rows.Scan( + &r.RoomID, &r.NumJoinedMembers, &aliases, &r.CanonicalAlias, + &r.Name, &r.Topic, &r.WorldReadable, &r.GuestCanJoin, &r.AvatarURL, + ) + if err != nil { + return rooms, err + } + + r.Aliases = aliases + + rooms = append(rooms, r) + } + + return rooms, nil +} + +func (s *publicRoomsStatements) selectRoomVisibility( + ctx context.Context, roomID string, +) (v bool, err error) { + err = s.selectRoomVisibilityStmt.QueryRowContext(ctx, roomID).Scan(&v) + return +} + +func (s *publicRoomsStatements) insertNewRoom( + ctx context.Context, roomID string, +) error { + _, err := s.insertNewRoomStmt.ExecContext(ctx, roomID) + return err +} + +func (s *publicRoomsStatements) incrementJoinedMembersInRoom( + ctx context.Context, roomID string, +) error { + _, err := s.incrementJoinedMembersInRoomStmt.ExecContext(ctx, roomID) + return err +} + +func (s *publicRoomsStatements) decrementJoinedMembersInRoom( + ctx context.Context, roomID string, +) error { + _, err := s.decrementJoinedMembersInRoomStmt.ExecContext(ctx, roomID) + return err +} + +func (s *publicRoomsStatements) updateRoomAttribute( + ctx context.Context, attrName string, attrValue attributeValue, roomID string, +) error { + stmt, isEditable := s.updateRoomAttributeStmts[attrName] + + if !isEditable { + return errors.New("Cannot edit " + attrName) + } + + var value interface{} + switch v := attrValue.(type) { + case []string: + value = pq.StringArray(v) + case bool, string: + value = attrValue + default: + return errors.New("Unsupported attribute type, must be bool, string or []string") + } + + _, err := stmt.ExecContext(ctx, value, roomID) + return err +} diff --git a/publicroomsapi/storage/sqlite3/storage.go b/publicroomsapi/storage/sqlite3/storage.go new file mode 100644 index 000000000..dcb8920f9 --- /dev/null +++ b/publicroomsapi/storage/sqlite3/storage.go @@ -0,0 +1,256 @@ +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + "encoding/json" + + _ "github.com/mattn/go-sqlite3" + + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/publicroomsapi/types" + + "github.com/matrix-org/gomatrixserverlib" +) + +// PublicRoomsServerDatabase represents a public rooms server database. +type PublicRoomsServerDatabase struct { + db *sql.DB + common.PartitionOffsetStatements + statements publicRoomsStatements +} + +type attributeValue interface{} + +// NewPublicRoomsServerDatabase creates a new public rooms server database. +func NewPublicRoomsServerDatabase(dataSourceName string) (*PublicRoomsServerDatabase, error) { + var db *sql.DB + var err error + if db, err = sql.Open("sqlite3", dataSourceName); err != nil { + return nil, err + } + storage := PublicRoomsServerDatabase{ + db: db, + } + if err = storage.PartitionOffsetStatements.Prepare(db, "publicroomsapi"); err != nil { + return nil, err + } + if err = storage.statements.prepare(db); err != nil { + return nil, err + } + return &storage, nil +} + +// GetRoomVisibility returns the room visibility as a boolean: true if the room +// is publicly visible, false if not. +// Returns an error if the retrieval failed. +func (d *PublicRoomsServerDatabase) GetRoomVisibility( + ctx context.Context, roomID string, +) (bool, error) { + return d.statements.selectRoomVisibility(ctx, roomID) +} + +// SetRoomVisibility updates the visibility attribute of a room. This attribute +// must be set to true if the room is publicly visible, false if not. +// Returns an error if the update failed. +func (d *PublicRoomsServerDatabase) SetRoomVisibility( + ctx context.Context, visible bool, roomID string, +) error { + return d.statements.updateRoomAttribute(ctx, "visibility", visible, roomID) +} + +// CountPublicRooms returns the number of room set as publicly visible on the server. +// Returns an error if the retrieval failed. +func (d *PublicRoomsServerDatabase) CountPublicRooms(ctx context.Context) (int64, error) { + return d.statements.countPublicRooms(ctx) +} + +// GetPublicRooms returns an array containing the local rooms set as publicly visible, ordered by their number +// of joined members. This array can be limited by a given number of elements, and offset by a given value. +// If the limit is 0, doesn't limit the number of results. If the offset is 0 too, the array contains all +// the rooms set as publicly visible on the server. +// Returns an error if the retrieval failed. +func (d *PublicRoomsServerDatabase) GetPublicRooms( + ctx context.Context, offset int64, limit int16, filter string, +) ([]types.PublicRoom, error) { + return d.statements.selectPublicRooms(ctx, offset, limit, filter) +} + +// UpdateRoomFromEvents iterate over a slice of state events and call +// UpdateRoomFromEvent on each of them to update the database representation of +// the rooms updated by each event. +// The slice of events to remove is used to update the number of joined members +// for the room in the database. +// If the update triggered by one of the events failed, aborts the process and +// returns an error. +func (d *PublicRoomsServerDatabase) UpdateRoomFromEvents( + ctx context.Context, + eventsToAdd []gomatrixserverlib.Event, + eventsToRemove []gomatrixserverlib.Event, +) error { + for _, event := range eventsToAdd { + if err := d.UpdateRoomFromEvent(ctx, event); err != nil { + return err + } + } + + for _, event := range eventsToRemove { + if event.Type() == "m.room.member" { + if err := d.updateNumJoinedUsers(ctx, event, true); err != nil { + return err + } + } + } + + return nil +} + +// UpdateRoomFromEvent updates the database representation of a room from a Matrix event, by +// checking the event's type to know which attribute to change and using the event's content +// to define the new value of the attribute. +// If the event doesn't match with any property used to compute the public room directory, +// does nothing. +// If something went wrong during the process, returns an error. +func (d *PublicRoomsServerDatabase) UpdateRoomFromEvent( + ctx context.Context, event gomatrixserverlib.Event, +) error { + // Process the event according to its type + switch event.Type() { + case "m.room.create": + return d.statements.insertNewRoom(ctx, event.RoomID()) + case "m.room.member": + return d.updateNumJoinedUsers(ctx, event, false) + case "m.room.aliases": + return d.updateRoomAliases(ctx, event) + case "m.room.canonical_alias": + var content common.CanonicalAliasContent + field := &(content.Alias) + attrName := "canonical_alias" + return d.updateStringAttribute(ctx, attrName, event, &content, field) + case "m.room.name": + var content common.NameContent + field := &(content.Name) + attrName := "name" + return d.updateStringAttribute(ctx, attrName, event, &content, field) + case "m.room.topic": + var content common.TopicContent + field := &(content.Topic) + attrName := "topic" + return d.updateStringAttribute(ctx, attrName, event, &content, field) + case "m.room.avatar": + var content common.AvatarContent + field := &(content.URL) + attrName := "avatar_url" + return d.updateStringAttribute(ctx, attrName, event, &content, field) + case "m.room.history_visibility": + var content common.HistoryVisibilityContent + field := &(content.HistoryVisibility) + attrName := "world_readable" + strForTrue := "world_readable" + return d.updateBooleanAttribute(ctx, attrName, event, &content, field, strForTrue) + case "m.room.guest_access": + var content common.GuestAccessContent + field := &(content.GuestAccess) + attrName := "guest_can_join" + strForTrue := "can_join" + return d.updateBooleanAttribute(ctx, attrName, event, &content, field, strForTrue) + } + + // If the event type didn't match, return with no error + return nil +} + +// updateNumJoinedUsers updates the number of joined user in the database representation +// of a room using a given "m.room.member" Matrix event. +// If the membership property of the event isn't "join", ignores it and returs nil. +// If the remove parameter is set to false, increments the joined members counter in the +// database, if set to truem decrements it. +// Returns an error if the update failed. +func (d *PublicRoomsServerDatabase) updateNumJoinedUsers( + ctx context.Context, membershipEvent gomatrixserverlib.Event, remove bool, +) error { + membership, err := membershipEvent.Membership() + if err != nil { + return err + } + + if membership != gomatrixserverlib.Join { + return nil + } + + if remove { + return d.statements.decrementJoinedMembersInRoom(ctx, membershipEvent.RoomID()) + } + return d.statements.incrementJoinedMembersInRoom(ctx, membershipEvent.RoomID()) +} + +// updateStringAttribute updates a given string attribute in the database +// representation of a room using a given string data field from content of the +// Matrix event triggering the update. +// Returns an error if decoding the Matrix event's content or updating the attribute +// failed. +func (d *PublicRoomsServerDatabase) updateStringAttribute( + ctx context.Context, attrName string, event gomatrixserverlib.Event, + content interface{}, field *string, +) error { + if err := json.Unmarshal(event.Content(), content); err != nil { + return err + } + + return d.statements.updateRoomAttribute(ctx, attrName, *field, event.RoomID()) +} + +// updateBooleanAttribute updates a given boolean attribute in the database +// representation of a room using a given string data field from content of the +// Matrix event triggering the update. +// The attribute is set to true if the field matches a given string, false if not. +// Returns an error if decoding the Matrix event's content or updating the attribute +// failed. +func (d *PublicRoomsServerDatabase) updateBooleanAttribute( + ctx context.Context, attrName string, event gomatrixserverlib.Event, + content interface{}, field *string, strForTrue string, +) error { + if err := json.Unmarshal(event.Content(), content); err != nil { + return err + } + + var attrValue bool + if *field == strForTrue { + attrValue = true + } else { + attrValue = false + } + + return d.statements.updateRoomAttribute(ctx, attrName, attrValue, event.RoomID()) +} + +// updateRoomAliases decodes the content of a "m.room.aliases" Matrix event and update the list of aliases of +// a given room with it. +// Returns an error if decoding the Matrix event or updating the list failed. +func (d *PublicRoomsServerDatabase) updateRoomAliases( + ctx context.Context, aliasesEvent gomatrixserverlib.Event, +) error { + var content common.AliasesContent + if err := json.Unmarshal(aliasesEvent.Content(), &content); err != nil { + return err + } + + return d.statements.updateRoomAttribute( + ctx, "aliases", content.Aliases, aliasesEvent.RoomID(), + ) +} diff --git a/roomserver/input/events.go b/roomserver/input/events.go index 03023a4af..a3b70753e 100644 --- a/roomserver/input/events.go +++ b/roomserver/input/events.go @@ -196,7 +196,12 @@ func processInviteEvent( return err } succeeded := false - defer common.EndTransaction(updater, &succeeded) + defer func() { + txerr := common.EndTransaction(updater, &succeeded) + if err == nil && txerr != nil { + err = txerr + } + }() if updater.IsJoin() { // If the user is joined to the room then that takes precedence over this diff --git a/roomserver/input/latest_events.go b/roomserver/input/latest_events.go index 7e03d544a..f9fd1d5d4 100644 --- a/roomserver/input/latest_events.go +++ b/roomserver/input/latest_events.go @@ -60,7 +60,12 @@ func updateLatestEvents( return } succeeded := false - defer common.EndTransaction(updater, &succeeded) + defer func() { + txerr := common.EndTransaction(updater, &succeeded) + if err == nil && txerr != nil { + err = txerr + } + }() u := latestEventsUpdater{ ctx: ctx, db: db, updater: updater, ow: ow, roomNID: roomNID, diff --git a/roomserver/storage/sqlite3/event_json_table.go b/roomserver/storage/sqlite3/event_json_table.go new file mode 100644 index 000000000..f6c83906a --- /dev/null +++ b/roomserver/storage/sqlite3/event_json_table.go @@ -0,0 +1,108 @@ +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + "strings" + + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/roomserver/types" +) + +const eventJSONSchema = ` + CREATE TABLE IF NOT EXISTS roomserver_event_json ( + event_nid INTEGER NOT NULL PRIMARY KEY, + event_json TEXT NOT NULL + ); +` + +const insertEventJSONSQL = ` + INSERT INTO roomserver_event_json (event_nid, event_json) VALUES ($1, $2) + ON CONFLICT DO NOTHING +` + +// Bulk event JSON lookup by numeric event ID. +// Sort by the numeric event ID. +// This means that we can use binary search to lookup by numeric event ID. +const bulkSelectEventJSONSQL = ` + SELECT event_nid, event_json FROM roomserver_event_json + WHERE event_nid IN ($1) + ORDER BY event_nid ASC +` + +type eventJSONStatements struct { + db *sql.DB + insertEventJSONStmt *sql.Stmt + bulkSelectEventJSONStmt *sql.Stmt +} + +func (s *eventJSONStatements) prepare(db *sql.DB) (err error) { + s.db = db + _, err = db.Exec(eventJSONSchema) + if err != nil { + return + } + return statementList{ + {&s.insertEventJSONStmt, insertEventJSONSQL}, + {&s.bulkSelectEventJSONStmt, bulkSelectEventJSONSQL}, + }.prepare(db) +} + +func (s *eventJSONStatements) insertEventJSON( + ctx context.Context, txn *sql.Tx, eventNID types.EventNID, eventJSON []byte, +) error { + _, err := common.TxStmt(txn, s.insertEventJSONStmt).ExecContext(ctx, int64(eventNID), eventJSON) + return err +} + +type eventJSONPair struct { + EventNID types.EventNID + EventJSON []byte +} + +func (s *eventJSONStatements) bulkSelectEventJSON( + ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID, +) ([]eventJSONPair, error) { + iEventNIDs := make([]interface{}, len(eventNIDs)) + for k, v := range eventNIDs { + iEventNIDs[k] = v + } + selectOrig := strings.Replace(bulkSelectEventJSONSQL, "($1)", common.QueryVariadic(len(iEventNIDs)), 1) + + rows, err := txn.QueryContext(ctx, selectOrig, iEventNIDs...) + if err != nil { + return nil, err + } + defer rows.Close() // nolint: errcheck + + // We know that we will only get as many results as event NIDs + // because of the unique constraint on event NIDs. + // So we can allocate an array of the correct size now. + // We might get fewer results than NIDs so we adjust the length of the slice before returning it. + results := make([]eventJSONPair, len(eventNIDs)) + i := 0 + for ; rows.Next(); i++ { + result := &results[i] + var eventNID int64 + if err := rows.Scan(&eventNID, &result.EventJSON); err != nil { + return nil, err + } + result.EventNID = types.EventNID(eventNID) + } + return results[:i], nil +} diff --git a/roomserver/storage/sqlite3/event_state_keys_table.go b/roomserver/storage/sqlite3/event_state_keys_table.go new file mode 100644 index 000000000..b8bc6c02d --- /dev/null +++ b/roomserver/storage/sqlite3/event_state_keys_table.go @@ -0,0 +1,156 @@ +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + "strings" + + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/roomserver/types" +) + +const eventStateKeysSchema = ` + CREATE TABLE IF NOT EXISTS roomserver_event_state_keys ( + event_state_key_nid INTEGER PRIMARY KEY AUTOINCREMENT, + event_state_key TEXT NOT NULL UNIQUE + ); + INSERT INTO roomserver_event_state_keys (event_state_key_nid, event_state_key) + VALUES (1, '') + ON CONFLICT DO NOTHING; +` + +// Same as insertEventTypeNIDSQL +const insertEventStateKeyNIDSQL = ` + INSERT INTO roomserver_event_state_keys (event_state_key) VALUES ($1) + ON CONFLICT DO NOTHING; +` + +const selectEventStateKeyNIDSQL = ` + SELECT event_state_key_nid FROM roomserver_event_state_keys + WHERE event_state_key = $1 +` + +// Bulk lookup from string state key to numeric ID for that state key. +// Takes an array of strings as the query parameter. +const bulkSelectEventStateKeyNIDSQL = ` + SELECT event_state_key, event_state_key_nid FROM roomserver_event_state_keys + WHERE event_state_key IN ($1) +` + +// Bulk lookup from numeric ID to string state key for that state key. +// Takes an array of strings as the query parameter. +const bulkSelectEventStateKeySQL = ` + SELECT event_state_key, event_state_key_nid FROM roomserver_event_state_keys + WHERE event_state_key_nid IN ($1) +` + +type eventStateKeyStatements struct { + db *sql.DB + insertEventStateKeyNIDStmt *sql.Stmt + selectEventStateKeyNIDStmt *sql.Stmt + bulkSelectEventStateKeyNIDStmt *sql.Stmt + bulkSelectEventStateKeyStmt *sql.Stmt +} + +func (s *eventStateKeyStatements) prepare(db *sql.DB) (err error) { + s.db = db + _, err = db.Exec(eventStateKeysSchema) + if err != nil { + return + } + return statementList{ + {&s.insertEventStateKeyNIDStmt, insertEventStateKeyNIDSQL}, + {&s.selectEventStateKeyNIDStmt, selectEventStateKeyNIDSQL}, + {&s.bulkSelectEventStateKeyNIDStmt, bulkSelectEventStateKeyNIDSQL}, + {&s.bulkSelectEventStateKeyStmt, bulkSelectEventStateKeySQL}, + }.prepare(db) +} + +func (s *eventStateKeyStatements) insertEventStateKeyNID( + ctx context.Context, txn *sql.Tx, eventStateKey string, +) (types.EventStateKeyNID, error) { + var eventStateKeyNID int64 + var err error + var res sql.Result + insertStmt := txn.Stmt(s.insertEventStateKeyNIDStmt) + if res, err = insertStmt.ExecContext(ctx, eventStateKey); err == nil { + eventStateKeyNID, err = res.LastInsertId() + } + return types.EventStateKeyNID(eventStateKeyNID), err +} + +func (s *eventStateKeyStatements) selectEventStateKeyNID( + ctx context.Context, txn *sql.Tx, eventStateKey string, +) (types.EventStateKeyNID, error) { + var eventStateKeyNID int64 + stmt := txn.Stmt(s.selectEventStateKeyNIDStmt) + err := stmt.QueryRowContext(ctx, eventStateKey).Scan(&eventStateKeyNID) + return types.EventStateKeyNID(eventStateKeyNID), err +} + +func (s *eventStateKeyStatements) bulkSelectEventStateKeyNID( + ctx context.Context, txn *sql.Tx, eventStateKeys []string, +) (map[string]types.EventStateKeyNID, error) { + iEventStateKeys := make([]interface{}, len(eventStateKeys)) + for k, v := range eventStateKeys { + iEventStateKeys[k] = v + } + selectOrig := strings.Replace(bulkSelectEventStateKeyNIDSQL, "($1)", common.QueryVariadic(len(eventStateKeys)), 1) + + rows, err := txn.QueryContext(ctx, selectOrig, iEventStateKeys...) + if err != nil { + return nil, err + } + defer rows.Close() // nolint: errcheck + result := make(map[string]types.EventStateKeyNID, len(eventStateKeys)) + for rows.Next() { + var stateKey string + var stateKeyNID int64 + if err := rows.Scan(&stateKey, &stateKeyNID); err != nil { + return nil, err + } + result[stateKey] = types.EventStateKeyNID(stateKeyNID) + } + return result, nil +} + +func (s *eventStateKeyStatements) bulkSelectEventStateKey( + ctx context.Context, txn *sql.Tx, eventStateKeyNIDs []types.EventStateKeyNID, +) (map[types.EventStateKeyNID]string, error) { + iEventStateKeyNIDs := make([]interface{}, len(eventStateKeyNIDs)) + for k, v := range eventStateKeyNIDs { + iEventStateKeyNIDs[k] = v + } + selectOrig := strings.Replace(bulkSelectEventStateKeyNIDSQL, "($1)", common.QueryVariadic(len(eventStateKeyNIDs)), 1) + + rows, err := txn.QueryContext(ctx, selectOrig, iEventStateKeyNIDs...) + if err != nil { + return nil, err + } + defer rows.Close() // nolint: errcheck + result := make(map[types.EventStateKeyNID]string, len(eventStateKeyNIDs)) + for rows.Next() { + var stateKey string + var stateKeyNID int64 + if err := rows.Scan(&stateKey, &stateKeyNID); err != nil { + return nil, err + } + result[types.EventStateKeyNID(stateKeyNID)] = stateKey + } + return result, nil +} diff --git a/roomserver/storage/sqlite3/event_types_table.go b/roomserver/storage/sqlite3/event_types_table.go new file mode 100644 index 000000000..edc06d4c6 --- /dev/null +++ b/roomserver/storage/sqlite3/event_types_table.go @@ -0,0 +1,153 @@ +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + "strings" + + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/roomserver/types" +) + +const eventTypesSchema = ` + CREATE TABLE IF NOT EXISTS roomserver_event_types ( + event_type_nid INTEGER PRIMARY KEY AUTOINCREMENT, + event_type TEXT NOT NULL UNIQUE + ); + INSERT INTO roomserver_event_types (event_type_nid, event_type) VALUES + (1, 'm.room.create'), + (2, 'm.room.power_levels'), + (3, 'm.room.join_rules'), + (4, 'm.room.third_party_invite'), + (5, 'm.room.member'), + (6, 'm.room.redaction'), + (7, 'm.room.history_visibility') ON CONFLICT DO NOTHING; +` + +// Assign a new numeric event type ID. +// The usual case is that the event type is not in the database. +// In that case the ID will be assigned using the next value from the sequence. +// We use `RETURNING` to tell postgres to return the assigned ID. +// But it's possible that the type was added in a query that raced with us. +// This will result in a conflict on the event_type_unique constraint, in this +// case we do nothing. Postgresql won't return a row in that case so we rely on +// the caller catching the sql.ErrNoRows error and running a select to get the row. +// We could get postgresql to return the row on a conflict by updating the row +// but it doesn't seem like a good idea to modify the rows just to make postgresql +// return it. Modifying the rows will cause postgres to assign a new tuple for the +// row even though the data doesn't change resulting in unncesssary modifications +// to the indexes. +const insertEventTypeNIDSQL = ` + INSERT INTO roomserver_event_types (event_type) VALUES ($1) + ON CONFLICT DO NOTHING; +` + +const insertEventTypeNIDResultSQL = ` + SELECT event_type_nid FROM roomserver_event_types + WHERE rowid = last_insert_rowid(); +` + +const selectEventTypeNIDSQL = ` + SELECT event_type_nid FROM roomserver_event_types WHERE event_type = $1 +` + +// Bulk lookup from string event type to numeric ID for that event type. +// Takes an array of strings as the query parameter. +const bulkSelectEventTypeNIDSQL = ` + SELECT event_type, event_type_nid FROM roomserver_event_types + WHERE event_type IN ($1) +` + +type eventTypeStatements struct { + db *sql.DB + insertEventTypeNIDStmt *sql.Stmt + insertEventTypeNIDResultStmt *sql.Stmt + selectEventTypeNIDStmt *sql.Stmt + bulkSelectEventTypeNIDStmt *sql.Stmt +} + +func (s *eventTypeStatements) prepare(db *sql.DB) (err error) { + s.db = db + _, err = db.Exec(eventTypesSchema) + if err != nil { + return + } + + return statementList{ + {&s.insertEventTypeNIDStmt, insertEventTypeNIDSQL}, + {&s.insertEventTypeNIDResultStmt, insertEventTypeNIDResultSQL}, + {&s.selectEventTypeNIDStmt, selectEventTypeNIDSQL}, + {&s.bulkSelectEventTypeNIDStmt, bulkSelectEventTypeNIDSQL}, + }.prepare(db) +} + +func (s *eventTypeStatements) insertEventTypeNID( + ctx context.Context, tx *sql.Tx, eventType string, +) (types.EventTypeNID, error) { + var eventTypeNID int64 + var err error + insertStmt := common.TxStmt(tx, s.insertEventTypeNIDStmt) + resultStmt := common.TxStmt(tx, s.insertEventTypeNIDResultStmt) + if _, err = insertStmt.ExecContext(ctx, eventType); err == nil { + err = resultStmt.QueryRowContext(ctx).Scan(&eventTypeNID) + } + return types.EventTypeNID(eventTypeNID), err +} + +func (s *eventTypeStatements) selectEventTypeNID( + ctx context.Context, tx *sql.Tx, eventType string, +) (types.EventTypeNID, error) { + var eventTypeNID int64 + selectStmt := common.TxStmt(tx, s.selectEventTypeNIDStmt) + err := selectStmt.QueryRowContext(ctx, eventType).Scan(&eventTypeNID) + return types.EventTypeNID(eventTypeNID), err +} + +func (s *eventTypeStatements) bulkSelectEventTypeNID( + ctx context.Context, tx *sql.Tx, eventTypes []string, +) (map[string]types.EventTypeNID, error) { + /////////////// + iEventTypes := make([]interface{}, len(eventTypes)) + for k, v := range eventTypes { + iEventTypes[k] = v + } + selectOrig := strings.Replace(bulkSelectEventTypeNIDSQL, "($1)", common.QueryVariadic(len(iEventTypes)), 1) + selectPrep, err := s.db.Prepare(selectOrig) + if err != nil { + return nil, err + } + /////////////// + + selectStmt := common.TxStmt(tx, selectPrep) + rows, err := selectStmt.QueryContext(ctx, iEventTypes...) + if err != nil { + return nil, err + } + defer rows.Close() // nolint: errcheck + + result := make(map[string]types.EventTypeNID, len(eventTypes)) + for rows.Next() { + var eventType string + var eventTypeNID int64 + if err := rows.Scan(&eventType, &eventTypeNID); err != nil { + return nil, err + } + result[eventType] = types.EventTypeNID(eventTypeNID) + } + return result, nil +} diff --git a/roomserver/storage/sqlite3/events_table.go b/roomserver/storage/sqlite3/events_table.go new file mode 100644 index 000000000..4ed1395da --- /dev/null +++ b/roomserver/storage/sqlite3/events_table.go @@ -0,0 +1,479 @@ +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + "fmt" + "strings" + + "github.com/lib/pq" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/gomatrixserverlib" +) + +const eventsSchema = ` + CREATE TABLE IF NOT EXISTS roomserver_events ( + event_nid INTEGER PRIMARY KEY AUTOINCREMENT, + room_nid INTEGER NOT NULL, + event_type_nid INTEGER NOT NULL, + event_state_key_nid INTEGER NOT NULL, + sent_to_output BOOLEAN NOT NULL DEFAULT FALSE, + state_snapshot_nid INTEGER NOT NULL DEFAULT 0, + depth INTEGER NOT NULL, + event_id TEXT NOT NULL UNIQUE, + reference_sha256 BLOB NOT NULL, + auth_event_nids TEXT NOT NULL DEFAULT '{}' + ); +` + +const insertEventSQL = ` + INSERT INTO roomserver_events (room_nid, event_type_nid, event_state_key_nid, event_id, reference_sha256, auth_event_nids, depth) + VALUES ($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT DO NOTHING; +` + +const insertEventResultSQL = ` + SELECT event_nid, state_snapshot_nid FROM roomserver_events + WHERE rowid = last_insert_rowid(); +` + +const selectEventSQL = "" + + "SELECT event_nid, state_snapshot_nid FROM roomserver_events WHERE event_id = $1" + +// Bulk lookup of events by string ID. +// Sort by the numeric IDs for event type and state key. +// This means we can use binary search to lookup entries by type and state key. +const bulkSelectStateEventByIDSQL = "" + + "SELECT event_type_nid, event_state_key_nid, event_nid FROM roomserver_events" + + " WHERE event_id IN ($1)" + + " ORDER BY event_type_nid, event_state_key_nid ASC" + +const bulkSelectStateAtEventByIDSQL = "" + + "SELECT event_type_nid, event_state_key_nid, event_nid, state_snapshot_nid FROM roomserver_events" + + " WHERE event_id IN ($1)" + +const updateEventStateSQL = "" + + "UPDATE roomserver_events SET state_snapshot_nid = $1 WHERE event_nid = $2" + +const selectEventSentToOutputSQL = "" + + "SELECT sent_to_output FROM roomserver_events WHERE event_nid = $1" + +const updateEventSentToOutputSQL = "" + + "UPDATE roomserver_events SET sent_to_output = TRUE WHERE event_nid = $1" + +const selectEventIDSQL = "" + + "SELECT event_id FROM roomserver_events WHERE event_nid = $1" + +const bulkSelectStateAtEventAndReferenceSQL = "" + + "SELECT event_type_nid, event_state_key_nid, event_nid, state_snapshot_nid, event_id, reference_sha256" + + " FROM roomserver_events WHERE event_nid IN ($1)" + +const bulkSelectEventReferenceSQL = "" + + "SELECT event_id, reference_sha256 FROM roomserver_events WHERE event_nid IN ($1)" + +const bulkSelectEventIDSQL = "" + + "SELECT event_nid, event_id FROM roomserver_events WHERE event_nid IN ($1)" + +const bulkSelectEventNIDSQL = "" + + "SELECT event_id, event_nid FROM roomserver_events WHERE event_id IN ($1)" + +const selectMaxEventDepthSQL = "" + + "SELECT COALESCE(MAX(depth) + 1, 0) FROM roomserver_events WHERE event_nid IN ($1)" + +type eventStatements struct { + db *sql.DB + insertEventStmt *sql.Stmt + insertEventResultStmt *sql.Stmt + selectEventStmt *sql.Stmt + bulkSelectStateEventByIDStmt *sql.Stmt + bulkSelectStateAtEventByIDStmt *sql.Stmt + updateEventStateStmt *sql.Stmt + selectEventSentToOutputStmt *sql.Stmt + updateEventSentToOutputStmt *sql.Stmt + selectEventIDStmt *sql.Stmt + bulkSelectStateAtEventAndReferenceStmt *sql.Stmt + bulkSelectEventReferenceStmt *sql.Stmt + bulkSelectEventIDStmt *sql.Stmt + bulkSelectEventNIDStmt *sql.Stmt + selectMaxEventDepthStmt *sql.Stmt +} + +func (s *eventStatements) prepare(db *sql.DB) (err error) { + s.db = db + _, err = db.Exec(eventsSchema) + if err != nil { + return + } + + return statementList{ + {&s.insertEventStmt, insertEventSQL}, + {&s.insertEventResultStmt, insertEventResultSQL}, + {&s.selectEventStmt, selectEventSQL}, + {&s.bulkSelectStateEventByIDStmt, bulkSelectStateEventByIDSQL}, + {&s.bulkSelectStateAtEventByIDStmt, bulkSelectStateAtEventByIDSQL}, + {&s.updateEventStateStmt, updateEventStateSQL}, + {&s.updateEventSentToOutputStmt, updateEventSentToOutputSQL}, + {&s.selectEventSentToOutputStmt, selectEventSentToOutputSQL}, + {&s.selectEventIDStmt, selectEventIDSQL}, + {&s.bulkSelectStateAtEventAndReferenceStmt, bulkSelectStateAtEventAndReferenceSQL}, + {&s.bulkSelectEventReferenceStmt, bulkSelectEventReferenceSQL}, + {&s.bulkSelectEventIDStmt, bulkSelectEventIDSQL}, + {&s.bulkSelectEventNIDStmt, bulkSelectEventNIDSQL}, + {&s.selectMaxEventDepthStmt, selectMaxEventDepthSQL}, + }.prepare(db) +} + +func (s *eventStatements) insertEvent( + ctx context.Context, + txn *sql.Tx, + roomNID types.RoomNID, + eventTypeNID types.EventTypeNID, + eventStateKeyNID types.EventStateKeyNID, + eventID string, + referenceSHA256 []byte, + authEventNIDs []types.EventNID, + depth int64, +) (types.EventNID, types.StateSnapshotNID, error) { + var eventNID int64 + var stateNID int64 + var err error + insertStmt := common.TxStmt(txn, s.insertEventStmt) + resultStmt := common.TxStmt(txn, s.insertEventResultStmt) + if _, err = insertStmt.ExecContext( + ctx, int64(roomNID), int64(eventTypeNID), int64(eventStateKeyNID), + eventID, referenceSHA256, eventNIDsAsArray(authEventNIDs), depth, + ); err == nil { + err = resultStmt.QueryRowContext(ctx).Scan(&eventNID, &stateNID) + } + return types.EventNID(eventNID), types.StateSnapshotNID(stateNID), err +} + +func (s *eventStatements) selectEvent( + ctx context.Context, txn *sql.Tx, eventID string, +) (types.EventNID, types.StateSnapshotNID, error) { + var eventNID int64 + var stateNID int64 + selectStmt := common.TxStmt(txn, s.selectEventStmt) + err := selectStmt.QueryRowContext(ctx, eventID).Scan(&eventNID, &stateNID) + return types.EventNID(eventNID), types.StateSnapshotNID(stateNID), err +} + +// bulkSelectStateEventByID lookups a list of state events by event ID. +// If any of the requested events are missing from the database it returns a types.MissingEventError +func (s *eventStatements) bulkSelectStateEventByID( + ctx context.Context, txn *sql.Tx, eventIDs []string, +) ([]types.StateEntry, error) { + /////////////// + iEventIDs := make([]interface{}, len(eventIDs)) + for k, v := range eventIDs { + iEventIDs[k] = v + } + selectOrig := strings.Replace(bulkSelectStateEventByIDSQL, "($1)", common.QueryVariadic(len(iEventIDs)), 1) + selectPrep, err := txn.Prepare(selectOrig) + if err != nil { + return nil, err + } + /////////////// + + selectStmt := common.TxStmt(txn, selectPrep) + rows, err := selectStmt.QueryContext(ctx, iEventIDs...) + if err != nil { + return nil, err + } + defer rows.Close() // nolint: errcheck + // We know that we will only get as many results as event IDs + // because of the unique constraint on event IDs. + // So we can allocate an array of the correct size now. + // We might get fewer results than IDs so we adjust the length of the slice before returning it. + results := make([]types.StateEntry, len(eventIDs)) + i := 0 + for ; rows.Next(); i++ { + result := &results[i] + if err = rows.Scan( + &result.EventTypeNID, + &result.EventStateKeyNID, + &result.EventNID, + ); err != nil { + return nil, err + } + } + if i != len(eventIDs) { + // If there are fewer rows returned than IDs then we were asked to lookup event IDs we don't have. + // We don't know which ones were missing because we don't return the string IDs in the query. + // However it should be possible debug this by replaying queries or entries from the input kafka logs. + // If this turns out to be impossible and we do need the debug information here, it would be better + // to do it as a separate query rather than slowing down/complicating the common case. + return nil, types.MissingEventError( + fmt.Sprintf("storage: state event IDs missing from the database (%d != %d)", i, len(eventIDs)), + ) + } + return results, err +} + +// bulkSelectStateAtEventByID lookups the state at a list of events by event ID. +// If any of the requested events are missing from the database it returns a types.MissingEventError. +// If we do not have the state for any of the requested events it returns a types.MissingEventError. +func (s *eventStatements) bulkSelectStateAtEventByID( + ctx context.Context, txn *sql.Tx, eventIDs []string, +) ([]types.StateAtEvent, error) { + /////////////// + iEventIDs := make([]interface{}, len(eventIDs)) + for k, v := range eventIDs { + iEventIDs[k] = v + } + selectOrig := strings.Replace(bulkSelectStateAtEventByIDSQL, "($1)", common.QueryVariadic(len(iEventIDs)), 1) + selectPrep, err := txn.Prepare(selectOrig) + if err != nil { + return nil, err + } + /////////////// + + selectStmt := common.TxStmt(txn, selectPrep) + rows, err := selectStmt.QueryContext(ctx, iEventIDs...) + if err != nil { + return nil, err + } + defer rows.Close() // nolint: errcheck + results := make([]types.StateAtEvent, len(eventIDs)) + i := 0 + for ; rows.Next(); i++ { + result := &results[i] + if err = rows.Scan( + &result.EventTypeNID, + &result.EventStateKeyNID, + &result.EventNID, + &result.BeforeStateSnapshotNID, + ); err != nil { + return nil, err + } + if result.BeforeStateSnapshotNID == 0 { + return nil, types.MissingEventError( + fmt.Sprintf("storage: missing state for event NID %d", result.EventNID), + ) + } + } + if i != len(eventIDs) { + return nil, types.MissingEventError( + fmt.Sprintf("storage: event IDs missing from the database (%d != %d)", i, len(eventIDs)), + ) + } + return results, err +} + +func (s *eventStatements) updateEventState( + ctx context.Context, txn *sql.Tx, eventNID types.EventNID, stateNID types.StateSnapshotNID, +) error { + updateStmt := common.TxStmt(txn, s.updateEventStateStmt) + _, err := updateStmt.ExecContext(ctx, int64(stateNID), int64(eventNID)) + return err +} + +func (s *eventStatements) selectEventSentToOutput( + ctx context.Context, txn *sql.Tx, eventNID types.EventNID, +) (sentToOutput bool, err error) { + selectStmt := common.TxStmt(txn, s.selectEventSentToOutputStmt) + err = selectStmt.QueryRowContext(ctx, int64(eventNID)).Scan(&sentToOutput) + //err = s.selectEventSentToOutputStmt.QueryRowContext(ctx, int64(eventNID)).Scan(&sentToOutput) + if err != nil { + } + return +} + +func (s *eventStatements) updateEventSentToOutput(ctx context.Context, txn *sql.Tx, eventNID types.EventNID) error { + updateStmt := common.TxStmt(txn, s.updateEventSentToOutputStmt) + _, err := updateStmt.ExecContext(ctx, int64(eventNID)) + //_, err := s.updateEventSentToOutputStmt.ExecContext(ctx, int64(eventNID)) + return err +} + +func (s *eventStatements) selectEventID( + ctx context.Context, txn *sql.Tx, eventNID types.EventNID, +) (eventID string, err error) { + selectStmt := common.TxStmt(txn, s.selectEventIDStmt) + err = selectStmt.QueryRowContext(ctx, int64(eventNID)).Scan(&eventID) + return +} + +func (s *eventStatements) bulkSelectStateAtEventAndReference( + ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID, +) ([]types.StateAtEventAndReference, error) { + /////////////// + iEventNIDs := make([]interface{}, len(eventNIDs)) + for k, v := range eventNIDs { + iEventNIDs[k] = v + } + selectOrig := strings.Replace(bulkSelectStateAtEventAndReferenceSQL, "($1)", common.QueryVariadic(len(iEventNIDs)), 1) + ////////////// + + rows, err := txn.QueryContext(ctx, selectOrig, iEventNIDs...) + if err != nil { + return nil, err + } + defer rows.Close() // nolint: errcheck + results := make([]types.StateAtEventAndReference, len(eventNIDs)) + i := 0 + for ; rows.Next(); i++ { + var ( + eventTypeNID int64 + eventStateKeyNID int64 + eventNID int64 + stateSnapshotNID int64 + eventID string + eventSHA256 []byte + ) + if err = rows.Scan( + &eventTypeNID, &eventStateKeyNID, &eventNID, &stateSnapshotNID, &eventID, &eventSHA256, + ); err != nil { + return nil, err + } + result := &results[i] + result.EventTypeNID = types.EventTypeNID(eventTypeNID) + result.EventStateKeyNID = types.EventStateKeyNID(eventStateKeyNID) + result.EventNID = types.EventNID(eventNID) + result.BeforeStateSnapshotNID = types.StateSnapshotNID(stateSnapshotNID) + result.EventID = eventID + result.EventSHA256 = eventSHA256 + } + if i != len(eventNIDs) { + return nil, fmt.Errorf("storage: event NIDs missing from the database (%d != %d)", i, len(eventNIDs)) + } + return results, nil +} + +func (s *eventStatements) bulkSelectEventReference( + ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID, +) ([]gomatrixserverlib.EventReference, error) { + /////////////// + iEventNIDs := make([]interface{}, len(eventNIDs)) + for k, v := range eventNIDs { + iEventNIDs[k] = v + } + selectOrig := strings.Replace(bulkSelectEventReferenceSQL, "($1)", common.QueryVariadic(len(iEventNIDs)), 1) + selectPrep, err := txn.Prepare(selectOrig) + if err != nil { + return nil, err + } + /////////////// + + selectStmt := common.TxStmt(txn, selectPrep) + rows, err := selectStmt.QueryContext(ctx, iEventNIDs...) + if err != nil { + return nil, err + } + defer rows.Close() // nolint: errcheck + results := make([]gomatrixserverlib.EventReference, len(eventNIDs)) + i := 0 + for ; rows.Next(); i++ { + result := &results[i] + if err = rows.Scan(&result.EventID, &result.EventSHA256); err != nil { + return nil, err + } + } + if i != len(eventNIDs) { + return nil, fmt.Errorf("storage: event NIDs missing from the database (%d != %d)", i, len(eventNIDs)) + } + return results, nil +} + +// bulkSelectEventID returns a map from numeric event ID to string event ID. +func (s *eventStatements) bulkSelectEventID(ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID) (map[types.EventNID]string, error) { + /////////////// + iEventNIDs := make([]interface{}, len(eventNIDs)) + for k, v := range eventNIDs { + iEventNIDs[k] = v + } + selectOrig := strings.Replace(bulkSelectEventIDSQL, "($1)", common.QueryVariadic(len(iEventNIDs)), 1) + selectPrep, err := txn.Prepare(selectOrig) + if err != nil { + return nil, err + } + /////////////// + + selectStmt := common.TxStmt(txn, selectPrep) + rows, err := selectStmt.QueryContext(ctx, iEventNIDs...) + if err != nil { + return nil, err + } + defer rows.Close() // nolint: errcheck + results := make(map[types.EventNID]string, len(eventNIDs)) + i := 0 + for ; rows.Next(); i++ { + var eventNID int64 + var eventID string + if err = rows.Scan(&eventNID, &eventID); err != nil { + return nil, err + } + results[types.EventNID(eventNID)] = eventID + } + if i != len(eventNIDs) { + return nil, fmt.Errorf("storage: event NIDs missing from the database (%d != %d)", i, len(eventNIDs)) + } + return results, nil +} + +// bulkSelectEventNIDs returns a map from string event ID to numeric event ID. +// If an event ID is not in the database then it is omitted from the map. +func (s *eventStatements) bulkSelectEventNID(ctx context.Context, txn *sql.Tx, eventIDs []string) (map[string]types.EventNID, error) { + /////////////// + iEventIDs := make([]interface{}, len(eventIDs)) + for k, v := range eventIDs { + iEventIDs[k] = v + } + selectOrig := strings.Replace(bulkSelectEventNIDSQL, "($1)", common.QueryVariadic(len(iEventIDs)), 1) + selectPrep, err := txn.Prepare(selectOrig) + if err != nil { + return nil, err + } + /////////////// + + selectStmt := common.TxStmt(txn, selectPrep) + rows, err := selectStmt.QueryContext(ctx, iEventIDs...) + if err != nil { + return nil, err + } + defer rows.Close() // nolint: errcheck + results := make(map[string]types.EventNID, len(eventIDs)) + for rows.Next() { + var eventID string + var eventNID int64 + if err = rows.Scan(&eventID, &eventNID); err != nil { + return nil, err + } + results[eventID] = types.EventNID(eventNID) + } + return results, nil +} + +func (s *eventStatements) selectMaxEventDepth(ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID) (int64, error) { + var result int64 + selectStmt := common.TxStmt(txn, s.selectMaxEventDepthStmt) + err := selectStmt.QueryRowContext(ctx, sqliteIn(eventNIDsAsArray(eventNIDs))).Scan(&result) + if err != nil { + return 0, err + } + return result, nil +} + +func eventNIDsAsArray(eventNIDs []types.EventNID) pq.Int64Array { + nids := make([]int64, len(eventNIDs)) + for i := range eventNIDs { + nids[i] = int64(eventNIDs[i]) + } + return nids +} diff --git a/roomserver/storage/sqlite3/invite_table.go b/roomserver/storage/sqlite3/invite_table.go new file mode 100644 index 000000000..5a0f0bf79 --- /dev/null +++ b/roomserver/storage/sqlite3/invite_table.go @@ -0,0 +1,142 @@ +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/roomserver/types" +) + +const inviteSchema = ` + CREATE TABLE IF NOT EXISTS roomserver_invites ( + invite_event_id TEXT PRIMARY KEY, + room_nid INTEGER NOT NULL, + target_nid INTEGER NOT NULL, + sender_nid INTEGER NOT NULL DEFAULT 0, + retired BOOLEAN NOT NULL DEFAULT FALSE, + invite_event_json TEXT NOT NULL + ); + + CREATE INDEX IF NOT EXISTS roomserver_invites_active_idx ON roomserver_invites (target_nid, room_nid) + WHERE NOT retired; +` +const insertInviteEventSQL = "" + + "INSERT INTO roomserver_invites (invite_event_id, room_nid, target_nid," + + " sender_nid, invite_event_json) VALUES ($1, $2, $3, $4, $5)" + + " ON CONFLICT DO NOTHING" + +const selectInviteActiveForUserInRoomSQL = "" + + "SELECT sender_nid FROM roomserver_invites" + + " WHERE target_nid = $1 AND room_nid = $2" + + " AND NOT retired" + +// Retire every active invite for a user in a room. +// Ideally we'd know which invite events were retired by a given update so we +// wouldn't need to remove every active invite. +// However the matrix protocol doesn't give us a way to reliably identify the +// invites that were retired, so we are forced to retire all of them. +const updateInviteRetiredSQL = ` + UPDATE roomserver_invites SET retired = TRUE + WHERE room_nid = $1 AND target_nid = $2 AND NOT retired; + SELECT invite_event_id FROM roomserver_invites + WHERE rowid = last_insert_rowid(); +` + +type inviteStatements struct { + insertInviteEventStmt *sql.Stmt + selectInviteActiveForUserInRoomStmt *sql.Stmt + updateInviteRetiredStmt *sql.Stmt +} + +func (s *inviteStatements) prepare(db *sql.DB) (err error) { + _, err = db.Exec(inviteSchema) + if err != nil { + return + } + + return statementList{ + {&s.insertInviteEventStmt, insertInviteEventSQL}, + {&s.selectInviteActiveForUserInRoomStmt, selectInviteActiveForUserInRoomSQL}, + {&s.updateInviteRetiredStmt, updateInviteRetiredSQL}, + }.prepare(db) +} + +func (s *inviteStatements) insertInviteEvent( + ctx context.Context, + txn *sql.Tx, inviteEventID string, roomNID types.RoomNID, + targetUserNID, senderUserNID types.EventStateKeyNID, + inviteEventJSON []byte, +) (bool, error) { + stmt := common.TxStmt(txn, s.insertInviteEventStmt) + defer stmt.Close() // nolint: errcheck + result, err := stmt.ExecContext( + ctx, inviteEventID, roomNID, targetUserNID, senderUserNID, inviteEventJSON, + ) + if err != nil { + return false, err + } + count, err := result.RowsAffected() + if err != nil { + return false, err + } + return count != 0, nil +} + +func (s *inviteStatements) updateInviteRetired( + ctx context.Context, + txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, +) (eventIDs []string, err error) { + stmt := common.TxStmt(txn, s.updateInviteRetiredStmt) + rows, err := stmt.QueryContext(ctx, roomNID, targetUserNID) + if err != nil { + return nil, err + } + defer (func() { err = rows.Close() })() + for rows.Next() { + var inviteEventID string + if err := rows.Scan(&inviteEventID); err != nil { + return nil, err + } + eventIDs = append(eventIDs, inviteEventID) + } + return +} + +// selectInviteActiveForUserInRoom returns a list of sender state key NIDs +func (s *inviteStatements) selectInviteActiveForUserInRoom( + ctx context.Context, + targetUserNID types.EventStateKeyNID, roomNID types.RoomNID, +) ([]types.EventStateKeyNID, error) { + rows, err := s.selectInviteActiveForUserInRoomStmt.QueryContext( + ctx, targetUserNID, roomNID, + ) + if err != nil { + return nil, err + } + defer rows.Close() // nolint: errcheck + var result []types.EventStateKeyNID + for rows.Next() { + var senderUserNID int64 + if err := rows.Scan(&senderUserNID); err != nil { + return nil, err + } + result = append(result, types.EventStateKeyNID(senderUserNID)) + } + return result, nil +} diff --git a/roomserver/storage/sqlite3/list.go b/roomserver/storage/sqlite3/list.go new file mode 100644 index 000000000..4fe4e334b --- /dev/null +++ b/roomserver/storage/sqlite3/list.go @@ -0,0 +1,18 @@ +package sqlite3 + +import ( + "strconv" + "strings" + + "github.com/lib/pq" +) + +type SqliteList string + +func sqliteIn(a pq.Int64Array) string { + var b []string + for _, n := range a { + b = append(b, strconv.FormatInt(n, 10)) + } + return strings.Join(b, ",") +} diff --git a/roomserver/storage/sqlite3/membership_table.go b/roomserver/storage/sqlite3/membership_table.go new file mode 100644 index 000000000..978776738 --- /dev/null +++ b/roomserver/storage/sqlite3/membership_table.go @@ -0,0 +1,180 @@ +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/roomserver/types" +) + +type membershipState int64 + +const ( + membershipStateLeaveOrBan membershipState = 1 + membershipStateInvite membershipState = 2 + membershipStateJoin membershipState = 3 +) + +const membershipSchema = ` + CREATE TABLE IF NOT EXISTS roomserver_membership ( + room_nid INTEGER NOT NULL, + target_nid INTEGER NOT NULL, + sender_nid INTEGER NOT NULL DEFAULT 0, + membership_nid INTEGER NOT NULL DEFAULT 1, + event_nid INTEGER NOT NULL DEFAULT 0, + UNIQUE (room_nid, target_nid) + ); +` + +// Insert a row in to membership table so that it can be locked by the +// SELECT FOR UPDATE +const insertMembershipSQL = "" + + "INSERT INTO roomserver_membership (room_nid, target_nid)" + + " VALUES ($1, $2)" + + " ON CONFLICT DO NOTHING" + +const selectMembershipFromRoomAndTargetSQL = "" + + "SELECT membership_nid, event_nid FROM roomserver_membership" + + " WHERE room_nid = $1 AND target_nid = $2" + +const selectMembershipsFromRoomAndMembershipSQL = "" + + "SELECT event_nid FROM roomserver_membership" + + " WHERE room_nid = $1 AND membership_nid = $2" + +const selectMembershipsFromRoomSQL = "" + + "SELECT event_nid FROM roomserver_membership" + + " WHERE room_nid = $1" + +const selectMembershipForUpdateSQL = "" + + "SELECT membership_nid FROM roomserver_membership" + + " WHERE room_nid = $1 AND target_nid = $2" + +const updateMembershipSQL = "" + + "UPDATE roomserver_membership SET sender_nid = $1, membership_nid = $2, event_nid = $3" + + " WHERE room_nid = $4 AND target_nid = $5" + +type membershipStatements struct { + insertMembershipStmt *sql.Stmt + selectMembershipForUpdateStmt *sql.Stmt + selectMembershipFromRoomAndTargetStmt *sql.Stmt + selectMembershipsFromRoomAndMembershipStmt *sql.Stmt + selectMembershipsFromRoomStmt *sql.Stmt + updateMembershipStmt *sql.Stmt +} + +func (s *membershipStatements) prepare(db *sql.DB) (err error) { + _, err = db.Exec(membershipSchema) + if err != nil { + return + } + + return statementList{ + {&s.insertMembershipStmt, insertMembershipSQL}, + {&s.selectMembershipForUpdateStmt, selectMembershipForUpdateSQL}, + {&s.selectMembershipFromRoomAndTargetStmt, selectMembershipFromRoomAndTargetSQL}, + {&s.selectMembershipsFromRoomAndMembershipStmt, selectMembershipsFromRoomAndMembershipSQL}, + {&s.selectMembershipsFromRoomStmt, selectMembershipsFromRoomSQL}, + {&s.updateMembershipStmt, updateMembershipSQL}, + }.prepare(db) +} + +func (s *membershipStatements) insertMembership( + ctx context.Context, txn *sql.Tx, + roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, +) error { + stmt := common.TxStmt(txn, s.insertMembershipStmt) + _, err := stmt.ExecContext(ctx, roomNID, targetUserNID) + return err +} + +func (s *membershipStatements) selectMembershipForUpdate( + ctx context.Context, txn *sql.Tx, + roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, +) (membership membershipState, err error) { + stmt := common.TxStmt(txn, s.selectMembershipForUpdateStmt) + err = stmt.QueryRowContext( + ctx, roomNID, targetUserNID, + ).Scan(&membership) + return +} + +func (s *membershipStatements) selectMembershipFromRoomAndTarget( + ctx context.Context, txn *sql.Tx, + roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, +) (eventNID types.EventNID, membership membershipState, err error) { + selectStmt := common.TxStmt(txn, s.selectMembershipFromRoomAndTargetStmt) + err = selectStmt.QueryRowContext( + ctx, roomNID, targetUserNID, + ).Scan(&membership, &eventNID) + return +} + +func (s *membershipStatements) selectMembershipsFromRoom( + ctx context.Context, txn *sql.Tx, + roomNID types.RoomNID, +) (eventNIDs []types.EventNID, err error) { + selectStmt := common.TxStmt(txn, s.selectMembershipsFromRoomStmt) + rows, err := selectStmt.QueryContext(ctx, roomNID) + if err != nil { + return nil, err + } + defer rows.Close() // nolint: errcheck + + for rows.Next() { + var eNID types.EventNID + if err = rows.Scan(&eNID); err != nil { + return + } + eventNIDs = append(eventNIDs, eNID) + } + return +} +func (s *membershipStatements) selectMembershipsFromRoomAndMembership( + ctx context.Context, txn *sql.Tx, + roomNID types.RoomNID, membership membershipState, +) (eventNIDs []types.EventNID, err error) { + stmt := common.TxStmt(txn, s.selectMembershipsFromRoomAndMembershipStmt) + rows, err := stmt.QueryContext(ctx, roomNID, membership) + if err != nil { + return + } + defer rows.Close() // nolint: errcheck + + for rows.Next() { + var eNID types.EventNID + if err = rows.Scan(&eNID); err != nil { + return + } + eventNIDs = append(eventNIDs, eNID) + } + return +} + +func (s *membershipStatements) updateMembership( + ctx context.Context, txn *sql.Tx, + roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, + senderUserNID types.EventStateKeyNID, membership membershipState, + eventNID types.EventNID, +) error { + stmt := common.TxStmt(txn, s.updateMembershipStmt) + _, err := stmt.ExecContext( + ctx, senderUserNID, membership, eventNID, roomNID, targetUserNID, + ) + return err +} diff --git a/roomserver/storage/sqlite3/prepare.go b/roomserver/storage/sqlite3/prepare.go new file mode 100644 index 000000000..482dfa2b9 --- /dev/null +++ b/roomserver/storage/sqlite3/prepare.go @@ -0,0 +1,36 @@ +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "database/sql" +) + +// a statementList is a list of SQL statements to prepare and a pointer to where to store the resulting prepared statement. +type statementList []struct { + statement **sql.Stmt + sql string +} + +// prepare the SQL for each statement in the list and assign the result to the prepared statement. +func (s statementList) prepare(db *sql.DB) (err error) { + for _, statement := range s { + if *statement.statement, err = db.Prepare(statement.sql); err != nil { + return + } + } + return +} diff --git a/roomserver/storage/sqlite3/previous_events_table.go b/roomserver/storage/sqlite3/previous_events_table.go new file mode 100644 index 000000000..9ed64a38e --- /dev/null +++ b/roomserver/storage/sqlite3/previous_events_table.go @@ -0,0 +1,92 @@ +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/roomserver/types" +) + +const previousEventSchema = ` + CREATE TABLE IF NOT EXISTS roomserver_previous_events ( + previous_event_id TEXT NOT NULL, + previous_reference_sha256 BLOB NOT NULL, + event_nids TEXT NOT NULL, + UNIQUE (previous_event_id, previous_reference_sha256) + ); +` + +// Insert an entry into the previous_events table. +// If there is already an entry indicating that an event references that previous event then +// add the event NID to the list to indicate that this event references that previous event as well. +// This should only be modified while holding a "FOR UPDATE" lock on the row in the rooms table for this room. +// The lock is necessary to avoid data races when checking whether an event is already referenced by another event. +const insertPreviousEventSQL = ` + INSERT OR REPLACE INTO roomserver_previous_events + (previous_event_id, previous_reference_sha256, event_nids) + VALUES ($1, $2, $3) +` + +// Check if the event is referenced by another event in the table. +// This should only be done while holding a "FOR UPDATE" lock on the row in the rooms table for this room. +const selectPreviousEventExistsSQL = ` + SELECT 1 FROM roomserver_previous_events + WHERE previous_event_id = $1 AND previous_reference_sha256 = $2 +` + +type previousEventStatements struct { + insertPreviousEventStmt *sql.Stmt + selectPreviousEventExistsStmt *sql.Stmt +} + +func (s *previousEventStatements) prepare(db *sql.DB) (err error) { + _, err = db.Exec(previousEventSchema) + if err != nil { + return + } + + return statementList{ + {&s.insertPreviousEventStmt, insertPreviousEventSQL}, + {&s.selectPreviousEventExistsStmt, selectPreviousEventExistsSQL}, + }.prepare(db) +} + +func (s *previousEventStatements) insertPreviousEvent( + ctx context.Context, + txn *sql.Tx, + previousEventID string, + previousEventReferenceSHA256 []byte, + eventNID types.EventNID, +) error { + stmt := common.TxStmt(txn, s.insertPreviousEventStmt) + _, err := stmt.ExecContext( + ctx, previousEventID, previousEventReferenceSHA256, int64(eventNID), + ) + return err +} + +// Check if the event reference exists +// Returns sql.ErrNoRows if the event reference doesn't exist. +func (s *previousEventStatements) selectPreviousEventExists( + ctx context.Context, txn *sql.Tx, eventID string, eventReferenceSHA256 []byte, +) error { + var ok int64 + stmt := common.TxStmt(txn, s.selectPreviousEventExistsStmt) + return stmt.QueryRowContext(ctx, eventID, eventReferenceSHA256).Scan(&ok) +} diff --git a/roomserver/storage/sqlite3/room_aliases_table.go b/roomserver/storage/sqlite3/room_aliases_table.go new file mode 100644 index 000000000..a5fd5449a --- /dev/null +++ b/roomserver/storage/sqlite3/room_aliases_table.go @@ -0,0 +1,135 @@ +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + + "github.com/matrix-org/dendrite/common" +) + +const roomAliasesSchema = ` + CREATE TABLE IF NOT EXISTS roomserver_room_aliases ( + alias TEXT NOT NULL PRIMARY KEY, + room_id TEXT NOT NULL, + creator_id TEXT NOT NULL + ); + + CREATE INDEX IF NOT EXISTS roomserver_room_id_idx ON roomserver_room_aliases(room_id); +` + +const insertRoomAliasSQL = ` + INSERT INTO roomserver_room_aliases (alias, room_id, creator_id) VALUES ($1, $2, $3) +` + +const selectRoomIDFromAliasSQL = ` + SELECT room_id FROM roomserver_room_aliases WHERE alias = $1 +` + +const selectAliasesFromRoomIDSQL = ` + SELECT alias FROM roomserver_room_aliases WHERE room_id = $1 +` + +const selectCreatorIDFromAliasSQL = ` + SELECT creator_id FROM roomserver_room_aliases WHERE alias = $1 +` + +const deleteRoomAliasSQL = ` + DELETE FROM roomserver_room_aliases WHERE alias = $1 +` + +type roomAliasesStatements struct { + insertRoomAliasStmt *sql.Stmt + selectRoomIDFromAliasStmt *sql.Stmt + selectAliasesFromRoomIDStmt *sql.Stmt + selectCreatorIDFromAliasStmt *sql.Stmt + deleteRoomAliasStmt *sql.Stmt +} + +func (s *roomAliasesStatements) prepare(db *sql.DB) (err error) { + _, err = db.Exec(roomAliasesSchema) + if err != nil { + return + } + return statementList{ + {&s.insertRoomAliasStmt, insertRoomAliasSQL}, + {&s.selectRoomIDFromAliasStmt, selectRoomIDFromAliasSQL}, + {&s.selectAliasesFromRoomIDStmt, selectAliasesFromRoomIDSQL}, + {&s.selectCreatorIDFromAliasStmt, selectCreatorIDFromAliasSQL}, + {&s.deleteRoomAliasStmt, deleteRoomAliasSQL}, + }.prepare(db) +} + +func (s *roomAliasesStatements) insertRoomAlias( + ctx context.Context, txn *sql.Tx, alias string, roomID string, creatorUserID string, +) (err error) { + insertStmt := common.TxStmt(txn, s.insertRoomAliasStmt) + _, err = insertStmt.ExecContext(ctx, alias, roomID, creatorUserID) + return +} + +func (s *roomAliasesStatements) selectRoomIDFromAlias( + ctx context.Context, txn *sql.Tx, alias string, +) (roomID string, err error) { + selectStmt := common.TxStmt(txn, s.selectRoomIDFromAliasStmt) + err = selectStmt.QueryRowContext(ctx, alias).Scan(&roomID) + if err == sql.ErrNoRows { + return "", nil + } + return +} + +func (s *roomAliasesStatements) selectAliasesFromRoomID( + ctx context.Context, txn *sql.Tx, roomID string, +) (aliases []string, err error) { + aliases = []string{} + selectStmt := common.TxStmt(txn, s.selectAliasesFromRoomIDStmt) + rows, err := selectStmt.QueryContext(ctx, roomID) + if err != nil { + return + } + + for rows.Next() { + var alias string + if err = rows.Scan(&alias); err != nil { + return + } + + aliases = append(aliases, alias) + } + + return +} + +func (s *roomAliasesStatements) selectCreatorIDFromAlias( + ctx context.Context, txn *sql.Tx, alias string, +) (creatorID string, err error) { + selectStmt := common.TxStmt(txn, s.selectCreatorIDFromAliasStmt) + err = selectStmt.QueryRowContext(ctx, alias).Scan(&creatorID) + if err == sql.ErrNoRows { + return "", nil + } + return +} + +func (s *roomAliasesStatements) deleteRoomAlias( + ctx context.Context, txn *sql.Tx, alias string, +) (err error) { + deleteStmt := common.TxStmt(txn, s.deleteRoomAliasStmt) + _, err = deleteStmt.ExecContext(ctx, alias) + return +} diff --git a/roomserver/storage/sqlite3/rooms_table.go b/roomserver/storage/sqlite3/rooms_table.go new file mode 100644 index 000000000..bf237728d --- /dev/null +++ b/roomserver/storage/sqlite3/rooms_table.go @@ -0,0 +1,165 @@ +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + + "github.com/lib/pq" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/roomserver/types" +) + +const roomsSchema = ` + CREATE TABLE IF NOT EXISTS roomserver_rooms ( + room_nid INTEGER PRIMARY KEY AUTOINCREMENT, + room_id TEXT NOT NULL UNIQUE, + latest_event_nids TEXT NOT NULL DEFAULT '{}', + last_event_sent_nid INTEGER NOT NULL DEFAULT 0, + state_snapshot_nid INTEGER NOT NULL DEFAULT 0, + room_version INTEGER NOT NULL DEFAULT 1 + ); +` + +// Same as insertEventTypeNIDSQL +const insertRoomNIDSQL = ` + INSERT INTO roomserver_rooms (room_id) VALUES ($1) + ON CONFLICT DO NOTHING; +` + +const selectRoomNIDSQL = "" + + "SELECT room_nid FROM roomserver_rooms WHERE room_id = $1" + +const selectLatestEventNIDsSQL = "" + + "SELECT latest_event_nids, state_snapshot_nid FROM roomserver_rooms WHERE room_nid = $1" + +const selectLatestEventNIDsForUpdateSQL = "" + + "SELECT latest_event_nids, last_event_sent_nid, state_snapshot_nid FROM roomserver_rooms WHERE room_nid = $1" + +const updateLatestEventNIDsSQL = "" + + "UPDATE roomserver_rooms SET latest_event_nids = $1, last_event_sent_nid = $2, state_snapshot_nid = $3 WHERE room_nid = $4" + +const selectRoomVersionForRoomNIDSQL = "" + + "SELECT room_version FROM roomserver_rooms WHERE room_nid = $1" + +type roomStatements struct { + insertRoomNIDStmt *sql.Stmt + selectRoomNIDStmt *sql.Stmt + selectLatestEventNIDsStmt *sql.Stmt + selectLatestEventNIDsForUpdateStmt *sql.Stmt + updateLatestEventNIDsStmt *sql.Stmt + selectRoomVersionForRoomNIDStmt *sql.Stmt +} + +func (s *roomStatements) prepare(db *sql.DB) (err error) { + _, err = db.Exec(roomsSchema) + if err != nil { + return + } + return statementList{ + {&s.insertRoomNIDStmt, insertRoomNIDSQL}, + {&s.selectRoomNIDStmt, selectRoomNIDSQL}, + {&s.selectLatestEventNIDsStmt, selectLatestEventNIDsSQL}, + {&s.selectLatestEventNIDsForUpdateStmt, selectLatestEventNIDsForUpdateSQL}, + {&s.updateLatestEventNIDsStmt, updateLatestEventNIDsSQL}, + {&s.selectRoomVersionForRoomNIDStmt, selectRoomVersionForRoomNIDSQL}, + }.prepare(db) +} + +func (s *roomStatements) insertRoomNID( + ctx context.Context, txn *sql.Tx, roomID string, +) (types.RoomNID, error) { + var err error + insertStmt := common.TxStmt(txn, s.insertRoomNIDStmt) + if _, err = insertStmt.ExecContext(ctx, roomID); err == nil { + return s.selectRoomNID(ctx, txn, roomID) + } else { + return types.RoomNID(0), err + } +} + +func (s *roomStatements) selectRoomNID( + ctx context.Context, txn *sql.Tx, roomID string, +) (types.RoomNID, error) { + var roomNID int64 + stmt := common.TxStmt(txn, s.selectRoomNIDStmt) + err := stmt.QueryRowContext(ctx, roomID).Scan(&roomNID) + return types.RoomNID(roomNID), err +} + +func (s *roomStatements) selectLatestEventNIDs( + ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, +) ([]types.EventNID, types.StateSnapshotNID, error) { + var nids pq.Int64Array + var stateSnapshotNID int64 + stmt := common.TxStmt(txn, s.selectLatestEventNIDsStmt) + err := stmt.QueryRowContext(ctx, int64(roomNID)).Scan(&nids, &stateSnapshotNID) + if err != nil { + return nil, 0, err + } + eventNIDs := make([]types.EventNID, len(nids)) + for i := range nids { + eventNIDs[i] = types.EventNID(nids[i]) + } + return eventNIDs, types.StateSnapshotNID(stateSnapshotNID), nil +} + +func (s *roomStatements) selectLatestEventsNIDsForUpdate( + ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, +) ([]types.EventNID, types.EventNID, types.StateSnapshotNID, error) { + var nids pq.Int64Array + var lastEventSentNID int64 + var stateSnapshotNID int64 + stmt := common.TxStmt(txn, s.selectLatestEventNIDsForUpdateStmt) + err := stmt.QueryRowContext(ctx, int64(roomNID)).Scan(&nids, &lastEventSentNID, &stateSnapshotNID) + if err != nil { + return nil, 0, 0, err + } + eventNIDs := make([]types.EventNID, len(nids)) + for i := range nids { + eventNIDs[i] = types.EventNID(nids[i]) + } + return eventNIDs, types.EventNID(lastEventSentNID), types.StateSnapshotNID(stateSnapshotNID), nil +} + +func (s *roomStatements) updateLatestEventNIDs( + ctx context.Context, + txn *sql.Tx, + roomNID types.RoomNID, + eventNIDs []types.EventNID, + lastEventSentNID types.EventNID, + stateSnapshotNID types.StateSnapshotNID, +) error { + stmt := common.TxStmt(txn, s.updateLatestEventNIDsStmt) + _, err := stmt.ExecContext( + ctx, + eventNIDsAsArray(eventNIDs), + int64(lastEventSentNID), + int64(stateSnapshotNID), + roomNID, + ) + return err +} + +func (s *roomStatements) selectRoomVersionForRoomNID( + ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, +) (int64, error) { + var roomVersion int64 + stmt := common.TxStmt(txn, s.selectRoomVersionForRoomNIDStmt) + err := stmt.QueryRowContext(ctx, roomNID).Scan(&roomVersion) + return roomVersion, err +} diff --git a/roomserver/storage/sqlite3/sql.go b/roomserver/storage/sqlite3/sql.go new file mode 100644 index 000000000..0d49432b8 --- /dev/null +++ b/roomserver/storage/sqlite3/sql.go @@ -0,0 +1,60 @@ +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "database/sql" +) + +type statements struct { + eventTypeStatements + eventStateKeyStatements + roomStatements + eventStatements + eventJSONStatements + stateSnapshotStatements + stateBlockStatements + previousEventStatements + roomAliasesStatements + inviteStatements + membershipStatements + transactionStatements +} + +func (s *statements) prepare(db *sql.DB) error { + var err error + + for _, prepare := range []func(db *sql.DB) error{ + s.eventTypeStatements.prepare, + s.eventStateKeyStatements.prepare, + s.roomStatements.prepare, + s.eventStatements.prepare, + s.eventJSONStatements.prepare, + s.stateSnapshotStatements.prepare, + s.stateBlockStatements.prepare, + s.previousEventStatements.prepare, + s.roomAliasesStatements.prepare, + s.inviteStatements.prepare, + s.membershipStatements.prepare, + s.transactionStatements.prepare, + } { + if err = prepare(db); err != nil { + return err + } + } + + return nil +} diff --git a/roomserver/storage/sqlite3/state_block_table.go b/roomserver/storage/sqlite3/state_block_table.go new file mode 100644 index 000000000..ac593546a --- /dev/null +++ b/roomserver/storage/sqlite3/state_block_table.go @@ -0,0 +1,292 @@ +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + "fmt" + "sort" + "strings" + + "github.com/lib/pq" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/util" +) + +const stateDataSchema = ` + CREATE TABLE IF NOT EXISTS roomserver_state_block ( + state_block_nid INTEGER PRIMARY KEY AUTOINCREMENT, + event_type_nid INTEGER NOT NULL, + event_state_key_nid INTEGER NOT NULL, + event_nid INTEGER NOT NULL, + UNIQUE (state_block_nid, event_type_nid, event_state_key_nid) + ); +` + +const insertStateDataSQL = "" + + "INSERT INTO roomserver_state_block (state_block_nid, event_type_nid, event_state_key_nid, event_nid)" + + " VALUES ($1, $2, $3, $4)" + +const selectNextStateBlockNIDSQL = ` + SELECT COALESCE(( + SELECT seq+1 AS state_block_nid FROM sqlite_sequence + WHERE name = 'roomserver_state_block'), 1 + ) AS state_block_nid +` + +// Bulk state lookup by numeric state block ID. +// Sort by the state_block_nid, event_type_nid, event_state_key_nid +// This means that all the entries for a given state_block_nid will appear +// together in the list and those entries will sorted by event_type_nid +// and event_state_key_nid. This property makes it easier to merge two +// state data blocks together. +const bulkSelectStateBlockEntriesSQL = "" + + "SELECT state_block_nid, event_type_nid, event_state_key_nid, event_nid" + + " FROM roomserver_state_block WHERE state_block_nid IN ($1)" + + " ORDER BY state_block_nid, event_type_nid, event_state_key_nid" + +// Bulk state lookup by numeric state block ID. +// Filters the rows in each block to the requested types and state keys. +// We would like to restrict to particular type state key pairs but we are +// restricted by the query language to pull the cross product of a list +// of types and a list state_keys. So we have to filter the result in the +// application to restrict it to the list of event types and state keys we +// actually wanted. +const bulkSelectFilteredStateBlockEntriesSQL = "" + + "SELECT state_block_nid, event_type_nid, event_state_key_nid, event_nid" + + " FROM roomserver_state_block WHERE state_block_nid IN ($1)" + + " AND event_type_nid IN ($2) AND event_state_key_nid IN ($3)" + + " ORDER BY state_block_nid, event_type_nid, event_state_key_nid" + +type stateBlockStatements struct { + db *sql.DB + insertStateDataStmt *sql.Stmt + selectNextStateBlockNIDStmt *sql.Stmt + bulkSelectStateBlockEntriesStmt *sql.Stmt + bulkSelectFilteredStateBlockEntriesStmt *sql.Stmt +} + +func (s *stateBlockStatements) prepare(db *sql.DB) (err error) { + s.db = db + _, err = db.Exec(stateDataSchema) + if err != nil { + return + } + + return statementList{ + {&s.insertStateDataStmt, insertStateDataSQL}, + {&s.selectNextStateBlockNIDStmt, selectNextStateBlockNIDSQL}, + {&s.bulkSelectStateBlockEntriesStmt, bulkSelectStateBlockEntriesSQL}, + {&s.bulkSelectFilteredStateBlockEntriesStmt, bulkSelectFilteredStateBlockEntriesSQL}, + }.prepare(db) +} + +func (s *stateBlockStatements) bulkInsertStateData( + ctx context.Context, txn *sql.Tx, + stateBlockNID types.StateBlockNID, + entries []types.StateEntry, +) error { + for _, entry := range entries { + _, err := common.TxStmt(txn, s.insertStateDataStmt).ExecContext( + ctx, + int64(stateBlockNID), + int64(entry.EventTypeNID), + int64(entry.EventStateKeyNID), + int64(entry.EventNID), + ) + if err != nil { + return err + } + } + return nil +} + +func (s *stateBlockStatements) selectNextStateBlockNID( + ctx context.Context, + txn *sql.Tx, +) (types.StateBlockNID, error) { + var stateBlockNID int64 + selectStmt := common.TxStmt(txn, s.selectNextStateBlockNIDStmt) + err := selectStmt.QueryRowContext(ctx).Scan(&stateBlockNID) + return types.StateBlockNID(stateBlockNID), err +} + +func (s *stateBlockStatements) bulkSelectStateBlockEntries( + ctx context.Context, txn *sql.Tx, stateBlockNIDs []types.StateBlockNID, +) ([]types.StateEntryList, error) { + nids := make([]interface{}, len(stateBlockNIDs)) + for k, v := range stateBlockNIDs { + nids[k] = v + } + selectOrig := strings.Replace(bulkSelectStateBlockEntriesSQL, "($1)", common.QueryVariadic(len(nids)), 1) + selectPrep, err := s.db.Prepare(selectOrig) + if err != nil { + return nil, err + } + selectStmt := common.TxStmt(txn, selectPrep) + rows, err := selectStmt.QueryContext(ctx, nids...) + if err != nil { + return nil, err + } + defer rows.Close() // nolint: errcheck + + results := make([]types.StateEntryList, len(stateBlockNIDs)) + // current is a pointer to the StateEntryList to append the state entries to. + var current *types.StateEntryList + i := 0 + for rows.Next() { + var ( + stateBlockNID int64 + eventTypeNID int64 + eventStateKeyNID int64 + eventNID int64 + entry types.StateEntry + ) + if err := rows.Scan( + &stateBlockNID, &eventTypeNID, &eventStateKeyNID, &eventNID, + ); err != nil { + return nil, err + } + entry.EventTypeNID = types.EventTypeNID(eventTypeNID) + entry.EventStateKeyNID = types.EventStateKeyNID(eventStateKeyNID) + entry.EventNID = types.EventNID(eventNID) + if current == nil || types.StateBlockNID(stateBlockNID) != current.StateBlockNID { + // The state entry row is for a different state data block to the current one. + // So we start appending to the next entry in the list. + current = &results[i] + current.StateBlockNID = types.StateBlockNID(stateBlockNID) + i++ + } + current.StateEntries = append(current.StateEntries, entry) + } + if i != len(nids) { + return nil, fmt.Errorf("storage: state data NIDs missing from the database (%d != %d)", i, len(nids)) + } + return results, nil +} + +func (s *stateBlockStatements) bulkSelectFilteredStateBlockEntries( + ctx context.Context, txn *sql.Tx, // nolint: unparam + stateBlockNIDs []types.StateBlockNID, + stateKeyTuples []types.StateKeyTuple, +) ([]types.StateEntryList, error) { + tuples := stateKeyTupleSorter(stateKeyTuples) + // Sort the tuples so that we can run binary search against them as we filter the rows returned by the db. + sort.Sort(tuples) + + eventTypeNIDArray, eventStateKeyNIDArray := tuples.typesAndStateKeysAsArrays() + sqlStatement := strings.Replace(bulkSelectFilteredStateBlockEntriesSQL, "($1)", common.QueryVariadic(len(stateBlockNIDs)), 1) + sqlStatement = strings.Replace(sqlStatement, "($2)", common.QueryVariadicOffset(len(eventTypeNIDArray), len(stateBlockNIDs)), 1) + sqlStatement = strings.Replace(sqlStatement, "($3)", common.QueryVariadicOffset(len(eventStateKeyNIDArray), len(stateBlockNIDs)+len(eventTypeNIDArray)), 1) + + var params []interface{} + for _, val := range stateBlockNIDs { + params = append(params, int64(val)) + } + for _, val := range eventTypeNIDArray { + params = append(params, val) + } + for _, val := range eventStateKeyNIDArray { + params = append(params, val) + } + + rows, err := s.db.QueryContext( + ctx, + sqlStatement, + params..., + ) + if err != nil { + return nil, err + } + defer rows.Close() // nolint: errcheck + + var results []types.StateEntryList + var current types.StateEntryList + for rows.Next() { + var ( + stateBlockNID int64 + eventTypeNID int64 + eventStateKeyNID int64 + eventNID int64 + entry types.StateEntry + ) + if err := rows.Scan( + &stateBlockNID, &eventTypeNID, &eventStateKeyNID, &eventNID, + ); err != nil { + return nil, err + } + entry.EventTypeNID = types.EventTypeNID(eventTypeNID) + entry.EventStateKeyNID = types.EventStateKeyNID(eventStateKeyNID) + entry.EventNID = types.EventNID(eventNID) + + // We can use binary search here because we sorted the tuples earlier + if !tuples.contains(entry.StateKeyTuple) { + // The select will return the cross product of types and state keys. + // So we need to check if type of the entry is in the list. + continue + } + + if types.StateBlockNID(stateBlockNID) != current.StateBlockNID { + // The state entry row is for a different state data block to the current one. + // So we append the current entry to the results and start adding to a new one. + // The first time through the loop current will be empty. + if current.StateEntries != nil { + results = append(results, current) + } + current = types.StateEntryList{StateBlockNID: types.StateBlockNID(stateBlockNID)} + } + current.StateEntries = append(current.StateEntries, entry) + } + // Add the last entry to the list if it is not empty. + if current.StateEntries != nil { + results = append(results, current) + } + return results, nil +} + +type stateKeyTupleSorter []types.StateKeyTuple + +func (s stateKeyTupleSorter) Len() int { return len(s) } +func (s stateKeyTupleSorter) Less(i, j int) bool { return s[i].LessThan(s[j]) } +func (s stateKeyTupleSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +// Check whether a tuple is in the list. Assumes that the list is sorted. +func (s stateKeyTupleSorter) contains(value types.StateKeyTuple) bool { + i := sort.Search(len(s), func(i int) bool { return !s[i].LessThan(value) }) + return i < len(s) && s[i] == value +} + +// List the unique eventTypeNIDs and eventStateKeyNIDs. +// Assumes that the list is sorted. +func (s stateKeyTupleSorter) typesAndStateKeysAsArrays() (eventTypeNIDs pq.Int64Array, eventStateKeyNIDs pq.Int64Array) { + eventTypeNIDs = make(pq.Int64Array, len(s)) + eventStateKeyNIDs = make(pq.Int64Array, len(s)) + for i := range s { + eventTypeNIDs[i] = int64(s[i].EventTypeNID) + eventStateKeyNIDs[i] = int64(s[i].EventStateKeyNID) + } + eventTypeNIDs = eventTypeNIDs[:util.SortAndUnique(int64Sorter(eventTypeNIDs))] + eventStateKeyNIDs = eventStateKeyNIDs[:util.SortAndUnique(int64Sorter(eventStateKeyNIDs))] + return +} + +type int64Sorter []int64 + +func (s int64Sorter) Len() int { return len(s) } +func (s int64Sorter) Less(i, j int) bool { return s[i] < s[j] } +func (s int64Sorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } diff --git a/roomserver/storage/sqlite3/state_block_table_test.go b/roomserver/storage/sqlite3/state_block_table_test.go new file mode 100644 index 000000000..98439f5c0 --- /dev/null +++ b/roomserver/storage/sqlite3/state_block_table_test.go @@ -0,0 +1,86 @@ +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "sort" + "testing" + + "github.com/matrix-org/dendrite/roomserver/types" +) + +func TestStateKeyTupleSorter(t *testing.T) { + input := stateKeyTupleSorter{ + {EventTypeNID: 1, EventStateKeyNID: 2}, + {EventTypeNID: 1, EventStateKeyNID: 4}, + {EventTypeNID: 2, EventStateKeyNID: 2}, + {EventTypeNID: 1, EventStateKeyNID: 1}, + } + want := []types.StateKeyTuple{ + {EventTypeNID: 1, EventStateKeyNID: 1}, + {EventTypeNID: 1, EventStateKeyNID: 2}, + {EventTypeNID: 1, EventStateKeyNID: 4}, + {EventTypeNID: 2, EventStateKeyNID: 2}, + } + doNotWant := []types.StateKeyTuple{ + {EventTypeNID: 0, EventStateKeyNID: 0}, + {EventTypeNID: 1, EventStateKeyNID: 3}, + {EventTypeNID: 2, EventStateKeyNID: 1}, + {EventTypeNID: 3, EventStateKeyNID: 1}, + } + wantTypeNIDs := []int64{1, 2} + wantStateKeyNIDs := []int64{1, 2, 4} + + // Sort the input and check it's in the right order. + sort.Sort(input) + gotTypeNIDs, gotStateKeyNIDs := input.typesAndStateKeysAsArrays() + + for i := range want { + if input[i] != want[i] { + t.Errorf("Wanted %#v at index %d got %#v", want[i], i, input[i]) + } + + if !input.contains(want[i]) { + t.Errorf("Wanted %#v.contains(%#v) to be true but got false", input, want[i]) + } + } + + for i := range doNotWant { + if input.contains(doNotWant[i]) { + t.Errorf("Wanted %#v.contains(%#v) to be false but got true", input, doNotWant[i]) + } + } + + if len(wantTypeNIDs) != len(gotTypeNIDs) { + t.Fatalf("Wanted type NIDs %#v got %#v", wantTypeNIDs, gotTypeNIDs) + } + + for i := range wantTypeNIDs { + if wantTypeNIDs[i] != gotTypeNIDs[i] { + t.Fatalf("Wanted type NIDs %#v got %#v", wantTypeNIDs, gotTypeNIDs) + } + } + + if len(wantStateKeyNIDs) != len(gotStateKeyNIDs) { + t.Fatalf("Wanted state key NIDs %#v got %#v", wantStateKeyNIDs, gotStateKeyNIDs) + } + + for i := range wantStateKeyNIDs { + if wantStateKeyNIDs[i] != gotStateKeyNIDs[i] { + t.Fatalf("Wanted type NIDs %#v got %#v", wantTypeNIDs, gotTypeNIDs) + } + } +} diff --git a/roomserver/storage/sqlite3/state_snapshot_table.go b/roomserver/storage/sqlite3/state_snapshot_table.go new file mode 100644 index 000000000..df97aa419 --- /dev/null +++ b/roomserver/storage/sqlite3/state_snapshot_table.go @@ -0,0 +1,120 @@ +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + "fmt" + "strings" + + "github.com/lib/pq" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/roomserver/types" +) + +const stateSnapshotSchema = ` + CREATE TABLE IF NOT EXISTS roomserver_state_snapshots ( + state_snapshot_nid INTEGER PRIMARY KEY AUTOINCREMENT, + room_nid INTEGER NOT NULL, + state_block_nids TEXT NOT NULL DEFAULT '{}' + ); +` + +const insertStateSQL = ` + INSERT INTO roomserver_state_snapshots (room_nid, state_block_nids) + VALUES ($1, $2);` + +// Bulk state data NID lookup. +// Sorting by state_snapshot_nid means we can use binary search over the result +// to lookup the state data NIDs for a state snapshot NID. +const bulkSelectStateBlockNIDsSQL = "" + + "SELECT state_snapshot_nid, state_block_nids FROM roomserver_state_snapshots" + + " WHERE state_snapshot_nid IN ($1) ORDER BY state_snapshot_nid ASC" + +type stateSnapshotStatements struct { + db *sql.DB + insertStateStmt *sql.Stmt + bulkSelectStateBlockNIDsStmt *sql.Stmt +} + +func (s *stateSnapshotStatements) prepare(db *sql.DB) (err error) { + s.db = db + _, err = db.Exec(stateSnapshotSchema) + if err != nil { + return + } + + return statementList{ + {&s.insertStateStmt, insertStateSQL}, + {&s.bulkSelectStateBlockNIDsStmt, bulkSelectStateBlockNIDsSQL}, + }.prepare(db) +} + +func (s *stateSnapshotStatements) insertState( + ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, stateBlockNIDs []types.StateBlockNID, +) (stateNID types.StateSnapshotNID, err error) { + nids := make([]int64, len(stateBlockNIDs)) + for i := range stateBlockNIDs { + nids[i] = int64(stateBlockNIDs[i]) + } + insertStmt := txn.Stmt(s.insertStateStmt) + if res, err2 := insertStmt.ExecContext(ctx, int64(roomNID), pq.Int64Array(nids)); err2 == nil { + lastRowID, err3 := res.LastInsertId() + if err3 != nil { + err = err3 + } + stateNID = types.StateSnapshotNID(lastRowID) + } + return +} + +func (s *stateSnapshotStatements) bulkSelectStateBlockNIDs( + ctx context.Context, txn *sql.Tx, stateNIDs []types.StateSnapshotNID, +) ([]types.StateBlockNIDList, error) { + nids := make([]interface{}, len(stateNIDs)) + for k, v := range stateNIDs { + nids[k] = v + } + selectOrig := strings.Replace(bulkSelectStateBlockNIDsSQL, "($1)", common.QueryVariadic(len(nids)), 1) + selectStmt, err := txn.Prepare(selectOrig) + if err != nil { + return nil, err + } + + rows, err := selectStmt.QueryContext(ctx, nids...) + if err != nil { + return nil, err + } + defer rows.Close() // nolint: errcheck + results := make([]types.StateBlockNIDList, len(stateNIDs)) + i := 0 + for ; rows.Next(); i++ { + result := &results[i] + var stateBlockNIDs pq.Int64Array + if err := rows.Scan(&result.StateSnapshotNID, &stateBlockNIDs); err != nil { + return nil, err + } + result.StateBlockNIDs = make([]types.StateBlockNID, len(stateBlockNIDs)) + for k := range stateBlockNIDs { + result.StateBlockNIDs[k] = types.StateBlockNID(stateBlockNIDs[k]) + } + } + if i != len(stateNIDs) { + return nil, fmt.Errorf("storage: state NIDs missing from the database (%d != %d)", i, len(stateNIDs)) + } + return results, nil +} diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go new file mode 100644 index 000000000..e20e8aed7 --- /dev/null +++ b/roomserver/storage/sqlite3/storage.go @@ -0,0 +1,864 @@ +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + "errors" + "net/url" + + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/gomatrixserverlib" + _ "github.com/mattn/go-sqlite3" +) + +// A Database is used to store room events and stream offsets. +type Database struct { + statements statements + db *sql.DB +} + +// Open a postgres database. +func Open(dataSourceName string) (*Database, error) { + var d Database + uri, err := url.Parse(dataSourceName) + if err != nil { + return nil, err + } + var cs string + if uri.Opaque != "" { // file:filename.db + cs = uri.Opaque + } else if uri.Path != "" { // file:///path/to/filename.db + cs = uri.Path + } else { + return nil, errors.New("no filename or path in connect string") + } + if d.db, err = sql.Open("sqlite3", cs); err != nil { + return nil, err + } + //d.db.Exec("PRAGMA journal_mode=WAL;") + //d.db.Exec("PRAGMA read_uncommitted = true;") + d.db.SetMaxOpenConns(2) + if err = d.statements.prepare(d.db); err != nil { + return nil, err + } + return &d, nil +} + +// StoreEvent implements input.EventDatabase +func (d *Database) StoreEvent( + ctx context.Context, event gomatrixserverlib.Event, + txnAndSessionID *api.TransactionID, authEventNIDs []types.EventNID, +) (types.RoomNID, types.StateAtEvent, error) { + var ( + roomNID types.RoomNID + eventTypeNID types.EventTypeNID + eventStateKeyNID types.EventStateKeyNID + eventNID types.EventNID + stateNID types.StateSnapshotNID + err error + ) + + err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + if txnAndSessionID != nil { + if err = d.statements.insertTransaction( + ctx, txn, txnAndSessionID.TransactionID, + txnAndSessionID.SessionID, event.Sender(), event.EventID(), + ); err != nil { + return err + } + } + + if roomNID, err = d.assignRoomNID(ctx, txn, event.RoomID()); err != nil { + return err + } + + if eventTypeNID, err = d.assignEventTypeNID(ctx, txn, event.Type()); err != nil { + return err + } + + eventStateKey := event.StateKey() + // Assigned a numeric ID for the state_key if there is one present. + // Otherwise set the numeric ID for the state_key to 0. + if eventStateKey != nil { + if eventStateKeyNID, err = d.assignStateKeyNID(ctx, txn, *eventStateKey); err != nil { + return err + } + } + + if eventNID, stateNID, err = d.statements.insertEvent( + ctx, + txn, + roomNID, + eventTypeNID, + eventStateKeyNID, + event.EventID(), + event.EventReference().EventSHA256, + authEventNIDs, + event.Depth(), + ); err != nil { + if err == sql.ErrNoRows { + // We've already inserted the event so select the numeric event ID + eventNID, stateNID, err = d.statements.selectEvent(ctx, txn, event.EventID()) + } + if err != nil { + return err + } + } + + if err = d.statements.insertEventJSON(ctx, txn, eventNID, event.JSON()); err != nil { + return err + } + + return nil + }) + if err != nil { + return 0, types.StateAtEvent{}, err + } + + return roomNID, types.StateAtEvent{ + BeforeStateSnapshotNID: stateNID, + StateEntry: types.StateEntry{ + StateKeyTuple: types.StateKeyTuple{ + EventTypeNID: eventTypeNID, + EventStateKeyNID: eventStateKeyNID, + }, + EventNID: eventNID, + }, + }, nil +} + +func (d *Database) assignRoomNID( + ctx context.Context, txn *sql.Tx, roomID string, +) (roomNID types.RoomNID, err error) { + // Check if we already have a numeric ID in the database. + roomNID, err = d.statements.selectRoomNID(ctx, txn, roomID) + if err == sql.ErrNoRows { + // We don't have a numeric ID so insert one into the database. + roomNID, err = d.statements.insertRoomNID(ctx, txn, roomID) + if err == nil { + // Now get the numeric ID back out of the database + roomNID, err = d.statements.selectRoomNID(ctx, txn, roomID) + } + } + return +} + +func (d *Database) assignEventTypeNID( + ctx context.Context, txn *sql.Tx, eventType string, +) (eventTypeNID types.EventTypeNID, err error) { + // Check if we already have a numeric ID in the database. + eventTypeNID, err = d.statements.selectEventTypeNID(ctx, txn, eventType) + if err == sql.ErrNoRows { + // We don't have a numeric ID so insert one into the database. + eventTypeNID, err = d.statements.insertEventTypeNID(ctx, txn, eventType) + if err == sql.ErrNoRows { + // We raced with another insert so run the select again. + eventTypeNID, err = d.statements.selectEventTypeNID(ctx, txn, eventType) + } + } + return +} + +func (d *Database) assignStateKeyNID( + ctx context.Context, txn *sql.Tx, eventStateKey string, +) (eventStateKeyNID types.EventStateKeyNID, err error) { + // Check if we already have a numeric ID in the database. + eventStateKeyNID, err = d.statements.selectEventStateKeyNID(ctx, txn, eventStateKey) + if err == sql.ErrNoRows { + // We don't have a numeric ID so insert one into the database. + eventStateKeyNID, err = d.statements.insertEventStateKeyNID(ctx, txn, eventStateKey) + if err == sql.ErrNoRows { + // We raced with another insert so run the select again. + eventStateKeyNID, err = d.statements.selectEventStateKeyNID(ctx, txn, eventStateKey) + } + } + return +} + +// StateEntriesForEventIDs implements input.EventDatabase +func (d *Database) StateEntriesForEventIDs( + ctx context.Context, eventIDs []string, +) (se []types.StateEntry, err error) { + err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + se, err = d.statements.bulkSelectStateEventByID(ctx, txn, eventIDs) + return err + }) + return +} + +// EventTypeNIDs implements state.RoomStateDatabase +func (d *Database) EventTypeNIDs( + ctx context.Context, eventTypes []string, +) (etnids map[string]types.EventTypeNID, err error) { + err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + etnids, err = d.statements.bulkSelectEventTypeNID(ctx, txn, eventTypes) + return err + }) + return +} + +// EventStateKeyNIDs implements state.RoomStateDatabase +func (d *Database) EventStateKeyNIDs( + ctx context.Context, eventStateKeys []string, +) (esknids map[string]types.EventStateKeyNID, err error) { + err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + esknids, err = d.statements.bulkSelectEventStateKeyNID(ctx, txn, eventStateKeys) + return err + }) + return +} + +// EventStateKeys implements query.RoomserverQueryAPIDatabase +func (d *Database) EventStateKeys( + ctx context.Context, eventStateKeyNIDs []types.EventStateKeyNID, +) (out map[types.EventStateKeyNID]string, err error) { + err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + out, err = d.statements.bulkSelectEventStateKey(ctx, txn, eventStateKeyNIDs) + return err + }) + return +} + +// EventNIDs implements query.RoomserverQueryAPIDatabase +func (d *Database) EventNIDs( + ctx context.Context, eventIDs []string, +) (out map[string]types.EventNID, err error) { + err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + out, err = d.statements.bulkSelectEventNID(ctx, txn, eventIDs) + return err + }) + return +} + +// Events implements input.EventDatabase +func (d *Database) Events( + ctx context.Context, eventNIDs []types.EventNID, +) ([]types.Event, error) { + var eventJSONs []eventJSONPair + var err error + results := make([]types.Event, len(eventNIDs)) + err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + eventJSONs, err = d.statements.bulkSelectEventJSON(ctx, txn, eventNIDs) + if err != nil || len(eventJSONs) == 0 { + return nil + } + for i, eventJSON := range eventJSONs { + result := &results[i] + result.EventNID = eventJSON.EventNID + // TODO: Use NewEventFromTrustedJSON for efficiency + result.Event, err = gomatrixserverlib.NewEventFromUntrustedJSON(eventJSON.EventJSON) + if err != nil { + return nil + } + } + return nil + }) + if err != nil { + return []types.Event{}, err + } + return results, nil +} + +// AddState implements input.EventDatabase +func (d *Database) AddState( + ctx context.Context, + roomNID types.RoomNID, + stateBlockNIDs []types.StateBlockNID, + state []types.StateEntry, +) (stateNID types.StateSnapshotNID, err error) { + err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + if len(state) > 0 { + var stateBlockNID types.StateBlockNID + stateBlockNID, err = d.statements.selectNextStateBlockNID(ctx, txn) + if err != nil { + return err + } + if err = d.statements.bulkInsertStateData(ctx, txn, stateBlockNID, state); err != nil { + return err + } + stateBlockNIDs = append(stateBlockNIDs[:len(stateBlockNIDs):len(stateBlockNIDs)], stateBlockNID) + } + stateNID, err = d.statements.insertState(ctx, txn, roomNID, stateBlockNIDs) + return err + }) + if err != nil { + return 0, err + } + return +} + +// SetState implements input.EventDatabase +func (d *Database) SetState( + ctx context.Context, eventNID types.EventNID, stateNID types.StateSnapshotNID, +) error { + e := common.WithTransaction(d.db, func(txn *sql.Tx) error { + return d.statements.updateEventState(ctx, txn, eventNID, stateNID) + }) + return e +} + +// StateAtEventIDs implements input.EventDatabase +func (d *Database) StateAtEventIDs( + ctx context.Context, eventIDs []string, +) (se []types.StateAtEvent, err error) { + err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + se, err = d.statements.bulkSelectStateAtEventByID(ctx, txn, eventIDs) + return err + }) + return +} + +// StateBlockNIDs implements state.RoomStateDatabase +func (d *Database) StateBlockNIDs( + ctx context.Context, stateNIDs []types.StateSnapshotNID, +) (sl []types.StateBlockNIDList, err error) { + err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + sl, err = d.statements.bulkSelectStateBlockNIDs(ctx, txn, stateNIDs) + return err + }) + return +} + +// StateEntries implements state.RoomStateDatabase +func (d *Database) StateEntries( + ctx context.Context, stateBlockNIDs []types.StateBlockNID, +) (sel []types.StateEntryList, err error) { + err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + sel, err = d.statements.bulkSelectStateBlockEntries(ctx, txn, stateBlockNIDs) + return err + }) + return +} + +// SnapshotNIDFromEventID implements state.RoomStateDatabase +func (d *Database) SnapshotNIDFromEventID( + ctx context.Context, eventID string, +) (stateNID types.StateSnapshotNID, err error) { + err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + _, stateNID, err = d.statements.selectEvent(ctx, txn, eventID) + return err + }) + return +} + +// EventIDs implements input.RoomEventDatabase +func (d *Database) EventIDs( + ctx context.Context, eventNIDs []types.EventNID, +) (out map[types.EventNID]string, err error) { + err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + out, err = d.statements.bulkSelectEventID(ctx, txn, eventNIDs) + return err + }) + return +} + +// GetLatestEventsForUpdate implements input.EventDatabase +func (d *Database) GetLatestEventsForUpdate( + ctx context.Context, roomNID types.RoomNID, +) (types.RoomRecentEventsUpdater, error) { + txn, err := d.db.Begin() + if err != nil { + return nil, err + } + eventNIDs, lastEventNIDSent, currentStateSnapshotNID, err := + d.statements.selectLatestEventsNIDsForUpdate(ctx, txn, roomNID) + if err != nil { + txn.Rollback() // nolint: errcheck + return nil, err + } + stateAndRefs, err := d.statements.bulkSelectStateAtEventAndReference(ctx, txn, eventNIDs) + if err != nil { + txn.Rollback() // nolint: errcheck + return nil, err + } + var lastEventIDSent string + if lastEventNIDSent != 0 { + lastEventIDSent, err = d.statements.selectEventID(ctx, txn, lastEventNIDSent) + if err != nil { + txn.Rollback() // nolint: errcheck + return nil, err + } + } + + // FIXME: we probably want to support long-lived txns in sqlite somehow, but we don't because we get + // 'database is locked' errors caused by multiple write txns (one being the long-lived txn created here) + // so for now let's not use a long-lived txn at all, and just commit it here and set the txn to nil so + // we fail fast if someone tries to use the underlying txn object. + err = txn.Commit() + if err != nil { + return nil, err + } + return &roomRecentEventsUpdater{ + transaction{ctx, nil}, d, roomNID, stateAndRefs, lastEventIDSent, currentStateSnapshotNID, + }, nil +} + +// GetTransactionEventID implements input.EventDatabase +func (d *Database) GetTransactionEventID( + ctx context.Context, transactionID string, + sessionID int64, userID string, +) (string, error) { + eventID, err := d.statements.selectTransactionEventID(ctx, nil, transactionID, sessionID, userID) + if err == sql.ErrNoRows { + return "", nil + } + return eventID, err +} + +type roomRecentEventsUpdater struct { + transaction + d *Database + roomNID types.RoomNID + latestEvents []types.StateAtEventAndReference + lastEventIDSent string + currentStateSnapshotNID types.StateSnapshotNID +} + +// LatestEvents implements types.RoomRecentEventsUpdater +func (u *roomRecentEventsUpdater) LatestEvents() []types.StateAtEventAndReference { + return u.latestEvents +} + +// LastEventIDSent implements types.RoomRecentEventsUpdater +func (u *roomRecentEventsUpdater) LastEventIDSent() string { + return u.lastEventIDSent +} + +// CurrentStateSnapshotNID implements types.RoomRecentEventsUpdater +func (u *roomRecentEventsUpdater) CurrentStateSnapshotNID() types.StateSnapshotNID { + return u.currentStateSnapshotNID +} + +// StorePreviousEvents implements types.RoomRecentEventsUpdater +func (u *roomRecentEventsUpdater) StorePreviousEvents(eventNID types.EventNID, previousEventReferences []gomatrixserverlib.EventReference) error { + err := common.WithTransaction(u.d.db, func(txn *sql.Tx) error { + for _, ref := range previousEventReferences { + if err := u.d.statements.insertPreviousEvent(u.ctx, txn, ref.EventID, ref.EventSHA256, eventNID); err != nil { + return err + } + } + return nil + }) + return err +} + +// IsReferenced implements types.RoomRecentEventsUpdater +func (u *roomRecentEventsUpdater) IsReferenced(eventReference gomatrixserverlib.EventReference) (res bool, err error) { + err = common.WithTransaction(u.d.db, func(txn *sql.Tx) error { + err := u.d.statements.selectPreviousEventExists(u.ctx, txn, eventReference.EventID, eventReference.EventSHA256) + if err == nil { + res = true + err = nil + } + if err == sql.ErrNoRows { + res = false + err = nil + } + return err + }) + return +} + +// SetLatestEvents implements types.RoomRecentEventsUpdater +func (u *roomRecentEventsUpdater) SetLatestEvents( + roomNID types.RoomNID, latest []types.StateAtEventAndReference, lastEventNIDSent types.EventNID, + currentStateSnapshotNID types.StateSnapshotNID, +) error { + err := common.WithTransaction(u.d.db, func(txn *sql.Tx) error { + eventNIDs := make([]types.EventNID, len(latest)) + for i := range latest { + eventNIDs[i] = latest[i].EventNID + } + return u.d.statements.updateLatestEventNIDs(u.ctx, txn, roomNID, eventNIDs, lastEventNIDSent, currentStateSnapshotNID) + }) + return err +} + +// HasEventBeenSent implements types.RoomRecentEventsUpdater +func (u *roomRecentEventsUpdater) HasEventBeenSent(eventNID types.EventNID) (res bool, err error) { + err = common.WithTransaction(u.d.db, func(txn *sql.Tx) error { + res, err = u.d.statements.selectEventSentToOutput(u.ctx, txn, eventNID) + return err + }) + return +} + +// MarkEventAsSent implements types.RoomRecentEventsUpdater +func (u *roomRecentEventsUpdater) MarkEventAsSent(eventNID types.EventNID) error { + err := common.WithTransaction(u.d.db, func(txn *sql.Tx) error { + return u.d.statements.updateEventSentToOutput(u.ctx, txn, eventNID) + }) + return err +} + +func (u *roomRecentEventsUpdater) MembershipUpdater(targetUserNID types.EventStateKeyNID) (mu types.MembershipUpdater, err error) { + err = common.WithTransaction(u.d.db, func(txn *sql.Tx) error { + mu, err = u.d.membershipUpdaterTxn(u.ctx, txn, u.roomNID, targetUserNID) + return err + }) + return +} + +// RoomNID implements query.RoomserverQueryAPIDB +func (d *Database) RoomNID(ctx context.Context, roomID string) (roomNID types.RoomNID, err error) { + err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + roomNID, err = d.statements.selectRoomNID(ctx, txn, roomID) + if err == sql.ErrNoRows { + roomNID = 0 + err = nil + } + return err + }) + return +} + +// LatestEventIDs implements query.RoomserverQueryAPIDatabase +func (d *Database) LatestEventIDs( + ctx context.Context, roomNID types.RoomNID, +) (references []gomatrixserverlib.EventReference, currentStateSnapshotNID types.StateSnapshotNID, depth int64, err error) { + err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + var eventNIDs []types.EventNID + eventNIDs, currentStateSnapshotNID, err = d.statements.selectLatestEventNIDs(ctx, txn, roomNID) + if err != nil { + return err + } + references, err = d.statements.bulkSelectEventReference(ctx, txn, eventNIDs) + if err != nil { + return err + } + depth, err = d.statements.selectMaxEventDepth(ctx, txn, eventNIDs) + if err != nil { + return err + } + return nil + }) + return +} + +// GetInvitesForUser implements query.RoomserverQueryAPIDatabase +func (d *Database) GetInvitesForUser( + ctx context.Context, + roomNID types.RoomNID, + targetUserNID types.EventStateKeyNID, +) (senderUserIDs []types.EventStateKeyNID, err error) { + return d.statements.selectInviteActiveForUserInRoom(ctx, targetUserNID, roomNID) +} + +// SetRoomAlias implements alias.RoomserverAliasAPIDB +func (d *Database) SetRoomAlias(ctx context.Context, alias string, roomID string, creatorUserID string) error { + return d.statements.insertRoomAlias(ctx, nil, alias, roomID, creatorUserID) +} + +// GetRoomIDForAlias implements alias.RoomserverAliasAPIDB +func (d *Database) GetRoomIDForAlias(ctx context.Context, alias string) (string, error) { + return d.statements.selectRoomIDFromAlias(ctx, nil, alias) +} + +// GetAliasesForRoomID implements alias.RoomserverAliasAPIDB +func (d *Database) GetAliasesForRoomID(ctx context.Context, roomID string) ([]string, error) { + return d.statements.selectAliasesFromRoomID(ctx, nil, roomID) +} + +// GetCreatorIDForAlias implements alias.RoomserverAliasAPIDB +func (d *Database) GetCreatorIDForAlias( + ctx context.Context, alias string, +) (string, error) { + return d.statements.selectCreatorIDFromAlias(ctx, nil, alias) +} + +// RemoveRoomAlias implements alias.RoomserverAliasAPIDB +func (d *Database) RemoveRoomAlias(ctx context.Context, alias string) error { + return d.statements.deleteRoomAlias(ctx, nil, alias) +} + +// StateEntriesForTuples implements state.RoomStateDatabase +func (d *Database) StateEntriesForTuples( + ctx context.Context, + stateBlockNIDs []types.StateBlockNID, + stateKeyTuples []types.StateKeyTuple, +) ([]types.StateEntryList, error) { + return d.statements.bulkSelectFilteredStateBlockEntries( + ctx, nil, stateBlockNIDs, stateKeyTuples, + ) +} + +// MembershipUpdater implements input.RoomEventDatabase +func (d *Database) MembershipUpdater( + ctx context.Context, roomID, targetUserID string, +) (types.MembershipUpdater, error) { + txn, err := d.db.Begin() + if err != nil { + return nil, err + } + succeeded := false + defer func() { + if !succeeded { + txn.Rollback() // nolint: errcheck + } + }() + + roomNID, err := d.assignRoomNID(ctx, txn, roomID) + if err != nil { + return nil, err + } + + targetUserNID, err := d.assignStateKeyNID(ctx, txn, targetUserID) + if err != nil { + return nil, err + } + + updater, err := d.membershipUpdaterTxn(ctx, txn, roomNID, targetUserNID) + if err != nil { + return nil, err + } + + succeeded = true + return updater, nil +} + +type membershipUpdater struct { + transaction + d *Database + roomNID types.RoomNID + targetUserNID types.EventStateKeyNID + membership membershipState +} + +func (d *Database) membershipUpdaterTxn( + ctx context.Context, + txn *sql.Tx, + roomNID types.RoomNID, + targetUserNID types.EventStateKeyNID, +) (types.MembershipUpdater, error) { + + if err := d.statements.insertMembership(ctx, txn, roomNID, targetUserNID); err != nil { + return nil, err + } + + membership, err := d.statements.selectMembershipForUpdate(ctx, txn, roomNID, targetUserNID) + if err != nil { + return nil, err + } + + return &membershipUpdater{ + transaction{ctx, txn}, d, roomNID, targetUserNID, membership, + }, nil +} + +// IsInvite implements types.MembershipUpdater +func (u *membershipUpdater) IsInvite() bool { + return u.membership == membershipStateInvite +} + +// IsJoin implements types.MembershipUpdater +func (u *membershipUpdater) IsJoin() bool { + return u.membership == membershipStateJoin +} + +// IsLeave implements types.MembershipUpdater +func (u *membershipUpdater) IsLeave() bool { + return u.membership == membershipStateLeaveOrBan +} + +// SetToInvite implements types.MembershipUpdater +func (u *membershipUpdater) SetToInvite(event gomatrixserverlib.Event) (inserted bool, err error) { + err = common.WithTransaction(u.d.db, func(txn *sql.Tx) error { + senderUserNID, err := u.d.assignStateKeyNID(u.ctx, txn, event.Sender()) + if err != nil { + return err + } + inserted, err = u.d.statements.insertInviteEvent( + u.ctx, txn, event.EventID(), u.roomNID, u.targetUserNID, senderUserNID, event.JSON(), + ) + if err != nil { + return err + } + if u.membership != membershipStateInvite { + if err = u.d.statements.updateMembership( + u.ctx, txn, u.roomNID, u.targetUserNID, senderUserNID, membershipStateInvite, 0, + ); err != nil { + return err + } + } + return nil + }) + return +} + +// SetToJoin implements types.MembershipUpdater +func (u *membershipUpdater) SetToJoin(senderUserID string, eventID string, isUpdate bool) (inviteEventIDs []string, err error) { + err = common.WithTransaction(u.d.db, func(txn *sql.Tx) error { + senderUserNID, err := u.d.assignStateKeyNID(u.ctx, txn, senderUserID) + if err != nil { + return err + } + + // If this is a join event update, there is no invite to update + if !isUpdate { + inviteEventIDs, err = u.d.statements.updateInviteRetired( + u.ctx, txn, u.roomNID, u.targetUserNID, + ) + if err != nil { + return err + } + } + + // Look up the NID of the new join event + nIDs, err := u.d.EventNIDs(u.ctx, []string{eventID}) + if err != nil { + return err + } + + if u.membership != membershipStateJoin || isUpdate { + if err = u.d.statements.updateMembership( + u.ctx, txn, u.roomNID, u.targetUserNID, senderUserNID, + membershipStateJoin, nIDs[eventID], + ); err != nil { + return err + } + } + return nil + }) + + return +} + +// SetToLeave implements types.MembershipUpdater +func (u *membershipUpdater) SetToLeave(senderUserID string, eventID string) (inviteEventIDs []string, err error) { + err = common.WithTransaction(u.d.db, func(txn *sql.Tx) error { + senderUserNID, err := u.d.assignStateKeyNID(u.ctx, txn, senderUserID) + if err != nil { + return err + } + inviteEventIDs, err = u.d.statements.updateInviteRetired( + u.ctx, txn, u.roomNID, u.targetUserNID, + ) + if err != nil { + return err + } + + // Look up the NID of the new leave event + nIDs, err := u.d.EventNIDs(u.ctx, []string{eventID}) + if err != nil { + return err + } + + if u.membership != membershipStateLeaveOrBan { + if err = u.d.statements.updateMembership( + u.ctx, txn, u.roomNID, u.targetUserNID, senderUserNID, + membershipStateLeaveOrBan, nIDs[eventID], + ); err != nil { + return err + } + } + return nil + }) + return +} + +// GetMembership implements query.RoomserverQueryAPIDB +func (d *Database) GetMembership( + ctx context.Context, roomNID types.RoomNID, requestSenderUserID string, +) (membershipEventNID types.EventNID, stillInRoom bool, err error) { + err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + requestSenderUserNID, err := d.assignStateKeyNID(ctx, txn, requestSenderUserID) + if err != nil { + return err + } + + membershipEventNID, _, err = + d.statements.selectMembershipFromRoomAndTarget( + ctx, txn, roomNID, requestSenderUserNID, + ) + if err == sql.ErrNoRows { + // The user has never been a member of that room + return nil + } + if err != nil { + return err + } + stillInRoom = true + return nil + }) + + return +} + +// GetMembershipEventNIDsForRoom implements query.RoomserverQueryAPIDB +func (d *Database) GetMembershipEventNIDsForRoom( + ctx context.Context, roomNID types.RoomNID, joinOnly bool, +) (eventNIDs []types.EventNID, err error) { + err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + if joinOnly { + eventNIDs, err = d.statements.selectMembershipsFromRoomAndMembership( + ctx, txn, roomNID, membershipStateJoin, + ) + return nil + } + + eventNIDs, err = d.statements.selectMembershipsFromRoom(ctx, txn, roomNID) + return nil + }) + return +} + +// EventsFromIDs implements query.RoomserverQueryAPIEventDB +func (d *Database) EventsFromIDs(ctx context.Context, eventIDs []string) ([]types.Event, error) { + nidMap, err := d.EventNIDs(ctx, eventIDs) + if err != nil { + return nil, err + } + + var nids []types.EventNID + for _, nid := range nidMap { + nids = append(nids, nid) + } + + return d.Events(ctx, nids) +} + +func (d *Database) GetRoomVersionForRoom( + ctx context.Context, roomNID types.RoomNID, +) (int64, error) { + return d.statements.selectRoomVersionForRoomNID( + ctx, nil, roomNID, + ) +} + +type transaction struct { + ctx context.Context + txn *sql.Tx +} + +// Commit implements types.Transaction +func (t *transaction) Commit() error { + if t.txn == nil { + return nil + } + return t.txn.Commit() +} + +// Rollback implements types.Transaction +func (t *transaction) Rollback() error { + if t.txn == nil { + return nil + } + return t.txn.Rollback() +} diff --git a/roomserver/storage/sqlite3/transactions_table.go b/roomserver/storage/sqlite3/transactions_table.go new file mode 100644 index 000000000..7740e5f07 --- /dev/null +++ b/roomserver/storage/sqlite3/transactions_table.go @@ -0,0 +1,86 @@ +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + + "github.com/matrix-org/dendrite/common" +) + +const transactionsSchema = ` + CREATE TABLE IF NOT EXISTS roomserver_transactions ( + transaction_id TEXT NOT NULL, + session_id INTEGER NOT NULL, + user_id TEXT NOT NULL, + event_id TEXT NOT NULL, + PRIMARY KEY (transaction_id, session_id, user_id) + ); +` +const insertTransactionSQL = ` + INSERT INTO roomserver_transactions (transaction_id, session_id, user_id, event_id) + VALUES ($1, $2, $3, $4) +` + +const selectTransactionEventIDSQL = ` + SELECT event_id FROM roomserver_transactions + WHERE transaction_id = $1 AND session_id = $2 AND user_id = $3 +` + +type transactionStatements struct { + insertTransactionStmt *sql.Stmt + selectTransactionEventIDStmt *sql.Stmt +} + +func (s *transactionStatements) prepare(db *sql.DB) (err error) { + _, err = db.Exec(transactionsSchema) + if err != nil { + return + } + + return statementList{ + {&s.insertTransactionStmt, insertTransactionSQL}, + {&s.selectTransactionEventIDStmt, selectTransactionEventIDSQL}, + }.prepare(db) +} + +func (s *transactionStatements) insertTransaction( + ctx context.Context, txn *sql.Tx, + transactionID string, + sessionID int64, + userID string, + eventID string, +) (err error) { + stmt := common.TxStmt(txn, s.insertTransactionStmt) + _, err = stmt.ExecContext( + ctx, transactionID, sessionID, userID, eventID, + ) + return +} + +func (s *transactionStatements) selectTransactionEventID( + ctx context.Context, txn *sql.Tx, + transactionID string, + sessionID int64, + userID string, +) (eventID string, err error) { + stmt := common.TxStmt(txn, s.selectTransactionEventIDStmt) + err = stmt.QueryRowContext( + ctx, transactionID, sessionID, userID, + ).Scan(&eventID) + return +} diff --git a/roomserver/storage/storage.go b/roomserver/storage/storage.go index 908411681..551d97cd1 100644 --- a/roomserver/storage/storage.go +++ b/roomserver/storage/storage.go @@ -19,25 +19,20 @@ import ( "net/url" "github.com/matrix-org/dendrite/roomserver/api" + statedb "github.com/matrix-org/dendrite/roomserver/state/database" "github.com/matrix-org/dendrite/roomserver/storage/postgres" + "github.com/matrix-org/dendrite/roomserver/storage/sqlite3" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" ) type Database interface { + statedb.RoomStateDatabase StoreEvent(ctx context.Context, event gomatrixserverlib.Event, txnAndSessionID *api.TransactionID, authEventNIDs []types.EventNID) (types.RoomNID, types.StateAtEvent, error) StateEntriesForEventIDs(ctx context.Context, eventIDs []string) ([]types.StateEntry, error) - EventTypeNIDs(ctx context.Context, eventTypes []string) (map[string]types.EventTypeNID, error) - EventStateKeyNIDs(ctx context.Context, eventStateKeys []string) (map[string]types.EventStateKeyNID, error) EventStateKeys(ctx context.Context, eventStateKeyNIDs []types.EventStateKeyNID) (map[types.EventStateKeyNID]string, error) EventNIDs(ctx context.Context, eventIDs []string) (map[string]types.EventNID, error) - Events(ctx context.Context, eventNIDs []types.EventNID) ([]types.Event, error) - AddState(ctx context.Context, roomNID types.RoomNID, stateBlockNIDs []types.StateBlockNID, state []types.StateEntry) (types.StateSnapshotNID, error) SetState(ctx context.Context, eventNID types.EventNID, stateNID types.StateSnapshotNID) error - StateAtEventIDs(ctx context.Context, eventIDs []string) ([]types.StateAtEvent, error) - StateBlockNIDs(ctx context.Context, stateNIDs []types.StateSnapshotNID) ([]types.StateBlockNIDList, error) - StateEntries(ctx context.Context, stateBlockNIDs []types.StateBlockNID) ([]types.StateEntryList, error) - SnapshotNIDFromEventID(ctx context.Context, eventID string) (types.StateSnapshotNID, error) EventIDs(ctx context.Context, eventNIDs []types.EventNID) (map[types.EventNID]string, error) GetLatestEventsForUpdate(ctx context.Context, roomNID types.RoomNID) (types.RoomRecentEventsUpdater, error) GetTransactionEventID(ctx context.Context, transactionID string, sessionID int64, userID string) (string, error) @@ -49,7 +44,6 @@ type Database interface { GetAliasesForRoomID(ctx context.Context, roomID string) ([]string, error) GetCreatorIDForAlias(ctx context.Context, alias string) (string, error) RemoveRoomAlias(ctx context.Context, alias string) error - StateEntriesForTuples(ctx context.Context, stateBlockNIDs []types.StateBlockNID, stateKeyTuples []types.StateKeyTuple) ([]types.StateEntryList, error) MembershipUpdater(ctx context.Context, roomID, targetUserID string) (types.MembershipUpdater, error) GetMembership(ctx context.Context, roomNID types.RoomNID, requestSenderUserID string) (membershipEventNID types.EventNID, stillInRoom bool, err error) GetMembershipEventNIDsForRoom(ctx context.Context, roomNID types.RoomNID, joinOnly bool) ([]types.EventNID, error) @@ -66,6 +60,8 @@ func Open(dataSourceName string) (Database, error) { switch uri.Scheme { case "postgres": return postgres.Open(dataSourceName) + case "file": + return sqlite3.Open(dataSourceName) default: return postgres.Open(dataSourceName) } diff --git a/syncapi/routing/routing.go b/syncapi/routing/routing.go index 8916565dc..be90e0a07 100644 --- a/syncapi/routing/routing.go +++ b/syncapi/routing/routing.go @@ -39,7 +39,7 @@ const pathPrefixR0 = "/_matrix/client/r0" // nolint: gocyclo func Setup( apiMux *mux.Router, srp *sync.RequestPool, syncDB storage.Database, - deviceDB *devices.Database, federation *gomatrixserverlib.FederationClient, + deviceDB devices.Database, federation *gomatrixserverlib.FederationClient, queryAPI api.RoomserverQueryAPI, cfg *config.Dendrite, ) { diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go index aec37185d..6a33a8b4f 100644 --- a/syncapi/storage/postgres/syncserver.go +++ b/syncapi/storage/postgres/syncserver.go @@ -413,13 +413,18 @@ func (d *SyncServerDatasource) addPDUDeltaToResponse( numRecentEventsPerRoom int, wantFullState bool, res *types.Response, -) ([]string, error) { +) (joinedRoomIDs []string, err error) { txn, err := d.db.BeginTx(ctx, &txReadOnlySnapshot) if err != nil { return nil, err } var succeeded bool - defer common.EndTransaction(txn, &succeeded) + defer func() { + txerr := common.EndTransaction(txn, &succeeded) + if err == nil && txerr != nil { + err = txerr + } + }() stateFilter := gomatrixserverlib.DefaultStateFilter() // TODO: use filter provided in request @@ -428,7 +433,6 @@ func (d *SyncServerDatasource) addPDUDeltaToResponse( // This works out what the 'state' key should be for each room as well as which membership block // to put the room into. var deltas []stateDelta - var joinedRoomIDs []string if !wantFullState { deltas, joinedRoomIDs, err = d.getStateDeltas( ctx, &device, txn, fromPos, toPos, device.UserID, &stateFilter, @@ -570,7 +574,12 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( return } var succeeded bool - defer common.EndTransaction(txn, &succeeded) + defer func() { + txerr := common.EndTransaction(txn, &succeeded) + if err == nil && txerr != nil { + err = txerr + } + }() // Get the current sync position which we will base the sync response on. toPos, err = d.syncPositionTx(ctx, txn) diff --git a/syncapi/storage/sqlite3/account_data_table.go b/syncapi/storage/sqlite3/account_data_table.go new file mode 100644 index 000000000..3274e66ea --- /dev/null +++ b/syncapi/storage/sqlite3/account_data_table.go @@ -0,0 +1,143 @@ +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + + "github.com/lib/pq" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/gomatrixserverlib" +) + +const accountDataSchema = ` +CREATE TABLE IF NOT EXISTS syncapi_account_data_type ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id TEXT NOT NULL, + room_id TEXT NOT NULL, + type TEXT NOT NULL, + UNIQUE (user_id, room_id, type) +); +` + +const insertAccountDataSQL = "" + + "INSERT INTO syncapi_account_data_type (id, user_id, room_id, type) VALUES ($1, $2, $3, $4)" + + " ON CONFLICT (user_id, room_id, type) DO UPDATE" + + " SET id = EXCLUDED.id" + +const selectAccountDataInRangeSQL = "" + + "SELECT room_id, type FROM syncapi_account_data_type" + + " WHERE user_id = $1 AND id > $2 AND id <= $3" + + " AND ( $4 IS NULL OR type IN ($4) )" + + " AND ( $5 IS NULL OR NOT(type IN ($5)) )" + + " ORDER BY id ASC LIMIT $6" + +const selectMaxAccountDataIDSQL = "" + + "SELECT MAX(id) FROM syncapi_account_data_type" + +type accountDataStatements struct { + streamIDStatements *streamIDStatements + insertAccountDataStmt *sql.Stmt + selectAccountDataInRangeStmt *sql.Stmt + selectMaxAccountDataIDStmt *sql.Stmt +} + +func (s *accountDataStatements) prepare(db *sql.DB, streamID *streamIDStatements) (err error) { + s.streamIDStatements = streamID + _, err = db.Exec(accountDataSchema) + if err != nil { + return + } + if s.insertAccountDataStmt, err = db.Prepare(insertAccountDataSQL); err != nil { + return + } + if s.selectAccountDataInRangeStmt, err = db.Prepare(selectAccountDataInRangeSQL); err != nil { + return + } + if s.selectMaxAccountDataIDStmt, err = db.Prepare(selectMaxAccountDataIDSQL); err != nil { + return + } + return +} + +func (s *accountDataStatements) insertAccountData( + ctx context.Context, txn *sql.Tx, + userID, roomID, dataType string, +) (pos types.StreamPosition, err error) { + pos, err = s.streamIDStatements.nextStreamID(ctx, txn) + if err != nil { + return + } + insertStmt := common.TxStmt(txn, s.insertAccountDataStmt) + _, err = insertStmt.ExecContext(ctx, pos, userID, roomID, dataType) + return +} + +func (s *accountDataStatements) selectAccountDataInRange( + ctx context.Context, + userID string, + oldPos, newPos types.StreamPosition, + accountDataFilterPart *gomatrixserverlib.EventFilter, +) (data map[string][]string, err error) { + data = make(map[string][]string) + + // If both positions are the same, it means that the data was saved after the + // latest room event. In that case, we need to decrement the old position as + // it would prevent the SQL request from returning anything. + if oldPos == newPos { + oldPos-- + } + + rows, err := s.selectAccountDataInRangeStmt.QueryContext(ctx, userID, oldPos, newPos, + pq.StringArray(filterConvertTypeWildcardToSQL(accountDataFilterPart.Types)), + pq.StringArray(filterConvertTypeWildcardToSQL(accountDataFilterPart.NotTypes)), + accountDataFilterPart.Limit, + ) + if err != nil { + return + } + + for rows.Next() { + var dataType string + var roomID string + + if err = rows.Scan(&roomID, &dataType); err != nil { + return + } + + if len(data[roomID]) > 0 { + data[roomID] = append(data[roomID], dataType) + } else { + data[roomID] = []string{dataType} + } + } + + return +} + +func (s *accountDataStatements) selectMaxAccountDataID( + ctx context.Context, txn *sql.Tx, +) (id int64, err error) { + var nullableID sql.NullInt64 + stmt := common.TxStmt(txn, s.selectMaxAccountDataIDStmt) + err = stmt.QueryRowContext(ctx).Scan(&nullableID) + if nullableID.Valid { + id = nullableID.Int64 + } + return +} diff --git a/syncapi/storage/sqlite3/backward_extremities_table.go b/syncapi/storage/sqlite3/backward_extremities_table.go new file mode 100644 index 000000000..fcf15da25 --- /dev/null +++ b/syncapi/storage/sqlite3/backward_extremities_table.go @@ -0,0 +1,124 @@ +// Copyright 2018 New Vector Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + + "github.com/matrix-org/dendrite/common" +) + +const backwardExtremitiesSchema = ` +-- Stores output room events received from the roomserver. +CREATE TABLE IF NOT EXISTS syncapi_backward_extremities ( + room_id TEXT NOT NULL, + event_id TEXT NOT NULL, + + PRIMARY KEY(room_id, event_id) +); +` + +const insertBackwardExtremitySQL = "" + + "INSERT INTO syncapi_backward_extremities (room_id, event_id)" + + " VALUES ($1, $2)" + + " ON CONFLICT (room_id, event_id) DO NOTHING" + +const selectBackwardExtremitiesForRoomSQL = "" + + "SELECT event_id FROM syncapi_backward_extremities WHERE room_id = $1" + +const isBackwardExtremitySQL = "" + + "SELECT EXISTS (" + + " SELECT TRUE FROM syncapi_backward_extremities" + + " WHERE room_id = $1 AND event_id = $2" + + ")" + +const deleteBackwardExtremitySQL = "" + + "DELETE FROM syncapi_backward_extremities" + + " WHERE room_id = $1 AND event_id = $2" + +type backwardExtremitiesStatements struct { + insertBackwardExtremityStmt *sql.Stmt + selectBackwardExtremitiesForRoomStmt *sql.Stmt + isBackwardExtremityStmt *sql.Stmt + deleteBackwardExtremityStmt *sql.Stmt +} + +func (s *backwardExtremitiesStatements) prepare(db *sql.DB) (err error) { + _, err = db.Exec(backwardExtremitiesSchema) + if err != nil { + return + } + if s.insertBackwardExtremityStmt, err = db.Prepare(insertBackwardExtremitySQL); err != nil { + return + } + if s.selectBackwardExtremitiesForRoomStmt, err = db.Prepare(selectBackwardExtremitiesForRoomSQL); err != nil { + return + } + if s.isBackwardExtremityStmt, err = db.Prepare(isBackwardExtremitySQL); err != nil { + return + } + if s.deleteBackwardExtremityStmt, err = db.Prepare(deleteBackwardExtremitySQL); err != nil { + return + } + return +} + +func (s *backwardExtremitiesStatements) insertsBackwardExtremity( + ctx context.Context, txn *sql.Tx, roomID, eventID string, +) (err error) { + stmt := common.TxStmt(txn, s.insertBackwardExtremityStmt) + _, err = stmt.ExecContext(ctx, roomID, eventID) + return +} + +func (s *backwardExtremitiesStatements) selectBackwardExtremitiesForRoom( + ctx context.Context, txn *sql.Tx, roomID string, +) (eventIDs []string, err error) { + eventIDs = make([]string, 0) + + stmt := common.TxStmt(txn, s.selectBackwardExtremitiesForRoomStmt) + rows, err := stmt.QueryContext(ctx, roomID) + if err != nil { + return + } + + for rows.Next() { + var eID string + if err = rows.Scan(&eID); err != nil { + return + } + + eventIDs = append(eventIDs, eID) + } + + return +} + +func (s *backwardExtremitiesStatements) isBackwardExtremity( + ctx context.Context, txn *sql.Tx, roomID, eventID string, +) (isBE bool, err error) { + stmt := common.TxStmt(txn, s.isBackwardExtremityStmt) + err = stmt.QueryRowContext(ctx, roomID, eventID).Scan(&isBE) + return +} + +func (s *backwardExtremitiesStatements) deleteBackwardExtremity( + ctx context.Context, txn *sql.Tx, roomID, eventID string, +) (err error) { + stmt := common.TxStmt(txn, s.deleteBackwardExtremityStmt) + _, err = stmt.ExecContext(ctx, roomID, eventID) + return +} diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go new file mode 100644 index 000000000..4ce946667 --- /dev/null +++ b/syncapi/storage/sqlite3/current_room_state_table.go @@ -0,0 +1,276 @@ +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + "encoding/json" + + "github.com/lib/pq" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/gomatrixserverlib" +) + +const currentRoomStateSchema = ` +-- Stores the current room state for every room. +CREATE TABLE IF NOT EXISTS syncapi_current_room_state ( + room_id TEXT NOT NULL, + event_id TEXT NOT NULL, + type TEXT NOT NULL, + sender TEXT NOT NULL, + contains_url BOOL NOT NULL DEFAULT false, + state_key TEXT NOT NULL, + event_json TEXT NOT NULL, + membership TEXT, + added_at BIGINT, + UNIQUE (room_id, type, state_key) +); +-- for event deletion +CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_id_idx ON syncapi_current_room_state(event_id, room_id, type, sender, contains_url); +-- for querying membership states of users +-- CREATE INDEX IF NOT EXISTS syncapi_membership_idx ON syncapi_current_room_state(type, state_key, membership) WHERE membership IS NOT NULL AND membership != 'leave'; +` + +const upsertRoomStateSQL = "" + + "INSERT INTO syncapi_current_room_state (room_id, event_id, type, sender, contains_url, state_key, event_json, membership, added_at)" + + " VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)" + + " ON CONFLICT (event_id, room_id, type, sender, contains_url)" + + " DO UPDATE SET event_id = $2, sender=$4, contains_url=$5, event_json = $7, membership = $8, added_at = $9" + +const deleteRoomStateByEventIDSQL = "" + + "DELETE FROM syncapi_current_room_state WHERE event_id = $1" + +const selectRoomIDsWithMembershipSQL = "" + + "SELECT room_id FROM syncapi_current_room_state WHERE type = 'm.room.member' AND state_key = $1 AND membership = $2" + +const selectCurrentStateSQL = "" + + "SELECT event_json FROM syncapi_current_room_state WHERE room_id = $1" + + " AND ( $2 IS NULL OR sender IN ($2) )" + + " AND ( $3 IS NULL OR NOT(sender IN ($3)) )" + + " AND ( $4 IS NULL OR type IN ($4) )" + + " AND ( $5 IS NULL OR NOT(type IN ($5)) )" + + " AND ( $6 IS NULL OR contains_url = $6 )" + + " LIMIT $7" + +const selectJoinedUsersSQL = "" + + "SELECT room_id, state_key FROM syncapi_current_room_state WHERE type = 'm.room.member' AND membership = 'join'" + +const selectStateEventSQL = "" + + "SELECT event_json FROM syncapi_current_room_state WHERE room_id = $1 AND type = $2 AND state_key = $3" + +const selectEventsWithEventIDsSQL = "" + + // TODO: The session_id and transaction_id blanks are here because otherwise + // the rowsToStreamEvents expects there to be exactly five columns. We need to + // figure out if these really need to be in the DB, and if so, we need a + // better permanent fix for this. - neilalexander, 2 Jan 2020 + "SELECT added_at, event_json, 0 AS session_id, false AS exclude_from_sync, '' AS transaction_id" + + " FROM syncapi_current_room_state WHERE event_id IN ($1)" + +type currentRoomStateStatements struct { + streamIDStatements *streamIDStatements + upsertRoomStateStmt *sql.Stmt + deleteRoomStateByEventIDStmt *sql.Stmt + selectRoomIDsWithMembershipStmt *sql.Stmt + selectCurrentStateStmt *sql.Stmt + selectJoinedUsersStmt *sql.Stmt + selectEventsWithEventIDsStmt *sql.Stmt + selectStateEventStmt *sql.Stmt +} + +func (s *currentRoomStateStatements) prepare(db *sql.DB, streamID *streamIDStatements) (err error) { + s.streamIDStatements = streamID + _, err = db.Exec(currentRoomStateSchema) + if err != nil { + return + } + if s.upsertRoomStateStmt, err = db.Prepare(upsertRoomStateSQL); err != nil { + return + } + if s.deleteRoomStateByEventIDStmt, err = db.Prepare(deleteRoomStateByEventIDSQL); err != nil { + return + } + if s.selectRoomIDsWithMembershipStmt, err = db.Prepare(selectRoomIDsWithMembershipSQL); err != nil { + return + } + if s.selectCurrentStateStmt, err = db.Prepare(selectCurrentStateSQL); err != nil { + return + } + if s.selectJoinedUsersStmt, err = db.Prepare(selectJoinedUsersSQL); err != nil { + return + } + if s.selectEventsWithEventIDsStmt, err = db.Prepare(selectEventsWithEventIDsSQL); err != nil { + return + } + if s.selectStateEventStmt, err = db.Prepare(selectStateEventSQL); err != nil { + return + } + return +} + +// JoinedMemberLists returns a map of room ID to a list of joined user IDs. +func (s *currentRoomStateStatements) selectJoinedUsers( + ctx context.Context, +) (map[string][]string, error) { + rows, err := s.selectJoinedUsersStmt.QueryContext(ctx) + if err != nil { + return nil, err + } + defer rows.Close() // nolint: errcheck + + result := make(map[string][]string) + for rows.Next() { + var roomID string + var userID string + if err := rows.Scan(&roomID, &userID); err != nil { + return nil, err + } + users := result[roomID] + users = append(users, userID) + result[roomID] = users + } + return result, nil +} + +// SelectRoomIDsWithMembership returns the list of room IDs which have the given user in the given membership state. +func (s *currentRoomStateStatements) selectRoomIDsWithMembership( + ctx context.Context, + txn *sql.Tx, + userID string, + membership string, // nolint: unparam +) ([]string, error) { + stmt := common.TxStmt(txn, s.selectRoomIDsWithMembershipStmt) + rows, err := stmt.QueryContext(ctx, userID, membership) + if err != nil { + return nil, err + } + defer rows.Close() // nolint: errcheck + + var result []string + for rows.Next() { + var roomID string + if err := rows.Scan(&roomID); err != nil { + return nil, err + } + result = append(result, roomID) + } + return result, nil +} + +// CurrentState returns all the current state events for the given room. +func (s *currentRoomStateStatements) selectCurrentState( + ctx context.Context, txn *sql.Tx, roomID string, + stateFilterPart *gomatrixserverlib.StateFilter, +) ([]gomatrixserverlib.Event, error) { + stmt := common.TxStmt(txn, s.selectCurrentStateStmt) + rows, err := stmt.QueryContext(ctx, roomID, + pq.StringArray(stateFilterPart.Senders), + pq.StringArray(stateFilterPart.NotSenders), + pq.StringArray(filterConvertTypeWildcardToSQL(stateFilterPart.Types)), + pq.StringArray(filterConvertTypeWildcardToSQL(stateFilterPart.NotTypes)), + stateFilterPart.ContainsURL, + stateFilterPart.Limit, + ) + if err != nil { + return nil, err + } + defer rows.Close() // nolint: errcheck + + return rowsToEvents(rows) +} + +func (s *currentRoomStateStatements) deleteRoomStateByEventID( + ctx context.Context, txn *sql.Tx, eventID string, +) error { + stmt := common.TxStmt(txn, s.deleteRoomStateByEventIDStmt) + _, err := stmt.ExecContext(ctx, eventID) + return err +} + +func (s *currentRoomStateStatements) upsertRoomState( + ctx context.Context, txn *sql.Tx, + event gomatrixserverlib.Event, membership *string, addedAt types.StreamPosition, +) error { + // Parse content as JSON and search for an "url" key + containsURL := false + var content map[string]interface{} + if json.Unmarshal(event.Content(), &content) != nil { + // Set containsURL to true if url is present + _, containsURL = content["url"] + } + + // upsert state event + stmt := common.TxStmt(txn, s.upsertRoomStateStmt) + _, err := stmt.ExecContext( + ctx, + event.RoomID(), + event.EventID(), + event.Type(), + event.Sender(), + containsURL, + *event.StateKey(), + event.JSON(), + membership, + addedAt, + ) + return err +} + +func (s *currentRoomStateStatements) selectEventsWithEventIDs( + ctx context.Context, txn *sql.Tx, eventIDs []string, +) ([]types.StreamEvent, error) { + stmt := common.TxStmt(txn, s.selectEventsWithEventIDsStmt) + rows, err := stmt.QueryContext(ctx, pq.StringArray(eventIDs)) + if err != nil { + return nil, err + } + defer rows.Close() // nolint: errcheck + return rowsToStreamEvents(rows) +} + +func rowsToEvents(rows *sql.Rows) ([]gomatrixserverlib.Event, error) { + result := []gomatrixserverlib.Event{} + for rows.Next() { + var eventBytes []byte + if err := rows.Scan(&eventBytes); err != nil { + return nil, err + } + // TODO: Handle redacted events + ev, err := gomatrixserverlib.NewEventFromTrustedJSON(eventBytes, false) + if err != nil { + return nil, err + } + result = append(result, ev) + } + return result, nil +} + +func (s *currentRoomStateStatements) selectStateEvent( + ctx context.Context, roomID, evType, stateKey string, +) (*gomatrixserverlib.Event, error) { + stmt := s.selectStateEventStmt + var res []byte + err := stmt.QueryRowContext(ctx, roomID, evType, stateKey).Scan(&res) + if err == sql.ErrNoRows { + return nil, nil + } + if err != nil { + return nil, err + } + ev, err := gomatrixserverlib.NewEventFromTrustedJSON(res, false) + return &ev, err +} diff --git a/syncapi/storage/sqlite3/filtering.go b/syncapi/storage/sqlite3/filtering.go new file mode 100644 index 000000000..c4a2f4bf9 --- /dev/null +++ b/syncapi/storage/sqlite3/filtering.go @@ -0,0 +1,36 @@ +// Copyright 2017 Thibaut CHARLES +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "strings" +) + +// filterConvertWildcardToSQL converts wildcards as defined in +// https://matrix.org/docs/spec/client_server/r0.3.0.html#post-matrix-client-r0-user-userid-filter +// to SQL wildcards that can be used with LIKE() +func filterConvertTypeWildcardToSQL(values []string) []string { + if values == nil { + // Return nil instead of []string{} so IS NULL can work correctly when + // the return value is passed into SQL queries + return nil + } + + ret := make([]string, len(values)) + for i := range values { + ret[i] = strings.Replace(values[i], "*", "%", -1) + } + return ret +} diff --git a/syncapi/storage/sqlite3/invites_table.go b/syncapi/storage/sqlite3/invites_table.go new file mode 100644 index 000000000..74dba245b --- /dev/null +++ b/syncapi/storage/sqlite3/invites_table.go @@ -0,0 +1,157 @@ +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/gomatrixserverlib" +) + +const inviteEventsSchema = ` +CREATE TABLE IF NOT EXISTS syncapi_invite_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + target_user_id TEXT NOT NULL, + event_json TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS syncapi_invites_target_user_id_idx ON syncapi_invite_events (target_user_id, id); +CREATE INDEX IF NOT EXISTS syncapi_invites_event_id_idx ON syncapi_invite_events (event_id); +` + +const insertInviteEventSQL = "" + + "INSERT INTO syncapi_invite_events" + + " (room_id, event_id, target_user_id, event_json)" + + " VALUES ($1, $2, $3, $4)" + +const selectLastInsertedInviteEventSQL = "" + + "SELECT id FROM syncapi_invite_events WHERE rowid = last_insert_rowid()" + +const deleteInviteEventSQL = "" + + "DELETE FROM syncapi_invite_events WHERE event_id = $1" + +const selectInviteEventsInRangeSQL = "" + + "SELECT room_id, event_json FROM syncapi_invite_events" + + " WHERE target_user_id = $1 AND id > $2 AND id <= $3" + + " ORDER BY id DESC" + +const selectMaxInviteIDSQL = "" + + "SELECT MAX(id) FROM syncapi_invite_events" + +type inviteEventsStatements struct { + streamIDStatements *streamIDStatements + insertInviteEventStmt *sql.Stmt + selectLastInsertedInviteEventStmt *sql.Stmt + selectInviteEventsInRangeStmt *sql.Stmt + deleteInviteEventStmt *sql.Stmt + selectMaxInviteIDStmt *sql.Stmt +} + +func (s *inviteEventsStatements) prepare(db *sql.DB, streamID *streamIDStatements) (err error) { + s.streamIDStatements = streamID + _, err = db.Exec(inviteEventsSchema) + if err != nil { + return + } + if s.insertInviteEventStmt, err = db.Prepare(insertInviteEventSQL); err != nil { + return + } + if s.selectLastInsertedInviteEventStmt, err = db.Prepare(selectLastInsertedInviteEventSQL); err != nil { + return + } + if s.selectInviteEventsInRangeStmt, err = db.Prepare(selectInviteEventsInRangeSQL); err != nil { + return + } + if s.deleteInviteEventStmt, err = db.Prepare(deleteInviteEventSQL); err != nil { + return + } + if s.selectMaxInviteIDStmt, err = db.Prepare(selectMaxInviteIDSQL); err != nil { + return + } + return +} + +func (s *inviteEventsStatements) insertInviteEvent( + ctx context.Context, inviteEvent gomatrixserverlib.Event, +) (streamPos types.StreamPosition, err error) { + _, err = s.insertInviteEventStmt.ExecContext( + ctx, + inviteEvent.RoomID(), + inviteEvent.EventID(), + *inviteEvent.StateKey(), + inviteEvent.JSON(), + ) + if err != nil { + return + } + err = s.selectLastInsertedInviteEventStmt.QueryRowContext(ctx).Scan(&streamPos) + return +} + +func (s *inviteEventsStatements) deleteInviteEvent( + ctx context.Context, inviteEventID string, +) error { + _, err := s.deleteInviteEventStmt.ExecContext(ctx, inviteEventID) + return err +} + +// selectInviteEventsInRange returns a map of room ID to invite event for the +// active invites for the target user ID in the supplied range. +func (s *inviteEventsStatements) selectInviteEventsInRange( + ctx context.Context, txn *sql.Tx, targetUserID string, startPos, endPos types.StreamPosition, +) (map[string]gomatrixserverlib.Event, error) { + stmt := common.TxStmt(txn, s.selectInviteEventsInRangeStmt) + rows, err := stmt.QueryContext(ctx, targetUserID, startPos, endPos) + if err != nil { + return nil, err + } + defer rows.Close() // nolint: errcheck + result := map[string]gomatrixserverlib.Event{} + for rows.Next() { + var ( + roomID string + eventJSON []byte + ) + if err = rows.Scan(&roomID, &eventJSON); err != nil { + return nil, err + } + + event, err := gomatrixserverlib.NewEventFromTrustedJSON(eventJSON, false) + if err != nil { + return nil, err + } + + result[roomID] = event + } + return result, nil +} + +func (s *inviteEventsStatements) selectMaxInviteID( + ctx context.Context, txn *sql.Tx, +) (id int64, err error) { + var nullableID sql.NullInt64 + stmt := common.TxStmt(txn, s.selectMaxInviteIDStmt) + err = stmt.QueryRowContext(ctx).Scan(&nullableID) + if nullableID.Valid { + id = nullableID.Int64 + } + return +} diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go new file mode 100644 index 000000000..8c01f2ced --- /dev/null +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -0,0 +1,411 @@ +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + "encoding/json" + "sort" + + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/syncapi/types" + + "github.com/lib/pq" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/gomatrixserverlib" + log "github.com/sirupsen/logrus" +) + +const outputRoomEventsSchema = ` +-- Stores output room events received from the roomserver. +CREATE TABLE IF NOT EXISTS syncapi_output_room_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + event_id TEXT NOT NULL UNIQUE, + room_id TEXT NOT NULL, + event_json TEXT NOT NULL, + type TEXT NOT NULL, + sender TEXT NOT NULL, + contains_url BOOL NOT NULL, + add_state_ids TEXT[], + remove_state_ids TEXT[], + session_id BIGINT, + transaction_id TEXT, + exclude_from_sync BOOL DEFAULT FALSE +); +` + +const insertEventSQL = "" + + "INSERT INTO syncapi_output_room_events (" + + "id, room_id, event_id, event_json, type, sender, contains_url, add_state_ids, remove_state_ids, session_id, transaction_id, exclude_from_sync" + + ") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) " + + "ON CONFLICT (event_id) DO UPDATE SET exclude_from_sync = $11" + +const selectLastInsertedEventSQL = "" + + "SELECT id FROM syncapi_output_room_events WHERE rowid = last_insert_rowid()" + +const selectEventsSQL = "" + + "SELECT id, event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events WHERE event_id = $1" + +const selectRecentEventsSQL = "" + + "SELECT id, event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + + " WHERE room_id = $1 AND id > $2 AND id <= $3" + + " ORDER BY id DESC LIMIT $4" + +const selectRecentEventsForSyncSQL = "" + + "SELECT id, event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + + " WHERE room_id = $1 AND id > $2 AND id <= $3 AND exclude_from_sync = FALSE" + + " ORDER BY id DESC LIMIT $4" + +const selectEarlyEventsSQL = "" + + "SELECT id, event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + + " WHERE room_id = $1 AND id > $2 AND id <= $3" + + " ORDER BY id ASC LIMIT $4" + +const selectMaxEventIDSQL = "" + + "SELECT MAX(id) FROM syncapi_output_room_events" + +// In order for us to apply the state updates correctly, rows need to be ordered in the order they were received (id). +/* + $1 = oldPos, + $2 = newPos, + $3 = pq.StringArray(stateFilterPart.Senders), + $4 = pq.StringArray(stateFilterPart.NotSenders), + $5 = pq.StringArray(filterConvertTypeWildcardToSQL(stateFilterPart.Types)), + $6 = pq.StringArray(filterConvertTypeWildcardToSQL(stateFilterPart.NotTypes)), + $7 = stateFilterPart.ContainsURL, + $8 = stateFilterPart.Limit, +*/ +const selectStateInRangeSQL = "" + + "SELECT id, event_json, exclude_from_sync, add_state_ids, remove_state_ids" + + " FROM syncapi_output_room_events" + + " WHERE (id > $1 AND id <= $2)" + // old/new pos + " AND (add_state_ids IS NOT NULL OR remove_state_ids IS NOT NULL)" + + /* " AND ( $3 IS NULL OR sender IN ($3) )" + // sender + " AND ( $4 IS NULL OR NOT(sender IN ($4)) )" + // not sender + " AND ( $5 IS NULL OR type IN ($5) )" + // type + " AND ( $6 IS NULL OR NOT(type IN ($6)) )" + // not type + " AND ( $7 IS NULL OR contains_url = $7)" + // contains URL? */ + " ORDER BY id ASC" + + " LIMIT $8" // limit + +type outputRoomEventsStatements struct { + streamIDStatements *streamIDStatements + insertEventStmt *sql.Stmt + selectLastInsertedEventStmt *sql.Stmt + selectEventsStmt *sql.Stmt + selectMaxEventIDStmt *sql.Stmt + selectRecentEventsStmt *sql.Stmt + selectRecentEventsForSyncStmt *sql.Stmt + selectEarlyEventsStmt *sql.Stmt + selectStateInRangeStmt *sql.Stmt +} + +func (s *outputRoomEventsStatements) prepare(db *sql.DB, streamID *streamIDStatements) (err error) { + s.streamIDStatements = streamID + _, err = db.Exec(outputRoomEventsSchema) + if err != nil { + return + } + if s.insertEventStmt, err = db.Prepare(insertEventSQL); err != nil { + return + } + if s.selectLastInsertedEventStmt, err = db.Prepare(selectLastInsertedEventSQL); err != nil { + return + } + if s.selectEventsStmt, err = db.Prepare(selectEventsSQL); err != nil { + return + } + if s.selectMaxEventIDStmt, err = db.Prepare(selectMaxEventIDSQL); err != nil { + return + } + if s.selectRecentEventsStmt, err = db.Prepare(selectRecentEventsSQL); err != nil { + return + } + if s.selectRecentEventsForSyncStmt, err = db.Prepare(selectRecentEventsForSyncSQL); err != nil { + return + } + if s.selectEarlyEventsStmt, err = db.Prepare(selectEarlyEventsSQL); err != nil { + return + } + if s.selectStateInRangeStmt, err = db.Prepare(selectStateInRangeSQL); err != nil { + return + } + return +} + +// selectStateInRange returns the state events between the two given PDU stream positions, exclusive of oldPos, inclusive of newPos. +// Results are bucketed based on the room ID. If the same state is overwritten multiple times between the +// two positions, only the most recent state is returned. +func (s *outputRoomEventsStatements) selectStateInRange( + ctx context.Context, txn *sql.Tx, oldPos, newPos types.StreamPosition, + stateFilterPart *gomatrixserverlib.StateFilter, +) (map[string]map[string]bool, map[string]types.StreamEvent, error) { + stmt := common.TxStmt(txn, s.selectStateInRangeStmt) + + rows, err := stmt.QueryContext( + ctx, oldPos, newPos, + /*pq.StringArray(stateFilterPart.Senders), + pq.StringArray(stateFilterPart.NotSenders), + pq.StringArray(filterConvertTypeWildcardToSQL(stateFilterPart.Types)), + pq.StringArray(filterConvertTypeWildcardToSQL(stateFilterPart.NotTypes)), + stateFilterPart.ContainsURL,*/ + stateFilterPart.Limit, + ) + if err != nil { + return nil, nil, err + } + // Fetch all the state change events for all rooms between the two positions then loop each event and: + // - Keep a cache of the event by ID (99% of state change events are for the event itself) + // - For each room ID, build up an array of event IDs which represents cumulative adds/removes + // For each room, map cumulative event IDs to events and return. This may need to a batch SELECT based on event ID + // if they aren't in the event ID cache. We don't handle state deletion yet. + eventIDToEvent := make(map[string]types.StreamEvent) + + // RoomID => A set (map[string]bool) of state event IDs which are between the two positions + stateNeeded := make(map[string]map[string]bool) + + for rows.Next() { + var ( + streamPos types.StreamPosition + eventBytes []byte + excludeFromSync bool + addIDs pq.StringArray + delIDs pq.StringArray + ) + if err := rows.Scan(&streamPos, &eventBytes, &excludeFromSync, &addIDs, &delIDs); err != nil { + return nil, nil, err + } + // Sanity check for deleted state and whine if we see it. We don't need to do anything + // since it'll just mark the event as not being needed. + if len(addIDs) < len(delIDs) { + log.WithFields(log.Fields{ + "since": oldPos, + "current": newPos, + "adds": addIDs, + "dels": delIDs, + }).Warn("StateBetween: ignoring deleted state") + } + + // TODO: Handle redacted events + ev, err := gomatrixserverlib.NewEventFromTrustedJSON(eventBytes, false) + if err != nil { + return nil, nil, err + } + needSet := stateNeeded[ev.RoomID()] + if needSet == nil { // make set if required + needSet = make(map[string]bool) + } + for _, id := range delIDs { + needSet[id] = false + } + for _, id := range addIDs { + needSet[id] = true + } + stateNeeded[ev.RoomID()] = needSet + + eventIDToEvent[ev.EventID()] = types.StreamEvent{ + Event: ev, + StreamPosition: streamPos, + ExcludeFromSync: excludeFromSync, + } + } + + return stateNeeded, eventIDToEvent, nil +} + +// MaxID returns the ID of the last inserted event in this table. 'txn' is optional. If it is not supplied, +// then this function should only ever be used at startup, as it will race with inserting events if it is +// done afterwards. If there are no inserted events, 0 is returned. +func (s *outputRoomEventsStatements) selectMaxEventID( + ctx context.Context, txn *sql.Tx, +) (id int64, err error) { + var nullableID sql.NullInt64 + stmt := common.TxStmt(txn, s.selectMaxEventIDStmt) + err = stmt.QueryRowContext(ctx).Scan(&nullableID) + if nullableID.Valid { + id = nullableID.Int64 + } + return +} + +// InsertEvent into the output_room_events table. addState and removeState are an optional list of state event IDs. Returns the position +// of the inserted event. +func (s *outputRoomEventsStatements) insertEvent( + ctx context.Context, txn *sql.Tx, + event *gomatrixserverlib.Event, addState, removeState []string, + transactionID *api.TransactionID, excludeFromSync bool, +) (streamPos types.StreamPosition, err error) { + var txnID *string + var sessionID *int64 + if transactionID != nil { + sessionID = &transactionID.SessionID + txnID = &transactionID.TransactionID + } + + // Parse content as JSON and search for an "url" key + containsURL := false + var content map[string]interface{} + if json.Unmarshal(event.Content(), &content) != nil { + // Set containsURL to true if url is present + _, containsURL = content["url"] + } + + streamPos, err = s.streamIDStatements.nextStreamID(ctx, txn) + if err != nil { + return + } + + insertStmt := common.TxStmt(txn, s.insertEventStmt) + selectStmt := common.TxStmt(txn, s.selectLastInsertedEventStmt) + _, err = insertStmt.ExecContext( + ctx, + streamPos, + event.RoomID(), + event.EventID(), + event.JSON(), + event.Type(), + event.Sender(), + containsURL, + pq.StringArray(addState), + pq.StringArray(removeState), + sessionID, + txnID, + excludeFromSync, + ) + if err != nil { + return + } + err = selectStmt.QueryRowContext(ctx).Scan(&streamPos) + return +} + +// selectRecentEvents returns the most recent events in the given room, up to a maximum of 'limit'. +// If onlySyncEvents has a value of true, only returns the events that aren't marked as to exclude +// from sync. +func (s *outputRoomEventsStatements) selectRecentEvents( + ctx context.Context, txn *sql.Tx, + roomID string, fromPos, toPos types.StreamPosition, limit int, + chronologicalOrder bool, onlySyncEvents bool, +) ([]types.StreamEvent, error) { + var stmt *sql.Stmt + if onlySyncEvents { + stmt = common.TxStmt(txn, s.selectRecentEventsForSyncStmt) + } else { + stmt = common.TxStmt(txn, s.selectRecentEventsStmt) + } + + rows, err := stmt.QueryContext(ctx, roomID, fromPos, toPos, limit) + if err != nil { + return nil, err + } + defer rows.Close() // nolint: errcheck + events, err := rowsToStreamEvents(rows) + if err != nil { + return nil, err + } + if chronologicalOrder { + // The events need to be returned from oldest to latest, which isn't + // necessary the way the SQL query returns them, so a sort is necessary to + // ensure the events are in the right order in the slice. + sort.SliceStable(events, func(i int, j int) bool { + return events[i].StreamPosition < events[j].StreamPosition + }) + } + return events, nil +} + +// selectEarlyEvents returns the earliest events in the given room, starting +// from a given position, up to a maximum of 'limit'. +func (s *outputRoomEventsStatements) selectEarlyEvents( + ctx context.Context, txn *sql.Tx, + roomID string, fromPos, toPos types.StreamPosition, limit int, +) ([]types.StreamEvent, error) { + stmt := common.TxStmt(txn, s.selectEarlyEventsStmt) + rows, err := stmt.QueryContext(ctx, roomID, fromPos, toPos, limit) + if err != nil { + return nil, err + } + defer rows.Close() // nolint: errcheck + events, err := rowsToStreamEvents(rows) + if err != nil { + return nil, err + } + // The events need to be returned from oldest to latest, which isn't + // necessarily the way the SQL query returns them, so a sort is necessary to + // ensure the events are in the right order in the slice. + sort.SliceStable(events, func(i int, j int) bool { + return events[i].StreamPosition < events[j].StreamPosition + }) + return events, nil +} + +// selectEvents returns the events for the given event IDs. If an event is +// missing from the database, it will be omitted. +func (s *outputRoomEventsStatements) selectEvents( + ctx context.Context, txn *sql.Tx, eventIDs []string, +) ([]types.StreamEvent, error) { + var returnEvents []types.StreamEvent + stmt := common.TxStmt(txn, s.selectEventsStmt) + for _, eventID := range eventIDs { + rows, err := stmt.QueryContext(ctx, eventID) + if err != nil { + return nil, err + } + if streamEvents, err := rowsToStreamEvents(rows); err == nil { + returnEvents = append(returnEvents, streamEvents...) + } + rows.Close() // nolint: errcheck + } + return returnEvents, nil +} + +func rowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) { + var result []types.StreamEvent + for rows.Next() { + var ( + streamPos types.StreamPosition + eventBytes []byte + excludeFromSync bool + sessionID *int64 + txnID *string + transactionID *api.TransactionID + ) + if err := rows.Scan(&streamPos, &eventBytes, &sessionID, &excludeFromSync, &txnID); err != nil { + return nil, err + } + // TODO: Handle redacted events + ev, err := gomatrixserverlib.NewEventFromTrustedJSON(eventBytes, false) + if err != nil { + return nil, err + } + + if sessionID != nil && txnID != nil { + transactionID = &api.TransactionID{ + SessionID: *sessionID, + TransactionID: *txnID, + } + } + + result = append(result, types.StreamEvent{ + Event: ev, + StreamPosition: streamPos, + TransactionID: transactionID, + ExcludeFromSync: excludeFromSync, + }) + } + return result, nil +} diff --git a/syncapi/storage/sqlite3/output_room_events_topology_table.go b/syncapi/storage/sqlite3/output_room_events_topology_table.go new file mode 100644 index 000000000..f7075bd6f --- /dev/null +++ b/syncapi/storage/sqlite3/output_room_events_topology_table.go @@ -0,0 +1,192 @@ +// Copyright 2018 New Vector Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/gomatrixserverlib" +) + +const outputRoomEventsTopologySchema = ` +-- Stores output room events received from the roomserver. +CREATE TABLE IF NOT EXISTS syncapi_output_room_events_topology ( + event_id TEXT PRIMARY KEY, + topological_position BIGINT NOT NULL, + room_id TEXT NOT NULL, + + UNIQUE(topological_position, room_id) +); +-- The topological order will be used in events selection and ordering +-- CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_topological_position_idx ON syncapi_output_room_events_topology(topological_position, room_id); +` + +const insertEventInTopologySQL = "" + + "INSERT INTO syncapi_output_room_events_topology (event_id, topological_position, room_id)" + + " VALUES ($1, $2, $3)" + + " ON CONFLICT (topological_position, room_id) DO UPDATE SET event_id = $1" + +const selectEventIDsInRangeASCSQL = "" + + "SELECT event_id FROM syncapi_output_room_events_topology" + + " WHERE room_id = $1 AND topological_position > $2 AND topological_position <= $3" + + " ORDER BY topological_position ASC LIMIT $4" + +const selectEventIDsInRangeDESCSQL = "" + + "SELECT event_id FROM syncapi_output_room_events_topology" + + " WHERE room_id = $1 AND topological_position > $2 AND topological_position <= $3" + + " ORDER BY topological_position DESC LIMIT $4" + +const selectPositionInTopologySQL = "" + + "SELECT topological_position FROM syncapi_output_room_events_topology" + + " WHERE event_id = $1" + +const selectMaxPositionInTopologySQL = "" + + "SELECT MAX(topological_position) FROM syncapi_output_room_events_topology" + + " WHERE room_id = $1" + +const selectEventIDsFromPositionSQL = "" + + "SELECT event_id FROM syncapi_output_room_events_topology" + + " WHERE room_id = $1 AND topological_position = $2" + +type outputRoomEventsTopologyStatements struct { + insertEventInTopologyStmt *sql.Stmt + selectEventIDsInRangeASCStmt *sql.Stmt + selectEventIDsInRangeDESCStmt *sql.Stmt + selectPositionInTopologyStmt *sql.Stmt + selectMaxPositionInTopologyStmt *sql.Stmt + selectEventIDsFromPositionStmt *sql.Stmt +} + +func (s *outputRoomEventsTopologyStatements) prepare(db *sql.DB) (err error) { + _, err = db.Exec(outputRoomEventsTopologySchema) + if err != nil { + return + } + if s.insertEventInTopologyStmt, err = db.Prepare(insertEventInTopologySQL); err != nil { + return + } + if s.selectEventIDsInRangeASCStmt, err = db.Prepare(selectEventIDsInRangeASCSQL); err != nil { + return + } + if s.selectEventIDsInRangeDESCStmt, err = db.Prepare(selectEventIDsInRangeDESCSQL); err != nil { + return + } + if s.selectPositionInTopologyStmt, err = db.Prepare(selectPositionInTopologySQL); err != nil { + return + } + if s.selectMaxPositionInTopologyStmt, err = db.Prepare(selectMaxPositionInTopologySQL); err != nil { + return + } + if s.selectEventIDsFromPositionStmt, err = db.Prepare(selectEventIDsFromPositionSQL); err != nil { + return + } + return +} + +// insertEventInTopology inserts the given event in the room's topology, based +// on the event's depth. +func (s *outputRoomEventsTopologyStatements) insertEventInTopology( + ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.Event, +) (err error) { + stmt := common.TxStmt(txn, s.insertEventInTopologyStmt) + _, err = stmt.ExecContext( + ctx, event.EventID(), event.Depth(), event.RoomID(), + ) + return +} + +// selectEventIDsInRange selects the IDs of events which positions are within a +// given range in a given room's topological order. +// Returns an empty slice if no events match the given range. +func (s *outputRoomEventsTopologyStatements) selectEventIDsInRange( + ctx context.Context, txn *sql.Tx, roomID string, + fromPos, toPos types.StreamPosition, + limit int, chronologicalOrder bool, +) (eventIDs []string, err error) { + // Decide on the selection's order according to whether chronological order + // is requested or not. + var stmt *sql.Stmt + if chronologicalOrder { + stmt = common.TxStmt(txn, s.selectEventIDsInRangeASCStmt) + } else { + stmt = common.TxStmt(txn, s.selectEventIDsInRangeDESCStmt) + } + + // Query the event IDs. + rows, err := stmt.QueryContext(ctx, roomID, fromPos, toPos, limit) + if err == sql.ErrNoRows { + // If no event matched the request, return an empty slice. + return []string{}, nil + } else if err != nil { + return + } + + // Return the IDs. + var eventID string + for rows.Next() { + if err = rows.Scan(&eventID); err != nil { + return + } + eventIDs = append(eventIDs, eventID) + } + + return +} + +// selectPositionInTopology returns the position of a given event in the +// topology of the room it belongs to. +func (s *outputRoomEventsTopologyStatements) selectPositionInTopology( + ctx context.Context, txn *sql.Tx, eventID string, +) (pos types.StreamPosition, err error) { + stmt := common.TxStmt(txn, s.selectPositionInTopologyStmt) + err = stmt.QueryRowContext(ctx, eventID).Scan(&pos) + return +} + +func (s *outputRoomEventsTopologyStatements) selectMaxPositionInTopology( + ctx context.Context, txn *sql.Tx, roomID string, +) (pos types.StreamPosition, err error) { + stmt := common.TxStmt(txn, s.selectMaxPositionInTopologyStmt) + err = stmt.QueryRowContext(ctx, roomID).Scan(&pos) + return +} + +// selectEventIDsFromPosition returns the IDs of all events that have a given +// position in the topology of a given room. +func (s *outputRoomEventsTopologyStatements) selectEventIDsFromPosition( + ctx context.Context, txn *sql.Tx, roomID string, pos types.StreamPosition, +) (eventIDs []string, err error) { + // Query the event IDs. + stmt := common.TxStmt(txn, s.selectEventIDsFromPositionStmt) + rows, err := stmt.QueryContext(ctx, roomID, pos) + if err == sql.ErrNoRows { + // If no event matched the request, return an empty slice. + return []string{}, nil + } else if err != nil { + return + } + // Return the IDs. + var eventID string + for rows.Next() { + if err = rows.Scan(&eventID); err != nil { + return + } + eventIDs = append(eventIDs, eventID) + } + return +} diff --git a/syncapi/storage/sqlite3/stream_id_table.go b/syncapi/storage/sqlite3/stream_id_table.go new file mode 100644 index 000000000..260f7a95d --- /dev/null +++ b/syncapi/storage/sqlite3/stream_id_table.go @@ -0,0 +1,58 @@ +package sqlite3 + +import ( + "context" + "database/sql" + + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/syncapi/types" +) + +const streamIDTableSchema = ` +-- Global stream ID counter, used by other tables. +CREATE TABLE IF NOT EXISTS syncapi_stream_id ( + stream_name TEXT NOT NULL PRIMARY KEY, + stream_id INT DEFAULT 0, + + UNIQUE(stream_name) +); +INSERT INTO syncapi_stream_id (stream_name, stream_id) VALUES ("global", 0) + ON CONFLICT DO NOTHING; +` + +const increaseStreamIDStmt = "" + + "UPDATE syncapi_stream_id SET stream_id = stream_id + 1 WHERE stream_name = $1" + +const selectStreamIDStmt = "" + + "SELECT stream_id FROM syncapi_stream_id WHERE stream_name = $1" + +type streamIDStatements struct { + increaseStreamIDStmt *sql.Stmt + selectStreamIDStmt *sql.Stmt +} + +func (s *streamIDStatements) prepare(db *sql.DB) (err error) { + _, err = db.Exec(streamIDTableSchema) + if err != nil { + return + } + if s.increaseStreamIDStmt, err = db.Prepare(increaseStreamIDStmt); err != nil { + return + } + if s.selectStreamIDStmt, err = db.Prepare(selectStreamIDStmt); err != nil { + return + } + return +} + +func (s *streamIDStatements) nextStreamID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error) { + increaseStmt := common.TxStmt(txn, s.increaseStreamIDStmt) + selectStmt := common.TxStmt(txn, s.selectStreamIDStmt) + if _, err = increaseStmt.ExecContext(ctx, "global"); err != nil { + return + } + if err = selectStmt.QueryRowContext(ctx, "global").Scan(&pos); err != nil { + return + } + return +} diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go new file mode 100644 index 000000000..8cfc1884f --- /dev/null +++ b/syncapi/storage/sqlite3/syncserver.go @@ -0,0 +1,1197 @@ +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "net/url" + "time" + + "github.com/sirupsen/logrus" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/roomserver/api" + + // Import the postgres database driver. + _ "github.com/lib/pq" + _ "github.com/mattn/go-sqlite3" + + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/dendrite/typingserver/cache" + "github.com/matrix-org/gomatrixserverlib" +) + +type stateDelta struct { + roomID string + stateEvents []gomatrixserverlib.Event + membership string + // The PDU stream position of the latest membership event for this user, if applicable. + // Can be 0 if there is no membership event in this delta. + membershipPos types.StreamPosition +} + +// SyncServerDatasource represents a sync server datasource which manages +// both the database for PDUs and caches for EDUs. +type SyncServerDatasource struct { + db *sql.DB + common.PartitionOffsetStatements + streamID streamIDStatements + accountData accountDataStatements + events outputRoomEventsStatements + roomstate currentRoomStateStatements + invites inviteEventsStatements + typingCache *cache.TypingCache + topology outputRoomEventsTopologyStatements + backwardExtremities backwardExtremitiesStatements +} + +// NewSyncServerDatasource creates a new sync server database +// nolint: gocyclo +func NewSyncServerDatasource(dataSourceName string) (*SyncServerDatasource, error) { + var d SyncServerDatasource + uri, err := url.Parse(dataSourceName) + if err != nil { + return nil, err + } + var cs string + if uri.Opaque != "" { // file:filename.db + cs = uri.Opaque + } else if uri.Path != "" { // file:///path/to/filename.db + cs = uri.Path + } else { + return nil, errors.New("no filename or path in connect string") + } + if d.db, err = sql.Open("sqlite3", cs); err != nil { + return nil, err + } + if err = d.prepare(); err != nil { + return nil, err + } + d.typingCache = cache.NewTypingCache() + return &d, nil +} + +func (d *SyncServerDatasource) prepare() (err error) { + if err = d.PartitionOffsetStatements.Prepare(d.db, "syncapi"); err != nil { + return err + } + if err = d.streamID.prepare(d.db); err != nil { + return err + } + if err = d.accountData.prepare(d.db, &d.streamID); err != nil { + return err + } + if err = d.events.prepare(d.db, &d.streamID); err != nil { + return err + } + if err := d.roomstate.prepare(d.db, &d.streamID); err != nil { + return err + } + if err := d.invites.prepare(d.db, &d.streamID); err != nil { + return err + } + if err := d.topology.prepare(d.db); err != nil { + return err + } + if err := d.backwardExtremities.prepare(d.db); err != nil { + return err + } + return nil +} + +// AllJoinedUsersInRooms returns a map of room ID to a list of all joined user IDs. +func (d *SyncServerDatasource) AllJoinedUsersInRooms(ctx context.Context) (map[string][]string, error) { + return d.roomstate.selectJoinedUsers(ctx) +} + +// Events lookups a list of event by their event ID. +// Returns a list of events matching the requested IDs found in the database. +// If an event is not found in the database then it will be omitted from the list. +// Returns an error if there was a problem talking with the database. +// Does not include any transaction IDs in the returned events. +func (d *SyncServerDatasource) Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.Event, error) { + streamEvents, err := d.events.selectEvents(ctx, nil, eventIDs) + if err != nil { + return nil, err + } + + // We don't include a device here as we only include transaction IDs in + // incremental syncs. + return d.StreamEventsToEvents(nil, streamEvents), nil +} + +func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, txn *sql.Tx, ev *gomatrixserverlib.Event) error { + // If the event is already known as a backward extremity, don't consider + // it as such anymore now that we have it. + isBackwardExtremity, err := d.backwardExtremities.isBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID()) + if err != nil { + return err + } + if isBackwardExtremity { + if err = d.backwardExtremities.deleteBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID()); err != nil { + return err + } + } + + // Check if we have all of the event's previous events. If an event is + // missing, add it to the room's backward extremities. + prevEvents, err := d.events.selectEvents(ctx, txn, ev.PrevEventIDs()) + if err != nil { + return err + } + var found bool + for _, eID := range ev.PrevEventIDs() { + found = false + for _, prevEv := range prevEvents { + if eID == prevEv.EventID() { + found = true + } + } + + // If the event is missing, consider it a backward extremity. + if !found { + if err = d.backwardExtremities.insertsBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID()); err != nil { + return err + } + } + } + + return nil +} + +// WriteEvent into the database. It is not safe to call this function from multiple goroutines, as it would create races +// when generating the sync stream position for this event. Returns the sync stream position for the inserted event. +// Returns an error if there was a problem inserting this event. +func (d *SyncServerDatasource) WriteEvent( + ctx context.Context, + ev *gomatrixserverlib.Event, + addStateEvents []gomatrixserverlib.Event, + addStateEventIDs, removeStateEventIDs []string, + transactionID *api.TransactionID, excludeFromSync bool, +) (pduPosition types.StreamPosition, returnErr error) { + returnErr = common.WithTransaction(d.db, func(txn *sql.Tx) error { + var err error + pos, err := d.events.insertEvent( + ctx, txn, ev, addStateEventIDs, removeStateEventIDs, transactionID, excludeFromSync, + ) + if err != nil { + fmt.Println("d.events.insertEvent:", err) + return err + } + pduPosition = pos + + if err = d.topology.insertEventInTopology(ctx, txn, ev); err != nil { + fmt.Println("d.topology.insertEventInTopology:", err) + return err + } + + if err = d.handleBackwardExtremities(ctx, txn, ev); err != nil { + fmt.Println("d.handleBackwardExtremities:", err) + return err + } + + if len(addStateEvents) == 0 && len(removeStateEventIDs) == 0 { + // Nothing to do, the event may have just been a message event. + fmt.Println("nothing to do") + return nil + } + + return d.updateRoomState(ctx, txn, removeStateEventIDs, addStateEvents, pduPosition) + }) + + return pduPosition, returnErr +} + +func (d *SyncServerDatasource) updateRoomState( + ctx context.Context, txn *sql.Tx, + removedEventIDs []string, + addedEvents []gomatrixserverlib.Event, + pduPosition types.StreamPosition, +) error { + // remove first, then add, as we do not ever delete state, but do replace state which is a remove followed by an add. + for _, eventID := range removedEventIDs { + if err := d.roomstate.deleteRoomStateByEventID(ctx, txn, eventID); err != nil { + return err + } + } + + for _, event := range addedEvents { + if event.StateKey() == nil { + // ignore non state events + continue + } + var membership *string + if event.Type() == "m.room.member" { + value, err := event.Membership() + if err != nil { + return err + } + membership = &value + } + if err := d.roomstate.upsertRoomState(ctx, txn, event, membership, pduPosition); err != nil { + return err + } + } + + return nil +} + +// GetStateEvent returns the Matrix state event of a given type for a given room with a given state key +// If no event could be found, returns nil +// If there was an issue during the retrieval, returns an error +func (d *SyncServerDatasource) GetStateEvent( + ctx context.Context, roomID, evType, stateKey string, +) (*gomatrixserverlib.Event, error) { + return d.roomstate.selectStateEvent(ctx, roomID, evType, stateKey) +} + +// GetStateEventsForRoom fetches the state events for a given room. +// Returns an empty slice if no state events could be found for this room. +// Returns an error if there was an issue with the retrieval. +func (d *SyncServerDatasource) GetStateEventsForRoom( + ctx context.Context, roomID string, stateFilterPart *gomatrixserverlib.StateFilter, +) (stateEvents []gomatrixserverlib.Event, err error) { + err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + stateEvents, err = d.roomstate.selectCurrentState(ctx, txn, roomID, stateFilterPart) + return err + }) + return +} + +// GetEventsInRange retrieves all of the events on a given ordering using the +// given extremities and limit. +func (d *SyncServerDatasource) GetEventsInRange( + ctx context.Context, + from, to *types.PaginationToken, + roomID string, limit int, + backwardOrdering bool, +) (events []types.StreamEvent, err error) { + // If the pagination token's type is types.PaginationTokenTypeTopology, the + // events must be retrieved from the rooms' topology table rather than the + // table contaning the syncapi server's whole stream of events. + if from.Type == types.PaginationTokenTypeTopology { + // Determine the backward and forward limit, i.e. the upper and lower + // limits to the selection in the room's topology, from the direction. + var backwardLimit, forwardLimit types.StreamPosition + if backwardOrdering { + // Backward ordering is antichronological (latest event to oldest + // one). + backwardLimit = to.PDUPosition + forwardLimit = from.PDUPosition + } else { + // Forward ordering is chronological (oldest event to latest one). + backwardLimit = from.PDUPosition + forwardLimit = to.PDUPosition + } + + // Select the event IDs from the defined range. + var eIDs []string + eIDs, err = d.topology.selectEventIDsInRange( + ctx, nil, roomID, backwardLimit, forwardLimit, limit, !backwardOrdering, + ) + if err != nil { + return + } + + // Retrieve the events' contents using their IDs. + events, err = d.events.selectEvents(ctx, nil, eIDs) + return + } + + // If the pagination token's type is types.PaginationTokenTypeStream, the + // events must be retrieved from the table contaning the syncapi server's + // whole stream of events. + + if backwardOrdering { + // When using backward ordering, we want the most recent events first. + if events, err = d.events.selectRecentEvents( + ctx, nil, roomID, to.PDUPosition, from.PDUPosition, limit, false, false, + ); err != nil { + return + } + } else { + // When using forward ordering, we want the least recent events first. + if events, err = d.events.selectEarlyEvents( + ctx, nil, roomID, from.PDUPosition, to.PDUPosition, limit, + ); err != nil { + return + } + } + + return +} + +// SyncPosition returns the latest positions for syncing. +func (d *SyncServerDatasource) SyncPosition(ctx context.Context) (types.PaginationToken, error) { + return d.syncPositionTx(ctx, nil) +} + +// BackwardExtremitiesForRoom returns the event IDs of all of the backward +// extremities we know of for a given room. +func (d *SyncServerDatasource) BackwardExtremitiesForRoom( + ctx context.Context, roomID string, +) (backwardExtremities []string, err error) { + return d.backwardExtremities.selectBackwardExtremitiesForRoom(ctx, nil, roomID) +} + +// MaxTopologicalPosition returns the highest topological position for a given +// room. +func (d *SyncServerDatasource) MaxTopologicalPosition( + ctx context.Context, roomID string, +) (types.StreamPosition, error) { + return d.topology.selectMaxPositionInTopology(ctx, nil, roomID) +} + +// EventsAtTopologicalPosition returns all of the events matching a given +// position in the topology of a given room. +func (d *SyncServerDatasource) EventsAtTopologicalPosition( + ctx context.Context, roomID string, pos types.StreamPosition, +) ([]types.StreamEvent, error) { + eIDs, err := d.topology.selectEventIDsFromPosition(ctx, nil, roomID, pos) + if err != nil { + return nil, err + } + + return d.events.selectEvents(ctx, nil, eIDs) +} + +func (d *SyncServerDatasource) EventPositionInTopology( + ctx context.Context, eventID string, +) (types.StreamPosition, error) { + return d.topology.selectPositionInTopology(ctx, nil, eventID) +} + +// SyncStreamPosition returns the latest position in the sync stream. Returns 0 if there are no events yet. +func (d *SyncServerDatasource) SyncStreamPosition(ctx context.Context) (types.StreamPosition, error) { + return d.syncStreamPositionTx(ctx, nil) +} + +func (d *SyncServerDatasource) syncStreamPositionTx( + ctx context.Context, txn *sql.Tx, +) (types.StreamPosition, error) { + maxID, err := d.events.selectMaxEventID(ctx, txn) + if err != nil { + return 0, err + } + maxAccountDataID, err := d.accountData.selectMaxAccountDataID(ctx, txn) + if err != nil { + return 0, err + } + if maxAccountDataID > maxID { + maxID = maxAccountDataID + } + maxInviteID, err := d.invites.selectMaxInviteID(ctx, txn) + if err != nil { + return 0, err + } + if maxInviteID > maxID { + maxID = maxInviteID + } + return types.StreamPosition(maxID), nil +} + +func (d *SyncServerDatasource) syncPositionTx( + ctx context.Context, txn *sql.Tx, +) (sp types.PaginationToken, err error) { + + maxEventID, err := d.events.selectMaxEventID(ctx, txn) + if err != nil { + return sp, err + } + maxAccountDataID, err := d.accountData.selectMaxAccountDataID(ctx, txn) + if err != nil { + return sp, err + } + if maxAccountDataID > maxEventID { + maxEventID = maxAccountDataID + } + maxInviteID, err := d.invites.selectMaxInviteID(ctx, txn) + if err != nil { + return sp, err + } + if maxInviteID > maxEventID { + maxEventID = maxInviteID + } + sp.PDUPosition = types.StreamPosition(maxEventID) + sp.EDUTypingPosition = types.StreamPosition(d.typingCache.GetLatestSyncPosition()) + return +} + +// addPDUDeltaToResponse adds all PDU deltas to a sync response. +// IDs of all rooms the user joined are returned so EDU deltas can be added for them. +func (d *SyncServerDatasource) addPDUDeltaToResponse( + ctx context.Context, + device authtypes.Device, + fromPos, toPos types.StreamPosition, + numRecentEventsPerRoom int, + wantFullState bool, + res *types.Response, +) (joinedRoomIDs []string, err error) { + txn, err := d.db.BeginTx(ctx, &txReadOnlySnapshot) + if err != nil { + return nil, err + } + var succeeded bool + defer func() { + txerr := common.EndTransaction(txn, &succeeded) + if err == nil && txerr != nil { + err = txerr + } + }() + + stateFilterPart := gomatrixserverlib.DefaultStateFilter() // TODO: use filter provided in request + + // Work out which rooms to return in the response. This is done by getting not only the currently + // joined rooms, but also which rooms have membership transitions for this user between the 2 PDU stream positions. + // This works out what the 'state' key should be for each room as well as which membership block + // to put the room into. + var deltas []stateDelta + if !wantFullState { + deltas, joinedRoomIDs, err = d.getStateDeltas( + ctx, &device, txn, fromPos, toPos, device.UserID, &stateFilterPart, + ) + } else { + deltas, joinedRoomIDs, err = d.getStateDeltasForFullStateSync( + ctx, &device, txn, fromPos, toPos, device.UserID, &stateFilterPart, + ) + } + if err != nil { + return nil, err + } + + for _, delta := range deltas { + err = d.addRoomDeltaToResponse(ctx, &device, txn, fromPos, toPos, delta, numRecentEventsPerRoom, res) + if err != nil { + return nil, err + } + } + + // TODO: This should be done in getStateDeltas + if err = d.addInvitesToResponse(ctx, txn, device.UserID, fromPos, toPos, res); err != nil { + return nil, err + } + + succeeded = true + return joinedRoomIDs, nil +} + +// addTypingDeltaToResponse adds all typing notifications to a sync response +// since the specified position. +func (d *SyncServerDatasource) addTypingDeltaToResponse( + since types.PaginationToken, + joinedRoomIDs []string, + res *types.Response, +) error { + var jr types.JoinResponse + var ok bool + var err error + for _, roomID := range joinedRoomIDs { + if typingUsers, updated := d.typingCache.GetTypingUsersIfUpdatedAfter( + roomID, int64(since.EDUTypingPosition), + ); updated { + ev := gomatrixserverlib.ClientEvent{ + Type: gomatrixserverlib.MTyping, + } + ev.Content, err = json.Marshal(map[string]interface{}{ + "user_ids": typingUsers, + }) + if err != nil { + return err + } + + if jr, ok = res.Rooms.Join[roomID]; !ok { + jr = *types.NewJoinResponse() + } + jr.Ephemeral.Events = append(jr.Ephemeral.Events, ev) + res.Rooms.Join[roomID] = jr + } + } + return nil +} + +// addEDUDeltaToResponse adds updates for EDUs of each type since fromPos if +// the positions of that type are not equal in fromPos and toPos. +func (d *SyncServerDatasource) addEDUDeltaToResponse( + fromPos, toPos types.PaginationToken, + joinedRoomIDs []string, + res *types.Response, +) (err error) { + + if fromPos.EDUTypingPosition != toPos.EDUTypingPosition { + err = d.addTypingDeltaToResponse( + fromPos, joinedRoomIDs, res, + ) + } + + return +} + +// IncrementalSync returns all the data needed in order to create an incremental +// sync response for the given user. Events returned will include any client +// transaction IDs associated with the given device. These transaction IDs come +// from when the device sent the event via an API that included a transaction +// ID. +func (d *SyncServerDatasource) IncrementalSync( + ctx context.Context, + device authtypes.Device, + fromPos, toPos types.PaginationToken, + numRecentEventsPerRoom int, + wantFullState bool, +) (*types.Response, error) { + nextBatchPos := fromPos.WithUpdates(toPos) + res := types.NewResponse(nextBatchPos) + + var joinedRoomIDs []string + var err error + if fromPos.PDUPosition != toPos.PDUPosition || wantFullState { + joinedRoomIDs, err = d.addPDUDeltaToResponse( + ctx, device, fromPos.PDUPosition, toPos.PDUPosition, numRecentEventsPerRoom, wantFullState, res, + ) + } else { + joinedRoomIDs, err = d.roomstate.selectRoomIDsWithMembership( + ctx, nil, device.UserID, gomatrixserverlib.Join, + ) + } + if err != nil { + return nil, err + } + + err = d.addEDUDeltaToResponse( + fromPos, toPos, joinedRoomIDs, res, + ) + if err != nil { + return nil, err + } + + return res, nil +} + +// getResponseWithPDUsForCompleteSync creates a response and adds all PDUs needed +// to it. It returns toPos and joinedRoomIDs for use of adding EDUs. +func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( + ctx context.Context, + userID string, + numRecentEventsPerRoom int, +) ( + res *types.Response, + toPos types.PaginationToken, + joinedRoomIDs []string, + err error, +) { + // This needs to be all done in a transaction as we need to do multiple SELECTs, and we need to have + // a consistent view of the database throughout. This includes extracting the sync position. + // This does have the unfortunate side-effect that all the matrixy logic resides in this function, + // but it's better to not hide the fact that this is being done in a transaction. + txn, err := d.db.BeginTx(ctx, &txReadOnlySnapshot) + if err != nil { + return + } + var succeeded bool + defer func() { + txerr := common.EndTransaction(txn, &succeeded) + if err == nil && txerr != nil { + err = txerr + } + }() + + // Get the current sync position which we will base the sync response on. + toPos, err = d.syncPositionTx(ctx, txn) + if err != nil { + return + } + + res = types.NewResponse(toPos) + + // Extract room state and recent events for all rooms the user is joined to. + joinedRoomIDs, err = d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, gomatrixserverlib.Join) + if err != nil { + return + } + fmt.Println("Joined rooms:", joinedRoomIDs) + + stateFilterPart := gomatrixserverlib.DefaultStateFilter() // TODO: use filter provided in request + + // Build up a /sync response. Add joined rooms. + for _, roomID := range joinedRoomIDs { + fmt.Println("WE'RE ON", roomID) + + var stateEvents []gomatrixserverlib.Event + stateEvents, err = d.roomstate.selectCurrentState(ctx, txn, roomID, &stateFilterPart) + if err != nil { + fmt.Println("d.roomstate.selectCurrentState:", err) + return + } + //fmt.Println("State events:", stateEvents) + // TODO: When filters are added, we may need to call this multiple times to get enough events. + // See: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L316 + var recentStreamEvents []types.StreamEvent + recentStreamEvents, err = d.events.selectRecentEvents( + ctx, txn, roomID, types.StreamPosition(0), toPos.PDUPosition, + numRecentEventsPerRoom, true, true, + ) + if err != nil { + fmt.Println("d.events.selectRecentEvents:", err) + return + } + //fmt.Println("Recent stream events:", recentStreamEvents) + + // Retrieve the backward topology position, i.e. the position of the + // oldest event in the room's topology. + var backwardTopologyPos types.StreamPosition + backwardTopologyPos, err = d.topology.selectPositionInTopology(ctx, txn, recentStreamEvents[0].EventID()) + if err != nil { + fmt.Println("d.topology.selectPositionInTopology:", err) + return nil, types.PaginationToken{}, []string{}, err + } + fmt.Println("Backward topology position:", backwardTopologyPos) + if backwardTopologyPos-1 <= 0 { + backwardTopologyPos = types.StreamPosition(1) + } else { + backwardTopologyPos-- + } + + // We don't include a device here as we don't need to send down + // transaction IDs for complete syncs + recentEvents := d.StreamEventsToEvents(nil, recentStreamEvents) + stateEvents = removeDuplicates(stateEvents, recentEvents) + jr := types.NewJoinResponse() + jr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition( + types.PaginationTokenTypeTopology, backwardTopologyPos, 0, + ).String() + jr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync) + jr.Timeline.Limited = true + jr.State.Events = gomatrixserverlib.ToClientEvents(stateEvents, gomatrixserverlib.FormatSync) + res.Rooms.Join[roomID] = *jr + } + + if err = d.addInvitesToResponse(ctx, txn, userID, 0, toPos.PDUPosition, res); err != nil { + fmt.Println("d.addInvitesToResponse:", err) + return + } + + succeeded = true + return res, toPos, joinedRoomIDs, err +} + +// CompleteSync returns a complete /sync API response for the given user. +func (d *SyncServerDatasource) CompleteSync( + ctx context.Context, userID string, numRecentEventsPerRoom int, +) (*types.Response, error) { + res, toPos, joinedRoomIDs, err := d.getResponseWithPDUsForCompleteSync( + ctx, userID, numRecentEventsPerRoom, + ) + if err != nil { + return nil, err + } + + // Use a zero value SyncPosition for fromPos so all EDU states are added. + err = d.addEDUDeltaToResponse( + types.PaginationToken{}, toPos, joinedRoomIDs, res, + ) + if err != nil { + return nil, err + } + + return res, nil +} + +var txReadOnlySnapshot = sql.TxOptions{ + // Set the isolation level so that we see a snapshot of the database. + // In PostgreSQL repeatable read transactions will see a snapshot taken + // at the first query, and since the transaction is read-only it can't + // run into any serialisation errors. + // https://www.postgresql.org/docs/9.5/static/transaction-iso.html#XACT-REPEATABLE-READ + Isolation: sql.LevelRepeatableRead, + ReadOnly: true, +} + +// GetAccountDataInRange returns all account data for a given user inserted or +// updated between two given positions +// Returns a map following the format data[roomID] = []dataTypes +// If no data is retrieved, returns an empty map +// If there was an issue with the retrieval, returns an error +func (d *SyncServerDatasource) GetAccountDataInRange( + ctx context.Context, userID string, oldPos, newPos types.StreamPosition, + accountDataFilterPart *gomatrixserverlib.EventFilter, +) (map[string][]string, error) { + return d.accountData.selectAccountDataInRange(ctx, userID, oldPos, newPos, accountDataFilterPart) +} + +// UpsertAccountData keeps track of new or updated account data, by saving the type +// of the new/updated data, and the user ID and room ID the data is related to (empty) +// room ID means the data isn't specific to any room) +// If no data with the given type, user ID and room ID exists in the database, +// creates a new row, else update the existing one +// Returns an error if there was an issue with the upsert +func (d *SyncServerDatasource) UpsertAccountData( + ctx context.Context, userID, roomID, dataType string, +) (sp types.StreamPosition, err error) { + txn, err := d.db.BeginTx(ctx, nil) + if err != nil { + return types.StreamPosition(0), err + } + var succeeded bool + defer func() { + txerr := common.EndTransaction(txn, &succeeded) + if err == nil && txerr != nil { + err = txerr + } + }() + sp, err = d.accountData.insertAccountData(ctx, txn, userID, roomID, dataType) + return +} + +// AddInviteEvent stores a new invite event for a user. +// If the invite was successfully stored this returns the stream ID it was stored at. +// Returns an error if there was a problem communicating with the database. +func (d *SyncServerDatasource) AddInviteEvent( + ctx context.Context, inviteEvent gomatrixserverlib.Event, +) (types.StreamPosition, error) { + return d.invites.insertInviteEvent(ctx, inviteEvent) +} + +// RetireInviteEvent removes an old invite event from the database. +// Returns an error if there was a problem communicating with the database. +func (d *SyncServerDatasource) RetireInviteEvent( + ctx context.Context, inviteEventID string, +) error { + // TODO: Record that invite has been retired in a stream so that we can + // notify the user in an incremental sync. + err := d.invites.deleteInviteEvent(ctx, inviteEventID) + return err +} + +func (d *SyncServerDatasource) SetTypingTimeoutCallback(fn cache.TimeoutCallbackFn) { + d.typingCache.SetTimeoutCallback(fn) +} + +// AddTypingUser adds a typing user to the typing cache. +// Returns the newly calculated sync position for typing notifications. +func (d *SyncServerDatasource) AddTypingUser( + userID, roomID string, expireTime *time.Time, +) types.StreamPosition { + return types.StreamPosition(d.typingCache.AddTypingUser(userID, roomID, expireTime)) +} + +// RemoveTypingUser removes a typing user from the typing cache. +// Returns the newly calculated sync position for typing notifications. +func (d *SyncServerDatasource) RemoveTypingUser( + userID, roomID string, +) types.StreamPosition { + return types.StreamPosition(d.typingCache.RemoveUser(userID, roomID)) +} + +func (d *SyncServerDatasource) addInvitesToResponse( + ctx context.Context, txn *sql.Tx, + userID string, + fromPos, toPos types.StreamPosition, + res *types.Response, +) error { + invites, err := d.invites.selectInviteEventsInRange( + ctx, txn, userID, fromPos, toPos, + ) + if err != nil { + return err + } + for roomID, inviteEvent := range invites { + ir := types.NewInviteResponse() + ir.InviteState.Events = gomatrixserverlib.ToClientEvents( + []gomatrixserverlib.Event{inviteEvent}, gomatrixserverlib.FormatSync, + ) + // TODO: add the invite state from the invite event. + res.Rooms.Invite[roomID] = *ir + } + return nil +} + +// Retrieve the backward topology position, i.e. the position of the +// oldest event in the room's topology. +func (d *SyncServerDatasource) getBackwardTopologyPos( + ctx context.Context, txn *sql.Tx, + events []types.StreamEvent, +) (pos types.StreamPosition) { + if len(events) > 0 { + pos, _ = d.topology.selectPositionInTopology(ctx, txn, events[0].EventID()) + } + if pos-1 <= 0 { + pos = types.StreamPosition(1) + } else { + pos = pos - 1 + } + return +} + +// addRoomDeltaToResponse adds a room state delta to a sync response +func (d *SyncServerDatasource) addRoomDeltaToResponse( + ctx context.Context, + device *authtypes.Device, + txn *sql.Tx, + fromPos, toPos types.StreamPosition, + delta stateDelta, + numRecentEventsPerRoom int, + res *types.Response, +) error { + endPos := toPos + if delta.membershipPos > 0 && delta.membership == gomatrixserverlib.Leave { + // make sure we don't leak recent events after the leave event. + // TODO: History visibility makes this somewhat complex to handle correctly. For example: + // TODO: This doesn't work for join -> leave in a single /sync request (see events prior to join). + // TODO: This will fail on join -> leave -> sensitive msg -> join -> leave + // in a single /sync request + // This is all "okay" assuming history_visibility == "shared" which it is by default. + endPos = delta.membershipPos + } + recentStreamEvents, err := d.events.selectRecentEvents( + ctx, txn, delta.roomID, types.StreamPosition(fromPos), types.StreamPosition(endPos), + numRecentEventsPerRoom, true, true, + ) + if err != nil { + return err + } + recentEvents := d.StreamEventsToEvents(device, recentStreamEvents) + delta.stateEvents = removeDuplicates(delta.stateEvents, recentEvents) // roll back + backwardTopologyPos := d.getBackwardTopologyPos(ctx, txn, recentStreamEvents) + + switch delta.membership { + case gomatrixserverlib.Join: + jr := types.NewJoinResponse() + + jr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition( + types.PaginationTokenTypeTopology, backwardTopologyPos, 0, + ).String() + jr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync) + jr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true + jr.State.Events = gomatrixserverlib.ToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) + res.Rooms.Join[delta.roomID] = *jr + case gomatrixserverlib.Leave: + fallthrough // transitions to leave are the same as ban + case gomatrixserverlib.Ban: + // TODO: recentEvents may contain events that this user is not allowed to see because they are + // no longer in the room. + lr := types.NewLeaveResponse() + lr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition( + types.PaginationTokenTypeTopology, backwardTopologyPos, 0, + ).String() + lr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync) + lr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true + lr.State.Events = gomatrixserverlib.ToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) + res.Rooms.Leave[delta.roomID] = *lr + } + + return nil +} + +// fetchStateEvents converts the set of event IDs into a set of events. It will fetch any which are missing from the database. +// Returns a map of room ID to list of events. +func (d *SyncServerDatasource) fetchStateEvents( + ctx context.Context, txn *sql.Tx, + roomIDToEventIDSet map[string]map[string]bool, + eventIDToEvent map[string]types.StreamEvent, +) (map[string][]types.StreamEvent, error) { + stateBetween := make(map[string][]types.StreamEvent) + missingEvents := make(map[string][]string) + for roomID, ids := range roomIDToEventIDSet { + events := stateBetween[roomID] + for id, need := range ids { + if !need { + continue // deleted state + } + e, ok := eventIDToEvent[id] + if ok { + events = append(events, e) + } else { + m := missingEvents[roomID] + m = append(m, id) + missingEvents[roomID] = m + } + } + stateBetween[roomID] = events + } + + if len(missingEvents) > 0 { + // This happens when add_state_ids has an event ID which is not in the provided range. + // We need to explicitly fetch them. + allMissingEventIDs := []string{} + for _, missingEvIDs := range missingEvents { + allMissingEventIDs = append(allMissingEventIDs, missingEvIDs...) + } + evs, err := d.fetchMissingStateEvents(ctx, txn, allMissingEventIDs) + if err != nil { + return nil, err + } + // we know we got them all otherwise an error would've been returned, so just loop the events + for _, ev := range evs { + roomID := ev.RoomID() + stateBetween[roomID] = append(stateBetween[roomID], ev) + } + } + return stateBetween, nil +} + +func (d *SyncServerDatasource) fetchMissingStateEvents( + ctx context.Context, txn *sql.Tx, eventIDs []string, +) ([]types.StreamEvent, error) { + // Fetch from the events table first so we pick up the stream ID for the + // event. + events, err := d.events.selectEvents(ctx, txn, eventIDs) + if err != nil { + return nil, err + } + + have := map[string]bool{} + for _, event := range events { + have[event.EventID()] = true + } + var missing []string + for _, eventID := range eventIDs { + if !have[eventID] { + missing = append(missing, eventID) + } + } + if len(missing) == 0 { + return events, nil + } + + // If they are missing from the events table then they should be state + // events that we received from outside the main event stream. + // These should be in the room state table. + stateEvents, err := d.roomstate.selectEventsWithEventIDs(ctx, txn, missing) + + if err != nil { + return nil, err + } + if len(stateEvents) != len(missing) { + return nil, fmt.Errorf("failed to map all event IDs to events: (got %d, wanted %d)", len(stateEvents), len(missing)) + } + events = append(events, stateEvents...) + return events, nil +} + +// getStateDeltas returns the state deltas between fromPos and toPos, +// exclusive of oldPos, inclusive of newPos, for the rooms in which +// the user has new membership events. +// A list of joined room IDs is also returned in case the caller needs it. +func (d *SyncServerDatasource) getStateDeltas( + ctx context.Context, device *authtypes.Device, txn *sql.Tx, + fromPos, toPos types.StreamPosition, userID string, + stateFilterPart *gomatrixserverlib.StateFilter, +) ([]stateDelta, []string, error) { + // Implement membership change algorithm: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L821 + // - Get membership list changes for this user in this sync response + // - For each room which has membership list changes: + // * Check if the room is 'newly joined' (insufficient to just check for a join event because we allow dupe joins TODO). + // If it is, then we need to send the full room state down (and 'limited' is always true). + // * Check if user is still CURRENTLY invited to the room. If so, add room to 'invited' block. + // * Check if the user is CURRENTLY (TODO) left/banned. If so, add room to 'archived' block. + // - Get all CURRENTLY joined rooms, and add them to 'joined' block. + var deltas []stateDelta + + // get all the state events ever between these two positions + stateNeeded, eventMap, err := d.events.selectStateInRange(ctx, txn, fromPos, toPos, stateFilterPart) + if err != nil { + return nil, nil, err + } + state, err := d.fetchStateEvents(ctx, txn, stateNeeded, eventMap) + if err != nil { + return nil, nil, err + } + + for roomID, stateStreamEvents := range state { + for _, ev := range stateStreamEvents { + // TODO: Currently this will incorrectly add rooms which were ALREADY joined but they sent another no-op join event. + // We should be checking if the user was already joined at fromPos and not proceed if so. As a result of this, + // dupe join events will result in the entire room state coming down to the client again. This is added in + // the 'state' part of the response though, so is transparent modulo bandwidth concerns as it is not added to + // the timeline. + if membership := getMembershipFromEvent(&ev.Event, userID); membership != "" { + if membership == gomatrixserverlib.Join { + // send full room state down instead of a delta + var s []types.StreamEvent + s, err = d.currentStateStreamEventsForRoom(ctx, txn, roomID, stateFilterPart) + if err != nil { + return nil, nil, err + } + state[roomID] = s + continue // we'll add this room in when we do joined rooms + } + + deltas = append(deltas, stateDelta{ + membership: membership, + membershipPos: ev.StreamPosition, + stateEvents: d.StreamEventsToEvents(device, stateStreamEvents), + roomID: roomID, + }) + break + } + } + } + + // Add in currently joined rooms + joinedRoomIDs, err := d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, gomatrixserverlib.Join) + if err != nil { + return nil, nil, err + } + for _, joinedRoomID := range joinedRoomIDs { + deltas = append(deltas, stateDelta{ + membership: gomatrixserverlib.Join, + stateEvents: d.StreamEventsToEvents(device, state[joinedRoomID]), + roomID: joinedRoomID, + }) + } + + return deltas, joinedRoomIDs, nil +} + +// getStateDeltasForFullStateSync is a variant of getStateDeltas used for /sync +// requests with full_state=true. +// Fetches full state for all joined rooms and uses selectStateInRange to get +// updates for other rooms. +func (d *SyncServerDatasource) getStateDeltasForFullStateSync( + ctx context.Context, device *authtypes.Device, txn *sql.Tx, + fromPos, toPos types.StreamPosition, userID string, + stateFilterPart *gomatrixserverlib.StateFilter, +) ([]stateDelta, []string, error) { + joinedRoomIDs, err := d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, gomatrixserverlib.Join) + if err != nil { + return nil, nil, err + } + + // Use a reasonable initial capacity + deltas := make([]stateDelta, 0, len(joinedRoomIDs)) + + // Add full states for all joined rooms + for _, joinedRoomID := range joinedRoomIDs { + s, stateErr := d.currentStateStreamEventsForRoom(ctx, txn, joinedRoomID, stateFilterPart) + if stateErr != nil { + return nil, nil, stateErr + } + deltas = append(deltas, stateDelta{ + membership: gomatrixserverlib.Join, + stateEvents: d.StreamEventsToEvents(device, s), + roomID: joinedRoomID, + }) + } + + // Get all the state events ever between these two positions + stateNeeded, eventMap, err := d.events.selectStateInRange(ctx, txn, fromPos, toPos, stateFilterPart) + if err != nil { + return nil, nil, err + } + state, err := d.fetchStateEvents(ctx, txn, stateNeeded, eventMap) + if err != nil { + return nil, nil, err + } + + for roomID, stateStreamEvents := range state { + for _, ev := range stateStreamEvents { + if membership := getMembershipFromEvent(&ev.Event, userID); membership != "" { + if membership != gomatrixserverlib.Join { // We've already added full state for all joined rooms above. + deltas = append(deltas, stateDelta{ + membership: membership, + membershipPos: ev.StreamPosition, + stateEvents: d.StreamEventsToEvents(device, stateStreamEvents), + roomID: roomID, + }) + } + + break + } + } + } + + return deltas, joinedRoomIDs, nil +} + +func (d *SyncServerDatasource) currentStateStreamEventsForRoom( + ctx context.Context, txn *sql.Tx, roomID string, + stateFilterPart *gomatrixserverlib.StateFilter, +) ([]types.StreamEvent, error) { + allState, err := d.roomstate.selectCurrentState(ctx, txn, roomID, stateFilterPart) + if err != nil { + return nil, err + } + s := make([]types.StreamEvent, len(allState)) + for i := 0; i < len(s); i++ { + s[i] = types.StreamEvent{Event: allState[i], StreamPosition: 0} + } + return s, nil +} + +// StreamEventsToEvents converts streamEvent to Event. If device is non-nil and +// matches the streamevent.transactionID device then the transaction ID gets +// added to the unsigned section of the output event. +func (d *SyncServerDatasource) StreamEventsToEvents(device *authtypes.Device, in []types.StreamEvent) []gomatrixserverlib.Event { + out := make([]gomatrixserverlib.Event, len(in)) + for i := 0; i < len(in); i++ { + out[i] = in[i].Event + if device != nil && in[i].TransactionID != nil { + if device.UserID == in[i].Sender() && device.SessionID == in[i].TransactionID.SessionID { + err := out[i].SetUnsignedField( + "transaction_id", in[i].TransactionID.TransactionID, + ) + if err != nil { + logrus.WithFields(logrus.Fields{ + "event_id": out[i].EventID(), + }).WithError(err).Warnf("Failed to add transaction ID to event") + } + } + } + } + return out +} + +// There may be some overlap where events in stateEvents are already in recentEvents, so filter +// them out so we don't include them twice in the /sync response. They should be in recentEvents +// only, so clients get to the correct state once they have rolled forward. +func removeDuplicates(stateEvents, recentEvents []gomatrixserverlib.Event) []gomatrixserverlib.Event { + for _, recentEv := range recentEvents { + if recentEv.StateKey() == nil { + continue // not a state event + } + // TODO: This is a linear scan over all the current state events in this room. This will + // be slow for big rooms. We should instead sort the state events by event ID (ORDER BY) + // then do a binary search to find matching events, similar to what roomserver does. + for j := 0; j < len(stateEvents); j++ { + if stateEvents[j].EventID() == recentEv.EventID() { + // overwrite the element to remove with the last element then pop the last element. + // This is orders of magnitude faster than re-slicing, but doesn't preserve ordering + // (we don't care about the order of stateEvents) + stateEvents[j] = stateEvents[len(stateEvents)-1] + stateEvents = stateEvents[:len(stateEvents)-1] + break // there shouldn't be multiple events with the same event ID + } + } + } + return stateEvents +} + +// getMembershipFromEvent returns the value of content.membership iff the event is a state event +// with type 'm.room.member' and state_key of userID. Otherwise, an empty string is returned. +func getMembershipFromEvent(ev *gomatrixserverlib.Event, userID string) string { + if ev.Type() == "m.room.member" && ev.StateKeyEquals(userID) { + membership, err := ev.Membership() + if err != nil { + return "" + } + return membership + } + return "" +} diff --git a/syncapi/storage/storage.go b/syncapi/storage/storage.go index e63928440..c87024b29 100644 --- a/syncapi/storage/storage.go +++ b/syncapi/storage/storage.go @@ -23,6 +23,7 @@ import ( "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/storage/postgres" + "github.com/matrix-org/dendrite/syncapi/storage/sqlite3" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/dendrite/typingserver/cache" "github.com/matrix-org/gomatrixserverlib" @@ -63,6 +64,8 @@ func NewSyncServerDatasource(dataSourceName string) (Database, error) { switch uri.Scheme { case "postgres": return postgres.NewSyncServerDatasource(dataSourceName) + case "file": + return sqlite3.NewSyncServerDatasource(dataSourceName) default: return postgres.NewSyncServerDatasource(dataSourceName) } diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index 06a8d6d8a..22bd239fc 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -32,12 +32,12 @@ import ( // RequestPool manages HTTP long-poll connections for /sync type RequestPool struct { db storage.Database - accountDB *accounts.Database + accountDB accounts.Database notifier *Notifier } // NewRequestPool makes a new RequestPool -func NewRequestPool(db storage.Database, n *Notifier, adb *accounts.Database) *RequestPool { +func NewRequestPool(db storage.Database, n *Notifier, adb accounts.Database) *RequestPool { return &RequestPool{db, adb, n} } diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go index ecf532ca2..1535d2b13 100644 --- a/syncapi/syncapi.go +++ b/syncapi/syncapi.go @@ -36,8 +36,8 @@ import ( // component. func SetupSyncAPIComponent( base *basecomponent.BaseDendrite, - deviceDB *devices.Database, - accountsDB *accounts.Database, + deviceDB devices.Database, + accountsDB accounts.Database, queryAPI api.RoomserverQueryAPI, federation *gomatrixserverlib.FederationClient, cfg *config.Dendrite, From 409fec2a48f454600936eb674da0611566c5a286 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Fri, 14 Feb 2020 13:07:14 +0000 Subject: [PATCH 16/86] Add whitelist/blacklist maintenance complainer to buildkite annotations (#870) --- show-expected-fail-tests.sh | 20 ++++++++++++++++++-- sytest-whitelist | 1 - 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/show-expected-fail-tests.sh b/show-expected-fail-tests.sh index d3872ad59..9cd51b007 100755 --- a/show-expected-fail-tests.sh +++ b/show-expected-fail-tests.sh @@ -76,14 +76,30 @@ while read -r test_name; do fi done <<< "${passed_but_expected_fail}" +# TODO: Check that the same test doesn't exist in both the whitelist and blacklist +# TODO: Check that the same test doesn't appear twice in the whitelist|blacklist + +# Trim test output strings +tests_to_add=$(echo -e $tests_to_add | xargs) +already_in_whitelist=$(echo -e $already_in_whitelist | xargs) + +# Format output with markdown for buildkite annotation rendering purposes +if [ -n "${tests_to_add}" ] && [ -n "${already_in_whitelist}" ]; then + echo "### 📜 SyTest Whitelist Maintenance" +fi + if [ -n "${tests_to_add}" ]; then - echo "ERROR: The following passed tests are not present in $2. Please append them to the file:" + echo "**ERROR**: The following tests passed but are not present in \`$2\`. Please append them to the file:" + echo "\`\`\`" echo -e "${tests_to_add}" + echo "\`\`\`" fi if [ -n "${already_in_whitelist}" ]; then - echo "WARN: Tests in the whitelist still marked as expected fail:" + echo "**WARN**: Tests in the whitelist still marked as **expected fail**:" + echo "\`\`\`" echo -e "${already_in_whitelist}" + echo "\`\`\`" fi exit ${fail_build} diff --git a/sytest-whitelist b/sytest-whitelist index 47fd58286..eb91b3f0c 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -183,7 +183,6 @@ GET /directory/room/:room_alias yields room ID PUT /directory/room/:room_alias creates alias Room aliases can contain Unicode Creators can delete alias -Alias creators can delete canonical alias with no ops Regular users cannot create room aliases within the AS namespace Deleting a non-existent alias should return a 404 Users can't delete other's aliases From 3dabf4d4ed52459a4661db32c983018c2c73a431 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 14 Feb 2020 14:12:33 +0000 Subject: [PATCH 17/86] More SQLite (#871) * SQLite support for appservice * SQLite support for mediaapi * Copyright notices * SQLite for public rooms API (although with some slight differences in behaviour) * Lazy match aliases, add TODOs --- appservice/consumers/roomserver.go | 4 +- .../{ => postgres}/appservice_events_table.go | 3 +- appservice/storage/postgres/storage.go | 111 ++++++++ .../{ => postgres}/txn_id_counter_table.go | 3 +- .../sqlite3/appservice_events_table.go | 249 ++++++++++++++++++ appservice/storage/sqlite3/storage.go | 111 ++++++++ .../storage/sqlite3/txn_id_counter_table.go | 60 +++++ appservice/storage/storage.go | 106 ++------ appservice/workers/transaction_scheduler.go | 6 +- .../storage/sqlite3/media_repository_table.go | 115 ++++++++ mediaapi/storage/sqlite3/prepare.go | 38 +++ mediaapi/storage/sqlite3/sql.go | 36 +++ mediaapi/storage/sqlite3/storage.go | 106 ++++++++ mediaapi/storage/sqlite3/thumbnail_table.go | 162 ++++++++++++ mediaapi/storage/storage.go | 3 + .../storage/sqlite3/public_rooms_table.go | 29 +- publicroomsapi/storage/storage.go | 3 + 17 files changed, 1034 insertions(+), 111 deletions(-) rename appservice/storage/{ => postgres}/appservice_events_table.go (99%) create mode 100644 appservice/storage/postgres/storage.go rename appservice/storage/{ => postgres}/txn_id_counter_table.go (94%) create mode 100644 appservice/storage/sqlite3/appservice_events_table.go create mode 100644 appservice/storage/sqlite3/storage.go create mode 100644 appservice/storage/sqlite3/txn_id_counter_table.go create mode 100644 mediaapi/storage/sqlite3/media_repository_table.go create mode 100644 mediaapi/storage/sqlite3/prepare.go create mode 100644 mediaapi/storage/sqlite3/sql.go create mode 100644 mediaapi/storage/sqlite3/storage.go create mode 100644 mediaapi/storage/sqlite3/thumbnail_table.go diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go index b9a567954..6d3ea808f 100644 --- a/appservice/consumers/roomserver.go +++ b/appservice/consumers/roomserver.go @@ -34,7 +34,7 @@ import ( type OutputRoomEventConsumer struct { roomServerConsumer *common.ContinualConsumer db accounts.Database - asDB *storage.Database + asDB storage.Database query api.RoomserverQueryAPI alias api.RoomserverAliasAPI serverName string @@ -47,7 +47,7 @@ func NewOutputRoomEventConsumer( cfg *config.Dendrite, kafkaConsumer sarama.Consumer, store accounts.Database, - appserviceDB *storage.Database, + appserviceDB storage.Database, queryAPI api.RoomserverQueryAPI, aliasAPI api.RoomserverAliasAPI, workerStates []types.ApplicationServiceWorkerState, diff --git a/appservice/storage/appservice_events_table.go b/appservice/storage/postgres/appservice_events_table.go similarity index 99% rename from appservice/storage/appservice_events_table.go rename to appservice/storage/postgres/appservice_events_table.go index 285bbf483..d72faeeae 100644 --- a/appservice/storage/appservice_events_table.go +++ b/appservice/storage/postgres/appservice_events_table.go @@ -1,4 +1,5 @@ // Copyright 2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package storage +package postgres import ( "context" diff --git a/appservice/storage/postgres/storage.go b/appservice/storage/postgres/storage.go new file mode 100644 index 000000000..c47564689 --- /dev/null +++ b/appservice/storage/postgres/storage.go @@ -0,0 +1,111 @@ +// Copyright 2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package postgres + +import ( + "context" + "database/sql" + + // Import postgres database driver + _ "github.com/lib/pq" + "github.com/matrix-org/gomatrixserverlib" +) + +// Database stores events intended to be later sent to application services +type Database struct { + events eventsStatements + txnID txnStatements + db *sql.DB +} + +// NewDatabase opens a new database +func NewDatabase(dataSourceName string) (*Database, error) { + var result Database + var err error + if result.db, err = sql.Open("postgres", dataSourceName); err != nil { + return nil, err + } + if err = result.prepare(); err != nil { + return nil, err + } + return &result, nil +} + +func (d *Database) prepare() error { + if err := d.events.prepare(d.db); err != nil { + return err + } + + return d.txnID.prepare(d.db) +} + +// StoreEvent takes in a gomatrixserverlib.Event and stores it in the database +// for a transaction worker to pull and later send to an application service. +func (d *Database) StoreEvent( + ctx context.Context, + appServiceID string, + event *gomatrixserverlib.Event, +) error { + return d.events.insertEvent(ctx, appServiceID, event) +} + +// GetEventsWithAppServiceID returns a slice of events and their IDs intended to +// be sent to an application service given its ID. +func (d *Database) GetEventsWithAppServiceID( + ctx context.Context, + appServiceID string, + limit int, +) (int, int, []gomatrixserverlib.Event, bool, error) { + return d.events.selectEventsByApplicationServiceID(ctx, appServiceID, limit) +} + +// CountEventsWithAppServiceID returns the number of events destined for an +// application service given its ID. +func (d *Database) CountEventsWithAppServiceID( + ctx context.Context, + appServiceID string, +) (int, error) { + return d.events.countEventsByApplicationServiceID(ctx, appServiceID) +} + +// UpdateTxnIDForEvents takes in an application service ID and a +// and stores them in the DB, unless the pair already exists, in +// which case it updates them. +func (d *Database) UpdateTxnIDForEvents( + ctx context.Context, + appserviceID string, + maxID, txnID int, +) error { + return d.events.updateTxnIDForEvents(ctx, appserviceID, maxID, txnID) +} + +// RemoveEventsBeforeAndIncludingID removes all events from the database that +// are less than or equal to a given maximum ID. IDs here are implemented as a +// serial, thus this should always delete events in chronological order. +func (d *Database) RemoveEventsBeforeAndIncludingID( + ctx context.Context, + appserviceID string, + eventTableID int, +) error { + return d.events.deleteEventsBeforeAndIncludingID(ctx, appserviceID, eventTableID) +} + +// GetLatestTxnID returns the latest available transaction id +func (d *Database) GetLatestTxnID( + ctx context.Context, +) (int, error) { + return d.txnID.selectTxnID(ctx) +} diff --git a/appservice/storage/txn_id_counter_table.go b/appservice/storage/postgres/txn_id_counter_table.go similarity index 94% rename from appservice/storage/txn_id_counter_table.go rename to appservice/storage/postgres/txn_id_counter_table.go index 7b0fa3786..a96a0e360 100644 --- a/appservice/storage/txn_id_counter_table.go +++ b/appservice/storage/postgres/txn_id_counter_table.go @@ -1,4 +1,5 @@ // Copyright 2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package storage +package postgres import ( "context" diff --git a/appservice/storage/sqlite3/appservice_events_table.go b/appservice/storage/sqlite3/appservice_events_table.go new file mode 100644 index 000000000..846f09f7f --- /dev/null +++ b/appservice/storage/sqlite3/appservice_events_table.go @@ -0,0 +1,249 @@ +// Copyright 2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + "encoding/json" + "time" + + "github.com/matrix-org/gomatrixserverlib" + log "github.com/sirupsen/logrus" +) + +const appserviceEventsSchema = ` +-- Stores events to be sent to application services +CREATE TABLE IF NOT EXISTS appservice_events ( + -- An auto-incrementing id unique to each event in the table + id INTEGER PRIMARY KEY AUTOINCREMENT, + -- The ID of the application service the event will be sent to + as_id TEXT NOT NULL, + -- JSON representation of the event + event_json TEXT NOT NULL, + -- The ID of the transaction that this event is a part of + txn_id INTEGER NOT NULL +); + +CREATE INDEX IF NOT EXISTS appservice_events_as_id ON appservice_events(as_id); +` + +const selectEventsByApplicationServiceIDSQL = "" + + "SELECT id, event_json, txn_id " + + "FROM appservice_events WHERE as_id = $1 ORDER BY txn_id DESC, id ASC" + +const countEventsByApplicationServiceIDSQL = "" + + "SELECT COUNT(id) FROM appservice_events WHERE as_id = $1" + +const insertEventSQL = "" + + "INSERT INTO appservice_events(as_id, event_json, txn_id) " + + "VALUES ($1, $2, $3)" + +const updateTxnIDForEventsSQL = "" + + "UPDATE appservice_events SET txn_id = $1 WHERE as_id = $2 AND id <= $3" + +const deleteEventsBeforeAndIncludingIDSQL = "" + + "DELETE FROM appservice_events WHERE as_id = $1 AND id <= $2" + +const ( + // A transaction ID number that no transaction should ever have. Used for + // checking again the default value. + invalidTxnID = -2 +) + +type eventsStatements struct { + selectEventsByApplicationServiceIDStmt *sql.Stmt + countEventsByApplicationServiceIDStmt *sql.Stmt + insertEventStmt *sql.Stmt + updateTxnIDForEventsStmt *sql.Stmt + deleteEventsBeforeAndIncludingIDStmt *sql.Stmt +} + +func (s *eventsStatements) prepare(db *sql.DB) (err error) { + _, err = db.Exec(appserviceEventsSchema) + if err != nil { + return + } + + if s.selectEventsByApplicationServiceIDStmt, err = db.Prepare(selectEventsByApplicationServiceIDSQL); err != nil { + return + } + if s.countEventsByApplicationServiceIDStmt, err = db.Prepare(countEventsByApplicationServiceIDSQL); err != nil { + return + } + if s.insertEventStmt, err = db.Prepare(insertEventSQL); err != nil { + return + } + if s.updateTxnIDForEventsStmt, err = db.Prepare(updateTxnIDForEventsSQL); err != nil { + return + } + if s.deleteEventsBeforeAndIncludingIDStmt, err = db.Prepare(deleteEventsBeforeAndIncludingIDSQL); err != nil { + return + } + + return +} + +// selectEventsByApplicationServiceID takes in an application service ID and +// returns a slice of events that need to be sent to that application service, +// as well as an int later used to remove these same events from the database +// once successfully sent to an application service. +func (s *eventsStatements) selectEventsByApplicationServiceID( + ctx context.Context, + applicationServiceID string, + limit int, +) ( + txnID, maxID int, + events []gomatrixserverlib.Event, + eventsRemaining bool, + err error, +) { + // Retrieve events from the database. Unsuccessfully sent events first + eventRows, err := s.selectEventsByApplicationServiceIDStmt.QueryContext(ctx, applicationServiceID) + if err != nil { + return + } + defer func() { + err = eventRows.Close() + if err != nil { + log.WithFields(log.Fields{ + "appservice": applicationServiceID, + }).WithError(err).Fatalf("appservice unable to select new events to send") + } + }() + events, maxID, txnID, eventsRemaining, err = retrieveEvents(eventRows, limit) + if err != nil { + return + } + + return +} + +func retrieveEvents(eventRows *sql.Rows, limit int) (events []gomatrixserverlib.Event, maxID, txnID int, eventsRemaining bool, err error) { + // Get current time for use in calculating event age + nowMilli := time.Now().UnixNano() / int64(time.Millisecond) + + // Iterate through each row and store event contents + // If txn_id changes dramatically, we've switched from collecting old events to + // new ones. Send back those events first. + lastTxnID := invalidTxnID + for eventsProcessed := 0; eventRows.Next(); { + var event gomatrixserverlib.Event + var eventJSON []byte + var id int + err = eventRows.Scan( + &id, + &eventJSON, + &txnID, + ) + if err != nil { + return nil, 0, 0, false, err + } + + // Unmarshal eventJSON + if err = json.Unmarshal(eventJSON, &event); err != nil { + return nil, 0, 0, false, err + } + + // If txnID has changed on this event from the previous event, then we've + // reached the end of a transaction's events. Return only those events. + if lastTxnID > invalidTxnID && lastTxnID != txnID { + return events, maxID, lastTxnID, true, nil + } + lastTxnID = txnID + + // Limit events that aren't part of an old transaction + if txnID == -1 { + // Return if we've hit the limit + if eventsProcessed++; eventsProcessed > limit { + return events, maxID, lastTxnID, true, nil + } + } + + if id > maxID { + maxID = id + } + + // Portion of the event that is unsigned due to rapid change + // TODO: Consider removing age as not many app services use it + if err = event.SetUnsignedField("age", nowMilli-int64(event.OriginServerTS())); err != nil { + return nil, 0, 0, false, err + } + + events = append(events, event) + } + + return +} + +// countEventsByApplicationServiceID inserts an event mapped to its corresponding application service +// IDs into the db. +func (s *eventsStatements) countEventsByApplicationServiceID( + ctx context.Context, + appServiceID string, +) (int, error) { + var count int + err := s.countEventsByApplicationServiceIDStmt.QueryRowContext(ctx, appServiceID).Scan(&count) + if err != nil && err != sql.ErrNoRows { + return 0, err + } + + return count, nil +} + +// insertEvent inserts an event mapped to its corresponding application service +// IDs into the db. +func (s *eventsStatements) insertEvent( + ctx context.Context, + appServiceID string, + event *gomatrixserverlib.Event, +) (err error) { + // Convert event to JSON before inserting + eventJSON, err := json.Marshal(event) + if err != nil { + return err + } + + _, err = s.insertEventStmt.ExecContext( + ctx, + appServiceID, + eventJSON, + -1, // No transaction ID yet + ) + return +} + +// updateTxnIDForEvents sets the transactionID for a collection of events. Done +// before sending them to an AppService. Referenced before sending to make sure +// we aren't constructing multiple transactions with the same events. +func (s *eventsStatements) updateTxnIDForEvents( + ctx context.Context, + appserviceID string, + maxID, txnID int, +) (err error) { + _, err = s.updateTxnIDForEventsStmt.ExecContext(ctx, txnID, appserviceID, maxID) + return +} + +// deleteEventsBeforeAndIncludingID removes events matching given IDs from the database. +func (s *eventsStatements) deleteEventsBeforeAndIncludingID( + ctx context.Context, + appserviceID string, + eventTableID int, +) (err error) { + _, err = s.deleteEventsBeforeAndIncludingIDStmt.ExecContext(ctx, appserviceID, eventTableID) + return +} diff --git a/appservice/storage/sqlite3/storage.go b/appservice/storage/sqlite3/storage.go new file mode 100644 index 000000000..56ab55f2f --- /dev/null +++ b/appservice/storage/sqlite3/storage.go @@ -0,0 +1,111 @@ +// Copyright 2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + + // Import SQLite database driver + "github.com/matrix-org/gomatrixserverlib" + _ "github.com/mattn/go-sqlite3" +) + +// Database stores events intended to be later sent to application services +type Database struct { + events eventsStatements + txnID txnStatements + db *sql.DB +} + +// NewDatabase opens a new database +func NewDatabase(dataSourceName string) (*Database, error) { + var result Database + var err error + if result.db, err = sql.Open("sqlite3", dataSourceName); err != nil { + return nil, err + } + if err = result.prepare(); err != nil { + return nil, err + } + return &result, nil +} + +func (d *Database) prepare() error { + if err := d.events.prepare(d.db); err != nil { + return err + } + + return d.txnID.prepare(d.db) +} + +// StoreEvent takes in a gomatrixserverlib.Event and stores it in the database +// for a transaction worker to pull and later send to an application service. +func (d *Database) StoreEvent( + ctx context.Context, + appServiceID string, + event *gomatrixserverlib.Event, +) error { + return d.events.insertEvent(ctx, appServiceID, event) +} + +// GetEventsWithAppServiceID returns a slice of events and their IDs intended to +// be sent to an application service given its ID. +func (d *Database) GetEventsWithAppServiceID( + ctx context.Context, + appServiceID string, + limit int, +) (int, int, []gomatrixserverlib.Event, bool, error) { + return d.events.selectEventsByApplicationServiceID(ctx, appServiceID, limit) +} + +// CountEventsWithAppServiceID returns the number of events destined for an +// application service given its ID. +func (d *Database) CountEventsWithAppServiceID( + ctx context.Context, + appServiceID string, +) (int, error) { + return d.events.countEventsByApplicationServiceID(ctx, appServiceID) +} + +// UpdateTxnIDForEvents takes in an application service ID and a +// and stores them in the DB, unless the pair already exists, in +// which case it updates them. +func (d *Database) UpdateTxnIDForEvents( + ctx context.Context, + appserviceID string, + maxID, txnID int, +) error { + return d.events.updateTxnIDForEvents(ctx, appserviceID, maxID, txnID) +} + +// RemoveEventsBeforeAndIncludingID removes all events from the database that +// are less than or equal to a given maximum ID. IDs here are implemented as a +// serial, thus this should always delete events in chronological order. +func (d *Database) RemoveEventsBeforeAndIncludingID( + ctx context.Context, + appserviceID string, + eventTableID int, +) error { + return d.events.deleteEventsBeforeAndIncludingID(ctx, appserviceID, eventTableID) +} + +// GetLatestTxnID returns the latest available transaction id +func (d *Database) GetLatestTxnID( + ctx context.Context, +) (int, error) { + return d.txnID.selectTxnID(ctx) +} diff --git a/appservice/storage/sqlite3/txn_id_counter_table.go b/appservice/storage/sqlite3/txn_id_counter_table.go new file mode 100644 index 000000000..b1ee60766 --- /dev/null +++ b/appservice/storage/sqlite3/txn_id_counter_table.go @@ -0,0 +1,60 @@ +// Copyright 2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" +) + +const txnIDSchema = ` +-- Keeps a count of the current transaction ID +CREATE TABLE IF NOT EXISTS appservice_counters ( + name TEXT PRIMARY KEY NOT NULL, + last_id INTEGER DEFAULT 1 +); +INSERT OR IGNORE INTO appservice_counters (name, last_id) VALUES('txn_id', 1); +` + +const selectTxnIDSQL = ` + SELECT last_id FROM appservice_counters WHERE name='txn_id'; + UPDATE appservice_counters SET last_id=last_id+1 WHERE name='txn_id'; +` + +type txnStatements struct { + selectTxnIDStmt *sql.Stmt +} + +func (s *txnStatements) prepare(db *sql.DB) (err error) { + _, err = db.Exec(txnIDSchema) + if err != nil { + return + } + + if s.selectTxnIDStmt, err = db.Prepare(selectTxnIDSQL); err != nil { + return + } + + return +} + +// selectTxnID selects the latest ascending transaction ID +func (s *txnStatements) selectTxnID( + ctx context.Context, +) (txnID int, err error) { + err = s.selectTxnIDStmt.QueryRowContext(ctx).Scan(&txnID) + return +} diff --git a/appservice/storage/storage.go b/appservice/storage/storage.go index b68989fb1..ce3776bc3 100644 --- a/appservice/storage/storage.go +++ b/appservice/storage/storage.go @@ -1,4 +1,4 @@ -// Copyright 2018 New Vector Ltd +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -16,95 +16,33 @@ package storage import ( "context" - "database/sql" + "net/url" - // Import postgres database driver - _ "github.com/lib/pq" + "github.com/matrix-org/dendrite/appservice/storage/postgres" + "github.com/matrix-org/dendrite/appservice/storage/sqlite3" "github.com/matrix-org/gomatrixserverlib" ) -// Database stores events intended to be later sent to application services -type Database struct { - events eventsStatements - txnID txnStatements - db *sql.DB +type Database interface { + StoreEvent(ctx context.Context, appServiceID string, event *gomatrixserverlib.Event) error + GetEventsWithAppServiceID(ctx context.Context, appServiceID string, limit int) (int, int, []gomatrixserverlib.Event, bool, error) + CountEventsWithAppServiceID(ctx context.Context, appServiceID string) (int, error) + UpdateTxnIDForEvents(ctx context.Context, appserviceID string, maxID, txnID int) error + RemoveEventsBeforeAndIncludingID(ctx context.Context, appserviceID string, eventTableID int) error + GetLatestTxnID(ctx context.Context) (int, error) } -// NewDatabase opens a new database -func NewDatabase(dataSourceName string) (*Database, error) { - var result Database - var err error - if result.db, err = sql.Open("postgres", dataSourceName); err != nil { - return nil, err +func NewDatabase(dataSourceName string) (Database, error) { + uri, err := url.Parse(dataSourceName) + if err != nil { + return postgres.NewDatabase(dataSourceName) } - if err = result.prepare(); err != nil { - return nil, err + switch uri.Scheme { + case "postgres": + return postgres.NewDatabase(dataSourceName) + case "file": + return sqlite3.NewDatabase(dataSourceName) + default: + return postgres.NewDatabase(dataSourceName) } - return &result, nil -} - -func (d *Database) prepare() error { - if err := d.events.prepare(d.db); err != nil { - return err - } - - return d.txnID.prepare(d.db) -} - -// StoreEvent takes in a gomatrixserverlib.Event and stores it in the database -// for a transaction worker to pull and later send to an application service. -func (d *Database) StoreEvent( - ctx context.Context, - appServiceID string, - event *gomatrixserverlib.Event, -) error { - return d.events.insertEvent(ctx, appServiceID, event) -} - -// GetEventsWithAppServiceID returns a slice of events and their IDs intended to -// be sent to an application service given its ID. -func (d *Database) GetEventsWithAppServiceID( - ctx context.Context, - appServiceID string, - limit int, -) (int, int, []gomatrixserverlib.Event, bool, error) { - return d.events.selectEventsByApplicationServiceID(ctx, appServiceID, limit) -} - -// CountEventsWithAppServiceID returns the number of events destined for an -// application service given its ID. -func (d *Database) CountEventsWithAppServiceID( - ctx context.Context, - appServiceID string, -) (int, error) { - return d.events.countEventsByApplicationServiceID(ctx, appServiceID) -} - -// UpdateTxnIDForEvents takes in an application service ID and a -// and stores them in the DB, unless the pair already exists, in -// which case it updates them. -func (d *Database) UpdateTxnIDForEvents( - ctx context.Context, - appserviceID string, - maxID, txnID int, -) error { - return d.events.updateTxnIDForEvents(ctx, appserviceID, maxID, txnID) -} - -// RemoveEventsBeforeAndIncludingID removes all events from the database that -// are less than or equal to a given maximum ID. IDs here are implemented as a -// serial, thus this should always delete events in chronological order. -func (d *Database) RemoveEventsBeforeAndIncludingID( - ctx context.Context, - appserviceID string, - eventTableID int, -) error { - return d.events.deleteEventsBeforeAndIncludingID(ctx, appserviceID, eventTableID) -} - -// GetLatestTxnID returns the latest available transaction id -func (d *Database) GetLatestTxnID( - ctx context.Context, -) (int, error) { - return d.txnID.selectTxnID(ctx) } diff --git a/appservice/workers/transaction_scheduler.go b/appservice/workers/transaction_scheduler.go index 0330eb9ea..faa1e4a96 100644 --- a/appservice/workers/transaction_scheduler.go +++ b/appservice/workers/transaction_scheduler.go @@ -43,7 +43,7 @@ var ( // size), then send that off to the AS's /transactions/{txnID} endpoint. It also // handles exponentially backing off in case the AS isn't currently available. func SetupTransactionWorkers( - appserviceDB *storage.Database, + appserviceDB storage.Database, workerStates []types.ApplicationServiceWorkerState, ) error { // Create a worker that handles transmitting events to a single homeserver @@ -58,7 +58,7 @@ func SetupTransactionWorkers( // worker is a goroutine that sends any queued events to the application service // it is given. -func worker(db *storage.Database, ws types.ApplicationServiceWorkerState) { +func worker(db storage.Database, ws types.ApplicationServiceWorkerState) { log.WithFields(log.Fields{ "appservice": ws.AppService.ID, }).Info("starting application service") @@ -149,7 +149,7 @@ func backoff(ws *types.ApplicationServiceWorkerState, err error) { // transaction, and JSON-encodes the results. func createTransaction( ctx context.Context, - db *storage.Database, + db storage.Database, appserviceID string, ) ( transactionJSON []byte, diff --git a/mediaapi/storage/sqlite3/media_repository_table.go b/mediaapi/storage/sqlite3/media_repository_table.go new file mode 100644 index 000000000..8e2e6236a --- /dev/null +++ b/mediaapi/storage/sqlite3/media_repository_table.go @@ -0,0 +1,115 @@ +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + "time" + + "github.com/matrix-org/dendrite/mediaapi/types" + "github.com/matrix-org/gomatrixserverlib" +) + +const mediaSchema = ` +-- The media_repository table holds metadata for each media file stored and accessible to the local server, +-- the actual file is stored separately. +CREATE TABLE IF NOT EXISTS mediaapi_media_repository ( + -- The id used to refer to the media. + -- For uploads to this server this is a base64-encoded sha256 hash of the file data + -- For media from remote servers, this can be any unique identifier string + media_id TEXT NOT NULL, + -- The origin of the media as requested by the client. Should be a homeserver domain. + media_origin TEXT NOT NULL, + -- The MIME-type of the media file as specified when uploading. + content_type TEXT NOT NULL, + -- Size of the media file in bytes. + file_size_bytes INTEGER NOT NULL, + -- When the content was uploaded in UNIX epoch ms. + creation_ts INTEGER NOT NULL, + -- The file name with which the media was uploaded. + upload_name TEXT NOT NULL, + -- Alternate RFC 4648 unpadded base64 encoding string representation of a SHA-256 hash sum of the file data. + base64hash TEXT NOT NULL, + -- The user who uploaded the file. Should be a Matrix user ID. + user_id TEXT NOT NULL +); +CREATE UNIQUE INDEX IF NOT EXISTS mediaapi_media_repository_index ON mediaapi_media_repository (media_id, media_origin); +` + +const insertMediaSQL = ` +INSERT INTO mediaapi_media_repository (media_id, media_origin, content_type, file_size_bytes, creation_ts, upload_name, base64hash, user_id) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) +` + +const selectMediaSQL = ` +SELECT content_type, file_size_bytes, creation_ts, upload_name, base64hash, user_id FROM mediaapi_media_repository WHERE media_id = $1 AND media_origin = $2 +` + +type mediaStatements struct { + insertMediaStmt *sql.Stmt + selectMediaStmt *sql.Stmt +} + +func (s *mediaStatements) prepare(db *sql.DB) (err error) { + _, err = db.Exec(mediaSchema) + if err != nil { + return + } + + return statementList{ + {&s.insertMediaStmt, insertMediaSQL}, + {&s.selectMediaStmt, selectMediaSQL}, + }.prepare(db) +} + +func (s *mediaStatements) insertMedia( + ctx context.Context, mediaMetadata *types.MediaMetadata, +) error { + mediaMetadata.CreationTimestamp = types.UnixMs(time.Now().UnixNano() / 1000000) + _, err := s.insertMediaStmt.ExecContext( + ctx, + mediaMetadata.MediaID, + mediaMetadata.Origin, + mediaMetadata.ContentType, + mediaMetadata.FileSizeBytes, + mediaMetadata.CreationTimestamp, + mediaMetadata.UploadName, + mediaMetadata.Base64Hash, + mediaMetadata.UserID, + ) + return err +} + +func (s *mediaStatements) selectMedia( + ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName, +) (*types.MediaMetadata, error) { + mediaMetadata := types.MediaMetadata{ + MediaID: mediaID, + Origin: mediaOrigin, + } + err := s.selectMediaStmt.QueryRowContext( + ctx, mediaMetadata.MediaID, mediaMetadata.Origin, + ).Scan( + &mediaMetadata.ContentType, + &mediaMetadata.FileSizeBytes, + &mediaMetadata.CreationTimestamp, + &mediaMetadata.UploadName, + &mediaMetadata.Base64Hash, + &mediaMetadata.UserID, + ) + return &mediaMetadata, err +} diff --git a/mediaapi/storage/sqlite3/prepare.go b/mediaapi/storage/sqlite3/prepare.go new file mode 100644 index 000000000..a6bc24c98 --- /dev/null +++ b/mediaapi/storage/sqlite3/prepare.go @@ -0,0 +1,38 @@ +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// FIXME: This should be made common! + +package sqlite3 + +import ( + "database/sql" +) + +// a statementList is a list of SQL statements to prepare and a pointer to where to store the resulting prepared statement. +type statementList []struct { + statement **sql.Stmt + sql string +} + +// prepare the SQL for each statement in the list and assign the result to the prepared statement. +func (s statementList) prepare(db *sql.DB) (err error) { + for _, statement := range s { + if *statement.statement, err = db.Prepare(statement.sql); err != nil { + return + } + } + return +} diff --git a/mediaapi/storage/sqlite3/sql.go b/mediaapi/storage/sqlite3/sql.go new file mode 100644 index 000000000..9cd78b8ee --- /dev/null +++ b/mediaapi/storage/sqlite3/sql.go @@ -0,0 +1,36 @@ +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "database/sql" +) + +type statements struct { + media mediaStatements + thumbnail thumbnailStatements +} + +func (s *statements) prepare(db *sql.DB) (err error) { + if err = s.media.prepare(db); err != nil { + return + } + if err = s.thumbnail.prepare(db); err != nil { + return + } + + return +} diff --git a/mediaapi/storage/sqlite3/storage.go b/mediaapi/storage/sqlite3/storage.go new file mode 100644 index 000000000..6477f8305 --- /dev/null +++ b/mediaapi/storage/sqlite3/storage.go @@ -0,0 +1,106 @@ +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + + // Import the postgres database driver. + "github.com/matrix-org/dendrite/mediaapi/types" + "github.com/matrix-org/gomatrixserverlib" + _ "github.com/mattn/go-sqlite3" +) + +// Database is used to store metadata about a repository of media files. +type Database struct { + statements statements + db *sql.DB +} + +// Open opens a postgres database. +func Open(dataSourceName string) (*Database, error) { + var d Database + var err error + if d.db, err = sql.Open("sqlite3", dataSourceName); err != nil { + return nil, err + } + if err = d.statements.prepare(d.db); err != nil { + return nil, err + } + return &d, nil +} + +// StoreMediaMetadata inserts the metadata about the uploaded media into the database. +// Returns an error if the combination of MediaID and Origin are not unique in the table. +func (d *Database) StoreMediaMetadata( + ctx context.Context, mediaMetadata *types.MediaMetadata, +) error { + return d.statements.media.insertMedia(ctx, mediaMetadata) +} + +// GetMediaMetadata returns metadata about media stored on this server. +// The media could have been uploaded to this server or fetched from another server and cached here. +// Returns nil metadata if there is no metadata associated with this media. +func (d *Database) GetMediaMetadata( + ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName, +) (*types.MediaMetadata, error) { + mediaMetadata, err := d.statements.media.selectMedia(ctx, mediaID, mediaOrigin) + if err != nil && err == sql.ErrNoRows { + return nil, nil + } + return mediaMetadata, err +} + +// StoreThumbnail inserts the metadata about the thumbnail into the database. +// Returns an error if the combination of MediaID and Origin are not unique in the table. +func (d *Database) StoreThumbnail( + ctx context.Context, thumbnailMetadata *types.ThumbnailMetadata, +) error { + return d.statements.thumbnail.insertThumbnail(ctx, thumbnailMetadata) +} + +// GetThumbnail returns metadata about a specific thumbnail. +// The media could have been uploaded to this server or fetched from another server and cached here. +// Returns nil metadata if there is no metadata associated with this thumbnail. +func (d *Database) GetThumbnail( + ctx context.Context, + mediaID types.MediaID, + mediaOrigin gomatrixserverlib.ServerName, + width, height int, + resizeMethod string, +) (*types.ThumbnailMetadata, error) { + thumbnailMetadata, err := d.statements.thumbnail.selectThumbnail( + ctx, mediaID, mediaOrigin, width, height, resizeMethod, + ) + if err != nil && err == sql.ErrNoRows { + return nil, nil + } + return thumbnailMetadata, err +} + +// GetThumbnails returns metadata about all thumbnails for a specific media stored on this server. +// The media could have been uploaded to this server or fetched from another server and cached here. +// Returns nil metadata if there are no thumbnails associated with this media. +func (d *Database) GetThumbnails( + ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName, +) ([]*types.ThumbnailMetadata, error) { + thumbnails, err := d.statements.thumbnail.selectThumbnails(ctx, mediaID, mediaOrigin) + if err != nil && err == sql.ErrNoRows { + return nil, nil + } + return thumbnails, err +} diff --git a/mediaapi/storage/sqlite3/thumbnail_table.go b/mediaapi/storage/sqlite3/thumbnail_table.go new file mode 100644 index 000000000..95332c9d4 --- /dev/null +++ b/mediaapi/storage/sqlite3/thumbnail_table.go @@ -0,0 +1,162 @@ +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + "time" + + "github.com/matrix-org/dendrite/mediaapi/types" + "github.com/matrix-org/gomatrixserverlib" +) + +const thumbnailSchema = ` +-- The mediaapi_thumbnail table holds metadata for each thumbnail file stored and accessible to the local server, +-- the actual file is stored separately. +CREATE TABLE IF NOT EXISTS mediaapi_thumbnail ( + media_id TEXT NOT NULL, + media_origin TEXT NOT NULL, + content_type TEXT NOT NULL, + file_size_bytes INTEGER NOT NULL, + creation_ts INTEGER NOT NULL, + width INTEGER NOT NULL, + height INTEGER NOT NULL, + resize_method TEXT NOT NULL +); +CREATE UNIQUE INDEX IF NOT EXISTS mediaapi_thumbnail_index ON mediaapi_thumbnail (media_id, media_origin, width, height, resize_method); +` + +const insertThumbnailSQL = ` +INSERT INTO mediaapi_thumbnail (media_id, media_origin, content_type, file_size_bytes, creation_ts, width, height, resize_method) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) +` + +// Note: this selects one specific thumbnail +const selectThumbnailSQL = ` +SELECT content_type, file_size_bytes, creation_ts FROM mediaapi_thumbnail WHERE media_id = $1 AND media_origin = $2 AND width = $3 AND height = $4 AND resize_method = $5 +` + +// Note: this selects all thumbnails for a media_origin and media_id +const selectThumbnailsSQL = ` +SELECT content_type, file_size_bytes, creation_ts, width, height, resize_method FROM mediaapi_thumbnail WHERE media_id = $1 AND media_origin = $2 +` + +type thumbnailStatements struct { + insertThumbnailStmt *sql.Stmt + selectThumbnailStmt *sql.Stmt + selectThumbnailsStmt *sql.Stmt +} + +func (s *thumbnailStatements) prepare(db *sql.DB) (err error) { + _, err = db.Exec(thumbnailSchema) + if err != nil { + return + } + + return statementList{ + {&s.insertThumbnailStmt, insertThumbnailSQL}, + {&s.selectThumbnailStmt, selectThumbnailSQL}, + {&s.selectThumbnailsStmt, selectThumbnailsSQL}, + }.prepare(db) +} + +func (s *thumbnailStatements) insertThumbnail( + ctx context.Context, thumbnailMetadata *types.ThumbnailMetadata, +) error { + thumbnailMetadata.MediaMetadata.CreationTimestamp = types.UnixMs(time.Now().UnixNano() / 1000000) + _, err := s.insertThumbnailStmt.ExecContext( + ctx, + thumbnailMetadata.MediaMetadata.MediaID, + thumbnailMetadata.MediaMetadata.Origin, + thumbnailMetadata.MediaMetadata.ContentType, + thumbnailMetadata.MediaMetadata.FileSizeBytes, + thumbnailMetadata.MediaMetadata.CreationTimestamp, + thumbnailMetadata.ThumbnailSize.Width, + thumbnailMetadata.ThumbnailSize.Height, + thumbnailMetadata.ThumbnailSize.ResizeMethod, + ) + return err +} + +func (s *thumbnailStatements) selectThumbnail( + ctx context.Context, + mediaID types.MediaID, + mediaOrigin gomatrixserverlib.ServerName, + width, height int, + resizeMethod string, +) (*types.ThumbnailMetadata, error) { + thumbnailMetadata := types.ThumbnailMetadata{ + MediaMetadata: &types.MediaMetadata{ + MediaID: mediaID, + Origin: mediaOrigin, + }, + ThumbnailSize: types.ThumbnailSize{ + Width: width, + Height: height, + ResizeMethod: resizeMethod, + }, + } + err := s.selectThumbnailStmt.QueryRowContext( + ctx, + thumbnailMetadata.MediaMetadata.MediaID, + thumbnailMetadata.MediaMetadata.Origin, + thumbnailMetadata.ThumbnailSize.Width, + thumbnailMetadata.ThumbnailSize.Height, + thumbnailMetadata.ThumbnailSize.ResizeMethod, + ).Scan( + &thumbnailMetadata.MediaMetadata.ContentType, + &thumbnailMetadata.MediaMetadata.FileSizeBytes, + &thumbnailMetadata.MediaMetadata.CreationTimestamp, + ) + return &thumbnailMetadata, err +} + +func (s *thumbnailStatements) selectThumbnails( + ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName, +) ([]*types.ThumbnailMetadata, error) { + rows, err := s.selectThumbnailsStmt.QueryContext( + ctx, mediaID, mediaOrigin, + ) + if err != nil { + return nil, err + } + defer rows.Close() // nolint: errcheck + + var thumbnails []*types.ThumbnailMetadata + for rows.Next() { + thumbnailMetadata := types.ThumbnailMetadata{ + MediaMetadata: &types.MediaMetadata{ + MediaID: mediaID, + Origin: mediaOrigin, + }, + } + err = rows.Scan( + &thumbnailMetadata.MediaMetadata.ContentType, + &thumbnailMetadata.MediaMetadata.FileSizeBytes, + &thumbnailMetadata.MediaMetadata.CreationTimestamp, + &thumbnailMetadata.ThumbnailSize.Width, + &thumbnailMetadata.ThumbnailSize.Height, + &thumbnailMetadata.ThumbnailSize.ResizeMethod, + ) + if err != nil { + return nil, err + } + thumbnails = append(thumbnails, &thumbnailMetadata) + } + + return thumbnails, rows.Err() +} diff --git a/mediaapi/storage/storage.go b/mediaapi/storage/storage.go index 2c7f937dd..1d35a95ed 100644 --- a/mediaapi/storage/storage.go +++ b/mediaapi/storage/storage.go @@ -19,6 +19,7 @@ import ( "net/url" "github.com/matrix-org/dendrite/mediaapi/storage/postgres" + "github.com/matrix-org/dendrite/mediaapi/storage/sqlite3" "github.com/matrix-org/dendrite/mediaapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -40,6 +41,8 @@ func Open(dataSourceName string) (Database, error) { switch uri.Scheme { case "postgres": return postgres.Open(dataSourceName) + case "file": + return sqlite3.Open(dataSourceName) default: return postgres.Open(dataSourceName) } diff --git a/publicroomsapi/storage/sqlite3/public_rooms_table.go b/publicroomsapi/storage/sqlite3/public_rooms_table.go index 06c74a331..3fe3d3cab 100644 --- a/publicroomsapi/storage/sqlite3/public_rooms_table.go +++ b/publicroomsapi/storage/sqlite3/public_rooms_table.go @@ -39,26 +39,15 @@ var editableAttributes = []string{ const publicRoomsSchema = ` -- Stores all of the rooms with data needed to create the server's room directory CREATE TABLE IF NOT EXISTS publicroomsapi_public_rooms( - -- The room's ID room_id TEXT NOT NULL PRIMARY KEY, - -- Number of joined members in the room joined_members INTEGER NOT NULL DEFAULT 0, - -- Aliases of the room (empty array if none) - aliases TEXT[] NOT NULL DEFAULT '{}'::TEXT[], - -- Canonical alias of the room (empty string if none) + aliases TEXT NOT NULL DEFAULT '', canonical_alias TEXT NOT NULL DEFAULT '', - -- Name of the room (empty string if none) name TEXT NOT NULL DEFAULT '', - -- Topic of the room (empty string if none) topic TEXT NOT NULL DEFAULT '', - -- Is the room world readable? world_readable BOOLEAN NOT NULL DEFAULT false, - -- Can guest join the room? guest_can_join BOOLEAN NOT NULL DEFAULT false, - -- URL of the room avatar (empty string if none) avatar_url TEXT NOT NULL DEFAULT '', - -- Visibility of the room: true means the room is publicly visible, false - -- means the room is private visibility BOOLEAN NOT NULL DEFAULT false ); ` @@ -71,13 +60,13 @@ const selectPublicRoomsSQL = "" + "SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" + " FROM publicroomsapi_public_rooms WHERE visibility = true" + " ORDER BY joined_members DESC" + - " OFFSET $1" + " LIMIT 30 OFFSET $1" const selectPublicRoomsWithLimitSQL = "" + "SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" + " FROM publicroomsapi_public_rooms WHERE visibility = true" + " ORDER BY joined_members DESC" + - " OFFSET $1 LIMIT $2" + " LIMIT $2 OFFSET $1" const selectPublicRoomsWithFilterSQL = "" + "SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" + @@ -85,9 +74,9 @@ const selectPublicRoomsWithFilterSQL = "" + " WHERE visibility = true" + " AND (LOWER(name) LIKE LOWER($1)" + " OR LOWER(topic) LIKE LOWER($1)" + - " OR LOWER(ARRAY_TO_STRING(aliases, ',')) LIKE LOWER($1))" + + " OR LOWER(aliases) LIKE LOWER($1))" + // TODO: Is there a better way to search aliases? " ORDER BY joined_members DESC" + - " OFFSET $2" + " LIMIT 30 OFFSET $2" const selectPublicRoomsWithLimitAndFilterSQL = "" + "SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" + @@ -95,9 +84,9 @@ const selectPublicRoomsWithLimitAndFilterSQL = "" + " WHERE visibility = true" + " AND (LOWER(name) LIKE LOWER($1)" + " OR LOWER(topic) LIKE LOWER($1)" + - " OR LOWER(ARRAY_TO_STRING(aliases, ',')) LIKE LOWER($1))" + + " OR LOWER(aliases) LIKE LOWER($1))" + // TODO: Is there a better way to search aliases? " ORDER BY joined_members DESC" + - " OFFSET $2 LIMIT $3" + " LIMIT $3 OFFSET $2" const selectRoomVisibilitySQL = "" + "SELECT visibility FROM publicroomsapi_public_rooms" + @@ -187,7 +176,7 @@ func (s *publicRoomsStatements) selectPublicRooms( ) } else { rows, err = s.selectPublicRoomsWithLimitAndFilterStmt.QueryContext( - ctx, pattern, offset, limit, + ctx, pattern, limit, offset, ) } } else { @@ -195,7 +184,7 @@ func (s *publicRoomsStatements) selectPublicRooms( rows, err = s.selectPublicRoomsStmt.QueryContext(ctx, offset) } else { rows, err = s.selectPublicRoomsWithLimitStmt.QueryContext( - ctx, offset, limit, + ctx, limit, offset, ) } } diff --git a/publicroomsapi/storage/storage.go b/publicroomsapi/storage/storage.go index a6e18fbcb..29a6619fa 100644 --- a/publicroomsapi/storage/storage.go +++ b/publicroomsapi/storage/storage.go @@ -20,6 +20,7 @@ import ( "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/publicroomsapi/storage/postgres" + "github.com/matrix-org/dendrite/publicroomsapi/storage/sqlite3" "github.com/matrix-org/dendrite/publicroomsapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -43,6 +44,8 @@ func NewPublicRoomsServerDatabase(dataSourceName string) (Database, error) { switch uri.Scheme { case "postgres": return postgres.NewPublicRoomsServerDatabase(dataSourceName) + case "file": + return sqlite3.NewPublicRoomsServerDatabase(dataSourceName) default: return postgres.NewPublicRoomsServerDatabase(dataSourceName) } From 5caae6f3a0fc56f5ae654dcc2a65822885cda0a0 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Thu, 20 Feb 2020 09:28:03 +0000 Subject: [PATCH 18/86] sqlite: fixes from sytest (#872) * bugfix: fix panic on new invite events from sytest I'm unsure why the previous code didn't work, but it's clearer, quicker and easier to read the `LastInsertID()` way. Previously, the code would panic as the SELECT would fail to find the last inserted row ID. * sqlite: Fix UNIQUE violations and close more cursors - Add missing `defer rows.Close()` - Do not have the state block NID as a PRIMARY KEY else it breaks for blocks with >1 state event in them. Instead, rejig the queries so we can still have monotonically increasing integers without using AUTOINCREMENT (which mandates PRIMARY KEY). * sqlite: Add missing variadic function * Use LastInsertId because empirically it works over the SELECT form (though I don't know why that is) * sqlite: Fix invite table by using the global stream pos rather than one specific to invites If we don't use the global, clients don't get notified about any invites because the position is too low. * linting: shadowing * sqlite: do not use last rowid, we already know the stream pos! * sqlite: Fix account data table in syncapi by commiting insert txns! * sqlite: Fix failing federation invite Was failing with 'database is locked' due to multiple write txns being taken out. * sqlite: Ensure we return exactly the number of events found in the database Previously we would return exactly the number of *requested* events, which meant that several zero-initialised events would bubble through the system, failing at JSON serialisation time. * sqlite: let's just ignore the problem for now.... * linting --- .../storage/accounts/sqlite3/filter_table.go | 22 +++----- roomserver/storage/sqlite3/invite_table.go | 20 +++++-- .../storage/sqlite3/room_aliases_table.go | 2 + .../storage/sqlite3/state_block_table.go | 35 +++++------- roomserver/storage/sqlite3/storage.go | 37 +++++++++---- syncapi/consumers/roomserver.go | 1 + syncapi/storage/sqlite3/account_data_table.go | 51 ++++++++++------- .../sqlite3/current_room_state_table.go | 13 +++-- syncapi/storage/sqlite3/invites_table.go | 34 ++++-------- .../sqlite3/output_room_events_table.go | 12 ---- syncapi/storage/sqlite3/syncserver.go | 55 +++++++++---------- 11 files changed, 145 insertions(+), 137 deletions(-) diff --git a/clientapi/auth/storage/accounts/sqlite3/filter_table.go b/clientapi/auth/storage/accounts/sqlite3/filter_table.go index 691ead775..7f1a0c249 100644 --- a/clientapi/auth/storage/accounts/sqlite3/filter_table.go +++ b/clientapi/auth/storage/accounts/sqlite3/filter_table.go @@ -18,6 +18,7 @@ import ( "context" "database/sql" "encoding/json" + "fmt" "github.com/matrix-org/gomatrixserverlib" ) @@ -47,14 +48,10 @@ const selectFilterIDByContentSQL = "" + const insertFilterSQL = "" + "INSERT INTO account_filter (filter, localpart) VALUES ($1, $2)" -const selectLastInsertedFilterIDSQL = "" + - "SELECT id FROM account_filter WHERE rowid = last_insert_rowid()" - type filterStatements struct { - selectFilterStmt *sql.Stmt - selectLastInsertedFilterIDStmt *sql.Stmt - selectFilterIDByContentStmt *sql.Stmt - insertFilterStmt *sql.Stmt + selectFilterStmt *sql.Stmt + selectFilterIDByContentStmt *sql.Stmt + insertFilterStmt *sql.Stmt } func (s *filterStatements) prepare(db *sql.DB) (err error) { @@ -65,9 +62,6 @@ func (s *filterStatements) prepare(db *sql.DB) (err error) { if s.selectFilterStmt, err = db.Prepare(selectFilterSQL); err != nil { return } - if s.selectLastInsertedFilterIDStmt, err = db.Prepare(selectLastInsertedFilterIDSQL); err != nil { - return - } if s.selectFilterIDByContentStmt, err = db.Prepare(selectFilterIDByContentSQL); err != nil { return } @@ -128,12 +122,14 @@ func (s *filterStatements) insertFilter( } // Otherwise insert the filter and return the new ID - if _, err = s.insertFilterStmt.ExecContext(ctx, filterJSON, localpart); err != nil { + res, err := s.insertFilterStmt.ExecContext(ctx, filterJSON, localpart) + if err != nil { return "", err } - row := s.selectLastInsertedFilterIDStmt.QueryRowContext(ctx) - if err := row.Scan(&filterID); err != nil { + rowid, err := res.LastInsertId() + if err != nil { return "", err } + filterID = fmt.Sprintf("%d", rowid) return } diff --git a/roomserver/storage/sqlite3/invite_table.go b/roomserver/storage/sqlite3/invite_table.go index 5a0f0bf79..641f80156 100644 --- a/roomserver/storage/sqlite3/invite_table.go +++ b/roomserver/storage/sqlite3/invite_table.go @@ -52,16 +52,18 @@ const selectInviteActiveForUserInRoomSQL = "" + // However the matrix protocol doesn't give us a way to reliably identify the // invites that were retired, so we are forced to retire all of them. const updateInviteRetiredSQL = ` - UPDATE roomserver_invites SET retired = TRUE - WHERE room_nid = $1 AND target_nid = $2 AND NOT retired; - SELECT invite_event_id FROM roomserver_invites - WHERE rowid = last_insert_rowid(); + UPDATE roomserver_invites SET retired = TRUE WHERE room_nid = $1 AND target_nid = $2 AND NOT retired +` + +const selectInvitesAboutToRetireSQL = ` +SELECT invite_event_id FROM roomserver_invites WHERE room_nid = $1 AND target_nid = $2 AND NOT retired ` type inviteStatements struct { insertInviteEventStmt *sql.Stmt selectInviteActiveForUserInRoomStmt *sql.Stmt updateInviteRetiredStmt *sql.Stmt + selectInvitesAboutToRetireStmt *sql.Stmt } func (s *inviteStatements) prepare(db *sql.DB) (err error) { @@ -74,6 +76,7 @@ func (s *inviteStatements) prepare(db *sql.DB) (err error) { {&s.insertInviteEventStmt, insertInviteEventSQL}, {&s.selectInviteActiveForUserInRoomStmt, selectInviteActiveForUserInRoomSQL}, {&s.updateInviteRetiredStmt, updateInviteRetiredSQL}, + {&s.selectInvitesAboutToRetireStmt, selectInvitesAboutToRetireSQL}, }.prepare(db) } @@ -102,7 +105,8 @@ func (s *inviteStatements) updateInviteRetired( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, ) (eventIDs []string, err error) { - stmt := common.TxStmt(txn, s.updateInviteRetiredStmt) + // gather all the event IDs we will retire + stmt := txn.Stmt(s.selectInvitesAboutToRetireStmt) rows, err := stmt.QueryContext(ctx, roomNID, targetUserNID) if err != nil { return nil, err @@ -110,11 +114,15 @@ func (s *inviteStatements) updateInviteRetired( defer (func() { err = rows.Close() })() for rows.Next() { var inviteEventID string - if err := rows.Scan(&inviteEventID); err != nil { + if err = rows.Scan(&inviteEventID); err != nil { return nil, err } eventIDs = append(eventIDs, inviteEventID) } + + // now retire the invites + stmt = txn.Stmt(s.updateInviteRetiredStmt) + _, err = stmt.ExecContext(ctx, roomNID, targetUserNID) return } diff --git a/roomserver/storage/sqlite3/room_aliases_table.go b/roomserver/storage/sqlite3/room_aliases_table.go index a5fd5449a..71238b0e4 100644 --- a/roomserver/storage/sqlite3/room_aliases_table.go +++ b/roomserver/storage/sqlite3/room_aliases_table.go @@ -103,6 +103,8 @@ func (s *roomAliasesStatements) selectAliasesFromRoomID( return } + defer rows.Close() // nolint: errcheck + for rows.Next() { var alias string if err = rows.Scan(&alias); err != nil { diff --git a/roomserver/storage/sqlite3/state_block_table.go b/roomserver/storage/sqlite3/state_block_table.go index ac593546a..d75abceec 100644 --- a/roomserver/storage/sqlite3/state_block_table.go +++ b/roomserver/storage/sqlite3/state_block_table.go @@ -30,7 +30,7 @@ import ( const stateDataSchema = ` CREATE TABLE IF NOT EXISTS roomserver_state_block ( - state_block_nid INTEGER PRIMARY KEY AUTOINCREMENT, + state_block_nid INTEGER NOT NULL, event_type_nid INTEGER NOT NULL, event_state_key_nid INTEGER NOT NULL, event_nid INTEGER NOT NULL, @@ -43,10 +43,7 @@ const insertStateDataSQL = "" + " VALUES ($1, $2, $3, $4)" const selectNextStateBlockNIDSQL = ` - SELECT COALESCE(( - SELECT seq+1 AS state_block_nid FROM sqlite_sequence - WHERE name = 'roomserver_state_block'), 1 - ) AS state_block_nid +SELECT IFNULL(MAX(state_block_nid), 0) + 1 FROM roomserver_state_block ` // Bulk state lookup by numeric state block ID. @@ -98,11 +95,19 @@ func (s *stateBlockStatements) prepare(db *sql.DB) (err error) { func (s *stateBlockStatements) bulkInsertStateData( ctx context.Context, txn *sql.Tx, - stateBlockNID types.StateBlockNID, entries []types.StateEntry, -) error { +) (types.StateBlockNID, error) { + if len(entries) == 0 { + return 0, nil + } + var stateBlockNID types.StateBlockNID + err := txn.Stmt(s.selectNextStateBlockNIDStmt).QueryRowContext(ctx).Scan(&stateBlockNID) + if err != nil { + return 0, err + } + for _, entry := range entries { - _, err := common.TxStmt(txn, s.insertStateDataStmt).ExecContext( + _, err := txn.Stmt(s.insertStateDataStmt).ExecContext( ctx, int64(stateBlockNID), int64(entry.EventTypeNID), @@ -110,20 +115,10 @@ func (s *stateBlockStatements) bulkInsertStateData( int64(entry.EventNID), ) if err != nil { - return err + return 0, err } } - return nil -} - -func (s *stateBlockStatements) selectNextStateBlockNID( - ctx context.Context, - txn *sql.Tx, -) (types.StateBlockNID, error) { - var stateBlockNID int64 - selectStmt := common.TxStmt(txn, s.selectNextStateBlockNIDStmt) - err := selectStmt.QueryRowContext(ctx).Scan(&stateBlockNID) - return types.StateBlockNID(stateBlockNID), err + return stateBlockNID, nil } func (s *stateBlockStatements) bulkSelectStateBlockEntries( diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index e20e8aed7..aebb308c4 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -54,7 +54,12 @@ func Open(dataSourceName string) (*Database, error) { } //d.db.Exec("PRAGMA journal_mode=WAL;") //d.db.Exec("PRAGMA read_uncommitted = true;") - d.db.SetMaxOpenConns(2) + + // FIXME: We are leaking connections somewhere. Setting this to 2 will eventually + // cause the roomserver to be unresponsive to new events because something will + // acquire the global mutex and never unlock it because it is waiting for a connection + // which it will never obtain. + d.db.SetMaxOpenConns(20) if err = d.statements.prepare(d.db); err != nil { return nil, err } @@ -253,12 +258,13 @@ func (d *Database) Events( ) ([]types.Event, error) { var eventJSONs []eventJSONPair var err error - results := make([]types.Event, len(eventNIDs)) + var results []types.Event err = common.WithTransaction(d.db, func(txn *sql.Tx) error { eventJSONs, err = d.statements.bulkSelectEventJSON(ctx, txn, eventNIDs) if err != nil || len(eventJSONs) == 0 { return nil } + results = make([]types.Event, len(eventJSONs)) for i, eventJSON := range eventJSONs { result := &results[i] result.EventNID = eventJSON.EventNID @@ -286,13 +292,10 @@ func (d *Database) AddState( err = common.WithTransaction(d.db, func(txn *sql.Tx) error { if len(state) > 0 { var stateBlockNID types.StateBlockNID - stateBlockNID, err = d.statements.selectNextStateBlockNID(ctx, txn) + stateBlockNID, err = d.statements.bulkInsertStateData(ctx, txn, state) if err != nil { return err } - if err = d.statements.bulkInsertStateData(ctx, txn, stateBlockNID, state); err != nil { - return err - } stateBlockNIDs = append(stateBlockNIDs[:len(stateBlockNIDs):len(stateBlockNIDs)], stateBlockNID) } stateNID, err = d.statements.insertState(ctx, txn, roomNID, stateBlockNIDs) @@ -602,8 +605,9 @@ func (d *Database) StateEntriesForTuples( // MembershipUpdater implements input.RoomEventDatabase func (d *Database) MembershipUpdater( ctx context.Context, roomID, targetUserID string, -) (types.MembershipUpdater, error) { - txn, err := d.db.Begin() +) (updater types.MembershipUpdater, err error) { + var txn *sql.Tx + txn, err = d.db.Begin() if err != nil { return nil, err } @@ -611,6 +615,18 @@ func (d *Database) MembershipUpdater( defer func() { if !succeeded { txn.Rollback() // nolint: errcheck + } else { + // TODO: We should be holding open this transaction but we cannot have + // multiple write transactions on sqlite. The code will perform additional + // write transactions independent of this one which will consistently cause + // 'database is locked' errors. For now, we'll break up the transaction and + // hope we don't race too catastrophically. Long term, we should be able to + // thread in txn objects where appropriate (either at the interface level or + // bring matrix business logic into the storage layer). + txerr := txn.Commit() + if err == nil && txerr != nil { + err = txerr + } } }() @@ -624,7 +640,7 @@ func (d *Database) MembershipUpdater( return nil, err } - updater, err := d.membershipUpdaterTxn(ctx, txn, roomNID, targetUserNID) + updater, err = d.membershipUpdaterTxn(ctx, txn, roomNID, targetUserNID) if err != nil { return nil, err } @@ -658,7 +674,8 @@ func (d *Database) membershipUpdaterTxn( } return &membershipUpdater{ - transaction{ctx, txn}, d, roomNID, targetUserNID, membership, + // purposefully set the txn to nil so if we try to use it we panic and fail fast + transaction{ctx, nil}, d, roomNID, targetUserNID, membership, }, nil } diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index ba1b7dc59..5dbef4b7d 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -158,6 +158,7 @@ func (s *OutputRoomEventConsumer) onNewInviteEvent( // panic rather than continue with an inconsistent database log.WithFields(log.Fields{ "event": string(msg.Event.JSON()), + "pdupos": pduPos, log.ErrorKey: err, }).Panicf("roomserver output log: write invite failure") return nil diff --git a/syncapi/storage/sqlite3/account_data_table.go b/syncapi/storage/sqlite3/account_data_table.go index 3274e66ea..71105d0c3 100644 --- a/syncapi/storage/sqlite3/account_data_table.go +++ b/syncapi/storage/sqlite3/account_data_table.go @@ -19,15 +19,13 @@ import ( "context" "database/sql" - "github.com/lib/pq" - "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" ) const accountDataSchema = ` CREATE TABLE IF NOT EXISTS syncapi_account_data_type ( - id INTEGER PRIMARY KEY AUTOINCREMENT, + id INTEGER PRIMARY KEY, user_id TEXT NOT NULL, room_id TEXT NOT NULL, type TEXT NOT NULL, @@ -43,9 +41,7 @@ const insertAccountDataSQL = "" + const selectAccountDataInRangeSQL = "" + "SELECT room_id, type FROM syncapi_account_data_type" + " WHERE user_id = $1 AND id > $2 AND id <= $3" + - " AND ( $4 IS NULL OR type IN ($4) )" + - " AND ( $5 IS NULL OR NOT(type IN ($5)) )" + - " ORDER BY id ASC LIMIT $6" + " ORDER BY id ASC" const selectMaxAccountDataIDSQL = "" + "SELECT MAX(id) FROM syncapi_account_data_type" @@ -53,8 +49,8 @@ const selectMaxAccountDataIDSQL = "" + type accountDataStatements struct { streamIDStatements *streamIDStatements insertAccountDataStmt *sql.Stmt - selectAccountDataInRangeStmt *sql.Stmt selectMaxAccountDataIDStmt *sql.Stmt + selectAccountDataInRangeStmt *sql.Stmt } func (s *accountDataStatements) prepare(db *sql.DB, streamID *streamIDStatements) (err error) { @@ -66,10 +62,10 @@ func (s *accountDataStatements) prepare(db *sql.DB, streamID *streamIDStatements if s.insertAccountDataStmt, err = db.Prepare(insertAccountDataSQL); err != nil { return } - if s.selectAccountDataInRangeStmt, err = db.Prepare(selectAccountDataInRangeSQL); err != nil { + if s.selectMaxAccountDataIDStmt, err = db.Prepare(selectMaxAccountDataIDSQL); err != nil { return } - if s.selectMaxAccountDataIDStmt, err = db.Prepare(selectMaxAccountDataIDSQL); err != nil { + if s.selectAccountDataInRangeStmt, err = db.Prepare(selectAccountDataInRangeSQL); err != nil { return } return @@ -83,8 +79,7 @@ func (s *accountDataStatements) insertAccountData( if err != nil { return } - insertStmt := common.TxStmt(txn, s.insertAccountDataStmt) - _, err = insertStmt.ExecContext(ctx, pos, userID, roomID, dataType) + _, err = txn.Stmt(s.insertAccountDataStmt).ExecContext(ctx, pos, userID, roomID, dataType) return } @@ -103,14 +98,13 @@ func (s *accountDataStatements) selectAccountDataInRange( oldPos-- } - rows, err := s.selectAccountDataInRangeStmt.QueryContext(ctx, userID, oldPos, newPos, - pq.StringArray(filterConvertTypeWildcardToSQL(accountDataFilterPart.Types)), - pq.StringArray(filterConvertTypeWildcardToSQL(accountDataFilterPart.NotTypes)), - accountDataFilterPart.Limit, - ) + rows, err := s.selectAccountDataInRangeStmt.QueryContext(ctx, userID, oldPos, newPos) if err != nil { return } + defer rows.Close() // nolint: errcheck + + var entries int for rows.Next() { var dataType string @@ -120,22 +114,41 @@ func (s *accountDataStatements) selectAccountDataInRange( return } + // check if we should add this by looking at the filter. + // It would be nice if we could do this in SQL-land, but the mix of variadic + // and positional parameters makes the query annoyingly hard to do, it's easier + // and clearer to do it in Go-land. If there are no filters for [not]types then + // this gets skipped. + for _, includeType := range accountDataFilterPart.Types { + if includeType != dataType { // TODO: wildcard support + continue + } + } + for _, excludeType := range accountDataFilterPart.NotTypes { + if excludeType == dataType { // TODO: wildcard support + continue + } + } + if len(data[roomID]) > 0 { data[roomID] = append(data[roomID], dataType) } else { data[roomID] = []string{dataType} } + entries++ + if entries >= accountDataFilterPart.Limit { + break + } } - return + return data, nil } func (s *accountDataStatements) selectMaxAccountDataID( ctx context.Context, txn *sql.Tx, ) (id int64, err error) { var nullableID sql.NullInt64 - stmt := common.TxStmt(txn, s.selectMaxAccountDataIDStmt) - err = stmt.QueryRowContext(ctx).Scan(&nullableID) + err = txn.Stmt(s.selectMaxAccountDataIDStmt).QueryRowContext(ctx).Scan(&nullableID) if nullableID.Valid { id = nullableID.Int64 } diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go index 4ce946667..eb969c956 100644 --- a/syncapi/storage/sqlite3/current_room_state_table.go +++ b/syncapi/storage/sqlite3/current_room_state_table.go @@ -19,6 +19,7 @@ import ( "context" "database/sql" "encoding/json" + "strings" "github.com/lib/pq" "github.com/matrix-org/dendrite/common" @@ -88,7 +89,6 @@ type currentRoomStateStatements struct { selectRoomIDsWithMembershipStmt *sql.Stmt selectCurrentStateStmt *sql.Stmt selectJoinedUsersStmt *sql.Stmt - selectEventsWithEventIDsStmt *sql.Stmt selectStateEventStmt *sql.Stmt } @@ -113,9 +113,6 @@ func (s *currentRoomStateStatements) prepare(db *sql.DB, streamID *streamIDState if s.selectJoinedUsersStmt, err = db.Prepare(selectJoinedUsersSQL); err != nil { return } - if s.selectEventsWithEventIDsStmt, err = db.Prepare(selectEventsWithEventIDsSQL); err != nil { - return - } if s.selectStateEventStmt, err = db.Prepare(selectStateEventSQL); err != nil { return } @@ -233,8 +230,12 @@ func (s *currentRoomStateStatements) upsertRoomState( func (s *currentRoomStateStatements) selectEventsWithEventIDs( ctx context.Context, txn *sql.Tx, eventIDs []string, ) ([]types.StreamEvent, error) { - stmt := common.TxStmt(txn, s.selectEventsWithEventIDsStmt) - rows, err := stmt.QueryContext(ctx, pq.StringArray(eventIDs)) + iEventIDs := make([]interface{}, len(eventIDs)) + for k, v := range eventIDs { + iEventIDs[k] = v + } + query := strings.Replace(selectEventsWithEventIDsSQL, "($1)", common.QueryVariadic(len(iEventIDs)), 1) + rows, err := txn.QueryContext(ctx, query, iEventIDs...) if err != nil { return nil, err } diff --git a/syncapi/storage/sqlite3/invites_table.go b/syncapi/storage/sqlite3/invites_table.go index 74dba245b..baf8871bd 100644 --- a/syncapi/storage/sqlite3/invites_table.go +++ b/syncapi/storage/sqlite3/invites_table.go @@ -26,7 +26,7 @@ import ( const inviteEventsSchema = ` CREATE TABLE IF NOT EXISTS syncapi_invite_events ( - id INTEGER PRIMARY KEY AUTOINCREMENT, + id INTEGER PRIMARY KEY, event_id TEXT NOT NULL, room_id TEXT NOT NULL, target_user_id TEXT NOT NULL, @@ -39,11 +39,8 @@ CREATE INDEX IF NOT EXISTS syncapi_invites_event_id_idx ON syncapi_invite_events const insertInviteEventSQL = "" + "INSERT INTO syncapi_invite_events" + - " (room_id, event_id, target_user_id, event_json)" + - " VALUES ($1, $2, $3, $4)" - -const selectLastInsertedInviteEventSQL = "" + - "SELECT id FROM syncapi_invite_events WHERE rowid = last_insert_rowid()" + " (id, room_id, event_id, target_user_id, event_json)" + + " VALUES ($1, $2, $3, $4, $5)" const deleteInviteEventSQL = "" + "DELETE FROM syncapi_invite_events WHERE event_id = $1" @@ -57,12 +54,11 @@ const selectMaxInviteIDSQL = "" + "SELECT MAX(id) FROM syncapi_invite_events" type inviteEventsStatements struct { - streamIDStatements *streamIDStatements - insertInviteEventStmt *sql.Stmt - selectLastInsertedInviteEventStmt *sql.Stmt - selectInviteEventsInRangeStmt *sql.Stmt - deleteInviteEventStmt *sql.Stmt - selectMaxInviteIDStmt *sql.Stmt + streamIDStatements *streamIDStatements + insertInviteEventStmt *sql.Stmt + selectInviteEventsInRangeStmt *sql.Stmt + deleteInviteEventStmt *sql.Stmt + selectMaxInviteIDStmt *sql.Stmt } func (s *inviteEventsStatements) prepare(db *sql.DB, streamID *streamIDStatements) (err error) { @@ -74,9 +70,6 @@ func (s *inviteEventsStatements) prepare(db *sql.DB, streamID *streamIDStatement if s.insertInviteEventStmt, err = db.Prepare(insertInviteEventSQL); err != nil { return } - if s.selectLastInsertedInviteEventStmt, err = db.Prepare(selectLastInsertedInviteEventSQL); err != nil { - return - } if s.selectInviteEventsInRangeStmt, err = db.Prepare(selectInviteEventsInRangeSQL); err != nil { return } @@ -90,19 +83,16 @@ func (s *inviteEventsStatements) prepare(db *sql.DB, streamID *streamIDStatement } func (s *inviteEventsStatements) insertInviteEvent( - ctx context.Context, inviteEvent gomatrixserverlib.Event, -) (streamPos types.StreamPosition, err error) { - _, err = s.insertInviteEventStmt.ExecContext( + ctx context.Context, txn *sql.Tx, inviteEvent gomatrixserverlib.Event, streamPos types.StreamPosition, +) (err error) { + _, err = txn.Stmt(s.insertInviteEventStmt).ExecContext( ctx, + streamPos, inviteEvent.RoomID(), inviteEvent.EventID(), *inviteEvent.StateKey(), inviteEvent.JSON(), ) - if err != nil { - return - } - err = s.selectLastInsertedInviteEventStmt.QueryRowContext(ctx).Scan(&streamPos) return } diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index 8c01f2ced..4535688df 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -54,9 +54,6 @@ const insertEventSQL = "" + ") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) " + "ON CONFLICT (event_id) DO UPDATE SET exclude_from_sync = $11" -const selectLastInsertedEventSQL = "" + - "SELECT id FROM syncapi_output_room_events WHERE rowid = last_insert_rowid()" - const selectEventsSQL = "" + "SELECT id, event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events WHERE event_id = $1" @@ -105,7 +102,6 @@ const selectStateInRangeSQL = "" + type outputRoomEventsStatements struct { streamIDStatements *streamIDStatements insertEventStmt *sql.Stmt - selectLastInsertedEventStmt *sql.Stmt selectEventsStmt *sql.Stmt selectMaxEventIDStmt *sql.Stmt selectRecentEventsStmt *sql.Stmt @@ -123,9 +119,6 @@ func (s *outputRoomEventsStatements) prepare(db *sql.DB, streamID *streamIDState if s.insertEventStmt, err = db.Prepare(insertEventSQL); err != nil { return } - if s.selectLastInsertedEventStmt, err = db.Prepare(selectLastInsertedEventSQL); err != nil { - return - } if s.selectEventsStmt, err = db.Prepare(selectEventsSQL); err != nil { return } @@ -270,7 +263,6 @@ func (s *outputRoomEventsStatements) insertEvent( } insertStmt := common.TxStmt(txn, s.insertEventStmt) - selectStmt := common.TxStmt(txn, s.selectLastInsertedEventStmt) _, err = insertStmt.ExecContext( ctx, streamPos, @@ -286,10 +278,6 @@ func (s *outputRoomEventsStatements) insertEvent( txnID, excludeFromSync, ) - if err != nil { - return - } - err = selectStmt.QueryRowContext(ctx).Scan(&streamPos) return } diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index 8cfc1884f..6ad3419c7 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -193,24 +193,20 @@ func (d *SyncServerDatasource) WriteEvent( ctx, txn, ev, addStateEventIDs, removeStateEventIDs, transactionID, excludeFromSync, ) if err != nil { - fmt.Println("d.events.insertEvent:", err) return err } pduPosition = pos if err = d.topology.insertEventInTopology(ctx, txn, ev); err != nil { - fmt.Println("d.topology.insertEventInTopology:", err) return err } if err = d.handleBackwardExtremities(ctx, txn, ev); err != nil { - fmt.Println("d.handleBackwardExtremities:", err) return err } if len(addStateEvents) == 0 && len(removeStateEventIDs) == 0 { // Nothing to do, the event may have just been a message event. - fmt.Println("nothing to do") return nil } @@ -340,8 +336,12 @@ func (d *SyncServerDatasource) GetEventsInRange( } // SyncPosition returns the latest positions for syncing. -func (d *SyncServerDatasource) SyncPosition(ctx context.Context) (types.PaginationToken, error) { - return d.syncPositionTx(ctx, nil) +func (d *SyncServerDatasource) SyncPosition(ctx context.Context) (tok types.PaginationToken, err error) { + err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + tok, err = d.syncPositionTx(ctx, txn) + return err + }) + return } // BackwardExtremitiesForRoom returns the event IDs of all of the backward @@ -380,8 +380,12 @@ func (d *SyncServerDatasource) EventPositionInTopology( } // SyncStreamPosition returns the latest position in the sync stream. Returns 0 if there are no events yet. -func (d *SyncServerDatasource) SyncStreamPosition(ctx context.Context) (types.StreamPosition, error) { - return d.syncStreamPositionTx(ctx, nil) +func (d *SyncServerDatasource) SyncStreamPosition(ctx context.Context) (pos types.StreamPosition, err error) { + err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + pos, err = d.syncStreamPositionTx(ctx, txn) + return err + }) + return } func (d *SyncServerDatasource) syncStreamPositionTx( @@ -625,18 +629,15 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( if err != nil { return } - fmt.Println("Joined rooms:", joinedRoomIDs) stateFilterPart := gomatrixserverlib.DefaultStateFilter() // TODO: use filter provided in request // Build up a /sync response. Add joined rooms. for _, roomID := range joinedRoomIDs { - fmt.Println("WE'RE ON", roomID) var stateEvents []gomatrixserverlib.Event stateEvents, err = d.roomstate.selectCurrentState(ctx, txn, roomID, &stateFilterPart) if err != nil { - fmt.Println("d.roomstate.selectCurrentState:", err) return } //fmt.Println("State events:", stateEvents) @@ -648,7 +649,6 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( numRecentEventsPerRoom, true, true, ) if err != nil { - fmt.Println("d.events.selectRecentEvents:", err) return } //fmt.Println("Recent stream events:", recentStreamEvents) @@ -658,10 +658,9 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( var backwardTopologyPos types.StreamPosition backwardTopologyPos, err = d.topology.selectPositionInTopology(ctx, txn, recentStreamEvents[0].EventID()) if err != nil { - fmt.Println("d.topology.selectPositionInTopology:", err) return nil, types.PaginationToken{}, []string{}, err } - fmt.Println("Backward topology position:", backwardTopologyPos) + if backwardTopologyPos-1 <= 0 { backwardTopologyPos = types.StreamPosition(1) } else { @@ -683,7 +682,6 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( } if err = d.addInvitesToResponse(ctx, txn, userID, 0, toPos.PDUPosition, res); err != nil { - fmt.Println("d.addInvitesToResponse:", err) return } @@ -744,18 +742,10 @@ func (d *SyncServerDatasource) GetAccountDataInRange( func (d *SyncServerDatasource) UpsertAccountData( ctx context.Context, userID, roomID, dataType string, ) (sp types.StreamPosition, err error) { - txn, err := d.db.BeginTx(ctx, nil) - if err != nil { - return types.StreamPosition(0), err - } - var succeeded bool - defer func() { - txerr := common.EndTransaction(txn, &succeeded) - if err == nil && txerr != nil { - err = txerr - } - }() - sp, err = d.accountData.insertAccountData(ctx, txn, userID, roomID, dataType) + err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + sp, err = d.accountData.insertAccountData(ctx, txn, userID, roomID, dataType) + return err + }) return } @@ -764,8 +754,15 @@ func (d *SyncServerDatasource) UpsertAccountData( // Returns an error if there was a problem communicating with the database. func (d *SyncServerDatasource) AddInviteEvent( ctx context.Context, inviteEvent gomatrixserverlib.Event, -) (types.StreamPosition, error) { - return d.invites.insertInviteEvent(ctx, inviteEvent) +) (streamPos types.StreamPosition, err error) { + err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + streamPos, err = d.streamID.nextStreamID(ctx, txn) + if err != nil { + return err + } + return d.invites.insertInviteEvent(ctx, txn, inviteEvent, streamPos) + }) + return } // RetireInviteEvent removes an old invite event from the database. From 0352f250b85721580055f4ba1321c2dda11f0b97 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 20 Feb 2020 13:54:50 +0000 Subject: [PATCH 19/86] Fix dockerfile --- docker/Dockerfile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 29b27dde2..5810825a4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,8 +1,4 @@ -<<<<<<< HEAD FROM docker.io/golang:1.13.7-alpine3.11 -======= -FROM docker.io/golang:1.13.6-alpine ->>>>>>> master RUN mkdir /build From 420ee15433aa2adcbc020875003c3d15c067b03e Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 28 Feb 2020 14:46:03 +0000 Subject: [PATCH 20/86] Kick CI --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 801d0e3ca..49f9ca840 100644 --- a/README.md +++ b/README.md @@ -28,3 +28,4 @@ discussion should happen in There's plenty still to do to make Dendrite usable! We're tracking progress in a [project board](https://github.com/matrix-org/dendrite/projects/2). + From 6460b3725d1d468c8f3353b99797661cfaf539e3 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 28 Feb 2020 14:54:51 +0000 Subject: [PATCH 21/86] Make sure PDUs and EDUs in transaction don't marshal to null (#876) --- federationsender/queue/destinationqueue.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/federationsender/queue/destinationqueue.go b/federationsender/queue/destinationqueue.go index c0afe3be2..f412cb19b 100644 --- a/federationsender/queue/destinationqueue.go +++ b/federationsender/queue/destinationqueue.go @@ -102,7 +102,10 @@ func (oq *destinationQueue) next() *gomatrixserverlib.Transaction { return nil } - var t gomatrixserverlib.Transaction + t := gomatrixserverlib.Transaction{ + PDUs: []gomatrixserverlib.Event{}, + EDUs: []gomatrixserverlib.EDU{}, + } now := gomatrixserverlib.AsTimestamp(time.Now()) t.TransactionID = gomatrixserverlib.TransactionID(fmt.Sprintf("%d-%d", now, oq.sentCounter)) t.Origin = oq.origin From 72565f2eeb8988af90590c90c5c8560186db9d62 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 28 Feb 2020 15:06:16 +0000 Subject: [PATCH 22/86] Fix bug in devices endpoint (#877) --- clientapi/auth/authtypes/device.go | 1 + clientapi/auth/storage/devices/postgres/devices_table.go | 2 +- clientapi/auth/storage/devices/sqlite3/devices_table.go | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/clientapi/auth/authtypes/device.go b/clientapi/auth/authtypes/device.go index 930ab3956..299eff036 100644 --- a/clientapi/auth/authtypes/device.go +++ b/clientapi/auth/authtypes/device.go @@ -26,4 +26,5 @@ type Device struct { // associated with access tokens. SessionID int64 // TODO: display name, last used timestamp, keys, etc + DisplayName string } diff --git a/clientapi/auth/storage/devices/postgres/devices_table.go b/clientapi/auth/storage/devices/postgres/devices_table.go index 349bf1ef7..6cae3743e 100644 --- a/clientapi/auth/storage/devices/postgres/devices_table.go +++ b/clientapi/auth/storage/devices/postgres/devices_table.go @@ -230,7 +230,7 @@ func (s *devicesStatements) selectDevicesByLocalpart( for rows.Next() { var dev authtypes.Device - err = rows.Scan(&dev.ID) + err = rows.Scan(&dev.ID, &dev.DisplayName) if err != nil { return devices, err } diff --git a/clientapi/auth/storage/devices/sqlite3/devices_table.go b/clientapi/auth/storage/devices/sqlite3/devices_table.go index dc88890d3..f69810b7d 100644 --- a/clientapi/auth/storage/devices/sqlite3/devices_table.go +++ b/clientapi/auth/storage/devices/sqlite3/devices_table.go @@ -231,7 +231,7 @@ func (s *devicesStatements) selectDevicesByLocalpart( for rows.Next() { var dev authtypes.Device - err = rows.Scan(&dev.ID) + err = rows.Scan(&dev.ID, &dev.DisplayName) if err != nil { return devices, err } From 59a1f4b8ed94267f4b4ad426192879dc3bef4f1e Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 2 Mar 2020 16:20:44 +0000 Subject: [PATCH 23/86] Remove httputil.LogThenError so that the line numbers are reported properly - make error reporting slightly more useful (#879) --- clientapi/auth/auth.go | 4 +- clientapi/httputil/httputil.go | 8 ---- clientapi/routing/account_data.go | 16 +++++--- clientapi/routing/auth_fallback.go | 7 ++-- clientapi/routing/capabilities.go | 5 ++- clientapi/routing/createroom.go | 21 ++++++---- clientapi/routing/device.go | 40 ++++++++++++------- clientapi/routing/directory.go | 18 ++++++--- clientapi/routing/filter.go | 9 +++-- clientapi/routing/getevent.go | 10 +++-- clientapi/routing/joinroom.go | 39 ++++++++++++------ clientapi/routing/login.go | 3 +- clientapi/routing/logout.go | 14 ++++--- clientapi/routing/membership.go | 9 +++-- clientapi/routing/memberships.go | 4 +- clientapi/routing/profile.go | 51 ++++++++++++++++-------- clientapi/routing/register.go | 12 ++++-- clientapi/routing/room_tagging.go | 21 ++++++---- clientapi/routing/sendevent.go | 9 +++-- clientapi/routing/sendtyping.go | 9 +++-- clientapi/routing/threepid.go | 27 ++++++++----- clientapi/routing/voip.go | 5 ++- federationapi/routing/backfill.go | 7 ++-- federationapi/routing/devices.go | 4 +- federationapi/routing/invite.go | 7 ++-- federationapi/routing/join.go | 16 +++++--- federationapi/routing/leave.go | 16 +++++--- federationapi/routing/missingevents.go | 4 +- federationapi/routing/profile.go | 10 +++-- federationapi/routing/query.go | 10 +++-- federationapi/routing/send.go | 4 +- federationapi/routing/threepid.go | 15 ++++--- publicroomsapi/directory/directory.go | 7 +++- publicroomsapi/directory/public_rooms.go | 12 ++++-- syncapi/routing/messages.go | 7 ++-- syncapi/routing/state.go | 10 +++-- syncapi/sync/requestpool.go | 10 +++-- 37 files changed, 302 insertions(+), 178 deletions(-) diff --git a/clientapi/auth/auth.go b/clientapi/auth/auth.go index f51cfea26..87a2f6677 100644 --- a/clientapi/auth/auth.go +++ b/clientapi/auth/auth.go @@ -26,7 +26,6 @@ import ( "github.com/matrix-org/dendrite/appservice/types" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/userutil" "github.com/matrix-org/dendrite/common/config" @@ -166,7 +165,8 @@ func verifyAccessToken(req *http.Request, deviceDB DeviceDatabase) (device *auth JSON: jsonerror.UnknownToken("Unknown token"), } } else { - jsonErr := httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("deviceDB.GetDeviceByAccessToken failed") + jsonErr := jsonerror.InternalServerError() resErr = &jsonErr } } diff --git a/clientapi/httputil/httputil.go b/clientapi/httputil/httputil.go index 11785f517..b0fe6a6cb 100644 --- a/clientapi/httputil/httputil.go +++ b/clientapi/httputil/httputil.go @@ -36,11 +36,3 @@ func UnmarshalJSONRequest(req *http.Request, iface interface{}) *util.JSONRespon } return nil } - -// LogThenError logs the given error then returns a matrix-compliant 500 internal server error response. -// This should be used to log fatal errors which require investigation. It should not be used -// to log client validation errors, etc. -func LogThenError(req *http.Request, err error) util.JSONResponse { - util.GetLogger(req.Context()).WithError(err).Error("request failed") - return jsonerror.InternalServerError() -} diff --git a/clientapi/routing/account_data.go b/clientapi/routing/account_data.go index 8ae9de2d5..24db41f5f 100644 --- a/clientapi/routing/account_data.go +++ b/clientapi/routing/account_data.go @@ -20,7 +20,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" - "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/gomatrixserverlib" @@ -42,7 +41,8 @@ func GetAccountData( localpart, _, err := gomatrixserverlib.SplitID('@', userID) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") + return jsonerror.InternalServerError() } if data, err := accountDB.GetAccountDataByType( @@ -74,24 +74,28 @@ func SaveAccountData( localpart, _, err := gomatrixserverlib.SplitID('@', userID) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") + return jsonerror.InternalServerError() } defer req.Body.Close() // nolint: errcheck body, err := ioutil.ReadAll(req.Body) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("ioutil.ReadAll failed") + return jsonerror.InternalServerError() } if err := accountDB.SaveAccountData( req.Context(), localpart, roomID, dataType, string(body), ); err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("accountDB.SaveAccountData failed") + return jsonerror.InternalServerError() } if err := syncProducer.SendData(userID, roomID, dataType); err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("syncProducer.SendData failed") + return jsonerror.InternalServerError() } return util.JSONResponse{ diff --git a/clientapi/routing/auth_fallback.go b/clientapi/routing/auth_fallback.go index 5332226c4..8cb6b3d9b 100644 --- a/clientapi/routing/auth_fallback.go +++ b/clientapi/routing/auth_fallback.go @@ -19,7 +19,6 @@ import ( "net/http" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/util" @@ -151,7 +150,8 @@ func AuthFallback( clientIP := req.RemoteAddr err := req.ParseForm() if err != nil { - res := httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("req.ParseForm failed") + res := jsonerror.InternalServerError() return &res } @@ -203,7 +203,8 @@ func writeHTTPMessage( w.WriteHeader(header) _, err := w.Write([]byte(message)) if err != nil { - res := httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("w.Write failed") + res := jsonerror.InternalServerError() return &res } return nil diff --git a/clientapi/routing/capabilities.go b/clientapi/routing/capabilities.go index c8743386f..0c583055e 100644 --- a/clientapi/routing/capabilities.go +++ b/clientapi/routing/capabilities.go @@ -17,7 +17,7 @@ package routing import ( "net/http" - "github.com/matrix-org/dendrite/clientapi/httputil" + "github.com/matrix-org/dendrite/clientapi/jsonerror" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/util" @@ -35,7 +35,8 @@ func GetCapabilities( &roomVersionsQueryReq, &roomVersionsQueryRes, ); err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("queryAPI.QueryRoomVersionCapabilities failed") + return jsonerror.InternalServerError() } response := map[string]interface{}{ diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index 2b1245b9a..c9623acb9 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -193,7 +193,8 @@ func createRoom( profile, err := appserviceAPI.RetrieveUserProfile(req.Context(), userID, asAPI, accountDB) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("appserviceAPI.RetrieveUserProfile failed") + return jsonerror.InternalServerError() } membershipContent := gomatrixserverlib.MemberContent{ @@ -276,7 +277,8 @@ func createRoom( } err = builder.SetContent(e.Content) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("builder.SetContent failed") + return jsonerror.InternalServerError() } if i > 0 { builder.PrevEvents = []gomatrixserverlib.EventReference{builtEvents[i-1].EventReference()} @@ -284,25 +286,29 @@ func createRoom( var ev *gomatrixserverlib.Event ev, err = buildEvent(&builder, &authEvents, cfg, evTime) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("buildEvent failed") + return jsonerror.InternalServerError() } if err = gomatrixserverlib.Allowed(*ev, &authEvents); err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.Allowed failed") + return jsonerror.InternalServerError() } // Add the event to the list of auth events builtEvents = append(builtEvents, *ev) err = authEvents.AddEvent(ev) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("authEvents.AddEvent failed") + return jsonerror.InternalServerError() } } // send events to the room server _, err = producer.SendEvents(req.Context(), builtEvents, cfg.Matrix.ServerName, nil) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("producer.SendEvents failed") + return jsonerror.InternalServerError() } // TODO(#269): Reserve room alias while we create the room. This stops us @@ -321,7 +327,8 @@ func createRoom( var aliasResp roomserverAPI.SetRoomAliasResponse err = aliasAPI.SetRoomAlias(req.Context(), &aliasReq, &aliasResp) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("aliasAPI.SetRoomAlias failed") + return jsonerror.InternalServerError() } if aliasResp.AliasExists { diff --git a/clientapi/routing/device.go b/clientapi/routing/device.go index 9b8647cd4..89c394913 100644 --- a/clientapi/routing/device.go +++ b/clientapi/routing/device.go @@ -21,7 +21,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" - "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -51,7 +50,8 @@ func GetDeviceByID( ) util.JSONResponse { localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") + return jsonerror.InternalServerError() } ctx := req.Context() @@ -62,7 +62,8 @@ func GetDeviceByID( JSON: jsonerror.NotFound("Unknown device"), } } else if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("deviceDB.GetDeviceByID failed") + return jsonerror.InternalServerError() } return util.JSONResponse{ @@ -80,14 +81,16 @@ func GetDevicesByLocalpart( ) util.JSONResponse { localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") + return jsonerror.InternalServerError() } ctx := req.Context() deviceList, err := deviceDB.GetDevicesByLocalpart(ctx, localpart) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("deviceDB.GetDevicesByLocalpart failed") + return jsonerror.InternalServerError() } res := devicesJSON{} @@ -112,7 +115,8 @@ func UpdateDeviceByID( ) util.JSONResponse { localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") + return jsonerror.InternalServerError() } ctx := req.Context() @@ -123,7 +127,8 @@ func UpdateDeviceByID( JSON: jsonerror.NotFound("Unknown device"), } } else if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("deviceDB.GetDeviceByID failed") + return jsonerror.InternalServerError() } if dev.UserID != device.UserID { @@ -138,11 +143,13 @@ func UpdateDeviceByID( payload := deviceUpdateJSON{} if err := json.NewDecoder(req.Body).Decode(&payload); err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("json.NewDecoder.Decode failed") + return jsonerror.InternalServerError() } if err := deviceDB.UpdateDevice(ctx, localpart, deviceID, payload.DisplayName); err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("deviceDB.UpdateDevice failed") + return jsonerror.InternalServerError() } return util.JSONResponse{ @@ -158,14 +165,16 @@ func DeleteDeviceById( ) util.JSONResponse { localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") + return jsonerror.InternalServerError() } ctx := req.Context() defer req.Body.Close() // nolint: errcheck if err := deviceDB.RemoveDevice(ctx, deviceID, localpart); err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("deviceDB.RemoveDevice failed") + return jsonerror.InternalServerError() } return util.JSONResponse{ @@ -180,20 +189,23 @@ func DeleteDevices( ) util.JSONResponse { localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") + return jsonerror.InternalServerError() } ctx := req.Context() payload := devicesDeleteJSON{} if err := json.NewDecoder(req.Body).Decode(&payload); err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("json.NewDecoder.Decode failed") + return jsonerror.InternalServerError() } defer req.Body.Close() // nolint: errcheck if err := deviceDB.RemoveDevices(ctx, localpart, payload.Devices); err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("deviceDB.RemoveDevices failed") + return jsonerror.InternalServerError() } return util.JSONResponse{ diff --git a/clientapi/routing/directory.go b/clientapi/routing/directory.go index 574b275d6..248696ab2 100644 --- a/clientapi/routing/directory.go +++ b/clientapi/routing/directory.go @@ -63,7 +63,8 @@ func DirectoryRoom( queryReq := roomserverAPI.GetRoomIDForAliasRequest{Alias: roomAlias} var queryRes roomserverAPI.GetRoomIDForAliasResponse if err = rsAPI.GetRoomIDForAlias(req.Context(), &queryReq, &queryRes); err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("rsAPI.GetRoomIDForAlias failed") + return jsonerror.InternalServerError() } res.RoomID = queryRes.RoomID @@ -76,7 +77,8 @@ func DirectoryRoom( if fedErr != nil { // TODO: Return 502 if the remote server errored. // TODO: Return 504 if the remote server timed out. - return httputil.LogThenError(req, fedErr) + util.GetLogger(req.Context()).WithError(err).Error("federation.LookupRoomAlias failed") + return jsonerror.InternalServerError() } res.RoomID = fedRes.RoomID res.fillServers(fedRes.Servers) @@ -94,7 +96,8 @@ func DirectoryRoom( joinedHostsReq := federationSenderAPI.QueryJoinedHostServerNamesInRoomRequest{RoomID: res.RoomID} var joinedHostsRes federationSenderAPI.QueryJoinedHostServerNamesInRoomResponse if err = fedSenderAPI.QueryJoinedHostServerNamesInRoom(req.Context(), &joinedHostsReq, &joinedHostsRes); err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("fedSenderAPI.QueryJoinedHostServerNamesInRoom failed") + return jsonerror.InternalServerError() } res.fillServers(joinedHostsRes.ServerNames) } @@ -165,7 +168,8 @@ func SetLocalAlias( } var queryRes roomserverAPI.SetRoomAliasResponse if err := aliasAPI.SetRoomAlias(req.Context(), &queryReq, &queryRes); err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("aliasAPI.SetRoomAlias failed") + return jsonerror.InternalServerError() } if queryRes.AliasExists { @@ -194,7 +198,8 @@ func RemoveLocalAlias( } var creatorQueryRes roomserverAPI.GetCreatorIDForAliasResponse if err := aliasAPI.GetCreatorIDForAlias(req.Context(), &creatorQueryReq, &creatorQueryRes); err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("aliasAPI.GetCreatorIDForAlias failed") + return jsonerror.InternalServerError() } if creatorQueryRes.UserID == "" { @@ -218,7 +223,8 @@ func RemoveLocalAlias( } var queryRes roomserverAPI.RemoveRoomAliasResponse if err := aliasAPI.RemoveRoomAlias(req.Context(), &queryReq, &queryRes); err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("aliasAPI.RemoveRoomAlias failed") + return jsonerror.InternalServerError() } return util.JSONResponse{ diff --git a/clientapi/routing/filter.go b/clientapi/routing/filter.go index 583b23957..505e09279 100644 --- a/clientapi/routing/filter.go +++ b/clientapi/routing/filter.go @@ -37,7 +37,8 @@ func GetFilter( } localpart, _, err := gomatrixserverlib.SplitID('@', userID) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") + return jsonerror.InternalServerError() } filter, err := accountDB.GetFilter(req.Context(), localpart, filterID) @@ -74,7 +75,8 @@ func PutFilter( localpart, _, err := gomatrixserverlib.SplitID('@', userID) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") + return jsonerror.InternalServerError() } var filter gomatrixserverlib.Filter @@ -93,7 +95,8 @@ func PutFilter( filterID, err := accountDB.PutFilter(req.Context(), localpart, &filter) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("accountDB.PutFilter failed") + return jsonerror.InternalServerError() } return util.JSONResponse{ diff --git a/clientapi/routing/getevent.go b/clientapi/routing/getevent.go index 115286bd6..1c1a7add9 100644 --- a/clientapi/routing/getevent.go +++ b/clientapi/routing/getevent.go @@ -18,7 +18,6 @@ import ( "net/http" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/roomserver/api" @@ -55,7 +54,8 @@ func GetEvent( var eventsResp api.QueryEventsByIDResponse err := queryAPI.QueryEventsByID(req.Context(), &eventsReq, &eventsResp) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("queryAPI.QueryEventsByID failed") + return jsonerror.InternalServerError() } if len(eventsResp.Events) == 0 { @@ -89,7 +89,8 @@ func GetEvent( } var stateResp api.QueryStateAfterEventsResponse if err := queryAPI.QueryStateAfterEvents(req.Context(), &stateReq, &stateResp); err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("queryAPI.QueryStateAfterEvents failed") + return jsonerror.InternalServerError() } if !stateResp.RoomExists { @@ -109,7 +110,8 @@ func GetEvent( if stateEvent.StateKeyEquals(r.device.UserID) { membership, err := stateEvent.Membership() if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("stateEvent.Membership failed") + return jsonerror.InternalServerError() } if membership == gomatrixserverlib.Join { return util.JSONResponse{ diff --git a/clientapi/routing/joinroom.go b/clientapi/routing/joinroom.go index 5e6f3e559..de9667e2e 100644 --- a/clientapi/routing/joinroom.go +++ b/clientapi/routing/joinroom.go @@ -62,12 +62,14 @@ func JoinRoomByIDOrAlias( localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") + return jsonerror.InternalServerError() } profile, err := accountDB.GetProfileByLocalpart(req.Context(), localpart) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("accountDB.GetProfileByLocalpart failed") + return jsonerror.InternalServerError() } content["membership"] = gomatrixserverlib.Join @@ -119,7 +121,8 @@ func (r joinRoomReq) joinRoomByID(roomID string) util.JSONResponse { } var queryRes roomserverAPI.QueryInvitesForUserResponse if err := r.queryAPI.QueryInvitesForUser(r.req.Context(), &queryReq, &queryRes); err != nil { - return httputil.LogThenError(r.req, err) + util.GetLogger(r.req.Context()).WithError(err).Error("r.queryAPI.QueryInvitesForUser failed") + return jsonerror.InternalServerError() } servers := []gomatrixserverlib.ServerName{} @@ -127,7 +130,8 @@ func (r joinRoomReq) joinRoomByID(roomID string) util.JSONResponse { for _, userID := range queryRes.InviteSenderUserIDs { _, domain, err := gomatrixserverlib.SplitID('@', userID) if err != nil { - return httputil.LogThenError(r.req, err) + util.GetLogger(r.req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") + return jsonerror.InternalServerError() } if !seenInInviterIDs[domain] { servers = append(servers, domain) @@ -141,7 +145,8 @@ func (r joinRoomReq) joinRoomByID(roomID string) util.JSONResponse { // Note: It's no guarantee we'll succeed because a room isn't bound to the domain in its ID _, domain, err := gomatrixserverlib.SplitID('!', roomID) if err != nil { - return httputil.LogThenError(r.req, err) + util.GetLogger(r.req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") + return jsonerror.InternalServerError() } if domain != r.cfg.Matrix.ServerName && !seenInInviterIDs[domain] { servers = append(servers, domain) @@ -164,7 +169,8 @@ func (r joinRoomReq) joinRoomByAlias(roomAlias string) util.JSONResponse { queryReq := roomserverAPI.GetRoomIDForAliasRequest{Alias: roomAlias} var queryRes roomserverAPI.GetRoomIDForAliasResponse if err = r.aliasAPI.GetRoomIDForAlias(r.req.Context(), &queryReq, &queryRes); err != nil { - return httputil.LogThenError(r.req, err) + util.GetLogger(r.req.Context()).WithError(err).Error("r.aliasAPI.GetRoomIDForAlias failed") + return jsonerror.InternalServerError() } if len(queryRes.RoomID) > 0 { @@ -194,7 +200,8 @@ func (r joinRoomReq) joinRoomByRemoteAlias( } } } - return httputil.LogThenError(r.req, err) + util.GetLogger(r.req.Context()).WithError(err).Error("r.federation.LookupRoomAlias failed") + return jsonerror.InternalServerError() } return r.joinRoomUsingServers(resp.RoomID, resp.Servers) @@ -227,14 +234,16 @@ func (r joinRoomReq) joinRoomUsingServers( var eb gomatrixserverlib.EventBuilder err := r.writeToBuilder(&eb, roomID) if err != nil { - return httputil.LogThenError(r.req, err) + util.GetLogger(r.req.Context()).WithError(err).Error("r.writeToBuilder failed") + return jsonerror.InternalServerError() } var queryRes roomserverAPI.QueryLatestEventsAndStateResponse event, err := common.BuildEvent(r.req.Context(), &eb, r.cfg, r.evTime, r.queryAPI, &queryRes) if err == nil { if _, err = r.producer.SendEvents(r.req.Context(), []gomatrixserverlib.Event{*event}, r.cfg.Matrix.ServerName, nil); err != nil { - return httputil.LogThenError(r.req, err) + util.GetLogger(r.req.Context()).WithError(err).Error("r.producer.SendEvents failed") + return jsonerror.InternalServerError() } return util.JSONResponse{ Code: http.StatusOK, @@ -244,7 +253,8 @@ func (r joinRoomReq) joinRoomUsingServers( } } if err != common.ErrRoomNoExists { - return httputil.LogThenError(r.req, err) + util.GetLogger(r.req.Context()).WithError(err).Error("common.BuildEvent failed") + return jsonerror.InternalServerError() } if len(servers) == 0 { @@ -280,7 +290,8 @@ func (r joinRoomReq) joinRoomUsingServers( // 4) We couldn't fetch the public keys needed to verify the // signatures on the state events. // 5) ... - return httputil.LogThenError(r.req, lastErr) + util.GetLogger(r.req.Context()).WithError(lastErr).Error("failed to join through any server") + return jsonerror.InternalServerError() } // joinRoomUsingServer tries to join a remote room using a given matrix server. @@ -306,7 +317,8 @@ func (r joinRoomReq) joinRoomUsingServer(roomID string, server gomatrixserverlib eventID, r.evTime, r.cfg.Matrix.ServerName, r.cfg.Matrix.KeyID, r.cfg.Matrix.PrivateKey, ) if err != nil { - res := httputil.LogThenError(r.req, err) + util.GetLogger(r.req.Context()).WithError(err).Error("respMakeJoin.JoinEvent.Build failed") + res := jsonerror.InternalServerError() return &res, nil } @@ -322,7 +334,8 @@ func (r joinRoomReq) joinRoomUsingServer(roomID string, server gomatrixserverlib if err = r.producer.SendEventWithState( r.req.Context(), gomatrixserverlib.RespState(respSendJoin.RespState), event, ); err != nil { - res := httputil.LogThenError(r.req, err) + util.GetLogger(r.req.Context()).WithError(err).Error("gomatrixserverlib.RespState failed") + res := jsonerror.InternalServerError() return &res, nil } diff --git a/clientapi/routing/login.go b/clientapi/routing/login.go index b8364ed9d..21b947200 100644 --- a/clientapi/routing/login.go +++ b/clientapi/routing/login.go @@ -122,7 +122,8 @@ func Login( token, err := auth.GenerateAccessToken() if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("auth.GenerateAccessToken failed") + return jsonerror.InternalServerError() } dev, err := getDevice(req.Context(), r, deviceDB, acc, token) diff --git a/clientapi/routing/logout.go b/clientapi/routing/logout.go index 0ac9ca4a5..26b7f117e 100644 --- a/clientapi/routing/logout.go +++ b/clientapi/routing/logout.go @@ -19,7 +19,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" - "github.com/matrix-org/dendrite/clientapi/httputil" + "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -30,11 +30,13 @@ func Logout( ) util.JSONResponse { localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") + return jsonerror.InternalServerError() } if err := deviceDB.RemoveDevice(req.Context(), device.ID, localpart); err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("deviceDB.RemoveDevice failed") + return jsonerror.InternalServerError() } return util.JSONResponse{ @@ -49,11 +51,13 @@ func LogoutAll( ) util.JSONResponse { localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") + return jsonerror.InternalServerError() } if err := deviceDB.RemoveAllDevices(req.Context(), localpart); err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("deviceDB.RemoveAllDevices failed") + return jsonerror.InternalServerError() } return util.JSONResponse{ diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index 68c131a2b..61d9e8e39 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -90,13 +90,15 @@ func SendMembership( JSON: jsonerror.NotFound(err.Error()), } } else if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("buildMembershipEvent failed") + return jsonerror.InternalServerError() } if _, err := producer.SendEvents( req.Context(), []gomatrixserverlib.Event{*event}, cfg.Matrix.ServerName, nil, ); err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("producer.SendEvents failed") + return jsonerror.InternalServerError() } var returnData interface{} = struct{}{} @@ -242,7 +244,8 @@ func checkAndProcessThreepid( JSON: jsonerror.NotFound(err.Error()), } } else if err != nil { - er := httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("threepid.CheckAndProcessInvite failed") + er := jsonerror.InternalServerError() return inviteStored, &er } return diff --git a/clientapi/routing/memberships.go b/clientapi/routing/memberships.go index e6fca505f..a6899eeb8 100644 --- a/clientapi/routing/memberships.go +++ b/clientapi/routing/memberships.go @@ -18,7 +18,6 @@ import ( "net/http" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/roomserver/api" @@ -43,7 +42,8 @@ func GetMemberships( } var queryRes api.QueryMembershipsForRoomResponse if err := queryAPI.QueryMembershipsForRoom(req.Context(), &queryReq, &queryRes); err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("queryAPI.QueryMembershipsForRoom failed") + return jsonerror.InternalServerError() } if !queryRes.HasBeenInRoom { diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index 9b091ddf7..b1909e7ab 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -50,7 +50,8 @@ func GetProfile( } } - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("getProfile failed") + return jsonerror.InternalServerError() } return util.JSONResponse{ @@ -77,7 +78,8 @@ func GetAvatarURL( } } - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("getProfile failed") + return jsonerror.InternalServerError() } return util.JSONResponse{ @@ -116,7 +118,8 @@ func SetAvatarURL( localpart, _, err := gomatrixserverlib.SplitID('@', userID) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") + return jsonerror.InternalServerError() } evTime, err := httputil.ParseTSParam(req) @@ -129,16 +132,19 @@ func SetAvatarURL( oldProfile, err := accountDB.GetProfileByLocalpart(req.Context(), localpart) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("accountDB.GetProfileByLocalpart failed") + return jsonerror.InternalServerError() } if err = accountDB.SetAvatarURL(req.Context(), localpart, r.AvatarURL); err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("accountDB.SetAvatarURL failed") + return jsonerror.InternalServerError() } memberships, err := accountDB.GetMembershipsByLocalpart(req.Context(), localpart) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("accountDB.GetMembershipsByLocalpart failed") + return jsonerror.InternalServerError() } newProfile := authtypes.Profile{ @@ -151,15 +157,18 @@ func SetAvatarURL( req.Context(), memberships, newProfile, userID, cfg, evTime, queryAPI, ) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("buildMembershipEvents failed") + return jsonerror.InternalServerError() } if _, err := rsProducer.SendEvents(req.Context(), events, cfg.Matrix.ServerName, nil); err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("rsProducer.SendEvents failed") + return jsonerror.InternalServerError() } if err := producer.SendUpdate(userID, changedKey, oldProfile.AvatarURL, r.AvatarURL); err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("producer.SendUpdate failed") + return jsonerror.InternalServerError() } return util.JSONResponse{ @@ -183,7 +192,8 @@ func GetDisplayName( } } - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("getProfile failed") + return jsonerror.InternalServerError() } return util.JSONResponse{ @@ -222,7 +232,8 @@ func SetDisplayName( localpart, _, err := gomatrixserverlib.SplitID('@', userID) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") + return jsonerror.InternalServerError() } evTime, err := httputil.ParseTSParam(req) @@ -235,16 +246,19 @@ func SetDisplayName( oldProfile, err := accountDB.GetProfileByLocalpart(req.Context(), localpart) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("accountDB.GetProfileByLocalpart failed") + return jsonerror.InternalServerError() } if err = accountDB.SetDisplayName(req.Context(), localpart, r.DisplayName); err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("accountDB.SetDisplayName failed") + return jsonerror.InternalServerError() } memberships, err := accountDB.GetMembershipsByLocalpart(req.Context(), localpart) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("accountDB.GetMembershipsByLocalpart failed") + return jsonerror.InternalServerError() } newProfile := authtypes.Profile{ @@ -257,15 +271,18 @@ func SetDisplayName( req.Context(), memberships, newProfile, userID, cfg, evTime, queryAPI, ) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("buildMembershipEvents failed") + return jsonerror.InternalServerError() } if _, err := rsProducer.SendEvents(req.Context(), events, cfg.Matrix.ServerName, nil); err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("rsProducer.SendEvents failed") + return jsonerror.InternalServerError() } if err := producer.SendUpdate(userID, changedKey, oldProfile.DisplayName, r.DisplayName); err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("producer.SendUpdate failed") + return jsonerror.InternalServerError() } return util.JSONResponse{ diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index 9d67d9982..6df0b28bc 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -472,7 +472,8 @@ func Register( if r.Username == "" { id, err := accountDB.GetNewNumericLocalpart(req.Context()) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("accountDB.GetNewNumericLocalpart failed") + return jsonerror.InternalServerError() } r.Username = strconv.FormatInt(id, 10) @@ -520,7 +521,8 @@ func handleGuestRegistration( //Generate numeric local part for guest user id, err := accountDB.GetNewNumericLocalpart(req.Context()) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("accountDB.GetNewNumericLocalpart failed") + return jsonerror.InternalServerError() } localpart := strconv.FormatInt(id, 10) @@ -602,7 +604,8 @@ func handleRegistrationFlow( valid, err := isValidMacLogin(cfg, r.Username, r.Password, r.Admin, r.Auth.Mac) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("isValidMacLogin failed") + return jsonerror.InternalServerError() } else if !valid { return util.MessageResponse(http.StatusForbidden, "HMAC incorrect") } @@ -758,7 +761,8 @@ func LegacyRegister( valid, err := isValidMacLogin(cfg, r.Username, r.Password, r.Admin, r.Mac) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("isValidMacLogin failed") + return jsonerror.InternalServerError() } if !valid { diff --git a/clientapi/routing/room_tagging.go b/clientapi/routing/room_tagging.go index aa5f13c4c..5c68668d0 100644 --- a/clientapi/routing/room_tagging.go +++ b/clientapi/routing/room_tagging.go @@ -56,7 +56,8 @@ func GetTags( _, data, err := obtainSavedTags(req, userID, roomID, accountDB) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("obtainSavedTags failed") + return jsonerror.InternalServerError() } if data == nil { @@ -99,20 +100,23 @@ func PutTag( localpart, data, err := obtainSavedTags(req, userID, roomID, accountDB) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("obtainSavedTags failed") + return jsonerror.InternalServerError() } var tagContent gomatrix.TagContent if data != nil { if err = json.Unmarshal(data.Content, &tagContent); err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("json.Unmarshal failed") + return jsonerror.InternalServerError() } } else { tagContent = newTag() } tagContent.Tags[tag] = properties if err = saveTagData(req, localpart, roomID, accountDB, tagContent); err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("saveTagData failed") + return jsonerror.InternalServerError() } // Send data to syncProducer in order to inform clients of changes @@ -151,7 +155,8 @@ func DeleteTag( localpart, data, err := obtainSavedTags(req, userID, roomID, accountDB) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("obtainSavedTags failed") + return jsonerror.InternalServerError() } // If there are no tags in the database, exit @@ -166,7 +171,8 @@ func DeleteTag( var tagContent gomatrix.TagContent err = json.Unmarshal(data.Content, &tagContent) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("json.Unmarshal failed") + return jsonerror.InternalServerError() } // Check whether the tag to be deleted exists @@ -180,7 +186,8 @@ func DeleteTag( } } if err = saveTagData(req, localpart, roomID, accountDB, tagContent); err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("saveTagData failed") + return jsonerror.InternalServerError() } // Send data to syncProducer in order to inform clients of changes diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go index e6de187f2..47ad18825 100644 --- a/clientapi/routing/sendevent.go +++ b/clientapi/routing/sendevent.go @@ -74,7 +74,8 @@ func SendEvent( req.Context(), []gomatrixserverlib.Event{*e}, cfg.Matrix.ServerName, txnAndSessionID, ) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("producer.SendEvents failed") + return jsonerror.InternalServerError() } res := util.JSONResponse{ @@ -121,7 +122,8 @@ func generateSendEvent( } err = builder.SetContent(r) if err != nil { - resErr := httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("builder.SetContent failed") + resErr := jsonerror.InternalServerError() return nil, &resErr } @@ -133,7 +135,8 @@ func generateSendEvent( JSON: jsonerror.NotFound("Room does not exist"), } } else if err != nil { - resErr := httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("common.BuildEvent failed") + resErr := jsonerror.InternalServerError() return nil, &resErr } diff --git a/clientapi/routing/sendtyping.go b/clientapi/routing/sendtyping.go index db3ab28b2..29953c32d 100644 --- a/clientapi/routing/sendtyping.go +++ b/clientapi/routing/sendtyping.go @@ -46,7 +46,8 @@ func SendTyping( localpart, err := userutil.ParseUsernameParam(userID, nil) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("userutil.ParseUsernameParam failed") + return jsonerror.InternalServerError() } // Verify that the user is a member of this room @@ -57,7 +58,8 @@ func SendTyping( JSON: jsonerror.Forbidden("User not in this room"), } } else if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("accountDB.GetMembershipInRoomByLocalPart failed") + return jsonerror.InternalServerError() } // parse the incoming http request @@ -70,7 +72,8 @@ func SendTyping( if err = typingProducer.Send( req.Context(), userID, roomID, r.Typing, r.Timeout, ); err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("typingProducer.Send failed") + return jsonerror.InternalServerError() } return util.JSONResponse{ diff --git a/clientapi/routing/threepid.go b/clientapi/routing/threepid.go index 69383cdf7..fed9ae32e 100644 --- a/clientapi/routing/threepid.go +++ b/clientapi/routing/threepid.go @@ -51,7 +51,8 @@ func RequestEmailToken(req *http.Request, accountDB accounts.Database, cfg *conf // Check if the 3PID is already in use locally localpart, err := accountDB.GetLocalpartForThreePID(req.Context(), body.Email, "email") if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("accountDB.GetLocalpartForThreePID failed") + return jsonerror.InternalServerError() } if len(localpart) > 0 { @@ -71,7 +72,8 @@ func RequestEmailToken(req *http.Request, accountDB accounts.Database, cfg *conf JSON: jsonerror.NotTrusted(body.IDServer), } } else if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("threepid.CreateSession failed") + return jsonerror.InternalServerError() } return util.JSONResponse{ @@ -98,7 +100,8 @@ func CheckAndSave3PIDAssociation( JSON: jsonerror.NotTrusted(body.Creds.IDServer), } } else if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("threepid.CheckAssociation failed") + return jsonerror.InternalServerError() } if !verified { @@ -120,18 +123,21 @@ func CheckAndSave3PIDAssociation( JSON: jsonerror.NotTrusted(body.Creds.IDServer), } } else if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("threepid.PublishAssociation failed") + return jsonerror.InternalServerError() } } // Save the association in the database localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") + return jsonerror.InternalServerError() } if err = accountDB.SaveThreePIDAssociation(req.Context(), address, localpart, medium); err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("accountsDB.SaveThreePIDAssociation failed") + return jsonerror.InternalServerError() } return util.JSONResponse{ @@ -146,12 +152,14 @@ func GetAssociated3PIDs( ) util.JSONResponse { localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") + return jsonerror.InternalServerError() } threepids, err := accountDB.GetThreePIDsForLocalpart(req.Context(), localpart) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("accountDB.GetThreePIDsForLocalpart failed") + return jsonerror.InternalServerError() } return util.JSONResponse{ @@ -168,7 +176,8 @@ func Forget3PID(req *http.Request, accountDB accounts.Database) util.JSONRespons } if err := accountDB.RemoveThreePIDAssociation(req.Context(), body.Address, body.Medium); err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("accountDB.RemoveThreePIDAssociation failed") + return jsonerror.InternalServerError() } return util.JSONResponse{ diff --git a/clientapi/routing/voip.go b/clientapi/routing/voip.go index 872e64473..c1fd741c9 100644 --- a/clientapi/routing/voip.go +++ b/clientapi/routing/voip.go @@ -23,7 +23,7 @@ import ( "time" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/clientapi/httputil" + "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/gomatrix" "github.com/matrix-org/util" @@ -56,7 +56,8 @@ func RequestTurnServer(req *http.Request, device *authtypes.Device, cfg *config. _, err := mac.Write([]byte(resp.Username)) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("mac.Write failed") + return jsonerror.InternalServerError() } resp.Username = fmt.Sprintf("%d:%s", expiry, device.UserID) diff --git a/federationapi/routing/backfill.go b/federationapi/routing/backfill.go index cb388f50d..b74bfe3ac 100644 --- a/federationapi/routing/backfill.go +++ b/federationapi/routing/backfill.go @@ -19,7 +19,6 @@ import ( "strconv" "time" - "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/roomserver/api" @@ -72,12 +71,14 @@ func Backfill( ServerName: request.Origin(), } if req.Limit, err = strconv.Atoi(limit); err != nil { - return httputil.LogThenError(httpReq, err) + util.GetLogger(httpReq.Context()).WithError(err).Error("strconv.Atoi failed") + return jsonerror.InternalServerError() } // Query the roomserver. if err = query.QueryBackfill(httpReq.Context(), &req, &res); err != nil { - return httputil.LogThenError(httpReq, err) + util.GetLogger(httpReq.Context()).WithError(err).Error("query.QueryBackfill failed") + return jsonerror.InternalServerError() } // Filter any event that's not from the requested room out. diff --git a/federationapi/routing/devices.go b/federationapi/routing/devices.go index 78021c12e..01647a61e 100644 --- a/federationapi/routing/devices.go +++ b/federationapi/routing/devices.go @@ -17,7 +17,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" - "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/userutil" "github.com/matrix-org/util" @@ -43,7 +42,8 @@ func GetUserDevices( devs, err := deviceDB.GetDevicesByLocalpart(req.Context(), localpart) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("deviceDB.GetDevicesByLocalPart failed") + return jsonerror.InternalServerError() } return util.JSONResponse{ diff --git a/federationapi/routing/invite.go b/federationapi/routing/invite.go index 9a04a0880..946103462 100644 --- a/federationapi/routing/invite.go +++ b/federationapi/routing/invite.go @@ -18,7 +18,6 @@ import ( "encoding/json" "net/http" - "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/common/config" @@ -78,7 +77,8 @@ func Invite( }} verifyResults, err := keys.VerifyJSONs(httpReq.Context(), verifyRequests) if err != nil { - return httputil.LogThenError(httpReq, err) + util.GetLogger(httpReq.Context()).WithError(err).Error("keys.VerifyJSONs failed") + return jsonerror.InternalServerError() } if verifyResults[0].Error != nil { return util.JSONResponse{ @@ -94,7 +94,8 @@ func Invite( // Add the invite event to the roomserver. if err = producer.SendInvite(httpReq.Context(), signedEvent); err != nil { - return httputil.LogThenError(httpReq, err) + util.GetLogger(httpReq.Context()).WithError(err).Error("producer.SendInvite failed") + return jsonerror.InternalServerError() } // Return the signed event to the originating server, it should then tell diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go index 325b99374..369e19aed 100644 --- a/federationapi/routing/join.go +++ b/federationapi/routing/join.go @@ -19,7 +19,6 @@ import ( "net/http" "time" - "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/common" @@ -60,7 +59,8 @@ func MakeJoin( } err = builder.SetContent(map[string]interface{}{"membership": gomatrixserverlib.Join}) if err != nil { - return httputil.LogThenError(httpReq, err) + util.GetLogger(httpReq.Context()).WithError(err).Error("builder.SetContent failed") + return jsonerror.InternalServerError() } var queryRes api.QueryLatestEventsAndStateResponse @@ -71,7 +71,8 @@ func MakeJoin( JSON: jsonerror.NotFound("Room does not exist"), } } else if err != nil { - return httputil.LogThenError(httpReq, err) + util.GetLogger(httpReq.Context()).WithError(err).Error("common.BuildEvent failed") + return jsonerror.InternalServerError() } // Check that the join is allowed or not @@ -143,7 +144,8 @@ func SendJoin( }} verifyResults, err := keys.VerifyJSONs(httpReq.Context(), verifyRequests) if err != nil { - return httputil.LogThenError(httpReq, err) + util.GetLogger(httpReq.Context()).WithError(err).Error("keys.VerifyJSONs failed") + return jsonerror.InternalServerError() } if verifyResults[0].Error != nil { return util.JSONResponse{ @@ -161,7 +163,8 @@ func SendJoin( RoomID: roomID, }, &stateAndAuthChainResponse) if err != nil { - return httputil.LogThenError(httpReq, err) + util.GetLogger(httpReq.Context()).WithError(err).Error("query.QueryStateAndAuthChain failed") + return jsonerror.InternalServerError() } if !stateAndAuthChainResponse.RoomExists { @@ -178,7 +181,8 @@ func SendJoin( httpReq.Context(), []gomatrixserverlib.Event{event}, cfg.Matrix.ServerName, nil, ) if err != nil { - return httputil.LogThenError(httpReq, err) + util.GetLogger(httpReq.Context()).WithError(err).Error("producer.SendEvents failed") + return jsonerror.InternalServerError() } return util.JSONResponse{ diff --git a/federationapi/routing/leave.go b/federationapi/routing/leave.go index 958158084..f27989ff7 100644 --- a/federationapi/routing/leave.go +++ b/federationapi/routing/leave.go @@ -17,7 +17,6 @@ import ( "net/http" "time" - "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/common" @@ -58,7 +57,8 @@ func MakeLeave( } err = builder.SetContent(map[string]interface{}{"membership": gomatrixserverlib.Leave}) if err != nil { - return httputil.LogThenError(httpReq, err) + util.GetLogger(httpReq.Context()).WithError(err).Error("builder.SetContent failed") + return jsonerror.InternalServerError() } var queryRes api.QueryLatestEventsAndStateResponse @@ -69,7 +69,8 @@ func MakeLeave( JSON: jsonerror.NotFound("Room does not exist"), } } else if err != nil { - return httputil.LogThenError(httpReq, err) + util.GetLogger(httpReq.Context()).WithError(err).Error("common.BuildEvent failed") + return jsonerror.InternalServerError() } // Check that the leave is allowed or not @@ -140,7 +141,8 @@ func SendLeave( }} verifyResults, err := keys.VerifyJSONs(httpReq.Context(), verifyRequests) if err != nil { - return httputil.LogThenError(httpReq, err) + util.GetLogger(httpReq.Context()).WithError(err).Error("keys.VerifyJSONs failed") + return jsonerror.InternalServerError() } if verifyResults[0].Error != nil { return util.JSONResponse{ @@ -152,7 +154,8 @@ func SendLeave( // check membership is set to leave mem, err := event.Membership() if err != nil { - return httputil.LogThenError(httpReq, err) + util.GetLogger(httpReq.Context()).WithError(err).Error("event.Membership failed") + return jsonerror.InternalServerError() } else if mem != gomatrixserverlib.Leave { return util.JSONResponse{ Code: http.StatusBadRequest, @@ -165,7 +168,8 @@ func SendLeave( // the room, so set SendAsServer to cfg.Matrix.ServerName _, err = producer.SendEvents(httpReq.Context(), []gomatrixserverlib.Event{event}, cfg.Matrix.ServerName, nil) if err != nil { - return httputil.LogThenError(httpReq, err) + util.GetLogger(httpReq.Context()).WithError(err).Error("producer.SendEvents failed") + return jsonerror.InternalServerError() } return util.JSONResponse{ diff --git a/federationapi/routing/missingevents.go b/federationapi/routing/missingevents.go index 0165b8a68..661ec33a4 100644 --- a/federationapi/routing/missingevents.go +++ b/federationapi/routing/missingevents.go @@ -16,7 +16,6 @@ import ( "encoding/json" "net/http" - "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" @@ -56,7 +55,8 @@ func GetMissingEvents( }, &eventsResponse, ); err != nil { - return httputil.LogThenError(httpReq, err) + util.GetLogger(httpReq.Context()).WithError(err).Error("query.QueryMissingEvents failed") + return jsonerror.InternalServerError() } eventsResponse.Events = filterEvents(eventsResponse.Events, gme.MinDepth, roomID) diff --git a/federationapi/routing/profile.go b/federationapi/routing/profile.go index 31b7a343f..01a70c01b 100644 --- a/federationapi/routing/profile.go +++ b/federationapi/routing/profile.go @@ -19,7 +19,6 @@ import ( appserviceAPI "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" - "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/config" @@ -46,16 +45,19 @@ func GetProfile( _, domain, err := gomatrixserverlib.SplitID('@', userID) if err != nil { - return httputil.LogThenError(httpReq, err) + util.GetLogger(httpReq.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") + return jsonerror.InternalServerError() } if domain != cfg.Matrix.ServerName { - return httputil.LogThenError(httpReq, err) + util.GetLogger(httpReq.Context()).WithError(err).Error("domain != cfg.Matrix.ServerName failed") + return jsonerror.InternalServerError() } profile, err := appserviceAPI.RetrieveUserProfile(httpReq.Context(), userID, asAPI, accountDB) if err != nil { - return httputil.LogThenError(httpReq, err) + util.GetLogger(httpReq.Context()).WithError(err).Error("appserviceAPI.RetrieveUserProfile failed") + return jsonerror.InternalServerError() } var res interface{} diff --git a/federationapi/routing/query.go b/federationapi/routing/query.go index 5277f0acd..7cb50e525 100644 --- a/federationapi/routing/query.go +++ b/federationapi/routing/query.go @@ -18,7 +18,6 @@ import ( "fmt" "net/http" - "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/common/config" federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" @@ -57,14 +56,16 @@ func RoomAliasToID( queryReq := roomserverAPI.GetRoomIDForAliasRequest{Alias: roomAlias} var queryRes roomserverAPI.GetRoomIDForAliasResponse if err = aliasAPI.GetRoomIDForAlias(httpReq.Context(), &queryReq, &queryRes); err != nil { - return httputil.LogThenError(httpReq, err) + util.GetLogger(httpReq.Context()).WithError(err).Error("aliasAPI.GetRoomIDForAlias failed") + return jsonerror.InternalServerError() } if queryRes.RoomID != "" { serverQueryReq := federationSenderAPI.QueryJoinedHostServerNamesInRoomRequest{RoomID: queryRes.RoomID} var serverQueryRes federationSenderAPI.QueryJoinedHostServerNamesInRoomResponse if err = senderAPI.QueryJoinedHostServerNamesInRoom(httpReq.Context(), &serverQueryReq, &serverQueryRes); err != nil { - return httputil.LogThenError(httpReq, err) + util.GetLogger(httpReq.Context()).WithError(err).Error("senderAPI.QueryJoinedHostServerNamesInRoom failed") + return jsonerror.InternalServerError() } resp = gomatrixserverlib.RespDirectory{ @@ -92,7 +93,8 @@ func RoomAliasToID( } // TODO: Return 502 if the remote server errored. // TODO: Return 504 if the remote server timed out. - return httputil.LogThenError(httpReq, err) + util.GetLogger(httpReq.Context()).WithError(err).Error("federation.LookupRoomAlias failed") + return jsonerror.InternalServerError() } } diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index 5513a088f..191e13ef7 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -20,7 +20,6 @@ import ( "fmt" "net/http" - "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/common/config" @@ -61,7 +60,8 @@ func Send( resp, err := t.processTransaction() if err != nil { - return httputil.LogThenError(httpReq, err) + util.GetLogger(httpReq.Context()).WithError(err).Error("t.processTransaction failed") + return jsonerror.InternalServerError() } return util.JSONResponse{ diff --git a/federationapi/routing/threepid.go b/federationapi/routing/threepid.go index a22685f25..03f3c5bba 100644 --- a/federationapi/routing/threepid.go +++ b/federationapi/routing/threepid.go @@ -74,7 +74,8 @@ func CreateInvitesFrom3PIDInvites( req.Context(), queryAPI, asAPI, cfg, inv, federation, accountDB, ) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("createInviteFrom3PIDInvite failed") + return jsonerror.InternalServerError() } if event != nil { evs = append(evs, *event) @@ -83,7 +84,8 @@ func CreateInvitesFrom3PIDInvites( // Send all the events if _, err := producer.SendEvents(req.Context(), evs, cfg.Matrix.ServerName, nil); err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("producer.SendEvents failed") + return jsonerror.InternalServerError() } return util.JSONResponse{ @@ -143,21 +145,24 @@ func ExchangeThirdPartyInvite( JSON: jsonerror.NotFound("Unknown room " + roomID), } } else if err != nil { - return httputil.LogThenError(httpReq, err) + util.GetLogger(httpReq.Context()).WithError(err).Error("buildMembershipEvent failed") + return jsonerror.InternalServerError() } // Ask the requesting server to sign the newly created event so we know it // acknowledged it signedEvent, err := federation.SendInvite(httpReq.Context(), request.Origin(), *event) if err != nil { - return httputil.LogThenError(httpReq, err) + util.GetLogger(httpReq.Context()).WithError(err).Error("federation.SendInvite failed") + return jsonerror.InternalServerError() } // Send the event to the roomserver if _, err = producer.SendEvents( httpReq.Context(), []gomatrixserverlib.Event{signedEvent.Event}, cfg.Matrix.ServerName, nil, ); err != nil { - return httputil.LogThenError(httpReq, err) + util.GetLogger(httpReq.Context()).WithError(err).Error("producer.SendEvents failed") + return jsonerror.InternalServerError() } return util.JSONResponse{ diff --git a/publicroomsapi/directory/directory.go b/publicroomsapi/directory/directory.go index 889815498..e56fc6cce 100644 --- a/publicroomsapi/directory/directory.go +++ b/publicroomsapi/directory/directory.go @@ -18,6 +18,7 @@ import ( "net/http" "github.com/matrix-org/dendrite/clientapi/httputil" + "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/gomatrixserverlib" @@ -35,7 +36,8 @@ func GetVisibility( ) util.JSONResponse { isPublic, err := publicRoomsDatabase.GetRoomVisibility(req.Context(), roomID) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("publicRoomsDatabase.GetRoomVisibility failed") + return jsonerror.InternalServerError() } var v roomVisibility @@ -64,7 +66,8 @@ func SetVisibility( isPublic := v.Visibility == gomatrixserverlib.Public if err := publicRoomsDatabase.SetRoomVisibility(req.Context(), isPublic, roomID); err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("publicRoomsDatabase.SetRoomVisibility failed") + return jsonerror.InternalServerError() } return util.JSONResponse{ diff --git a/publicroomsapi/directory/public_rooms.go b/publicroomsapi/directory/public_rooms.go index 10aaa0700..fd327942e 100644 --- a/publicroomsapi/directory/public_rooms.go +++ b/publicroomsapi/directory/public_rooms.go @@ -60,11 +60,13 @@ func GetPostPublicRooms( // ParseInt returns 0 and an error when trying to parse an empty string // In that case, we want to assign 0 so we ignore the error if err != nil && len(request.Since) > 0 { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("strconv.ParseInt failed") + return jsonerror.InternalServerError() } if response.Estimate, err = publicRoomDatabase.CountPublicRooms(req.Context()); err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("publicRoomDatabase.CountPublicRooms failed") + return jsonerror.InternalServerError() } if offset > 0 { @@ -78,7 +80,8 @@ func GetPostPublicRooms( if response.Chunk, err = publicRoomDatabase.GetPublicRooms( req.Context(), offset, limit, request.Filter.SearchTerms, ); err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("publicRoomDatabase.GetPublicRooms failed") + return jsonerror.InternalServerError() } return util.JSONResponse{ @@ -96,7 +99,8 @@ func fillPublicRoomsReq(httpReq *http.Request, request *publicRoomReq) *util.JSO // Atoi returns 0 and an error when trying to parse an empty string // In that case, we want to assign 0 so we ignore the error if err != nil && len(httpReq.FormValue("limit")) > 0 { - reqErr := httputil.LogThenError(httpReq, err) + util.GetLogger(httpReq.Context()).WithError(err).Error("strconv.Atoi failed") + reqErr := jsonerror.InternalServerError() return &reqErr } request.Limit = int16(limit) diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index 4fac2ba22..7bbe16f3c 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -20,7 +20,6 @@ import ( "sort" "strconv" - "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/roomserver/api" @@ -104,7 +103,8 @@ func OnIncomingMessagesRequest( // going forward). to, err = setToDefault(req.Context(), db, backwardOrdering, roomID) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("setToDefault failed") + return jsonerror.InternalServerError() } wasToProvided = false } @@ -147,7 +147,8 @@ func OnIncomingMessagesRequest( clientEvents, start, end, err := mReq.retrieveEvents() if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("mreq.retrieveEvents failed") + return jsonerror.InternalServerError() } // Respond with the events. diff --git a/syncapi/routing/state.go b/syncapi/routing/state.go index cf67f7522..5688086cd 100644 --- a/syncapi/routing/state.go +++ b/syncapi/routing/state.go @@ -18,7 +18,6 @@ import ( "encoding/json" "net/http" - "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/types" @@ -49,7 +48,8 @@ func OnIncomingStateRequest(req *http.Request, db storage.Database, roomID strin stateEvents, err := db.GetStateEventsForRoom(req.Context(), roomID, &stateFilter) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("db.GetStateEventsForRoom failed") + return jsonerror.InternalServerError() } resp := []stateEventInStateResp{} @@ -61,7 +61,8 @@ func OnIncomingStateRequest(req *http.Request, db storage.Database, roomID strin var prevEventRef types.PrevEventRef if len(event.Unsigned()) > 0 { if err := json.Unmarshal(event.Unsigned(), &prevEventRef); err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("json.Unmarshal failed") + return jsonerror.InternalServerError() } // Fills the previous state event ID if the state event replaces another // state event @@ -100,7 +101,8 @@ func OnIncomingStateTypeRequest(req *http.Request, db storage.Database, roomID s event, err := db.GetStateEvent(req.Context(), roomID, evType, stateKey) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("db.GetStateEvent failed") + return jsonerror.InternalServerError() } if event == nil { diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index 22bd239fc..b4ccbd272 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -20,7 +20,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" - "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/types" @@ -68,7 +67,8 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *authtype if shouldReturnImmediately(syncReq) { syncData, err = rp.currentSyncForUser(*syncReq, currPos) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("rp.currentSyncForUser failed") + return jsonerror.InternalServerError() } return util.JSONResponse{ Code: http.StatusOK, @@ -107,7 +107,8 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *authtype hasTimedOut = true // Or for the request to be cancelled case <-req.Context().Done(): - return httputil.LogThenError(req, req.Context().Err()) + util.GetLogger(req.Context()).WithError(err).Error("request cancelled") + return jsonerror.InternalServerError() } // Note that we don't time out during calculation of sync @@ -117,7 +118,8 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *authtype syncData, err = rp.currentSyncForUser(*syncReq, currPos) if err != nil { - return httputil.LogThenError(req, err) + util.GetLogger(req.Context()).WithError(err).Error("rp.currentSyncForUser failed") + return jsonerror.InternalServerError() } if !syncData.IsEmpty() || hasTimedOut { From 0cda3c52d0dc0462b204c8dc455762821bb74c3f Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 2 Mar 2020 17:47:39 +0000 Subject: [PATCH 24/86] Don't fail hard if backward topology position fails, just use 1 instead --- syncapi/storage/postgres/syncserver.go | 3 --- syncapi/storage/sqlite3/syncserver.go | 4 ---- 2 files changed, 7 deletions(-) diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go index 6a33a8b4f..abb5b4a4c 100644 --- a/syncapi/storage/postgres/syncserver.go +++ b/syncapi/storage/postgres/syncserver.go @@ -619,9 +619,6 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( // oldest event in the room's topology. var backwardTopologyPos types.StreamPosition backwardTopologyPos, err = d.topology.selectPositionInTopology(ctx, recentStreamEvents[0].EventID()) - if err != nil { - return nil, types.PaginationToken{}, []string{}, err - } if backwardTopologyPos-1 <= 0 { backwardTopologyPos = types.StreamPosition(1) } else { diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index 6ad3419c7..da580e3a2 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -657,10 +657,6 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( // oldest event in the room's topology. var backwardTopologyPos types.StreamPosition backwardTopologyPos, err = d.topology.selectPositionInTopology(ctx, txn, recentStreamEvents[0].EventID()) - if err != nil { - return nil, types.PaginationToken{}, []string{}, err - } - if backwardTopologyPos-1 <= 0 { backwardTopologyPos = types.StreamPosition(1) } else { From a97b8eafd459c6a5c742333e2d0d93385da6db95 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Fri, 6 Mar 2020 10:23:55 +0000 Subject: [PATCH 25/86] Add peer-to-peer support into Dendrite via libp2p and fetch (#880) * Use a fork of pq which supports userCurrent on wasm * Use sqlite3_js driver when running in JS * Add cmd/dendritejs to pull in sqlite3_js driver for wasm only * Update to latest go-sqlite-js version * Replace prometheus with a stub. sigh * Hard-code a config and don't use opentracing * Latest go-sqlite3-js version * Generate a key for now * Listen for fetch traffic rather than HTTP * Latest hacks for js * libp2p support * More libp2p * Fork gjson to allow us to enforce auth checks as before Previously, all events would come down redacted because the hash checks would fail. They would fail because sjson.DeleteBytes didn't remove keys not used for hashing. This didn't work because of a build tag which included a file which no-oped the index returned. See https://github.com/tidwall/gjson/issues/157 When it's resolved, let's go back to mainline. * Use gjson@1.6.0 as it fixes https://github.com/tidwall/gjson/issues/157 * Use latest gomatrixserverlib for sig checks * Fix a bug which could cause exclude_from_sync to not be set Caused when sending events over federation. * Use query variadic to make lookups actually work! * Latest gomatrixserverlib * Add notes on getting p2p up and running Partly so I don't forget myself! * refactor: Move p2p specific stuff to cmd/dendritejs This is important or else the normal build of dendrite will fail because the p2p libraries depend on syscall/js which doesn't work on normal builds. Also, clean up main.go to read a bit better. * Update ho-http-js-libp2p to return errors from RoundTrip * Add an LRU cache around the key DB We actually need this for P2P because otherwise we can *segfault* with things like: "runtime: unexpected return pc for runtime.handleEvent" where the event is a `syscall/js` event, caused by spamming sql.js caused by "Checking event signatures for 14 events of room state" which hammers the key DB repeatedly in quick succession. Using a cache fixes this, though the underlying cause is probably a bug in the version of Go I'm on (1.13.7) * breaking: Add Tracing.Enabled to toggle whether we do opentracing Defaults to false, which is why this is a breaking change. We need this flag because WASM builds cannot do opentracing. * Start adding conditional builds for wasm to handle lib/pq The general idea here is to have the wasm build have a `NewXXXDatabase` that doesn't import any postgres package and hence we never import `lib/pq`, which doesn't work under WASM (undefined `userCurrent`). * Remove lib/pq for wasm for syncapi * Add conditional building to remaining storage APIs * Update build script to set env vars correctly for dendritejs * sqlite bug fixes * Docs * Add a no-op main for dendritejs when not building under wasm * Use the real prometheus, even for WASM Instead, the dendrite-sw.js must mock out `process.pid` and `fs.stat` - which must invoke the callback with an error (e.g `EINVAL`) in order for it to work: ``` global.process = { pid: 1, }; global.fs.stat = function(path, cb) { cb({ code: "EINVAL", }); } ``` * Linting --- appservice/storage/interface.go | 30 ++++ appservice/storage/sqlite3/storage.go | 3 +- appservice/storage/storage.go | 13 +- appservice/storage/storage_wasm.go | 37 ++++ build.sh | 7 +- clientapi/auth/storage/accounts/interface.go | 52 ++++++ .../accounts/sqlite3/membership_table.go | 15 +- .../auth/storage/accounts/sqlite3/storage.go | 2 +- clientapi/auth/storage/accounts/storage.go | 48 ++---- .../auth/storage/accounts/storage_wasm.go | 38 +++++ clientapi/auth/storage/devices/interface.go | 32 ++++ .../auth/storage/devices/sqlite3/storage.go | 2 +- clientapi/auth/storage/devices/storage.go | 29 ++-- .../auth/storage/devices/storage_wasm.go | 38 +++++ clientapi/routing/register.go | 1 - cmd/dendritejs/main.go | 160 ++++++++++++++++++ cmd/dendritejs/main_noop.go | 23 +++ common/basecomponent/base.go | 2 +- common/config/config.go | 17 +- common/keydb/interface.go | 13 ++ common/keydb/keydb.go | 9 +- common/keydb/keydb_wasm.go | 46 +++++ common/keydb/sqlite3/keydb.go | 3 +- common/keydb/sqlite3/server_key_table.go | 40 ++++- common/postgres.go | 25 +++ common/postgres_wasm.go | 22 +++ common/sql.go | 16 +- federationsender/storage/interface.go | 28 +++ federationsender/storage/sqlite3/storage.go | 2 +- federationsender/storage/storage.go | 11 +- federationsender/storage/storage_wasm.go | 38 +++++ go.mod | 37 ++-- go.sum | 63 +++++-- mediaapi/storage/interface.go | 30 ++++ mediaapi/storage/sqlite3/storage.go | 3 +- mediaapi/storage/storage.go | 13 +- mediaapi/storage/storage_wasm.go | 38 +++++ p2p.md | 81 +++++++++ publicroomsapi/storage/interface.go | 33 ++++ .../storage/sqlite3/public_rooms_table.go | 15 +- publicroomsapi/storage/sqlite3/storage.go | 2 +- publicroomsapi/storage/storage.go | 16 +- publicroomsapi/storage/storage_wasm.go | 38 +++++ roomserver/storage/interface.go | 49 ++++++ roomserver/storage/sqlite3/events_table.go | 15 +- roomserver/storage/sqlite3/list.go | 18 -- roomserver/storage/sqlite3/rooms_table.go | 24 +-- .../storage/sqlite3/state_block_table.go | 7 +- .../storage/sqlite3/state_snapshot_table.go | 21 ++- roomserver/storage/sqlite3/storage.go | 2 +- roomserver/storage/storage.go | 32 +--- roomserver/storage/storage_wasm.go | 38 +++++ syncapi/storage/interface.go | 53 ++++++ .../sqlite3/current_room_state_table.go | 9 +- syncapi/storage/sqlite3/filtering.go | 36 ---- .../sqlite3/output_room_events_table.go | 53 ++++-- syncapi/storage/sqlite3/syncserver.go | 5 +- syncapi/storage/storage.go | 36 +--- syncapi/storage/storage_wasm.go | 38 +++++ 59 files changed, 1260 insertions(+), 347 deletions(-) create mode 100644 appservice/storage/interface.go create mode 100644 appservice/storage/storage_wasm.go create mode 100644 clientapi/auth/storage/accounts/interface.go create mode 100644 clientapi/auth/storage/accounts/storage_wasm.go create mode 100644 clientapi/auth/storage/devices/interface.go create mode 100644 clientapi/auth/storage/devices/storage_wasm.go create mode 100644 cmd/dendritejs/main.go create mode 100644 cmd/dendritejs/main_noop.go create mode 100644 common/keydb/interface.go create mode 100644 common/keydb/keydb_wasm.go create mode 100644 common/postgres.go create mode 100644 common/postgres_wasm.go create mode 100644 federationsender/storage/interface.go create mode 100644 federationsender/storage/storage_wasm.go create mode 100644 mediaapi/storage/interface.go create mode 100644 mediaapi/storage/storage_wasm.go create mode 100644 p2p.md create mode 100644 publicroomsapi/storage/interface.go create mode 100644 publicroomsapi/storage/storage_wasm.go create mode 100644 roomserver/storage/interface.go delete mode 100644 roomserver/storage/sqlite3/list.go create mode 100644 roomserver/storage/storage_wasm.go create mode 100644 syncapi/storage/interface.go delete mode 100644 syncapi/storage/sqlite3/filtering.go create mode 100644 syncapi/storage/storage_wasm.go diff --git a/appservice/storage/interface.go b/appservice/storage/interface.go new file mode 100644 index 000000000..4b75ff68e --- /dev/null +++ b/appservice/storage/interface.go @@ -0,0 +1,30 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "context" + + "github.com/matrix-org/gomatrixserverlib" +) + +type Database interface { + StoreEvent(ctx context.Context, appServiceID string, event *gomatrixserverlib.Event) error + GetEventsWithAppServiceID(ctx context.Context, appServiceID string, limit int) (int, int, []gomatrixserverlib.Event, bool, error) + CountEventsWithAppServiceID(ctx context.Context, appServiceID string) (int, error) + UpdateTxnIDForEvents(ctx context.Context, appserviceID string, maxID, txnID int) error + RemoveEventsBeforeAndIncludingID(ctx context.Context, appserviceID string, eventTableID int) error + GetLatestTxnID(ctx context.Context) (int, error) +} diff --git a/appservice/storage/sqlite3/storage.go b/appservice/storage/sqlite3/storage.go index 56ab55f2f..5040b61b2 100644 --- a/appservice/storage/sqlite3/storage.go +++ b/appservice/storage/sqlite3/storage.go @@ -20,6 +20,7 @@ import ( "database/sql" // Import SQLite database driver + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/gomatrixserverlib" _ "github.com/mattn/go-sqlite3" ) @@ -35,7 +36,7 @@ type Database struct { func NewDatabase(dataSourceName string) (*Database, error) { var result Database var err error - if result.db, err = sql.Open("sqlite3", dataSourceName); err != nil { + if result.db, err = sql.Open(common.SQLiteDriverName(), dataSourceName); err != nil { return nil, err } if err = result.prepare(); err != nil { diff --git a/appservice/storage/storage.go b/appservice/storage/storage.go index ce3776bc3..9fbd2a1f3 100644 --- a/appservice/storage/storage.go +++ b/appservice/storage/storage.go @@ -12,26 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build !wasm + package storage import ( - "context" "net/url" "github.com/matrix-org/dendrite/appservice/storage/postgres" "github.com/matrix-org/dendrite/appservice/storage/sqlite3" - "github.com/matrix-org/gomatrixserverlib" ) -type Database interface { - StoreEvent(ctx context.Context, appServiceID string, event *gomatrixserverlib.Event) error - GetEventsWithAppServiceID(ctx context.Context, appServiceID string, limit int) (int, int, []gomatrixserverlib.Event, bool, error) - CountEventsWithAppServiceID(ctx context.Context, appServiceID string) (int, error) - UpdateTxnIDForEvents(ctx context.Context, appserviceID string, maxID, txnID int) error - RemoveEventsBeforeAndIncludingID(ctx context.Context, appserviceID string, eventTableID int) error - GetLatestTxnID(ctx context.Context) (int, error) -} - func NewDatabase(dataSourceName string) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { diff --git a/appservice/storage/storage_wasm.go b/appservice/storage/storage_wasm.go new file mode 100644 index 000000000..2bd1433f9 --- /dev/null +++ b/appservice/storage/storage_wasm.go @@ -0,0 +1,37 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "fmt" + "net/url" + + "github.com/matrix-org/dendrite/appservice/storage/sqlite3" +) + +func NewDatabase(dataSourceName string) (Database, error) { + uri, err := url.Parse(dataSourceName) + if err != nil { + return nil, fmt.Errorf("Cannot use postgres implementation") + } + switch uri.Scheme { + case "postgres": + return nil, fmt.Errorf("Cannot use postgres implementation") + case "file": + return sqlite3.NewDatabase(dataSourceName) + default: + return nil, fmt.Errorf("Cannot use postgres implementation") + } +} diff --git a/build.sh b/build.sh index cb1091114..3ef148891 100755 --- a/build.sh +++ b/build.sh @@ -1,3 +1,6 @@ -#!/bin/sh +#!/bin/bash -eu -GOBIN=$PWD/`dirname $0`/bin go install -v $PWD/`dirname $0`/cmd/... +# Put installed packages into ./bin +export GOBIN=$PWD/`dirname $0`/bin + +go install -v $PWD/`dirname $0`/cmd/... \ No newline at end of file diff --git a/clientapi/auth/storage/accounts/interface.go b/clientapi/auth/storage/accounts/interface.go new file mode 100644 index 000000000..83d3ee725 --- /dev/null +++ b/clientapi/auth/storage/accounts/interface.go @@ -0,0 +1,52 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package accounts + +import ( + "context" + "errors" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/gomatrixserverlib" +) + +type Database interface { + common.PartitionStorer + GetAccountByPassword(ctx context.Context, localpart, plaintextPassword string) (*authtypes.Account, error) + GetProfileByLocalpart(ctx context.Context, localpart string) (*authtypes.Profile, error) + SetAvatarURL(ctx context.Context, localpart string, avatarURL string) error + SetDisplayName(ctx context.Context, localpart string, displayName string) error + CreateAccount(ctx context.Context, localpart, plaintextPassword, appserviceID string) (*authtypes.Account, error) + UpdateMemberships(ctx context.Context, eventsToAdd []gomatrixserverlib.Event, idsToRemove []string) error + GetMembershipInRoomByLocalpart(ctx context.Context, localpart, roomID string) (authtypes.Membership, error) + GetMembershipsByLocalpart(ctx context.Context, localpart string) (memberships []authtypes.Membership, err error) + SaveAccountData(ctx context.Context, localpart, roomID, dataType, content string) error + GetAccountData(ctx context.Context, localpart string) (global []gomatrixserverlib.ClientEvent, rooms map[string][]gomatrixserverlib.ClientEvent, err error) + GetAccountDataByType(ctx context.Context, localpart, roomID, dataType string) (data *gomatrixserverlib.ClientEvent, err error) + GetNewNumericLocalpart(ctx context.Context) (int64, error) + SaveThreePIDAssociation(ctx context.Context, threepid, localpart, medium string) (err error) + RemoveThreePIDAssociation(ctx context.Context, threepid string, medium string) (err error) + GetLocalpartForThreePID(ctx context.Context, threepid string, medium string) (localpart string, err error) + GetThreePIDsForLocalpart(ctx context.Context, localpart string) (threepids []authtypes.ThreePID, err error) + GetFilter(ctx context.Context, localpart string, filterID string) (*gomatrixserverlib.Filter, error) + PutFilter(ctx context.Context, localpart string, filter *gomatrixserverlib.Filter) (string, error) + CheckAccountAvailability(ctx context.Context, localpart string) (bool, error) + GetAccountByLocalpart(ctx context.Context, localpart string) (*authtypes.Account, error) +} + +// Err3PIDInUse is the error returned when trying to save an association involving +// a third-party identifier which is already associated to a local user. +var Err3PIDInUse = errors.New("This third-party identifier is already in use") diff --git a/clientapi/auth/storage/accounts/sqlite3/membership_table.go b/clientapi/auth/storage/accounts/sqlite3/membership_table.go index 8e5e69bad..38f21b7f3 100644 --- a/clientapi/auth/storage/accounts/sqlite3/membership_table.go +++ b/clientapi/auth/storage/accounts/sqlite3/membership_table.go @@ -17,9 +17,10 @@ package sqlite3 import ( "context" "database/sql" + "strings" - "github.com/lib/pq" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/common" ) const membershipSchema = ` @@ -54,7 +55,6 @@ const deleteMembershipsByEventIDsSQL = "" + "DELETE FROM account_memberships WHERE event_id IN ($1)" type membershipStatements struct { - deleteMembershipsByEventIDsStmt *sql.Stmt insertMembershipStmt *sql.Stmt selectMembershipInRoomByLocalpartStmt *sql.Stmt selectMembershipsByLocalpartStmt *sql.Stmt @@ -65,9 +65,6 @@ func (s *membershipStatements) prepare(db *sql.DB) (err error) { if err != nil { return } - if s.deleteMembershipsByEventIDsStmt, err = db.Prepare(deleteMembershipsByEventIDsSQL); err != nil { - return - } if s.insertMembershipStmt, err = db.Prepare(insertMembershipSQL); err != nil { return } @@ -91,8 +88,12 @@ func (s *membershipStatements) insertMembership( func (s *membershipStatements) deleteMembershipsByEventIDs( ctx context.Context, txn *sql.Tx, eventIDs []string, ) (err error) { - stmt := txn.Stmt(s.deleteMembershipsByEventIDsStmt) - _, err = stmt.ExecContext(ctx, pq.StringArray(eventIDs)) + sqlStr := strings.Replace(deleteMembershipsByEventIDsSQL, "($1)", common.QueryVariadic(len(eventIDs)), 1) + iEventIDs := make([]interface{}, len(eventIDs)) + for i, e := range eventIDs { + iEventIDs[i] = e + } + _, err = txn.ExecContext(ctx, sqlStr, iEventIDs...) return } diff --git a/clientapi/auth/storage/accounts/sqlite3/storage.go b/clientapi/auth/storage/accounts/sqlite3/storage.go index 199c4606e..3e62d10dd 100644 --- a/clientapi/auth/storage/accounts/sqlite3/storage.go +++ b/clientapi/auth/storage/accounts/sqlite3/storage.go @@ -45,7 +45,7 @@ type Database struct { func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) { var db *sql.DB var err error - if db, err = sql.Open("sqlite3", dataSourceName); err != nil { + if db, err = sql.Open(common.SQLiteDriverName(), dataSourceName); err != nil { return nil, err } partitions := common.PartitionOffsetStatements{} diff --git a/clientapi/auth/storage/accounts/storage.go b/clientapi/auth/storage/accounts/storage.go index 1dfd5f1f4..c643a4d0a 100644 --- a/clientapi/auth/storage/accounts/storage.go +++ b/clientapi/auth/storage/accounts/storage.go @@ -1,41 +1,29 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !wasm + package accounts import ( - "context" - "errors" "net/url" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/postgres" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/sqlite3" - "github.com/matrix-org/dendrite/common" "github.com/matrix-org/gomatrixserverlib" ) -type Database interface { - common.PartitionStorer - GetAccountByPassword(ctx context.Context, localpart, plaintextPassword string) (*authtypes.Account, error) - GetProfileByLocalpart(ctx context.Context, localpart string) (*authtypes.Profile, error) - SetAvatarURL(ctx context.Context, localpart string, avatarURL string) error - SetDisplayName(ctx context.Context, localpart string, displayName string) error - CreateAccount(ctx context.Context, localpart, plaintextPassword, appserviceID string) (*authtypes.Account, error) - UpdateMemberships(ctx context.Context, eventsToAdd []gomatrixserverlib.Event, idsToRemove []string) error - GetMembershipInRoomByLocalpart(ctx context.Context, localpart, roomID string) (authtypes.Membership, error) - GetMembershipsByLocalpart(ctx context.Context, localpart string) (memberships []authtypes.Membership, err error) - SaveAccountData(ctx context.Context, localpart, roomID, dataType, content string) error - GetAccountData(ctx context.Context, localpart string) (global []gomatrixserverlib.ClientEvent, rooms map[string][]gomatrixserverlib.ClientEvent, err error) - GetAccountDataByType(ctx context.Context, localpart, roomID, dataType string) (data *gomatrixserverlib.ClientEvent, err error) - GetNewNumericLocalpart(ctx context.Context) (int64, error) - SaveThreePIDAssociation(ctx context.Context, threepid, localpart, medium string) (err error) - RemoveThreePIDAssociation(ctx context.Context, threepid string, medium string) (err error) - GetLocalpartForThreePID(ctx context.Context, threepid string, medium string) (localpart string, err error) - GetThreePIDsForLocalpart(ctx context.Context, localpart string) (threepids []authtypes.ThreePID, err error) - GetFilter(ctx context.Context, localpart string, filterID string) (*gomatrixserverlib.Filter, error) - PutFilter(ctx context.Context, localpart string, filter *gomatrixserverlib.Filter) (string, error) - CheckAccountAvailability(ctx context.Context, localpart string) (bool, error) - GetAccountByLocalpart(ctx context.Context, localpart string) (*authtypes.Account, error) -} - func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { @@ -50,7 +38,3 @@ func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) return postgres.NewDatabase(dataSourceName, serverName) } } - -// Err3PIDInUse is the error returned when trying to save an association involving -// a third-party identifier which is already associated to a local user. -var Err3PIDInUse = errors.New("This third-party identifier is already in use") diff --git a/clientapi/auth/storage/accounts/storage_wasm.go b/clientapi/auth/storage/accounts/storage_wasm.go new file mode 100644 index 000000000..828afc6b4 --- /dev/null +++ b/clientapi/auth/storage/accounts/storage_wasm.go @@ -0,0 +1,38 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package accounts + +import ( + "fmt" + "net/url" + + "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/sqlite3" + "github.com/matrix-org/gomatrixserverlib" +) + +func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (Database, error) { + uri, err := url.Parse(dataSourceName) + if err != nil { + return nil, fmt.Errorf("Cannot use postgres implementation") + } + switch uri.Scheme { + case "postgres": + return nil, fmt.Errorf("Cannot use postgres implementation") + case "file": + return sqlite3.NewDatabase(dataSourceName, serverName) + default: + return nil, fmt.Errorf("Cannot use postgres implementation") + } +} diff --git a/clientapi/auth/storage/devices/interface.go b/clientapi/auth/storage/devices/interface.go new file mode 100644 index 000000000..95291e4a7 --- /dev/null +++ b/clientapi/auth/storage/devices/interface.go @@ -0,0 +1,32 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package devices + +import ( + "context" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" +) + +type Database interface { + GetDeviceByAccessToken(ctx context.Context, token string) (*authtypes.Device, error) + GetDeviceByID(ctx context.Context, localpart, deviceID string) (*authtypes.Device, error) + GetDevicesByLocalpart(ctx context.Context, localpart string) ([]authtypes.Device, error) + CreateDevice(ctx context.Context, localpart string, deviceID *string, accessToken string, displayName *string) (dev *authtypes.Device, returnErr error) + UpdateDevice(ctx context.Context, localpart, deviceID string, displayName *string) error + RemoveDevice(ctx context.Context, deviceID, localpart string) error + RemoveDevices(ctx context.Context, localpart string, devices []string) error + RemoveAllDevices(ctx context.Context, localpart string) error +} diff --git a/clientapi/auth/storage/devices/sqlite3/storage.go b/clientapi/auth/storage/devices/sqlite3/storage.go index e1ce6f00d..62c30322e 100644 --- a/clientapi/auth/storage/devices/sqlite3/storage.go +++ b/clientapi/auth/storage/devices/sqlite3/storage.go @@ -40,7 +40,7 @@ type Database struct { func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) { var db *sql.DB var err error - if db, err = sql.Open("sqlite3", dataSourceName); err != nil { + if db, err = sql.Open(common.SQLiteDriverName(), dataSourceName); err != nil { return nil, err } d := devicesStatements{} diff --git a/clientapi/auth/storage/devices/storage.go b/clientapi/auth/storage/devices/storage.go index 82f756401..99211db28 100644 --- a/clientapi/auth/storage/devices/storage.go +++ b/clientapi/auth/storage/devices/storage.go @@ -1,26 +1,29 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !wasm + package devices import ( - "context" "net/url" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices/postgres" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices/sqlite3" "github.com/matrix-org/gomatrixserverlib" ) -type Database interface { - GetDeviceByAccessToken(ctx context.Context, token string) (*authtypes.Device, error) - GetDeviceByID(ctx context.Context, localpart, deviceID string) (*authtypes.Device, error) - GetDevicesByLocalpart(ctx context.Context, localpart string) ([]authtypes.Device, error) - CreateDevice(ctx context.Context, localpart string, deviceID *string, accessToken string, displayName *string) (dev *authtypes.Device, returnErr error) - UpdateDevice(ctx context.Context, localpart, deviceID string, displayName *string) error - RemoveDevice(ctx context.Context, deviceID, localpart string) error - RemoveDevices(ctx context.Context, localpart string, devices []string) error - RemoveAllDevices(ctx context.Context, localpart string) error -} - func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { diff --git a/clientapi/auth/storage/devices/storage_wasm.go b/clientapi/auth/storage/devices/storage_wasm.go new file mode 100644 index 000000000..322852888 --- /dev/null +++ b/clientapi/auth/storage/devices/storage_wasm.go @@ -0,0 +1,38 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package devices + +import ( + "fmt" + "net/url" + + "github.com/matrix-org/dendrite/clientapi/auth/storage/devices/sqlite3" + "github.com/matrix-org/gomatrixserverlib" +) + +func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (Database, error) { + uri, err := url.Parse(dataSourceName) + if err != nil { + return nil, fmt.Errorf("Cannot use postgres implementation") + } + switch uri.Scheme { + case "postgres": + return nil, fmt.Errorf("Cannot use postgres implementation") + case "file": + return sqlite3.NewDatabase(dataSourceName, serverName) + default: + return nil, fmt.Errorf("Cannot use postgres implementation") + } +} diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index 6df0b28bc..ba24e5273 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -444,7 +444,6 @@ func Register( deviceDB devices.Database, cfg *config.Dendrite, ) util.JSONResponse { - var r registerRequest resErr := httputil.UnmarshalJSONRequest(req, &r) if resErr != nil { diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go new file mode 100644 index 000000000..347a52835 --- /dev/null +++ b/cmd/dendritejs/main.go @@ -0,0 +1,160 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build wasm + +package main + +import ( + "crypto/ed25519" + "fmt" + "net/http" + + "github.com/matrix-org/dendrite/appservice" + "github.com/matrix-org/dendrite/clientapi" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/common/basecomponent" + "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/common/keydb" + "github.com/matrix-org/dendrite/common/transactions" + "github.com/matrix-org/dendrite/federationapi" + "github.com/matrix-org/dendrite/federationsender" + "github.com/matrix-org/dendrite/mediaapi" + "github.com/matrix-org/dendrite/publicroomsapi" + "github.com/matrix-org/dendrite/roomserver" + "github.com/matrix-org/dendrite/syncapi" + "github.com/matrix-org/dendrite/typingserver" + "github.com/matrix-org/dendrite/typingserver/cache" + "github.com/matrix-org/go-http-js-libp2p/go_http_js_libp2p" + "github.com/matrix-org/gomatrixserverlib" + + "github.com/sirupsen/logrus" + + _ "github.com/matrix-org/go-sqlite3-js" +) + +func init() { + fmt.Println("dendrite.js starting...") +} + +func generateKey() ed25519.PrivateKey { + _, priv, err := ed25519.GenerateKey(nil) + if err != nil { + logrus.Fatalf("Failed to generate ed25519 key: %s", err) + } + return priv +} + +func createFederationClient(cfg *config.Dendrite, node *go_http_js_libp2p.P2pLocalNode) *gomatrixserverlib.FederationClient { + fmt.Println("Running in js-libp2p federation mode") + fmt.Println("Warning: Federation with non-libp2p homeservers will not work in this mode yet!") + tr := go_http_js_libp2p.NewP2pTransport(node) + + fed := gomatrixserverlib.NewFederationClient( + cfg.Matrix.ServerName, cfg.Matrix.KeyID, cfg.Matrix.PrivateKey, + ) + fed.Client = *gomatrixserverlib.NewClientWithTransport(tr) + + return fed +} + +func createP2PNode() (serverName string, node *go_http_js_libp2p.P2pLocalNode) { + hosted := "/dns4/ws-star.discovery.libp2p.io/tcp/443/wss/p2p-websocket-star" + _ = "/ip4/127.0.0.1/tcp/9090/ws/p2p-websocket-star/" + node = go_http_js_libp2p.NewP2pLocalNode("org.matrix.p2p.experiment", []string{hosted}) + serverName = node.Id + fmt.Println("p2p assigned ServerName: ", serverName) + return +} + +func main() { + cfg := &config.Dendrite{} + cfg.SetDefaults() + cfg.Kafka.UseNaffka = true + cfg.Database.Account = "file:dendritejs_account.db" + cfg.Database.AppService = "file:dendritejs_appservice.db" + cfg.Database.Device = "file:dendritejs_device.db" + cfg.Database.FederationSender = "file:dendritejs_fedsender.db" + cfg.Database.MediaAPI = "file:dendritejs_mediaapi.db" + cfg.Database.Naffka = "file:dendritejs_naffka.db" + cfg.Database.PublicRoomsAPI = "file:dendritejs_publicrooms.db" + cfg.Database.RoomServer = "file:dendritejs_roomserver.db" + cfg.Database.ServerKey = "file:dendritejs_serverkey.db" + cfg.Database.SyncAPI = "file:dendritejs_syncapi.db" + cfg.Matrix.TrustedIDServers = []string{ + "matrix.org", "vector.im", + } + cfg.Matrix.KeyID = "ed25519:1337" + cfg.Matrix.PrivateKey = generateKey() + + serverName, node := createP2PNode() + cfg.Matrix.ServerName = gomatrixserverlib.ServerName(serverName) + + if err := cfg.Derive(); err != nil { + logrus.Fatalf("Failed to derive values from config: %s", err) + } + base := basecomponent.NewBaseDendrite(cfg, "Monolith") + defer base.Close() // nolint: errcheck + + accountDB := base.CreateAccountsDB() + deviceDB := base.CreateDeviceDB() + keyDB := base.CreateKeyDB() + federation := createFederationClient(cfg, node) + keyRing := keydb.CreateKeyRing(federation.Client, keyDB) + + alias, input, query := roomserver.SetupRoomServerComponent(base) + typingInputAPI := typingserver.SetupTypingServerComponent(base, cache.NewTypingCache()) + asQuery := appservice.SetupAppServiceAPIComponent( + base, accountDB, deviceDB, federation, alias, query, transactions.New(), + ) + fedSenderAPI := federationsender.SetupFederationSenderComponent(base, federation, query) + + clientapi.SetupClientAPIComponent( + base, deviceDB, accountDB, + federation, &keyRing, alias, input, query, + typingInputAPI, asQuery, transactions.New(), fedSenderAPI, + ) + federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fedSenderAPI) + mediaapi.SetupMediaAPIComponent(base, deviceDB) + publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, query) + syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query, federation, cfg) + + httpHandler := common.WrapHandlerInCORS(base.APIMux) + + http.Handle("/", httpHandler) + + // Expose the matrix APIs via libp2p-js - for federation traffic + if node != nil { + go func() { + logrus.Info("Listening on libp2p-js host ID ", node.Id) + + listener := go_http_js_libp2p.NewP2pListener(node) + defer listener.Close() + s := &http.Server{} + s.Serve(listener) + }() + } + + // Expose the matrix APIs via fetch - for local traffic + go func() { + logrus.Info("Listening for service-worker fetch traffic") + + listener := go_http_js_libp2p.NewFetchListener() + s := &http.Server{} + go s.Serve(listener) + }() + + // We want to block forever to let the fetch and libp2p handler serve the APIs + select {} +} diff --git a/cmd/dendritejs/main_noop.go b/cmd/dendritejs/main_noop.go new file mode 100644 index 000000000..dcea032f2 --- /dev/null +++ b/cmd/dendritejs/main_noop.go @@ -0,0 +1,23 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !wasm + +package main + +import "fmt" + +func main() { + fmt.Println("dendritejs: no-op when not compiling for WebAssembly") +} diff --git a/common/basecomponent/base.go b/common/basecomponent/base.go index 4274de2b6..d1d953f7b 100644 --- a/common/basecomponent/base.go +++ b/common/basecomponent/base.go @@ -216,7 +216,7 @@ func setupNaffka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) { uri, err := url.Parse(string(cfg.Database.Naffka)) if err != nil || uri.Scheme == "file" { - db, err = sql.Open("sqlite3", string(cfg.Database.Naffka)) + db, err = sql.Open(common.SQLiteDriverName(), string(cfg.Database.Naffka)) if err != nil { logrus.WithError(err).Panic("Failed to open naffka database") } diff --git a/common/config/config.go b/common/config/config.go index 0332d0358..bd83cbf8b 100644 --- a/common/config/config.go +++ b/common/config/config.go @@ -224,6 +224,8 @@ type Dendrite struct { // The config for tracing the dendrite servers. Tracing struct { + // Set to true to enable tracer hooks. If false, no tracing is set up. + Enabled bool `yaml:"enabled"` // The config for the jaeger opentracing reporter. Jaeger jaegerconfig.Configuration `yaml:"jaeger"` } `yaml:"tracing"` @@ -365,7 +367,7 @@ func loadConfig( return nil, err } - config.setDefaults() + config.SetDefaults() if err = config.check(monolithic); err != nil { return nil, err @@ -398,7 +400,7 @@ func loadConfig( config.Media.AbsBasePath = Path(absPath(basePath, config.Media.BasePath)) // Generate data from config options - err = config.derive() + err = config.Derive() if err != nil { return nil, err } @@ -406,9 +408,9 @@ func loadConfig( return &config, nil } -// derive generates data that is derived from various values provided in +// Derive generates data that is derived from various values provided in // the config file. -func (config *Dendrite) derive() error { +func (config *Dendrite) Derive() error { // Determine registrations flows based off config values config.Derived.Registration.Params = make(map[string]interface{}) @@ -433,8 +435,8 @@ func (config *Dendrite) derive() error { return nil } -// setDefaults sets default config values if they are not explicitly set. -func (config *Dendrite) setDefaults() { +// SetDefaults sets default config values if they are not explicitly set. +func (config *Dendrite) SetDefaults() { if config.Matrix.KeyValidityPeriod == 0 { config.Matrix.KeyValidityPeriod = 24 * time.Hour } @@ -703,6 +705,9 @@ func (config *Dendrite) FederationSenderURL() string { // SetupTracing configures the opentracing using the supplied configuration. func (config *Dendrite) SetupTracing(serviceName string) (closer io.Closer, err error) { + if !config.Tracing.Enabled { + return ioutil.NopCloser(bytes.NewReader([]byte{})), nil + } return config.Tracing.Jaeger.InitGlobalTracer( serviceName, jaegerconfig.Logger(logrusLogger{logrus.StandardLogger()}), diff --git a/common/keydb/interface.go b/common/keydb/interface.go new file mode 100644 index 000000000..c9a20fdd9 --- /dev/null +++ b/common/keydb/interface.go @@ -0,0 +1,13 @@ +package keydb + +import ( + "context" + + "github.com/matrix-org/gomatrixserverlib" +) + +type Database interface { + FetcherName() string + FetchKeys(ctx context.Context, requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp) (map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, error) + StoreKeys(ctx context.Context, keyMap map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult) error +} diff --git a/common/keydb/keydb.go b/common/keydb/keydb.go index cf15c9f0c..fe6d87fc8 100644 --- a/common/keydb/keydb.go +++ b/common/keydb/keydb.go @@ -12,10 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build !wasm + package keydb import ( - "context" "net/url" "golang.org/x/crypto/ed25519" @@ -25,12 +26,6 @@ import ( "github.com/matrix-org/gomatrixserverlib" ) -type Database interface { - FetcherName() string - FetchKeys(ctx context.Context, requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp) (map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, error) - StoreKeys(ctx context.Context, keyMap map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult) error -} - // NewDatabase opens a database connection. func NewDatabase( dataSourceName string, diff --git a/common/keydb/keydb_wasm.go b/common/keydb/keydb_wasm.go new file mode 100644 index 000000000..807ed40b4 --- /dev/null +++ b/common/keydb/keydb_wasm.go @@ -0,0 +1,46 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package keydb + +import ( + "fmt" + "net/url" + + "golang.org/x/crypto/ed25519" + + "github.com/matrix-org/dendrite/common/keydb/sqlite3" + "github.com/matrix-org/gomatrixserverlib" +) + +// NewDatabase opens a database connection. +func NewDatabase( + dataSourceName string, + serverName gomatrixserverlib.ServerName, + serverKey ed25519.PublicKey, + serverKeyID gomatrixserverlib.KeyID, +) (Database, error) { + uri, err := url.Parse(dataSourceName) + if err != nil { + return nil, err + } + switch uri.Scheme { + case "postgres": + return nil, fmt.Errorf("Cannot use postgres implementation") + case "file": + return sqlite3.NewDatabase(dataSourceName, serverName, serverKey, serverKeyID) + default: + return nil, fmt.Errorf("Cannot use postgres implementation") + } +} diff --git a/common/keydb/sqlite3/keydb.go b/common/keydb/sqlite3/keydb.go index 88eb9d9fa..3c860d0c3 100644 --- a/common/keydb/sqlite3/keydb.go +++ b/common/keydb/sqlite3/keydb.go @@ -22,6 +22,7 @@ import ( "golang.org/x/crypto/ed25519" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/gomatrixserverlib" _ "github.com/mattn/go-sqlite3" @@ -43,7 +44,7 @@ func NewDatabase( serverKey ed25519.PublicKey, serverKeyID gomatrixserverlib.KeyID, ) (*Database, error) { - db, err := sql.Open("sqlite3", dataSourceName) + db, err := sql.Open(common.SQLiteDriverName(), dataSourceName) if err != nil { return nil, err } diff --git a/common/keydb/sqlite3/server_key_table.go b/common/keydb/sqlite3/server_key_table.go index 6c33f30a0..3caf4ea4c 100644 --- a/common/keydb/sqlite3/server_key_table.go +++ b/common/keydb/sqlite3/server_key_table.go @@ -18,9 +18,12 @@ package sqlite3 import ( "context" "database/sql" + "strings" - "github.com/lib/pq" + lru "github.com/hashicorp/golang-lru" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" ) const serverKeysSchema = ` @@ -60,11 +63,19 @@ const upsertServerKeysSQL = "" + " DO UPDATE SET valid_until_ts = $4, expired_ts = $5, server_key = $6" type serverKeyStatements struct { + db *sql.DB bulkSelectServerKeysStmt *sql.Stmt upsertServerKeysStmt *sql.Stmt + + cache *lru.Cache // nameAndKeyID => gomatrixserverlib.PublicKeyLookupResult } func (s *serverKeyStatements) prepare(db *sql.DB) (err error) { + s.db = db + s.cache, err = lru.New(64) + if err != nil { + return + } _, err = db.Exec(serverKeysSchema) if err != nil { return @@ -86,8 +97,30 @@ func (s *serverKeyStatements) bulkSelectServerKeys( for request := range requests { nameAndKeyIDs = append(nameAndKeyIDs, nameAndKeyID(request)) } - stmt := s.bulkSelectServerKeysStmt - rows, err := stmt.QueryContext(ctx, pq.StringArray(nameAndKeyIDs)) + + // If we can satisfy all of the requests from the cache, do so. TODO: Allow partial matches with merges. + cacheResults := map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult{} + for request := range requests { + r, ok := s.cache.Get(nameAndKeyID(request)) + if !ok { + break + } + cacheResult := r.(gomatrixserverlib.PublicKeyLookupResult) + cacheResults[request] = cacheResult + } + if len(cacheResults) == len(requests) { + util.GetLogger(ctx).Infof("KeyDB cache hit for %d keys", len(cacheResults)) + return cacheResults, nil + } + + query := strings.Replace(bulkSelectServerKeysSQL, "($1)", common.QueryVariadic(len(nameAndKeyIDs)), 1) + + iKeyIDs := make([]interface{}, len(nameAndKeyIDs)) + for i, v := range nameAndKeyIDs { + iKeyIDs[i] = v + } + + rows, err := s.db.QueryContext(ctx, query, iKeyIDs...) if err != nil { return nil, err } @@ -125,6 +158,7 @@ func (s *serverKeyStatements) upsertServerKeys( request gomatrixserverlib.PublicKeyLookupRequest, key gomatrixserverlib.PublicKeyLookupResult, ) error { + s.cache.Add(nameAndKeyID(request), key) _, err := s.upsertServerKeysStmt.ExecContext( ctx, string(request.ServerName), diff --git a/common/postgres.go b/common/postgres.go new file mode 100644 index 000000000..f8daf5783 --- /dev/null +++ b/common/postgres.go @@ -0,0 +1,25 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !wasm + +package common + +import "github.com/lib/pq" + +// IsUniqueConstraintViolationErr returns true if the error is a postgresql unique_violation error +func IsUniqueConstraintViolationErr(err error) bool { + pqErr, ok := err.(*pq.Error) + return ok && pqErr.Code == "23505" +} diff --git a/common/postgres_wasm.go b/common/postgres_wasm.go new file mode 100644 index 000000000..dcc07b31d --- /dev/null +++ b/common/postgres_wasm.go @@ -0,0 +1,22 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build wasm + +package common + +// IsUniqueConstraintViolationErr no-ops for this architecture +func IsUniqueConstraintViolationErr(err error) bool { + return false +} diff --git a/common/sql.go b/common/sql.go index 975930202..f50a58969 100644 --- a/common/sql.go +++ b/common/sql.go @@ -17,8 +17,7 @@ package common import ( "database/sql" "fmt" - - "github.com/lib/pq" + "runtime" ) // A Transaction is something that can be committed or rolledback. @@ -77,12 +76,6 @@ func TxStmt(transaction *sql.Tx, statement *sql.Stmt) *sql.Stmt { return statement } -// IsUniqueConstraintViolationErr returns true if the error is a postgresql unique_violation error -func IsUniqueConstraintViolationErr(err error) bool { - pqErr, ok := err.(*pq.Error) - return ok && pqErr.Code == "23505" -} - // Hack of the century func QueryVariadic(count int) string { return QueryVariadicOffset(count, 0) @@ -99,3 +92,10 @@ func QueryVariadicOffset(count, offset int) string { str += ")" return str } + +func SQLiteDriverName() string { + if runtime.GOOS == "js" { + return "sqlite3_js" + } + return "sqlite3" +} diff --git a/federationsender/storage/interface.go b/federationsender/storage/interface.go new file mode 100644 index 000000000..ae2956475 --- /dev/null +++ b/federationsender/storage/interface.go @@ -0,0 +1,28 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "context" + + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/federationsender/types" +) + +type Database interface { + common.PartitionStorer + UpdateRoom(ctx context.Context, roomID, oldEventID, newEventID string, addHosts []types.JoinedHost, removeHosts []string) (joinedHosts []types.JoinedHost, err error) + GetJoinedHosts(ctx context.Context, roomID string) ([]types.JoinedHost, error) +} diff --git a/federationsender/storage/sqlite3/storage.go b/federationsender/storage/sqlite3/storage.go index f9cfaa99d..6ab9ae00a 100644 --- a/federationsender/storage/sqlite3/storage.go +++ b/federationsender/storage/sqlite3/storage.go @@ -37,7 +37,7 @@ type Database struct { func NewDatabase(dataSourceName string) (*Database, error) { var result Database var err error - if result.db, err = sql.Open("sqlite3", dataSourceName); err != nil { + if result.db, err = sql.Open(common.SQLiteDriverName(), dataSourceName); err != nil { return nil, err } if err = result.prepare(); err != nil { diff --git a/federationsender/storage/storage.go b/federationsender/storage/storage.go index e83c1e9d2..2f018dff1 100644 --- a/federationsender/storage/storage.go +++ b/federationsender/storage/storage.go @@ -12,24 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build !wasm + package storage import ( - "context" "net/url" - "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/federationsender/storage/postgres" "github.com/matrix-org/dendrite/federationsender/storage/sqlite3" - "github.com/matrix-org/dendrite/federationsender/types" ) -type Database interface { - common.PartitionStorer - UpdateRoom(ctx context.Context, roomID, oldEventID, newEventID string, addHosts []types.JoinedHost, removeHosts []string) (joinedHosts []types.JoinedHost, err error) - GetJoinedHosts(ctx context.Context, roomID string) ([]types.JoinedHost, error) -} - // NewDatabase opens a new database func NewDatabase(dataSourceName string) (Database, error) { uri, err := url.Parse(dataSourceName) diff --git a/federationsender/storage/storage_wasm.go b/federationsender/storage/storage_wasm.go new file mode 100644 index 000000000..f2c8ae1b4 --- /dev/null +++ b/federationsender/storage/storage_wasm.go @@ -0,0 +1,38 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "fmt" + "net/url" + + "github.com/matrix-org/dendrite/federationsender/storage/sqlite3" +) + +// NewDatabase opens a new database +func NewDatabase(dataSourceName string) (Database, error) { + uri, err := url.Parse(dataSourceName) + if err != nil { + return nil, fmt.Errorf("Cannot use postgres implementation") + } + switch uri.Scheme { + case "file": + return sqlite3.NewDatabase(dataSourceName) + case "postgres": + return nil, fmt.Errorf("Cannot use postgres implementation") + default: + return nil, fmt.Errorf("Cannot use postgres implementation") + } +} diff --git a/go.mod b/go.mod index 2d442cd0d..7d59aeb1b 100644 --- a/go.mod +++ b/go.mod @@ -1,43 +1,30 @@ module github.com/matrix-org/dendrite require ( - github.com/DataDog/zstd v1.4.4 // indirect - github.com/Shopify/toxiproxy v2.1.4+incompatible // indirect - github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect - github.com/eapache/go-resiliency v1.2.0 // indirect - github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect - github.com/eapache/queue v1.1.0 // indirect - github.com/frankban/quicktest v1.7.2 // indirect - github.com/golang/snappy v0.0.1 // indirect github.com/gorilla/mux v1.7.3 - github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect + github.com/hashicorp/golang-lru v0.5.4 github.com/lib/pq v1.2.0 github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 + github.com/matrix-org/go-http-js-libp2p v0.0.0-20200304160008-4ec1129a00c4 + github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 + github.com/matrix-org/gomatrixserverlib v0.0.0-20200304110715-894c3c86ce3e github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 - github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 + github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible - github.com/miekg/dns v1.1.12 // indirect github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 - github.com/opentracing/opentracing-go v1.0.2 - github.com/pierrec/lz4 v2.4.1+incompatible // indirect + github.com/opentracing/opentracing-go v1.1.0 github.com/pkg/errors v0.8.1 - github.com/prometheus/client_golang v1.2.1 - github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 // indirect + github.com/prometheus/client_golang v1.4.1 github.com/sirupsen/logrus v1.4.2 - github.com/stretchr/testify v1.4.0 // indirect - github.com/uber-go/atomic v1.3.0 // indirect - github.com/uber/jaeger-client-go v2.15.0+incompatible - github.com/uber/jaeger-lib v1.5.0 - go.uber.org/atomic v1.3.0 // indirect + github.com/tidwall/gjson v1.6.0 + github.com/tidwall/pretty v1.0.1 // indirect + github.com/uber/jaeger-client-go v2.22.1+incompatible + github.com/uber/jaeger-lib v2.2.0+incompatible golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 - golang.org/x/net v0.0.0-20190620200207-3b0461eec859 // indirect - golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 // indirect gopkg.in/Shopify/sarama.v1 v1.20.1 - gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect gopkg.in/h2non/bimg.v1 v1.0.18 - gopkg.in/yaml.v2 v2.2.2 + gopkg.in/yaml.v2 v2.2.5 ) go 1.13 diff --git a/go.sum b/go.sum index 7c8732f63..19dc9662c 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,9 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/cespare/xxhash/v2 v2.1.0 h1:yTUvW7Vhb89inJ+8irsUqiWjh8iT6sQPZiQzI6ReGkA= github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= @@ -40,17 +41,20 @@ github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= +github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= @@ -63,18 +67,37 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/matrix-org/dendrite v0.0.0-20200220135450-0352f250b857/go.mod h1:DZ35IoR+ViBNVPe9umdlOSnjvKl7wfyRmZg4QfWGvTo= github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 h1:nMX2t7hbGF0NYDYySx0pCqEKGKAeZIiSqlWSspetlhY= github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg= +github.com/matrix-org/go-http-js-libp2p v0.0.0-20200225225149-e7191ca90a94 h1:gUqPXKWbuwmPnx3PJBxtQw5Ff8V229wZ7Ee0GaH/bg4= +github.com/matrix-org/go-http-js-libp2p v0.0.0-20200225225149-e7191ca90a94/go.mod h1:PJof7UbOKmVEEMsGqvbyIK0ldMMBjPH5EYia7MHR2RQ= +github.com/matrix-org/go-http-js-libp2p v0.0.0-20200304160008-4ec1129a00c4 h1:oFjG7X1jS473zPDix1/FBZ2qd0anM1Ko4AlJCB6MUZs= +github.com/matrix-org/go-http-js-libp2p v0.0.0-20200304160008-4ec1129a00c4/go.mod h1:PJof7UbOKmVEEMsGqvbyIK0ldMMBjPH5EYia7MHR2RQ= +github.com/matrix-org/go-sqlite3-js v0.0.0-20200226144546-ea6ed5b90074 h1:UWz6vfhmQVshBuE67X1BCsdMhEDtd+uOz8CJ48Fc0F4= +github.com/matrix-org/go-sqlite3-js v0.0.0-20200226144546-ea6ed5b90074/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= +github.com/matrix-org/go-sqlite3-js v0.0.0-20200304163011-cfb4884075db h1:ERuFJq4DI8fakfBZlvXHltHZ0ix3K5YsLG0tQfQn6TI= +github.com/matrix-org/go-sqlite3-js v0.0.0-20200304163011-cfb4884075db/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= +github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658 h1:UlhTKClOgWnSB25Rv+BS/Vc1mRinjNUErfyGEVOBP04= +github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20190130130140-385f072fe9af h1:piaIBNQGIHnni27xRB7VKkEwoWCgAmeuYf8pxAyG0bI= github.com/matrix-org/gomatrix v0.0.0-20190130130140-385f072fe9af/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bhrnp3Ky1qgx/fzCtCALOoGYylh2tpS9K4= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:kmRjpmFOenVpOaV/DRlo9p6z/IbOKlUC+hhKsAAh8Qg= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200228131708-347eb77397b8 h1:a9IV2iKMznKJ16MmcG/NU7qMcZ4jIKmPXC6RkOZZq+Q= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200228131708-347eb77397b8/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200304110715-894c3c86ce3e h1:DA1lP2mB2ddd2PhMOaNPwRJFi/3aL2Lj7bxnAhFxBFQ= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200304110715-894c3c86ce3e/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= +github.com/matrix-org/pq v1.3.2 h1:7Kh2Qz4xonRH8OcFpCIj2najpBxF2+j1ff0hC082L68= +github.com/matrix-org/pq v1.3.2/go.mod h1:l6tPTzDjcj8fhD5OD0+7dejtZrwyjJadAAYM+CntUVQ= github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 h1:W7l5CP4V7wPyPb4tYE11dbmeAOwtFQBTW0rf4OonOS8= github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5/go.mod h1:lePuOiXLNDott7NZfnQvJk0lAZ5HgvIuWGhel6J+RLA= +github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= +github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/mattn/go-sqlite3 v2.0.2+incompatible h1:qzw9c2GNT8UFrgWNDhCTqRqYUSmu/Dav/9Z58LGpk7U= github.com/mattn/go-sqlite3 v2.0.2+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= @@ -94,6 +117,8 @@ github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 h1:BvoENQQU+fZ9uukda/R github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pierrec/lz4 v2.4.1+incompatible h1:mFe7ttWaflA46Mhqh+jUfjp2qTbPYxLB2/OyBppH9dg= github.com/pierrec/lz4 v2.4.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -103,19 +128,23 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.2.1 h1:JnMpQc6ppsNgw9QPAGF6Dod479itz7lvlsMzzNayLOI= github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U= +github.com/prometheus/client_golang v1.4.1 h1:FFSuS004yOQEtDdTq+TAOLP5xUq63KqAFYyOi8zA+Y8= +github.com/prometheus/client_golang v1.4.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= +github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8= github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -126,22 +155,30 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/tidwall/gjson v1.1.5 h1:QysILxBeUEY3GTLA0fQVgkQG1zme8NxGvhh2SSqWNwI= github.com/tidwall/gjson v1.1.5/go.mod h1:c/nTNbUr0E0OrXEhq1pwa8iEgc2DOt4ZZqAt1HtCkPA= +github.com/tidwall/gjson v1.6.0 h1:9VEQWz6LLMUsUl6PueE49ir4Ka6CzLymOAZDxpFsTDc= +github.com/tidwall/gjson v1.6.0/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tidwall/pretty v1.0.1 h1:WE4RBSZ1x6McVVC8S/Md+Qse8YUv6HRObAx6ke00NY8= +github.com/tidwall/pretty v1.0.1/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/sjson v1.0.3 h1:DeF+0LZqvIt4fKYw41aPB29ZGlvwVkHKktoXJ1YW9Y8= github.com/tidwall/sjson v1.0.3/go.mod h1:bURseu1nuBkFpIES5cz6zBtjmYeOQmEESshn7VpF15Y= github.com/uber-go/atomic v1.3.0 h1:ylWoWcs+jXihgo3Us1Sdsatf2R6+OlBGm8fexR3oFG4= github.com/uber-go/atomic v1.3.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= github.com/uber/jaeger-client-go v2.15.0+incompatible h1:NP3qsSqNxh8VYr956ur1N/1C1PjvOJnJykCzcD5QHbk= github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= +github.com/uber/jaeger-client-go v2.22.1+incompatible h1:NHcubEkVbahf9t3p75TOCR83gdUHXjRJvjoBh1yACsM= +github.com/uber/jaeger-client-go v2.22.1+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v1.5.0 h1:OHbgr8l656Ub3Fw5k9SWnBfIEwvoHQ+W2y+Aa9D1Uyo= github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= +github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw= +github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= go.uber.org/atomic v1.3.0 h1:vs7fgriifsPbGdK3bNuMWapNn3qnZhCRXc19NRdq010= go.uber.org/atomic v1.3.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -156,24 +193,26 @@ golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190301231341-16b79f2e4e95 h1:fY7Dsw114eJN4boqzVSbpVHO6rTdhq6/GnXeu+PKnzU= golang.org/x/net v0.0.0-20190301231341-16b79f2e4e95/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0Is3p+EHBKNgquIzo1OI= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 h1:I6FyU15t786LL7oL/hn43zqTuEGr4PN7F4XJ1p4E3Y8= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY= golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 h1:ywK/j/KkyTHcdyYSZNXGjMwgmDSfjglYZ3vStQ/gSCU= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/Shopify/sarama.v1 v1.20.1 h1:Gi09A3fJXm0Jgt8kuKZ8YK+r60GfYn7MQuEmI3oq6hE= gopkg.in/Shopify/sarama.v1 v1.20.1/go.mod h1:AxnvoaevB2nBjNK17cG61A3LleFcWFwVBHBt+cot4Oc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -181,6 +220,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/h2non/bimg.v1 v1.0.18 h1:qn6/RpBHt+7WQqoBcK+aF2puc6nC78eZj5LexxoalT4= gopkg.in/h2non/bimg.v1 v1.0.18/go.mod h1:PgsZL7dLwUbsGm1NYps320GxGgvQNTnecMCZqxV11So= gopkg.in/h2non/gock.v1 v1.0.14 h1:fTeu9fcUvSnLNacYvYI54h+1/XEteDyHvrVCZEEEYNM= @@ -190,3 +230,6 @@ gopkg.in/macaroon.v2 v2.1.0/go.mod h1:OUb+TQP/OP0WOerC2Jp/3CwhIKyIa9kQjuc7H24e6/ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/mediaapi/storage/interface.go b/mediaapi/storage/interface.go new file mode 100644 index 000000000..672e8ef54 --- /dev/null +++ b/mediaapi/storage/interface.go @@ -0,0 +1,30 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "context" + + "github.com/matrix-org/dendrite/mediaapi/types" + "github.com/matrix-org/gomatrixserverlib" +) + +type Database interface { + StoreMediaMetadata(ctx context.Context, mediaMetadata *types.MediaMetadata) error + GetMediaMetadata(ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName) (*types.MediaMetadata, error) + StoreThumbnail(ctx context.Context, thumbnailMetadata *types.ThumbnailMetadata) error + GetThumbnail(ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName, width, height int, resizeMethod string) (*types.ThumbnailMetadata, error) + GetThumbnails(ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName) ([]*types.ThumbnailMetadata, error) +} diff --git a/mediaapi/storage/sqlite3/storage.go b/mediaapi/storage/sqlite3/storage.go index 6477f8305..cfa5bb986 100644 --- a/mediaapi/storage/sqlite3/storage.go +++ b/mediaapi/storage/sqlite3/storage.go @@ -20,6 +20,7 @@ import ( "database/sql" // Import the postgres database driver. + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/mediaapi/types" "github.com/matrix-org/gomatrixserverlib" _ "github.com/mattn/go-sqlite3" @@ -35,7 +36,7 @@ type Database struct { func Open(dataSourceName string) (*Database, error) { var d Database var err error - if d.db, err = sql.Open("sqlite3", dataSourceName); err != nil { + if d.db, err = sql.Open(common.SQLiteDriverName(), dataSourceName); err != nil { return nil, err } if err = d.statements.prepare(d.db); err != nil { diff --git a/mediaapi/storage/storage.go b/mediaapi/storage/storage.go index 1d35a95ed..c533477cd 100644 --- a/mediaapi/storage/storage.go +++ b/mediaapi/storage/storage.go @@ -12,26 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build !wasm + package storage import ( - "context" "net/url" "github.com/matrix-org/dendrite/mediaapi/storage/postgres" "github.com/matrix-org/dendrite/mediaapi/storage/sqlite3" - "github.com/matrix-org/dendrite/mediaapi/types" - "github.com/matrix-org/gomatrixserverlib" ) -type Database interface { - StoreMediaMetadata(ctx context.Context, mediaMetadata *types.MediaMetadata) error - GetMediaMetadata(ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName) (*types.MediaMetadata, error) - StoreThumbnail(ctx context.Context, thumbnailMetadata *types.ThumbnailMetadata) error - GetThumbnail(ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName, width, height int, resizeMethod string) (*types.ThumbnailMetadata, error) - GetThumbnails(ctx context.Context, mediaID types.MediaID, mediaOrigin gomatrixserverlib.ServerName) ([]*types.ThumbnailMetadata, error) -} - // Open opens a postgres database. func Open(dataSourceName string) (Database, error) { uri, err := url.Parse(dataSourceName) diff --git a/mediaapi/storage/storage_wasm.go b/mediaapi/storage/storage_wasm.go new file mode 100644 index 000000000..92f0ad134 --- /dev/null +++ b/mediaapi/storage/storage_wasm.go @@ -0,0 +1,38 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "fmt" + "net/url" + + "github.com/matrix-org/dendrite/mediaapi/storage/sqlite3" +) + +// Open opens a postgres database. +func Open(dataSourceName string) (Database, error) { + uri, err := url.Parse(dataSourceName) + if err != nil { + return nil, fmt.Errorf("Cannot use postgres implementation") + } + switch uri.Scheme { + case "postgres": + return nil, fmt.Errorf("Cannot use postgres implementation") + case "file": + return sqlite3.Open(dataSourceName) + default: + return nil, fmt.Errorf("Cannot use postgres implementation") + } +} diff --git a/p2p.md b/p2p.md new file mode 100644 index 000000000..0c3dc83bb --- /dev/null +++ b/p2p.md @@ -0,0 +1,81 @@ +## Peer-to-peer Matrix + +These are the instructions for setting up P2P Dendrite, current as of March 2020. There's both Go stuff and JS stuff to do to set this up. + + +### Dendrite + +- The `master` branch has a WASM-only binary for dendrite: `./cmd/dendritejs`. +- Build it and copy assets to riot-web. +``` +$ GOOS=js GOARCH=wasm go build -o main.wasm ./cmd/dendritejs +$ cp main.wasm ../riot-web/src/vector/dendrite.wasm +``` + +### Rendezvous + +This is how peers discover each other and communicate. + +By default, Dendrite uses the IPFS-hosted websocket star **Development** relay server at `/dns4/ws-star.discovery.libp2p.io/tcp/443/wss/p2p-websocket-star`. +This is currently hard-coded in `./cmd/dendritejs/main.go` - you can also use a local one if you run your own relay: + +``` +$ npm install --global libp2p-websocket-star-rendezvous +$ rendezvous --port=9090 --host=127.0.0.1 +``` + +Then use `/ip4/127.0.0.1/tcp/9090/ws/p2p-websocket-star/`. We'll probably run our own relay server at some point. + +### Riot-web + +You need to check out these repos: + +`` +$ git clone git@github.com:matrix-org/go-http-js-libp2p.git +$ git clone git@github.com:matrix-org/go-sqlite3-js.git +``` + +Make sure to `yarn install` in both of these repos. Then: + +- `$ cp "$(go env GOROOT)/misc/wasm/wasm_exec.js" ./src/vector/` +- Comment out the lines in `wasm_exec.js` which contains: +``` +if (!global.fs && global.require) { + global.fs = require("fs"); +} +``` +- Add the diff at https://github.com/vector-im/riot-web/compare/matthew/p2p?expand=1 - ignore the `package.json` stuff. +- Add the following symlinks: they HAVE to be symlinks as the diff in `webpack.config.js` references specific paths. +``` +$ cd node_modules +$ ln -s ../../go-sqlite-js # NB: NOT go-sqlite3-js +$ ln -s ../../go-http-js-libp2p +``` + +NB: If you don't run the server with `yarn start` you need to make sure your server is sending the header `Service-Worker-Allowed: /`. + +TODO: Make a Docker image with all of this in it and a volume mount for `dendrite.wasm`. + +## Running + +You need a Chrome and a Firefox running to test locally as service workers don't work in incognito tabs. +- For Chrome, use `chrome://serviceworker-internals/` to unregister/see logs. +- For Firefox, use `about:debugging#/runtime/this-firefox` to unregister. Use the console window to see logs. + +Assuming you've `yarn start`ed Riot-Web, go to `http://localhost:8080` and wait a bit. Then refresh the page (this is required +because the fetch interceptor races with setting up dendrite. If you don't refresh, you won't be able to contact your HS). After +the refresh, click Register and use `http://localhost:8080` as your HS URL. + +TODO: Fix the race so we don't need multiple refreshes. + +You can join rooms by room alias e.g `/join #foo:bar`. + +### Known issues + +- When registering you may be unable to find the server, it'll seem flakey. This happens because the SW, particularly in Firefox, + gets killed after 30s of inactivity. When you are not registered, you aren't doing `/sync` calls to keep the SW alive, so if you + don't register for a while and idle on the page, the HS will disappear. To fix, unregister the SW, and then refresh the page *twice*. + +- The libp2p layer has rate limits, so frequent Federation traffic may cause the connection to drop and messages to not be transferred. + I guess in other words, don't send too much traffic? + diff --git a/publicroomsapi/storage/interface.go b/publicroomsapi/storage/interface.go new file mode 100644 index 000000000..3f7e65892 --- /dev/null +++ b/publicroomsapi/storage/interface.go @@ -0,0 +1,33 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "context" + + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/publicroomsapi/types" + "github.com/matrix-org/gomatrixserverlib" +) + +type Database interface { + common.PartitionStorer + GetRoomVisibility(ctx context.Context, roomID string) (bool, error) + SetRoomVisibility(ctx context.Context, visible bool, roomID string) error + CountPublicRooms(ctx context.Context) (int64, error) + GetPublicRooms(ctx context.Context, offset int64, limit int16, filter string) ([]types.PublicRoom, error) + UpdateRoomFromEvents(ctx context.Context, eventsToAdd []gomatrixserverlib.Event, eventsToRemove []gomatrixserverlib.Event) error + UpdateRoomFromEvent(ctx context.Context, event gomatrixserverlib.Event) error +} diff --git a/publicroomsapi/storage/sqlite3/public_rooms_table.go b/publicroomsapi/storage/sqlite3/public_rooms_table.go index 3fe3d3cab..beba0d694 100644 --- a/publicroomsapi/storage/sqlite3/public_rooms_table.go +++ b/publicroomsapi/storage/sqlite3/public_rooms_table.go @@ -18,10 +18,10 @@ package sqlite3 import ( "context" "database/sql" + "encoding/json" "errors" "fmt" - "github.com/lib/pq" "github.com/matrix-org/dendrite/publicroomsapi/types" ) @@ -196,17 +196,21 @@ func (s *publicRoomsStatements) selectPublicRooms( rooms := []types.PublicRoom{} for rows.Next() { var r types.PublicRoom - var aliases pq.StringArray + var aliasesJSON string err = rows.Scan( - &r.RoomID, &r.NumJoinedMembers, &aliases, &r.CanonicalAlias, + &r.RoomID, &r.NumJoinedMembers, &aliasesJSON, &r.CanonicalAlias, &r.Name, &r.Topic, &r.WorldReadable, &r.GuestCanJoin, &r.AvatarURL, ) if err != nil { return rooms, err } - r.Aliases = aliases + if len(aliasesJSON) > 0 { + if err := json.Unmarshal([]byte(aliasesJSON), &r.Aliases); err != nil { + return rooms, err + } + } rooms = append(rooms, r) } @@ -254,7 +258,8 @@ func (s *publicRoomsStatements) updateRoomAttribute( var value interface{} switch v := attrValue.(type) { case []string: - value = pq.StringArray(v) + b, _ := json.Marshal(v) + value = string(b) case bool, string: value = attrValue default: diff --git a/publicroomsapi/storage/sqlite3/storage.go b/publicroomsapi/storage/sqlite3/storage.go index dcb8920f9..f8ba71a89 100644 --- a/publicroomsapi/storage/sqlite3/storage.go +++ b/publicroomsapi/storage/sqlite3/storage.go @@ -41,7 +41,7 @@ type attributeValue interface{} func NewPublicRoomsServerDatabase(dataSourceName string) (*PublicRoomsServerDatabase, error) { var db *sql.DB var err error - if db, err = sql.Open("sqlite3", dataSourceName); err != nil { + if db, err = sql.Open(common.SQLiteDriverName(), dataSourceName); err != nil { return nil, err } storage := PublicRoomsServerDatabase{ diff --git a/publicroomsapi/storage/storage.go b/publicroomsapi/storage/storage.go index 29a6619fa..41dbd7819 100644 --- a/publicroomsapi/storage/storage.go +++ b/publicroomsapi/storage/storage.go @@ -12,29 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build !wasm + package storage import ( - "context" "net/url" - "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/publicroomsapi/storage/postgres" "github.com/matrix-org/dendrite/publicroomsapi/storage/sqlite3" - "github.com/matrix-org/dendrite/publicroomsapi/types" - "github.com/matrix-org/gomatrixserverlib" ) -type Database interface { - common.PartitionStorer - GetRoomVisibility(ctx context.Context, roomID string) (bool, error) - SetRoomVisibility(ctx context.Context, visible bool, roomID string) error - CountPublicRooms(ctx context.Context) (int64, error) - GetPublicRooms(ctx context.Context, offset int64, limit int16, filter string) ([]types.PublicRoom, error) - UpdateRoomFromEvents(ctx context.Context, eventsToAdd []gomatrixserverlib.Event, eventsToRemove []gomatrixserverlib.Event) error - UpdateRoomFromEvent(ctx context.Context, event gomatrixserverlib.Event) error -} - // NewPublicRoomsServerDatabase opens a database connection. func NewPublicRoomsServerDatabase(dataSourceName string) (Database, error) { uri, err := url.Parse(dataSourceName) diff --git a/publicroomsapi/storage/storage_wasm.go b/publicroomsapi/storage/storage_wasm.go new file mode 100644 index 000000000..d00c339d8 --- /dev/null +++ b/publicroomsapi/storage/storage_wasm.go @@ -0,0 +1,38 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "fmt" + "net/url" + + "github.com/matrix-org/dendrite/publicroomsapi/storage/sqlite3" +) + +// NewPublicRoomsServerDatabase opens a database connection. +func NewPublicRoomsServerDatabase(dataSourceName string) (Database, error) { + uri, err := url.Parse(dataSourceName) + if err != nil { + return nil, err + } + switch uri.Scheme { + case "postgres": + return nil, fmt.Errorf("Cannot use postgres implementation") + case "file": + return sqlite3.NewPublicRoomsServerDatabase(dataSourceName) + default: + return nil, fmt.Errorf("Cannot use postgres implementation") + } +} diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go new file mode 100644 index 000000000..f03d6feab --- /dev/null +++ b/roomserver/storage/interface.go @@ -0,0 +1,49 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "context" + + "github.com/matrix-org/dendrite/roomserver/api" + statedb "github.com/matrix-org/dendrite/roomserver/state/database" + "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/gomatrixserverlib" +) + +type Database interface { + statedb.RoomStateDatabase + StoreEvent(ctx context.Context, event gomatrixserverlib.Event, txnAndSessionID *api.TransactionID, authEventNIDs []types.EventNID) (types.RoomNID, types.StateAtEvent, error) + StateEntriesForEventIDs(ctx context.Context, eventIDs []string) ([]types.StateEntry, error) + EventStateKeys(ctx context.Context, eventStateKeyNIDs []types.EventStateKeyNID) (map[types.EventStateKeyNID]string, error) + EventNIDs(ctx context.Context, eventIDs []string) (map[string]types.EventNID, error) + SetState(ctx context.Context, eventNID types.EventNID, stateNID types.StateSnapshotNID) error + EventIDs(ctx context.Context, eventNIDs []types.EventNID) (map[types.EventNID]string, error) + GetLatestEventsForUpdate(ctx context.Context, roomNID types.RoomNID) (types.RoomRecentEventsUpdater, error) + GetTransactionEventID(ctx context.Context, transactionID string, sessionID int64, userID string) (string, error) + RoomNID(ctx context.Context, roomID string) (types.RoomNID, error) + LatestEventIDs(ctx context.Context, roomNID types.RoomNID) ([]gomatrixserverlib.EventReference, types.StateSnapshotNID, int64, error) + GetInvitesForUser(ctx context.Context, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID) (senderUserIDs []types.EventStateKeyNID, err error) + SetRoomAlias(ctx context.Context, alias string, roomID string, creatorUserID string) error + GetRoomIDForAlias(ctx context.Context, alias string) (string, error) + GetAliasesForRoomID(ctx context.Context, roomID string) ([]string, error) + GetCreatorIDForAlias(ctx context.Context, alias string) (string, error) + RemoveRoomAlias(ctx context.Context, alias string) error + MembershipUpdater(ctx context.Context, roomID, targetUserID string) (types.MembershipUpdater, error) + GetMembership(ctx context.Context, roomNID types.RoomNID, requestSenderUserID string) (membershipEventNID types.EventNID, stillInRoom bool, err error) + GetMembershipEventNIDsForRoom(ctx context.Context, roomNID types.RoomNID, joinOnly bool) ([]types.EventNID, error) + EventsFromIDs(ctx context.Context, eventIDs []string) ([]types.Event, error) + GetRoomVersionForRoom(ctx context.Context, roomNID types.RoomNID) (int64, error) +} diff --git a/roomserver/storage/sqlite3/events_table.go b/roomserver/storage/sqlite3/events_table.go index 4ed1395da..56e379100 100644 --- a/roomserver/storage/sqlite3/events_table.go +++ b/roomserver/storage/sqlite3/events_table.go @@ -18,10 +18,10 @@ package sqlite3 import ( "context" "database/sql" + "encoding/json" "fmt" "strings" - "github.com/lib/pq" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" @@ -38,7 +38,7 @@ const eventsSchema = ` depth INTEGER NOT NULL, event_id TEXT NOT NULL UNIQUE, reference_sha256 BLOB NOT NULL, - auth_event_nids TEXT NOT NULL DEFAULT '{}' + auth_event_nids TEXT NOT NULL DEFAULT '[]' ); ` @@ -463,17 +463,14 @@ func (s *eventStatements) bulkSelectEventNID(ctx context.Context, txn *sql.Tx, e func (s *eventStatements) selectMaxEventDepth(ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID) (int64, error) { var result int64 selectStmt := common.TxStmt(txn, s.selectMaxEventDepthStmt) - err := selectStmt.QueryRowContext(ctx, sqliteIn(eventNIDsAsArray(eventNIDs))).Scan(&result) + err := selectStmt.QueryRowContext(ctx, eventNIDsAsArray(eventNIDs)).Scan(&result) if err != nil { return 0, err } return result, nil } -func eventNIDsAsArray(eventNIDs []types.EventNID) pq.Int64Array { - nids := make([]int64, len(eventNIDs)) - for i := range eventNIDs { - nids[i] = int64(eventNIDs[i]) - } - return nids +func eventNIDsAsArray(eventNIDs []types.EventNID) string { + b, _ := json.Marshal(eventNIDs) + return string(b) } diff --git a/roomserver/storage/sqlite3/list.go b/roomserver/storage/sqlite3/list.go deleted file mode 100644 index 4fe4e334b..000000000 --- a/roomserver/storage/sqlite3/list.go +++ /dev/null @@ -1,18 +0,0 @@ -package sqlite3 - -import ( - "strconv" - "strings" - - "github.com/lib/pq" -) - -type SqliteList string - -func sqliteIn(a pq.Int64Array) string { - var b []string - for _, n := range a { - b = append(b, strconv.FormatInt(n, 10)) - } - return strings.Join(b, ",") -} diff --git a/roomserver/storage/sqlite3/rooms_table.go b/roomserver/storage/sqlite3/rooms_table.go index bf237728d..99c03476f 100644 --- a/roomserver/storage/sqlite3/rooms_table.go +++ b/roomserver/storage/sqlite3/rooms_table.go @@ -18,8 +18,8 @@ package sqlite3 import ( "context" "database/sql" + "encoding/json" - "github.com/lib/pq" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -28,7 +28,7 @@ const roomsSchema = ` CREATE TABLE IF NOT EXISTS roomserver_rooms ( room_nid INTEGER PRIMARY KEY AUTOINCREMENT, room_id TEXT NOT NULL UNIQUE, - latest_event_nids TEXT NOT NULL DEFAULT '{}', + latest_event_nids TEXT NOT NULL DEFAULT '[]', last_event_sent_nid INTEGER NOT NULL DEFAULT 0, state_snapshot_nid INTEGER NOT NULL DEFAULT 0, room_version INTEGER NOT NULL DEFAULT 1 @@ -104,16 +104,16 @@ func (s *roomStatements) selectRoomNID( func (s *roomStatements) selectLatestEventNIDs( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, ) ([]types.EventNID, types.StateSnapshotNID, error) { - var nids pq.Int64Array + var eventNIDs []types.EventNID + var nidsJSON string var stateSnapshotNID int64 stmt := common.TxStmt(txn, s.selectLatestEventNIDsStmt) - err := stmt.QueryRowContext(ctx, int64(roomNID)).Scan(&nids, &stateSnapshotNID) + err := stmt.QueryRowContext(ctx, int64(roomNID)).Scan(&nidsJSON, &stateSnapshotNID) if err != nil { return nil, 0, err } - eventNIDs := make([]types.EventNID, len(nids)) - for i := range nids { - eventNIDs[i] = types.EventNID(nids[i]) + if err := json.Unmarshal([]byte(nidsJSON), &eventNIDs); err != nil { + return nil, 0, err } return eventNIDs, types.StateSnapshotNID(stateSnapshotNID), nil } @@ -121,17 +121,17 @@ func (s *roomStatements) selectLatestEventNIDs( func (s *roomStatements) selectLatestEventsNIDsForUpdate( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, ) ([]types.EventNID, types.EventNID, types.StateSnapshotNID, error) { - var nids pq.Int64Array + var eventNIDs []types.EventNID + var nidsJSON string var lastEventSentNID int64 var stateSnapshotNID int64 stmt := common.TxStmt(txn, s.selectLatestEventNIDsForUpdateStmt) - err := stmt.QueryRowContext(ctx, int64(roomNID)).Scan(&nids, &lastEventSentNID, &stateSnapshotNID) + err := stmt.QueryRowContext(ctx, int64(roomNID)).Scan(&nidsJSON, &lastEventSentNID, &stateSnapshotNID) if err != nil { return nil, 0, 0, err } - eventNIDs := make([]types.EventNID, len(nids)) - for i := range nids { - eventNIDs[i] = types.EventNID(nids[i]) + if err := json.Unmarshal([]byte(nidsJSON), &eventNIDs); err != nil { + return nil, 0, 0, err } return eventNIDs, types.EventNID(lastEventSentNID), types.StateSnapshotNID(stateSnapshotNID), nil } diff --git a/roomserver/storage/sqlite3/state_block_table.go b/roomserver/storage/sqlite3/state_block_table.go index d75abceec..c94ca2385 100644 --- a/roomserver/storage/sqlite3/state_block_table.go +++ b/roomserver/storage/sqlite3/state_block_table.go @@ -22,7 +22,6 @@ import ( "sort" "strings" - "github.com/lib/pq" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/util" @@ -268,9 +267,9 @@ func (s stateKeyTupleSorter) contains(value types.StateKeyTuple) bool { // List the unique eventTypeNIDs and eventStateKeyNIDs. // Assumes that the list is sorted. -func (s stateKeyTupleSorter) typesAndStateKeysAsArrays() (eventTypeNIDs pq.Int64Array, eventStateKeyNIDs pq.Int64Array) { - eventTypeNIDs = make(pq.Int64Array, len(s)) - eventStateKeyNIDs = make(pq.Int64Array, len(s)) +func (s stateKeyTupleSorter) typesAndStateKeysAsArrays() (eventTypeNIDs []int64, eventStateKeyNIDs []int64) { + eventTypeNIDs = make([]int64, len(s)) + eventStateKeyNIDs = make([]int64, len(s)) for i := range s { eventTypeNIDs[i] = int64(s[i].EventTypeNID) eventStateKeyNIDs[i] = int64(s[i].EventStateKeyNID) diff --git a/roomserver/storage/sqlite3/state_snapshot_table.go b/roomserver/storage/sqlite3/state_snapshot_table.go index df97aa419..0e1786eae 100644 --- a/roomserver/storage/sqlite3/state_snapshot_table.go +++ b/roomserver/storage/sqlite3/state_snapshot_table.go @@ -18,10 +18,10 @@ package sqlite3 import ( "context" "database/sql" + "encoding/json" "fmt" "strings" - "github.com/lib/pq" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -30,7 +30,7 @@ const stateSnapshotSchema = ` CREATE TABLE IF NOT EXISTS roomserver_state_snapshots ( state_snapshot_nid INTEGER PRIMARY KEY AUTOINCREMENT, room_nid INTEGER NOT NULL, - state_block_nids TEXT NOT NULL DEFAULT '{}' + state_block_nids TEXT NOT NULL DEFAULT '[]' ); ` @@ -67,12 +67,12 @@ func (s *stateSnapshotStatements) prepare(db *sql.DB) (err error) { func (s *stateSnapshotStatements) insertState( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, stateBlockNIDs []types.StateBlockNID, ) (stateNID types.StateSnapshotNID, err error) { - nids := make([]int64, len(stateBlockNIDs)) - for i := range stateBlockNIDs { - nids[i] = int64(stateBlockNIDs[i]) + stateBlockNIDsJSON, err := json.Marshal(stateBlockNIDs) + if err != nil { + return } insertStmt := txn.Stmt(s.insertStateStmt) - if res, err2 := insertStmt.ExecContext(ctx, int64(roomNID), pq.Int64Array(nids)); err2 == nil { + if res, err2 := insertStmt.ExecContext(ctx, int64(roomNID), string(stateBlockNIDsJSON)); err2 == nil { lastRowID, err3 := res.LastInsertId() if err3 != nil { err = err3 @@ -104,13 +104,12 @@ func (s *stateSnapshotStatements) bulkSelectStateBlockNIDs( i := 0 for ; rows.Next(); i++ { result := &results[i] - var stateBlockNIDs pq.Int64Array - if err := rows.Scan(&result.StateSnapshotNID, &stateBlockNIDs); err != nil { + var stateBlockNIDsJSON string + if err := rows.Scan(&result.StateSnapshotNID, &stateBlockNIDsJSON); err != nil { return nil, err } - result.StateBlockNIDs = make([]types.StateBlockNID, len(stateBlockNIDs)) - for k := range stateBlockNIDs { - result.StateBlockNIDs[k] = types.StateBlockNID(stateBlockNIDs[k]) + if err := json.Unmarshal([]byte(stateBlockNIDsJSON), &result.StateBlockNIDs); err != nil { + return nil, err } } if i != len(stateNIDs) { diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index aebb308c4..41d4c2a94 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -49,7 +49,7 @@ func Open(dataSourceName string) (*Database, error) { } else { return nil, errors.New("no filename or path in connect string") } - if d.db, err = sql.Open("sqlite3", cs); err != nil { + if d.db, err = sql.Open(common.SQLiteDriverName(), cs); err != nil { return nil, err } //d.db.Exec("PRAGMA journal_mode=WAL;") diff --git a/roomserver/storage/storage.go b/roomserver/storage/storage.go index 551d97cd1..7b9109aa0 100644 --- a/roomserver/storage/storage.go +++ b/roomserver/storage/storage.go @@ -12,45 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build !wasm + package storage import ( - "context" "net/url" - "github.com/matrix-org/dendrite/roomserver/api" - statedb "github.com/matrix-org/dendrite/roomserver/state/database" "github.com/matrix-org/dendrite/roomserver/storage/postgres" "github.com/matrix-org/dendrite/roomserver/storage/sqlite3" - "github.com/matrix-org/dendrite/roomserver/types" - "github.com/matrix-org/gomatrixserverlib" ) -type Database interface { - statedb.RoomStateDatabase - StoreEvent(ctx context.Context, event gomatrixserverlib.Event, txnAndSessionID *api.TransactionID, authEventNIDs []types.EventNID) (types.RoomNID, types.StateAtEvent, error) - StateEntriesForEventIDs(ctx context.Context, eventIDs []string) ([]types.StateEntry, error) - EventStateKeys(ctx context.Context, eventStateKeyNIDs []types.EventStateKeyNID) (map[types.EventStateKeyNID]string, error) - EventNIDs(ctx context.Context, eventIDs []string) (map[string]types.EventNID, error) - SetState(ctx context.Context, eventNID types.EventNID, stateNID types.StateSnapshotNID) error - EventIDs(ctx context.Context, eventNIDs []types.EventNID) (map[types.EventNID]string, error) - GetLatestEventsForUpdate(ctx context.Context, roomNID types.RoomNID) (types.RoomRecentEventsUpdater, error) - GetTransactionEventID(ctx context.Context, transactionID string, sessionID int64, userID string) (string, error) - RoomNID(ctx context.Context, roomID string) (types.RoomNID, error) - LatestEventIDs(ctx context.Context, roomNID types.RoomNID) ([]gomatrixserverlib.EventReference, types.StateSnapshotNID, int64, error) - GetInvitesForUser(ctx context.Context, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID) (senderUserIDs []types.EventStateKeyNID, err error) - SetRoomAlias(ctx context.Context, alias string, roomID string, creatorUserID string) error - GetRoomIDForAlias(ctx context.Context, alias string) (string, error) - GetAliasesForRoomID(ctx context.Context, roomID string) ([]string, error) - GetCreatorIDForAlias(ctx context.Context, alias string) (string, error) - RemoveRoomAlias(ctx context.Context, alias string) error - MembershipUpdater(ctx context.Context, roomID, targetUserID string) (types.MembershipUpdater, error) - GetMembership(ctx context.Context, roomNID types.RoomNID, requestSenderUserID string) (membershipEventNID types.EventNID, stillInRoom bool, err error) - GetMembershipEventNIDsForRoom(ctx context.Context, roomNID types.RoomNID, joinOnly bool) ([]types.EventNID, error) - EventsFromIDs(ctx context.Context, eventIDs []string) ([]types.Event, error) - GetRoomVersionForRoom(ctx context.Context, roomNID types.RoomNID) (int64, error) -} - // NewPublicRoomsServerDatabase opens a database connection. func Open(dataSourceName string) (Database, error) { uri, err := url.Parse(dataSourceName) diff --git a/roomserver/storage/storage_wasm.go b/roomserver/storage/storage_wasm.go new file mode 100644 index 000000000..d7fc352e8 --- /dev/null +++ b/roomserver/storage/storage_wasm.go @@ -0,0 +1,38 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "fmt" + "net/url" + + "github.com/matrix-org/dendrite/roomserver/storage/sqlite3" +) + +// NewPublicRoomsServerDatabase opens a database connection. +func Open(dataSourceName string) (Database, error) { + uri, err := url.Parse(dataSourceName) + if err != nil { + return nil, fmt.Errorf("Cannot use postgres implementation") + } + switch uri.Scheme { + case "postgres": + return nil, fmt.Errorf("Cannot use postgres implementation") + case "file": + return sqlite3.Open(dataSourceName) + default: + return nil, fmt.Errorf("Cannot use postgres implementation") + } +} diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go new file mode 100644 index 000000000..0abeac34d --- /dev/null +++ b/syncapi/storage/interface.go @@ -0,0 +1,53 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "context" + "time" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/dendrite/typingserver/cache" + "github.com/matrix-org/gomatrixserverlib" +) + +type Database interface { + common.PartitionStorer + AllJoinedUsersInRooms(ctx context.Context) (map[string][]string, error) + Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.Event, error) + WriteEvent(context.Context, *gomatrixserverlib.Event, []gomatrixserverlib.Event, []string, []string, *api.TransactionID, bool) (types.StreamPosition, error) + GetStateEvent(ctx context.Context, roomID, evType, stateKey string) (*gomatrixserverlib.Event, error) + GetStateEventsForRoom(ctx context.Context, roomID string, stateFilterPart *gomatrixserverlib.StateFilter) (stateEvents []gomatrixserverlib.Event, err error) + SyncPosition(ctx context.Context) (types.PaginationToken, error) + IncrementalSync(ctx context.Context, device authtypes.Device, fromPos, toPos types.PaginationToken, numRecentEventsPerRoom int, wantFullState bool) (*types.Response, error) + CompleteSync(ctx context.Context, userID string, numRecentEventsPerRoom int) (*types.Response, error) + GetAccountDataInRange(ctx context.Context, userID string, oldPos, newPos types.StreamPosition, accountDataFilterPart *gomatrixserverlib.EventFilter) (map[string][]string, error) + UpsertAccountData(ctx context.Context, userID, roomID, dataType string) (types.StreamPosition, error) + AddInviteEvent(ctx context.Context, inviteEvent gomatrixserverlib.Event) (types.StreamPosition, error) + RetireInviteEvent(ctx context.Context, inviteEventID string) error + SetTypingTimeoutCallback(fn cache.TimeoutCallbackFn) + AddTypingUser(userID, roomID string, expireTime *time.Time) types.StreamPosition + RemoveTypingUser(userID, roomID string) types.StreamPosition + GetEventsInRange(ctx context.Context, from, to *types.PaginationToken, roomID string, limit int, backwardOrdering bool) (events []types.StreamEvent, err error) + EventPositionInTopology(ctx context.Context, eventID string) (types.StreamPosition, error) + EventsAtTopologicalPosition(ctx context.Context, roomID string, pos types.StreamPosition) ([]types.StreamEvent, error) + BackwardExtremitiesForRoom(ctx context.Context, roomID string) (backwardExtremities []string, err error) + MaxTopologicalPosition(ctx context.Context, roomID string) (types.StreamPosition, error) + StreamEventsToEvents(device *authtypes.Device, in []types.StreamEvent) []gomatrixserverlib.Event + SyncStreamPosition(ctx context.Context) (types.StreamPosition, error) +} diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go index eb969c956..ed76177be 100644 --- a/syncapi/storage/sqlite3/current_room_state_table.go +++ b/syncapi/storage/sqlite3/current_room_state_table.go @@ -21,7 +21,6 @@ import ( "encoding/json" "strings" - "github.com/lib/pq" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -175,10 +174,10 @@ func (s *currentRoomStateStatements) selectCurrentState( ) ([]gomatrixserverlib.Event, error) { stmt := common.TxStmt(txn, s.selectCurrentStateStmt) rows, err := stmt.QueryContext(ctx, roomID, - pq.StringArray(stateFilterPart.Senders), - pq.StringArray(stateFilterPart.NotSenders), - pq.StringArray(filterConvertTypeWildcardToSQL(stateFilterPart.Types)), - pq.StringArray(filterConvertTypeWildcardToSQL(stateFilterPart.NotTypes)), + nil, // FIXME: pq.StringArray(stateFilterPart.Senders), + nil, // FIXME: pq.StringArray(stateFilterPart.NotSenders), + nil, // FIXME: pq.StringArray(filterConvertTypeWildcardToSQL(stateFilterPart.Types)), + nil, // FIXME: pq.StringArray(filterConvertTypeWildcardToSQL(stateFilterPart.NotTypes)), stateFilterPart.ContainsURL, stateFilterPart.Limit, ) diff --git a/syncapi/storage/sqlite3/filtering.go b/syncapi/storage/sqlite3/filtering.go deleted file mode 100644 index c4a2f4bf9..000000000 --- a/syncapi/storage/sqlite3/filtering.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2017 Thibaut CHARLES -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sqlite3 - -import ( - "strings" -) - -// filterConvertWildcardToSQL converts wildcards as defined in -// https://matrix.org/docs/spec/client_server/r0.3.0.html#post-matrix-client-r0-user-userid-filter -// to SQL wildcards that can be used with LIKE() -func filterConvertTypeWildcardToSQL(values []string) []string { - if values == nil { - // Return nil instead of []string{} so IS NULL can work correctly when - // the return value is passed into SQL queries - return nil - } - - ret := make([]string, len(values)) - for i := range values { - ret[i] = strings.Replace(values[i], "*", "%", -1) - } - return ret -} diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index 4535688df..be8937435 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -24,7 +24,6 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/types" - "github.com/lib/pq" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/gomatrixserverlib" log "github.com/sirupsen/logrus" @@ -40,11 +39,11 @@ CREATE TABLE IF NOT EXISTS syncapi_output_room_events ( type TEXT NOT NULL, sender TEXT NOT NULL, contains_url BOOL NOT NULL, - add_state_ids TEXT[], - remove_state_ids TEXT[], + add_state_ids TEXT, -- JSON encoded string array + remove_state_ids TEXT, -- JSON encoded string array session_id BIGINT, transaction_id TEXT, - exclude_from_sync BOOL DEFAULT FALSE + exclude_from_sync BOOL NOT NULL DEFAULT FALSE ); ` @@ -52,7 +51,7 @@ const insertEventSQL = "" + "INSERT INTO syncapi_output_room_events (" + "id, room_id, event_id, event_json, type, sender, contains_url, add_state_ids, remove_state_ids, session_id, transaction_id, exclude_from_sync" + ") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) " + - "ON CONFLICT (event_id) DO UPDATE SET exclude_from_sync = $11" + "ON CONFLICT (event_id) DO UPDATE SET exclude_from_sync = $13" const selectEventsSQL = "" + "SELECT id, event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events WHERE event_id = $1" @@ -176,20 +175,26 @@ func (s *outputRoomEventsStatements) selectStateInRange( streamPos types.StreamPosition eventBytes []byte excludeFromSync bool - addIDs pq.StringArray - delIDs pq.StringArray + addIDsJSON string + delIDsJSON string ) - if err := rows.Scan(&streamPos, &eventBytes, &excludeFromSync, &addIDs, &delIDs); err != nil { + if err := rows.Scan(&streamPos, &eventBytes, &excludeFromSync, &addIDsJSON, &delIDsJSON); err != nil { return nil, nil, err } + + addIDs, delIDs, err := unmarshalStateIDs(addIDsJSON, delIDsJSON) + if err != nil { + return nil, nil, err + } + // Sanity check for deleted state and whine if we see it. We don't need to do anything // since it'll just mark the event as not being needed. if len(addIDs) < len(delIDs) { log.WithFields(log.Fields{ "since": oldPos, "current": newPos, - "adds": addIDs, - "dels": delIDs, + "adds": addIDsJSON, + "dels": delIDsJSON, }).Warn("StateBetween: ignoring deleted state") } @@ -262,6 +267,15 @@ func (s *outputRoomEventsStatements) insertEvent( return } + addStateJSON, err := json.Marshal(addState) + if err != nil { + return + } + removeStateJSON, err := json.Marshal(removeState) + if err != nil { + return + } + insertStmt := common.TxStmt(txn, s.insertEventStmt) _, err = insertStmt.ExecContext( ctx, @@ -272,11 +286,12 @@ func (s *outputRoomEventsStatements) insertEvent( event.Type(), event.Sender(), containsURL, - pq.StringArray(addState), - pq.StringArray(removeState), + string(addStateJSON), + string(removeStateJSON), sessionID, txnID, excludeFromSync, + excludeFromSync, ) return } @@ -397,3 +412,17 @@ func rowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) { } return result, nil } + +func unmarshalStateIDs(addIDsJSON, delIDsJSON string) (addIDs []string, delIDs []string, err error) { + if len(addIDsJSON) > 0 { + if err = json.Unmarshal([]byte(addIDsJSON), &addIDs); err != nil { + return + } + } + if len(delIDsJSON) > 0 { + if err = json.Unmarshal([]byte(delIDsJSON), &delIDs); err != nil { + return + } + } + return +} diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index da580e3a2..0e84c8c86 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -29,8 +29,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/roomserver/api" - // Import the postgres database driver. - _ "github.com/lib/pq" + // Import the sqlite3 package _ "github.com/mattn/go-sqlite3" "github.com/matrix-org/dendrite/common" @@ -79,7 +78,7 @@ func NewSyncServerDatasource(dataSourceName string) (*SyncServerDatasource, erro } else { return nil, errors.New("no filename or path in connect string") } - if d.db, err = sql.Open("sqlite3", cs); err != nil { + if d.db, err = sql.Open(common.SQLiteDriverName(), cs); err != nil { return nil, err } if err = d.prepare(); err != nil { diff --git a/syncapi/storage/storage.go b/syncapi/storage/storage.go index c87024b29..c56db0635 100644 --- a/syncapi/storage/storage.go +++ b/syncapi/storage/storage.go @@ -12,49 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build !wasm + package storage import ( - "context" "net/url" - "time" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/storage/postgres" "github.com/matrix-org/dendrite/syncapi/storage/sqlite3" - "github.com/matrix-org/dendrite/syncapi/types" - "github.com/matrix-org/dendrite/typingserver/cache" - "github.com/matrix-org/gomatrixserverlib" ) -type Database interface { - common.PartitionStorer - AllJoinedUsersInRooms(ctx context.Context) (map[string][]string, error) - Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.Event, error) - WriteEvent(context.Context, *gomatrixserverlib.Event, []gomatrixserverlib.Event, []string, []string, *api.TransactionID, bool) (types.StreamPosition, error) - GetStateEvent(ctx context.Context, roomID, evType, stateKey string) (*gomatrixserverlib.Event, error) - GetStateEventsForRoom(ctx context.Context, roomID string, stateFilterPart *gomatrixserverlib.StateFilter) (stateEvents []gomatrixserverlib.Event, err error) - SyncPosition(ctx context.Context) (types.PaginationToken, error) - IncrementalSync(ctx context.Context, device authtypes.Device, fromPos, toPos types.PaginationToken, numRecentEventsPerRoom int, wantFullState bool) (*types.Response, error) - CompleteSync(ctx context.Context, userID string, numRecentEventsPerRoom int) (*types.Response, error) - GetAccountDataInRange(ctx context.Context, userID string, oldPos, newPos types.StreamPosition, accountDataFilterPart *gomatrixserverlib.EventFilter) (map[string][]string, error) - UpsertAccountData(ctx context.Context, userID, roomID, dataType string) (types.StreamPosition, error) - AddInviteEvent(ctx context.Context, inviteEvent gomatrixserverlib.Event) (types.StreamPosition, error) - RetireInviteEvent(ctx context.Context, inviteEventID string) error - SetTypingTimeoutCallback(fn cache.TimeoutCallbackFn) - AddTypingUser(userID, roomID string, expireTime *time.Time) types.StreamPosition - RemoveTypingUser(userID, roomID string) types.StreamPosition - GetEventsInRange(ctx context.Context, from, to *types.PaginationToken, roomID string, limit int, backwardOrdering bool) (events []types.StreamEvent, err error) - EventPositionInTopology(ctx context.Context, eventID string) (types.StreamPosition, error) - EventsAtTopologicalPosition(ctx context.Context, roomID string, pos types.StreamPosition) ([]types.StreamEvent, error) - BackwardExtremitiesForRoom(ctx context.Context, roomID string) (backwardExtremities []string, err error) - MaxTopologicalPosition(ctx context.Context, roomID string) (types.StreamPosition, error) - StreamEventsToEvents(device *authtypes.Device, in []types.StreamEvent) []gomatrixserverlib.Event - SyncStreamPosition(ctx context.Context) (types.StreamPosition, error) -} - // NewPublicRoomsServerDatabase opens a database connection. func NewSyncServerDatasource(dataSourceName string) (Database, error) { uri, err := url.Parse(dataSourceName) diff --git a/syncapi/storage/storage_wasm.go b/syncapi/storage/storage_wasm.go new file mode 100644 index 000000000..43806a012 --- /dev/null +++ b/syncapi/storage/storage_wasm.go @@ -0,0 +1,38 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "fmt" + "net/url" + + "github.com/matrix-org/dendrite/syncapi/storage/sqlite3" +) + +// NewPublicRoomsServerDatabase opens a database connection. +func NewSyncServerDatasource(dataSourceName string) (Database, error) { + uri, err := url.Parse(dataSourceName) + if err != nil { + return nil, fmt.Errorf("Cannot use postgres implementation") + } + switch uri.Scheme { + case "postgres": + return nil, fmt.Errorf("Cannot use postgres implementation") + case "file": + return sqlite3.NewSyncServerDatasource(dataSourceName) + default: + return nil, fmt.Errorf("Cannot use postgres implementation") + } +} From 87283e9de785f5153c5cf9b326d2640e202a36b3 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 6 Mar 2020 14:31:12 +0000 Subject: [PATCH 26/86] bugfix: fix sytest 155 by actually returning depth+1 and not 0 --- roomserver/storage/sqlite3/events_table.go | 10 ++++++---- syncapi/routing/messages.go | 4 ++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/roomserver/storage/sqlite3/events_table.go b/roomserver/storage/sqlite3/events_table.go index 56e379100..4fa095913 100644 --- a/roomserver/storage/sqlite3/events_table.go +++ b/roomserver/storage/sqlite3/events_table.go @@ -111,7 +111,6 @@ type eventStatements struct { bulkSelectEventReferenceStmt *sql.Stmt bulkSelectEventIDStmt *sql.Stmt bulkSelectEventNIDStmt *sql.Stmt - selectMaxEventDepthStmt *sql.Stmt } func (s *eventStatements) prepare(db *sql.DB) (err error) { @@ -135,7 +134,6 @@ func (s *eventStatements) prepare(db *sql.DB) (err error) { {&s.bulkSelectEventReferenceStmt, bulkSelectEventReferenceSQL}, {&s.bulkSelectEventIDStmt, bulkSelectEventIDSQL}, {&s.bulkSelectEventNIDStmt, bulkSelectEventNIDSQL}, - {&s.selectMaxEventDepthStmt, selectMaxEventDepthSQL}, }.prepare(db) } @@ -462,8 +460,12 @@ func (s *eventStatements) bulkSelectEventNID(ctx context.Context, txn *sql.Tx, e func (s *eventStatements) selectMaxEventDepth(ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID) (int64, error) { var result int64 - selectStmt := common.TxStmt(txn, s.selectMaxEventDepthStmt) - err := selectStmt.QueryRowContext(ctx, eventNIDsAsArray(eventNIDs)).Scan(&result) + iEventIDs := make([]interface{}, len(eventNIDs)) + for i, v := range eventNIDs { + iEventIDs[i] = v + } + sqlStr := strings.Replace(selectMaxEventDepthSQL, "($1)", common.QueryVariadic(len(iEventIDs)), 1) + err := txn.QueryRowContext(ctx, sqlStr, iEventIDs...).Scan(&result) if err != nil { return 0, err } diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index 7bbe16f3c..83bf75b2e 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -16,6 +16,7 @@ package routing import ( "context" + "fmt" "net/http" "sort" "strconv" @@ -176,6 +177,7 @@ func (r *messagesReq) retrieveEvents() ( r.ctx, r.from, r.to, r.roomID, r.limit, r.backwardOrdering, ) if err != nil { + err = fmt.Errorf("GetEventsInRange: %s", err) return } @@ -226,12 +228,14 @@ func (r *messagesReq) retrieveEvents() ( r.ctx, events[0].EventID(), ) if err != nil { + err = fmt.Errorf("EventPositionInTopology: for start event %s: %s", events[0].EventID(), err) return } endPos, err := r.db.EventPositionInTopology( r.ctx, events[len(events)-1].EventID(), ) if err != nil { + err = fmt.Errorf("EventPositionInTopology: for end event %s: %s", events[len(events)-1].EventID(), err) return } // Generate pagination tokens to send to the client using the positions From 6a1111c3d44cfa6b5301fed70045b1e555d39c65 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 6 Mar 2020 16:58:10 +0000 Subject: [PATCH 27/86] Try to recursively find auth events (to a point) if they are missing (#881) * Try to recursively find auth events (to a point) if they are missing * Remove recursion limit for now and other review fixes * Simplify error handling for recursion * Pass room version 1 only to MakeJoin until room version support comes later --- clientapi/routing/joinroom.go | 2 +- federationapi/routing/send.go | 17 +++++++++++++++++ go.mod | 4 ++-- go.sum | 18 ++++++------------ 4 files changed, 26 insertions(+), 15 deletions(-) diff --git a/clientapi/routing/joinroom.go b/clientapi/routing/joinroom.go index de9667e2e..7643c41f3 100644 --- a/clientapi/routing/joinroom.go +++ b/clientapi/routing/joinroom.go @@ -299,7 +299,7 @@ func (r joinRoomReq) joinRoomUsingServers( // server was invalid this returns an error. // Otherwise this returns a JSONResponse. func (r joinRoomReq) joinRoomUsingServer(roomID string, server gomatrixserverlib.ServerName) (*util.JSONResponse, error) { - respMakeJoin, err := r.federation.MakeJoin(r.req.Context(), server, roomID, r.userID) + respMakeJoin, err := r.federation.MakeJoin(r.req.Context(), server, roomID, r.userID, []int{1}) if err != nil { // TODO: Check if the user was not allowed to join the room. return nil, err diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index 191e13ef7..079e121f9 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -211,7 +211,24 @@ func (t *txnReq) processEventWithMissingState(e gomatrixserverlib.Event) error { return err } // Check that the event is allowed by the state. +retryAllowedState: if err := checkAllowedByState(e, state.StateEvents); err != nil { + switch missing := err.(type) { + case gomatrixserverlib.MissingAuthEventError: + // An auth event was missing so let's look up that event over federation + for _, s := range state.StateEvents { + if s.EventID() != missing.AuthEventID { + continue + } + err = t.processEventWithMissingState(s) + // If there was no error retrieving the event from federation then + // we assume that it succeeded, so retry the original state check + if err == nil { + goto retryAllowedState + } + } + default: + } return err } // pass the event along with the state to the roomserver diff --git a/go.mod b/go.mod index 7d59aeb1b..924c4b67d 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200304160008-4ec1129a00c4 github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200304110715-894c3c86ce3e + github.com/matrix-org/gomatrixserverlib v0.0.0-20200306154041-df6bb9a3e424 github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible @@ -17,7 +17,7 @@ require ( github.com/pkg/errors v0.8.1 github.com/prometheus/client_golang v1.4.1 github.com/sirupsen/logrus v1.4.2 - github.com/tidwall/gjson v1.6.0 + github.com/tidwall/gjson v1.6.0 // indirect github.com/tidwall/pretty v1.0.1 // indirect github.com/uber/jaeger-client-go v2.22.1+incompatible github.com/uber/jaeger-lib v2.2.0+incompatible diff --git a/go.sum b/go.sum index 19dc9662c..3a509debb 100644 --- a/go.sum +++ b/go.sum @@ -44,6 +44,7 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= @@ -70,14 +71,8 @@ github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/matrix-org/dendrite v0.0.0-20200220135450-0352f250b857/go.mod h1:DZ35IoR+ViBNVPe9umdlOSnjvKl7wfyRmZg4QfWGvTo= github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 h1:nMX2t7hbGF0NYDYySx0pCqEKGKAeZIiSqlWSspetlhY= github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg= -github.com/matrix-org/go-http-js-libp2p v0.0.0-20200225225149-e7191ca90a94 h1:gUqPXKWbuwmPnx3PJBxtQw5Ff8V229wZ7Ee0GaH/bg4= -github.com/matrix-org/go-http-js-libp2p v0.0.0-20200225225149-e7191ca90a94/go.mod h1:PJof7UbOKmVEEMsGqvbyIK0ldMMBjPH5EYia7MHR2RQ= github.com/matrix-org/go-http-js-libp2p v0.0.0-20200304160008-4ec1129a00c4 h1:oFjG7X1jS473zPDix1/FBZ2qd0anM1Ko4AlJCB6MUZs= github.com/matrix-org/go-http-js-libp2p v0.0.0-20200304160008-4ec1129a00c4/go.mod h1:PJof7UbOKmVEEMsGqvbyIK0ldMMBjPH5EYia7MHR2RQ= -github.com/matrix-org/go-sqlite3-js v0.0.0-20200226144546-ea6ed5b90074 h1:UWz6vfhmQVshBuE67X1BCsdMhEDtd+uOz8CJ48Fc0F4= -github.com/matrix-org/go-sqlite3-js v0.0.0-20200226144546-ea6ed5b90074/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= -github.com/matrix-org/go-sqlite3-js v0.0.0-20200304163011-cfb4884075db h1:ERuFJq4DI8fakfBZlvXHltHZ0ix3K5YsLG0tQfQn6TI= -github.com/matrix-org/go-sqlite3-js v0.0.0-20200304163011-cfb4884075db/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658 h1:UlhTKClOgWnSB25Rv+BS/Vc1mRinjNUErfyGEVOBP04= github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20190130130140-385f072fe9af h1:piaIBNQGIHnni27xRB7VKkEwoWCgAmeuYf8pxAyG0bI= @@ -86,14 +81,10 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:kmRjpmFOenVpOaV/DRlo9p6z/IbOKlUC+hhKsAAh8Qg= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200228131708-347eb77397b8 h1:a9IV2iKMznKJ16MmcG/NU7qMcZ4jIKmPXC6RkOZZq+Q= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200228131708-347eb77397b8/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200304110715-894c3c86ce3e h1:DA1lP2mB2ddd2PhMOaNPwRJFi/3aL2Lj7bxnAhFxBFQ= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200304110715-894c3c86ce3e/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200306154041-df6bb9a3e424 h1:H61lT6ckUFIDl9qb636qNomfo0B52lFt73ecioiqF10= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200306154041-df6bb9a3e424/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= -github.com/matrix-org/pq v1.3.2 h1:7Kh2Qz4xonRH8OcFpCIj2najpBxF2+j1ff0hC082L68= -github.com/matrix-org/pq v1.3.2/go.mod h1:l6tPTzDjcj8fhD5OD0+7dejtZrwyjJadAAYM+CntUVQ= github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 h1:W7l5CP4V7wPyPb4tYE11dbmeAOwtFQBTW0rf4OonOS8= github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5/go.mod h1:lePuOiXLNDott7NZfnQvJk0lAZ5HgvIuWGhel6J+RLA= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= @@ -200,6 +191,7 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0Is3p+EHBKNgquIzo1OI= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 h1:I6FyU15t786LL7oL/hn43zqTuEGr4PN7F4XJ1p4E3Y8= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -212,6 +204,7 @@ golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 h1:ywK/j/KkyTHcdyYSZNXGjMwgmDSfjglYZ3vStQ/gSCU= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/Shopify/sarama.v1 v1.20.1 h1:Gi09A3fJXm0Jgt8kuKZ8YK+r60GfYn7MQuEmI3oq6hE= gopkg.in/Shopify/sarama.v1 v1.20.1/go.mod h1:AxnvoaevB2nBjNK17cG61A3LleFcWFwVBHBt+cot4Oc= @@ -220,6 +213,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/h2non/bimg.v1 v1.0.18 h1:qn6/RpBHt+7WQqoBcK+aF2puc6nC78eZj5LexxoalT4= gopkg.in/h2non/bimg.v1 v1.0.18/go.mod h1:PgsZL7dLwUbsGm1NYps320GxGgvQNTnecMCZqxV11So= From c31cb0227111ab4441d6282081ebd6494a422ba8 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Fri, 6 Mar 2020 18:00:07 +0000 Subject: [PATCH 28/86] bugfix: Fix a race condition when creating guest accounts (#882) * bugfix: Fix a race condition when creating guest accounts It was possible to both select the same next numeric ID and then both attempt to INSERT this into the table. This would cause a UNIQUE violation which then presented itself as an error in sqlite because it does not implement `common.IsUniqueConstraintViolationErr`. The fix here is NOT to implement `common.IsUniqueConstraintViolationErr` otherwise the 2 users would get the SAME guest account. Instead, all of these operations should be done inside a transaction. This is what this PR does. * Update postgres * Typo * Actually use the txn when creating accounts * bugfix for database is locked on guest reg --- clientapi/auth/storage/accounts/interface.go | 1 + .../accounts/postgres/account_data_table.go | 4 +- .../accounts/postgres/accounts_table.go | 12 +++-- .../accounts/postgres/profile_table.go | 4 +- .../auth/storage/accounts/postgres/storage.go | 40 ++++++++++++-- .../accounts/sqlite3/account_data_table.go | 5 +- .../accounts/sqlite3/accounts_table.go | 14 +++-- .../storage/accounts/sqlite3/profile_table.go | 4 +- .../auth/storage/accounts/sqlite3/storage.go | 53 ++++++++++++++++--- clientapi/routing/register.go | 11 +--- 10 files changed, 108 insertions(+), 40 deletions(-) diff --git a/clientapi/auth/storage/accounts/interface.go b/clientapi/auth/storage/accounts/interface.go index 83d3ee725..9f6e3e1ea 100644 --- a/clientapi/auth/storage/accounts/interface.go +++ b/clientapi/auth/storage/accounts/interface.go @@ -30,6 +30,7 @@ type Database interface { SetAvatarURL(ctx context.Context, localpart string, avatarURL string) error SetDisplayName(ctx context.Context, localpart string, displayName string) error CreateAccount(ctx context.Context, localpart, plaintextPassword, appserviceID string) (*authtypes.Account, error) + CreateGuestAccount(ctx context.Context) (*authtypes.Account, error) UpdateMemberships(ctx context.Context, eventsToAdd []gomatrixserverlib.Event, idsToRemove []string) error GetMembershipInRoomByLocalpart(ctx context.Context, localpart, roomID string) (authtypes.Membership, error) GetMembershipsByLocalpart(ctx context.Context, localpart string) (memberships []authtypes.Membership, err error) diff --git a/clientapi/auth/storage/accounts/postgres/account_data_table.go b/clientapi/auth/storage/accounts/postgres/account_data_table.go index d0cfcc0cf..4573999b4 100644 --- a/clientapi/auth/storage/accounts/postgres/account_data_table.go +++ b/clientapi/auth/storage/accounts/postgres/account_data_table.go @@ -72,9 +72,9 @@ func (s *accountDataStatements) prepare(db *sql.DB) (err error) { } func (s *accountDataStatements) insertAccountData( - ctx context.Context, localpart, roomID, dataType, content string, + ctx context.Context, txn *sql.Tx, localpart, roomID, dataType, content string, ) (err error) { - stmt := s.insertAccountDataStmt + stmt := txn.Stmt(s.insertAccountDataStmt) _, err = stmt.ExecContext(ctx, localpart, roomID, dataType, content) return } diff --git a/clientapi/auth/storage/accounts/postgres/accounts_table.go b/clientapi/auth/storage/accounts/postgres/accounts_table.go index 6b8ed3728..85c1938a1 100644 --- a/clientapi/auth/storage/accounts/postgres/accounts_table.go +++ b/clientapi/auth/storage/accounts/postgres/accounts_table.go @@ -91,10 +91,10 @@ func (s *accountsStatements) prepare(db *sql.DB, server gomatrixserverlib.Server // this account will be passwordless. Returns an error if this account already exists. Returns the account // on success. func (s *accountsStatements) insertAccount( - ctx context.Context, localpart, hash, appserviceID string, + ctx context.Context, txn *sql.Tx, localpart, hash, appserviceID string, ) (*authtypes.Account, error) { createdTimeMS := time.Now().UnixNano() / 1000000 - stmt := s.insertAccountStmt + stmt := txn.Stmt(s.insertAccountStmt) var err error if appserviceID == "" { @@ -146,8 +146,12 @@ func (s *accountsStatements) selectAccountByLocalpart( } func (s *accountsStatements) selectNewNumericLocalpart( - ctx context.Context, + ctx context.Context, txn *sql.Tx, ) (id int64, err error) { - err = s.selectNewNumericLocalpartStmt.QueryRowContext(ctx).Scan(&id) + stmt := s.selectNewNumericLocalpartStmt + if txn != nil { + stmt = txn.Stmt(stmt) + } + err = stmt.QueryRowContext(ctx).Scan(&id) return } diff --git a/clientapi/auth/storage/accounts/postgres/profile_table.go b/clientapi/auth/storage/accounts/postgres/profile_table.go index 38c76c40f..d2cbeb8e6 100644 --- a/clientapi/auth/storage/accounts/postgres/profile_table.go +++ b/clientapi/auth/storage/accounts/postgres/profile_table.go @@ -73,9 +73,9 @@ func (s *profilesStatements) prepare(db *sql.DB) (err error) { } func (s *profilesStatements) insertProfile( - ctx context.Context, localpart string, + ctx context.Context, txn *sql.Tx, localpart string, ) (err error) { - _, err = s.insertProfileStmt.ExecContext(ctx, localpart, "", "") + _, err = txn.Stmt(s.insertProfileStmt).ExecContext(ctx, localpart, "", "") return } diff --git a/clientapi/auth/storage/accounts/postgres/storage.go b/clientapi/auth/storage/accounts/postgres/storage.go index cb74d1315..8115dca43 100644 --- a/clientapi/auth/storage/accounts/postgres/storage.go +++ b/clientapi/auth/storage/accounts/postgres/storage.go @@ -18,6 +18,7 @@ import ( "context" "database/sql" "errors" + "strconv" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/common" @@ -118,11 +119,37 @@ func (d *Database) SetDisplayName( return d.profiles.setDisplayName(ctx, localpart, displayName) } +// CreateGuestAccount makes a new guest account and creates an empty profile +// for this account. +func (d *Database) CreateGuestAccount(ctx context.Context) (acc *authtypes.Account, err error) { + err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + var numLocalpart int64 + numLocalpart, err = d.accounts.selectNewNumericLocalpart(ctx, txn) + if err != nil { + return err + } + localpart := strconv.FormatInt(numLocalpart, 10) + acc, err = d.createAccount(ctx, txn, localpart, "", "") + return err + }) + return acc, err +} + // CreateAccount makes a new account with the given login name and password, and creates an empty profile // for this account. If no password is supplied, the account will be a passwordless account. If the // account already exists, it will return nil, nil. func (d *Database) CreateAccount( ctx context.Context, localpart, plaintextPassword, appserviceID string, +) (acc *authtypes.Account, err error) { + err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + acc, err = d.createAccount(ctx, txn, localpart, plaintextPassword, appserviceID) + return err + }) + return +} + +func (d *Database) createAccount( + ctx context.Context, txn *sql.Tx, localpart, plaintextPassword, appserviceID string, ) (*authtypes.Account, error) { var err error @@ -134,13 +161,14 @@ func (d *Database) CreateAccount( return nil, err } } - if err := d.profiles.insertProfile(ctx, localpart); err != nil { + if err := d.profiles.insertProfile(ctx, txn, localpart); err != nil { if common.IsUniqueConstraintViolationErr(err) { return nil, nil } return nil, err } - if err := d.SaveAccountData(ctx, localpart, "", "m.push_rules", `{ + + if err := d.accountDatas.insertAccountData(ctx, txn, localpart, "", "m.push_rules", `{ "global": { "content": [], "override": [], @@ -151,7 +179,7 @@ func (d *Database) CreateAccount( }`); err != nil { return nil, err } - return d.accounts.insertAccount(ctx, localpart, hash, appserviceID) + return d.accounts.insertAccount(ctx, txn, localpart, hash, appserviceID) } // SaveMembership saves the user matching a given localpart as a member of a given @@ -258,7 +286,9 @@ func (d *Database) newMembership( func (d *Database) SaveAccountData( ctx context.Context, localpart, roomID, dataType, content string, ) error { - return d.accountDatas.insertAccountData(ctx, localpart, roomID, dataType, content) + return common.WithTransaction(d.db, func(txn *sql.Tx) error { + return d.accountDatas.insertAccountData(ctx, txn, localpart, roomID, dataType, content) + }) } // GetAccountData returns account data related to a given localpart @@ -288,7 +318,7 @@ func (d *Database) GetAccountDataByType( func (d *Database) GetNewNumericLocalpart( ctx context.Context, ) (int64, error) { - return d.accounts.selectNewNumericLocalpart(ctx) + return d.accounts.selectNewNumericLocalpart(ctx, nil) } func hashPassword(plaintext string) (hash string, err error) { diff --git a/clientapi/auth/storage/accounts/sqlite3/account_data_table.go b/clientapi/auth/storage/accounts/sqlite3/account_data_table.go index c2143881b..b6bb63617 100644 --- a/clientapi/auth/storage/accounts/sqlite3/account_data_table.go +++ b/clientapi/auth/storage/accounts/sqlite3/account_data_table.go @@ -72,10 +72,9 @@ func (s *accountDataStatements) prepare(db *sql.DB) (err error) { } func (s *accountDataStatements) insertAccountData( - ctx context.Context, localpart, roomID, dataType, content string, + ctx context.Context, txn *sql.Tx, localpart, roomID, dataType, content string, ) (err error) { - stmt := s.insertAccountDataStmt - _, err = stmt.ExecContext(ctx, localpart, roomID, dataType, content) + _, err = txn.Stmt(s.insertAccountDataStmt).ExecContext(ctx, localpart, roomID, dataType, content) return } diff --git a/clientapi/auth/storage/accounts/sqlite3/accounts_table.go b/clientapi/auth/storage/accounts/sqlite3/accounts_table.go index b029951f1..fd6a09cde 100644 --- a/clientapi/auth/storage/accounts/sqlite3/accounts_table.go +++ b/clientapi/auth/storage/accounts/sqlite3/accounts_table.go @@ -89,16 +89,16 @@ func (s *accountsStatements) prepare(db *sql.DB, server gomatrixserverlib.Server // this account will be passwordless. Returns an error if this account already exists. Returns the account // on success. func (s *accountsStatements) insertAccount( - ctx context.Context, localpart, hash, appserviceID string, + ctx context.Context, txn *sql.Tx, localpart, hash, appserviceID string, ) (*authtypes.Account, error) { createdTimeMS := time.Now().UnixNano() / 1000000 stmt := s.insertAccountStmt var err error if appserviceID == "" { - _, err = stmt.ExecContext(ctx, localpart, createdTimeMS, hash, nil) + _, err = txn.Stmt(stmt).ExecContext(ctx, localpart, createdTimeMS, hash, nil) } else { - _, err = stmt.ExecContext(ctx, localpart, createdTimeMS, hash, appserviceID) + _, err = txn.Stmt(stmt).ExecContext(ctx, localpart, createdTimeMS, hash, appserviceID) } if err != nil { return nil, err @@ -144,8 +144,12 @@ func (s *accountsStatements) selectAccountByLocalpart( } func (s *accountsStatements) selectNewNumericLocalpart( - ctx context.Context, + ctx context.Context, txn *sql.Tx, ) (id int64, err error) { - err = s.selectNewNumericLocalpartStmt.QueryRowContext(ctx).Scan(&id) + stmt := s.selectNewNumericLocalpartStmt + if txn != nil { + stmt = txn.Stmt(stmt) + } + err = stmt.QueryRowContext(ctx).Scan(&id) return } diff --git a/clientapi/auth/storage/accounts/sqlite3/profile_table.go b/clientapi/auth/storage/accounts/sqlite3/profile_table.go index 7af8307e1..9b5192a02 100644 --- a/clientapi/auth/storage/accounts/sqlite3/profile_table.go +++ b/clientapi/auth/storage/accounts/sqlite3/profile_table.go @@ -73,9 +73,9 @@ func (s *profilesStatements) prepare(db *sql.DB) (err error) { } func (s *profilesStatements) insertProfile( - ctx context.Context, localpart string, + ctx context.Context, txn *sql.Tx, localpart string, ) (err error) { - _, err = s.insertProfileStmt.ExecContext(ctx, localpart, "", "") + _, err = txn.Stmt(s.insertProfileStmt).ExecContext(ctx, localpart, "", "") return } diff --git a/clientapi/auth/storage/accounts/sqlite3/storage.go b/clientapi/auth/storage/accounts/sqlite3/storage.go index 3e62d10dd..9124640c6 100644 --- a/clientapi/auth/storage/accounts/sqlite3/storage.go +++ b/clientapi/auth/storage/accounts/sqlite3/storage.go @@ -18,6 +18,8 @@ import ( "context" "database/sql" "errors" + "strconv" + "sync" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/common" @@ -39,6 +41,8 @@ type Database struct { threepids threepidStatements filter filterStatements serverName gomatrixserverlib.ServerName + + createGuestAccountMu sync.Mutex } // NewDatabase creates a new accounts and profiles database @@ -76,7 +80,7 @@ func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) if err = f.prepare(db); err != nil { return nil, err } - return &Database{db, partitions, a, p, m, ac, t, f, serverName}, nil + return &Database{db, partitions, a, p, m, ac, t, f, serverName, sync.Mutex{}}, nil } // GetAccountByPassword returns the account associated with the given localpart and password. @@ -118,14 +122,46 @@ func (d *Database) SetDisplayName( return d.profiles.setDisplayName(ctx, localpart, displayName) } +// CreateGuestAccount makes a new guest account and creates an empty profile +// for this account. +func (d *Database) CreateGuestAccount(ctx context.Context) (acc *authtypes.Account, err error) { + err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + // We need to lock so we sequentially create numeric localparts. If we don't, two calls to + // this function will cause the same number to be selected and one will fail with 'database is locked' + // when the first txn upgrades to a write txn. + // We know we'll be the only process since this is sqlite ;) so a lock here will be all that is needed. + d.createGuestAccountMu.Lock() + defer d.createGuestAccountMu.Unlock() + + var numLocalpart int64 + numLocalpart, err = d.accounts.selectNewNumericLocalpart(ctx, txn) + if err != nil { + return err + } + localpart := strconv.FormatInt(numLocalpart, 10) + acc, err = d.createAccount(ctx, txn, localpart, "", "") + return err + }) + return acc, err +} + // CreateAccount makes a new account with the given login name and password, and creates an empty profile // for this account. If no password is supplied, the account will be a passwordless account. If the // account already exists, it will return nil, nil. func (d *Database) CreateAccount( ctx context.Context, localpart, plaintextPassword, appserviceID string, +) (acc *authtypes.Account, err error) { + err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + acc, err = d.createAccount(ctx, txn, localpart, plaintextPassword, appserviceID) + return err + }) + return +} + +func (d *Database) createAccount( + ctx context.Context, txn *sql.Tx, localpart, plaintextPassword, appserviceID string, ) (*authtypes.Account, error) { var err error - // Generate a password hash if this is not a password-less user hash := "" if plaintextPassword != "" { @@ -134,13 +170,14 @@ func (d *Database) CreateAccount( return nil, err } } - if err := d.profiles.insertProfile(ctx, localpart); err != nil { + if err := d.profiles.insertProfile(ctx, txn, localpart); err != nil { if common.IsUniqueConstraintViolationErr(err) { return nil, nil } return nil, err } - if err := d.SaveAccountData(ctx, localpart, "", "m.push_rules", `{ + + if err := d.accountDatas.insertAccountData(ctx, txn, localpart, "", "m.push_rules", `{ "global": { "content": [], "override": [], @@ -151,7 +188,7 @@ func (d *Database) CreateAccount( }`); err != nil { return nil, err } - return d.accounts.insertAccount(ctx, localpart, hash, appserviceID) + return d.accounts.insertAccount(ctx, txn, localpart, hash, appserviceID) } // SaveMembership saves the user matching a given localpart as a member of a given @@ -258,7 +295,9 @@ func (d *Database) newMembership( func (d *Database) SaveAccountData( ctx context.Context, localpart, roomID, dataType, content string, ) error { - return d.accountDatas.insertAccountData(ctx, localpart, roomID, dataType, content) + return common.WithTransaction(d.db, func(txn *sql.Tx) error { + return d.accountDatas.insertAccountData(ctx, txn, localpart, roomID, dataType, content) + }) } // GetAccountData returns account data related to a given localpart @@ -288,7 +327,7 @@ func (d *Database) GetAccountDataByType( func (d *Database) GetNewNumericLocalpart( ctx context.Context, ) (int64, error) { - return d.accounts.selectNewNumericLocalpart(ctx) + return d.accounts.selectNewNumericLocalpart(ctx, nil) } func hashPassword(plaintext string) (hash string, err error) { diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index ba24e5273..2de7b2733 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -516,16 +516,7 @@ func handleGuestRegistration( accountDB accounts.Database, deviceDB devices.Database, ) util.JSONResponse { - - //Generate numeric local part for guest user - id, err := accountDB.GetNewNumericLocalpart(req.Context()) - if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("accountDB.GetNewNumericLocalpart failed") - return jsonerror.InternalServerError() - } - - localpart := strconv.FormatInt(id, 10) - acc, err := accountDB.CreateAccount(req.Context(), localpart, "", "") + acc, err := accountDB.CreateGuestAccount(req.Context()) if err != nil { return util.JSONResponse{ Code: http.StatusInternalServerError, From cdc115778551af69b4c9c39f758adcae61701aa3 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Mon, 9 Mar 2020 14:37:51 +0000 Subject: [PATCH 29/86] Improve logging when sending events (#883) We have some failing sytests on sqlite but it's very difficult to debug due to lack of useful logging. This adds a log line for when a new event is sent (incl. logging the event ID) as well as adding a user_id field for all contextual logs so we know who initiated certain actions. --- clientapi/routing/sendevent.go | 1 + common/httpapi.go | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go index 47ad18825..da96c5c65 100644 --- a/clientapi/routing/sendevent.go +++ b/clientapi/routing/sendevent.go @@ -77,6 +77,7 @@ func SendEvent( util.GetLogger(req.Context()).WithError(err).Error("producer.SendEvents failed") return jsonerror.InternalServerError() } + util.GetLogger(req.Context()).WithField("event_id", eventID).Info("Sent event") res := util.JSONResponse{ Code: http.StatusOK, diff --git a/common/httpapi.go b/common/httpapi.go index 59b303b6d..22c774475 100644 --- a/common/httpapi.go +++ b/common/httpapi.go @@ -25,6 +25,10 @@ func MakeAuthAPI( if err != nil { return *err } + // add the user ID to the logger + logger := util.GetLogger((req.Context())) + logger = logger.WithField("user_id", device.UserID) + req = req.WithContext(util.ContextWithLogger(req.Context(), logger)) return f(req, device) } From 176f722d53fb47f68b77752aad56f03ac8c3ed4b Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 10 Mar 2020 11:42:40 +0000 Subject: [PATCH 30/86] Update .gitignore --- .gitignore | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitignore b/.gitignore index a36cb820a..1de8887ce 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,9 @@ _testmain.go # Default configuration file dendrite.yaml + +# Database files +*.db + +# Log files +*.log* From 09dfd4839a75c0018269355f7f4918b3dd3a503a Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 10 Mar 2020 14:13:15 +0000 Subject: [PATCH 31/86] p2p: Use a single ed25519 key for event signing/verifying and p2p peer IDs (#885) * Pass in the private key seed to NewP2PLocalNode to use the same key for p2p * Extract the ed25519 key from the server name and use it for event checks This allows HSes which no longer exist to still have events verified for them. This means that new members can join rooms that has missing servers. * Remove no-op ipfs uri which we don't use anymore --- cmd/dendritejs/keyfetcher.go | 84 ++++++++++++++++++++++++ cmd/dendritejs/main.go | 19 +++--- go.mod | 6 +- go.sum | 122 +++++++++++++++++++++++++++++++++++ 4 files changed, 221 insertions(+), 10 deletions(-) create mode 100644 cmd/dendritejs/keyfetcher.go diff --git a/cmd/dendritejs/keyfetcher.go b/cmd/dendritejs/keyfetcher.go new file mode 100644 index 000000000..2b0a54fd9 --- /dev/null +++ b/cmd/dendritejs/keyfetcher.go @@ -0,0 +1,84 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build wasm + +package main + +import ( + "context" + "fmt" + "time" + + "github.com/libp2p/go-libp2p-core/peer" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" +) + +const libp2pMatrixKeyID = "ed25519:libp2p-dendrite" + +type libp2pKeyFetcher struct { +} + +// FetchKeys looks up a batch of public keys. +// Takes a map from (server name, key ID) pairs to timestamp. +// The timestamp is when the keys need to be vaild up to. +// Returns a map from (server name, key ID) pairs to server key objects for +// that server name containing that key ID +// The result may have fewer (server name, key ID) pairs than were in the request. +// The result may have more (server name, key ID) pairs than were in the request. +// Returns an error if there was a problem fetching the keys. +func (f *libp2pKeyFetcher) FetchKeys( + ctx context.Context, + requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp, +) (map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, error) { + res := make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult) + for req := range requests { + if req.KeyID != libp2pMatrixKeyID { + return nil, fmt.Errorf("FetchKeys: cannot fetch key with ID %s, should be %s", req.KeyID, libp2pMatrixKeyID) + } + + // The server name is a libp2p peer ID + peerIDStr := string(req.ServerName) + peerID, err := peer.Decode(peerIDStr) + if err != nil { + return nil, fmt.Errorf("Failed to decode peer ID from server name '%s': %s", peerIDStr, err) + } + pubKey, err := peerID.ExtractPublicKey() + if err != nil { + return nil, fmt.Errorf("Failed to extract public key from peer ID: %s", err) + } + pubKeyBytes, err := pubKey.Raw() + if err != nil { + return nil, fmt.Errorf("Failed to extract raw bytes from public key: %s", err) + } + util.GetLogger(ctx).Info("libp2pKeyFetcher.FetchKeys: Using public key %v for server name %s", pubKeyBytes, req.ServerName) + + b64Key := gomatrixserverlib.Base64String(pubKeyBytes) + res[req] = gomatrixserverlib.PublicKeyLookupResult{ + VerifyKey: gomatrixserverlib.VerifyKey{ + Key: b64Key, + }, + ExpiredTS: gomatrixserverlib.PublicKeyNotExpired, + ValidUntilTS: gomatrixserverlib.AsTimestamp(time.Now().Add(24 * time.Hour * 365)), + } + } + return res, nil +} + +// FetcherName returns the name of this fetcher, which can then be used for +// logging errors etc. +func (f *libp2pKeyFetcher) FetcherName() string { + return "libp2pKeyFetcher" +} diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 347a52835..a8791de67 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -26,7 +26,6 @@ import ( "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/basecomponent" "github.com/matrix-org/dendrite/common/config" - "github.com/matrix-org/dendrite/common/keydb" "github.com/matrix-org/dendrite/common/transactions" "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/federationsender" @@ -69,10 +68,9 @@ func createFederationClient(cfg *config.Dendrite, node *go_http_js_libp2p.P2pLoc return fed } -func createP2PNode() (serverName string, node *go_http_js_libp2p.P2pLocalNode) { - hosted := "/dns4/ws-star.discovery.libp2p.io/tcp/443/wss/p2p-websocket-star" - _ = "/ip4/127.0.0.1/tcp/9090/ws/p2p-websocket-star/" - node = go_http_js_libp2p.NewP2pLocalNode("org.matrix.p2p.experiment", []string{hosted}) +func createP2PNode(privKey ed25519.PrivateKey) (serverName string, node *go_http_js_libp2p.P2pLocalNode) { + hosted := "/ip4/127.0.0.1/tcp/9090/ws/p2p-websocket-star/" + node = go_http_js_libp2p.NewP2pLocalNode("org.matrix.p2p.experiment", privKey.Seed(), []string{hosted}) serverName = node.Id fmt.Println("p2p assigned ServerName: ", serverName) return @@ -95,10 +93,10 @@ func main() { cfg.Matrix.TrustedIDServers = []string{ "matrix.org", "vector.im", } - cfg.Matrix.KeyID = "ed25519:1337" + cfg.Matrix.KeyID = libp2pMatrixKeyID cfg.Matrix.PrivateKey = generateKey() - serverName, node := createP2PNode() + serverName, node := createP2PNode(cfg.Matrix.PrivateKey) cfg.Matrix.ServerName = gomatrixserverlib.ServerName(serverName) if err := cfg.Derive(); err != nil { @@ -111,7 +109,12 @@ func main() { deviceDB := base.CreateDeviceDB() keyDB := base.CreateKeyDB() federation := createFederationClient(cfg, node) - keyRing := keydb.CreateKeyRing(federation.Client, keyDB) + keyRing := gomatrixserverlib.KeyRing{ + KeyFetchers: []gomatrixserverlib.KeyFetcher{ + &libp2pKeyFetcher{}, + }, + KeyDatabase: keyDB, + } alias, input, query := roomserver.SetupRoomServerComponent(base) typingInputAPI := typingserver.SetupTypingServerComponent(base, cache.NewTypingCache()) diff --git a/go.mod b/go.mod index 924c4b67d..cda7bdf1b 100644 --- a/go.mod +++ b/go.mod @@ -1,11 +1,13 @@ module github.com/matrix-org/dendrite require ( + github.com/btcsuite/btcutil v1.0.1 github.com/gorilla/mux v1.7.3 github.com/hashicorp/golang-lru v0.5.4 github.com/lib/pq v1.2.0 + github.com/libp2p/go-libp2p-core v0.5.0 github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 - github.com/matrix-org/go-http-js-libp2p v0.0.0-20200304160008-4ec1129a00c4 + github.com/matrix-org/go-http-js-libp2p v0.0.0-20200306192008-b9e71eeaa437 github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 github.com/matrix-org/gomatrixserverlib v0.0.0-20200306154041-df6bb9a3e424 @@ -21,7 +23,7 @@ require ( github.com/tidwall/pretty v1.0.1 // indirect github.com/uber/jaeger-client-go v2.22.1+incompatible github.com/uber/jaeger-lib v2.2.0+incompatible - golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 + golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d gopkg.in/Shopify/sarama.v1 v1.20.1 gopkg.in/h2non/bimg.v1 v1.0.18 gopkg.in/yaml.v2 v2.2.5 diff --git a/go.sum b/go.sum index 3a509debb..70be0cbb3 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,10 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/zstd v1.4.4 h1:+IawcoXhCBylN7ccwdwf8LOH2jKq7NavGpEPanrlTzE= github.com/DataDog/zstd v1.4.4/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -10,11 +13,25 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw= +github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= +github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= +github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= +github.com/btcsuite/btcutil v1.0.1 h1:GKOz8BnRjYrb/JTKgaOk+zh26NWNdSNvdvv0xoAZMSA= +github.com/btcsuite/btcutil v1.0.1/go.mod h1:j9HUFwoQRsZL3V4n+qG+CUnEGHOarIxfC3Le2Yhbcts= +github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= +github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= +github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= +github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= +github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -28,12 +45,18 @@ github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFP github.com/frankban/quicktest v1.0.0/go.mod h1:R98jIehRai+d1/3Hv2//jOVCTJhW1VBavT6B6CuGq2k= github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3BTYk= github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= @@ -53,10 +76,23 @@ github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslC github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ipfs/go-cid v0.0.5 h1:o0Ix8e/ql7Zb5UVUJEUfjsWCIY8t48++9lR8qi6oiJU= +github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog= +github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= +github.com/jbenet/goprocess v0.1.3 h1:YKyIEECS/XvcfHtBzxtjBBbWK+MbvA6dG8ASiqwvr10= +github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= +github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -68,11 +104,28 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/libp2p/go-buffer-pool v0.0.2 h1:QNK2iAFa8gjAe1SPz6mHSMuCcjs+X1wlHzeOSqcmlfs= +github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM= +github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= +github.com/libp2p/go-libp2p-core v0.5.0 h1:FBQ1fpq2Fo/ClyjojVJ5AKXlKhvNc/B6U0O+7AN1ffE= +github.com/libp2p/go-libp2p-core v0.5.0/go.mod h1:49XGI+kc38oGVwqSBhDEwytaAxgZasHhFfQKibzTls0= +github.com/libp2p/go-openssl v0.0.4/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= +github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/matrix-org/dendrite v0.0.0-20200220135450-0352f250b857/go.mod h1:DZ35IoR+ViBNVPe9umdlOSnjvKl7wfyRmZg4QfWGvTo= github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 h1:nMX2t7hbGF0NYDYySx0pCqEKGKAeZIiSqlWSspetlhY= github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg= github.com/matrix-org/go-http-js-libp2p v0.0.0-20200304160008-4ec1129a00c4 h1:oFjG7X1jS473zPDix1/FBZ2qd0anM1Ko4AlJCB6MUZs= github.com/matrix-org/go-http-js-libp2p v0.0.0-20200304160008-4ec1129a00c4/go.mod h1:PJof7UbOKmVEEMsGqvbyIK0ldMMBjPH5EYia7MHR2RQ= +github.com/matrix-org/go-http-js-libp2p v0.0.0-20200306184112-c0b07c81904f h1:08U2fGZgEMzRYM2QmL2p9r+QmGryYUuZCJOSZWmxzN8= +github.com/matrix-org/go-http-js-libp2p v0.0.0-20200306184112-c0b07c81904f/go.mod h1:/giSXVd8D6DZGSfTmhQrLEoZZwsfkC14kSqP9MiLqIY= +github.com/matrix-org/go-http-js-libp2p v0.0.0-20200306190227-af44d7013315 h1:tExW6I0wSXy0CWz8QvvkQEuvtBO88CFKYf0MYkpKwek= +github.com/matrix-org/go-http-js-libp2p v0.0.0-20200306190227-af44d7013315/go.mod h1:/giSXVd8D6DZGSfTmhQrLEoZZwsfkC14kSqP9MiLqIY= +github.com/matrix-org/go-http-js-libp2p v0.0.0-20200306192008-b9e71eeaa437 h1:zcGpWvVV6swXw9LBMRsdDHPOugQYSwesH2RByUfBx2I= +github.com/matrix-org/go-http-js-libp2p v0.0.0-20200306192008-b9e71eeaa437/go.mod h1:/giSXVd8D6DZGSfTmhQrLEoZZwsfkC14kSqP9MiLqIY= +github.com/matrix-org/go-sqlite3-js v0.0.0-20200226144546-ea6ed5b90074 h1:UWz6vfhmQVshBuE67X1BCsdMhEDtd+uOz8CJ48Fc0F4= +github.com/matrix-org/go-sqlite3-js v0.0.0-20200226144546-ea6ed5b90074/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= +github.com/matrix-org/go-sqlite3-js v0.0.0-20200304163011-cfb4884075db h1:ERuFJq4DI8fakfBZlvXHltHZ0ix3K5YsLG0tQfQn6TI= +github.com/matrix-org/go-sqlite3-js v0.0.0-20200304163011-cfb4884075db/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658 h1:UlhTKClOgWnSB25Rv+BS/Vc1mRinjNUErfyGEVOBP04= github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20190130130140-385f072fe9af h1:piaIBNQGIHnni27xRB7VKkEwoWCgAmeuYf8pxAyG0bI= @@ -89,23 +142,48 @@ github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 h1:W7l5CP4V7wPyPb4 github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5/go.mod h1:lePuOiXLNDott7NZfnQvJk0lAZ5HgvIuWGhel6J+RLA= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-sqlite3 v2.0.2+incompatible h1:qzw9c2GNT8UFrgWNDhCTqRqYUSmu/Dav/9Z58LGpk7U= github.com/mattn/go-sqlite3 v2.0.2+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/dns v1.1.4 h1:rCMZsU2ScVSYcAsOXgmC6+AKOK+6pmQTOcw03nfwYV0= github.com/miekg/dns v1.1.4/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.12 h1:WMhc1ik4LNkTg8U9l3hI1LvxKmIL+f1+WV/SZtCbDDA= github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g= +github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= +github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= +github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= +github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc= +github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI= +github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= +github.com/multiformats/go-multiaddr v0.2.1 h1:SgG/cw5vqyB5QQe5FPe2TqggU9WtrA9X4nZw7LlVqOI= +github.com/multiformats/go-multiaddr v0.2.1/go.mod h1:s/Apk6IyxfvMjDafnhJgJ3/46z7tZ04iMk5wP4QMGGE= +github.com/multiformats/go-multibase v0.0.1 h1:PN9/v21eLywrFWdFNsFKaU04kLJzuYzmrJR+ubhT9qA= +github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= +github.com/multiformats/go-multihash v0.0.13 h1:06x+mk/zj1FoMsgNejLpy6QTvJqlSt/BhLEy87zidlc= +github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= +github.com/multiformats/go-varint v0.0.2/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= +github.com/multiformats/go-varint v0.0.5 h1:XVZwSo04Cs3j/jS0uAEPpT3JY6DzMcVLLoWOSnCxOjg= +github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 h1:BvoENQQU+fZ9uukda/RzCAL/191HHwJA5b13R6diVlY= github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= @@ -143,6 +221,11 @@ github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY= +github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -170,42 +253,76 @@ github.com/uber/jaeger-lib v1.5.0 h1:OHbgr8l656Ub3Fw5k9SWnBfIEwvoHQ+W2y+Aa9D1Uyo github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw= github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= +github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.0 h1:vs7fgriifsPbGdK3bNuMWapNn3qnZhCRXc19NRdq010= go.uber.org/atomic v1.3.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613 h1:MQ/ZZiDsUapFFiMS+vzwXkCTeEKaum+Do5rINYJDmxc= golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d h1:2+ZP7EfsZV7Vvmx3TIqSlSzATMkTAKqM14YGFPoSKjI= +golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190301231341-16b79f2e4e95 h1:fY7Dsw114eJN4boqzVSbpVHO6rTdhq6/GnXeu+PKnzU= golang.org/x/net v0.0.0-20190301231341-16b79f2e4e95/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0Is3p+EHBKNgquIzo1OI= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 h1:I6FyU15t786LL7oL/hn43zqTuEGr4PN7F4XJ1p4E3Y8= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 h1:ywK/j/KkyTHcdyYSZNXGjMwgmDSfjglYZ3vStQ/gSCU= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= gopkg.in/Shopify/sarama.v1 v1.20.1 h1:Gi09A3fJXm0Jgt8kuKZ8YK+r60GfYn7MQuEmI3oq6hE= gopkg.in/Shopify/sarama.v1 v1.20.1/go.mod h1:AxnvoaevB2nBjNK17cG61A3LleFcWFwVBHBt+cot4Oc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -215,15 +332,20 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/h2non/bimg.v1 v1.0.18 h1:qn6/RpBHt+7WQqoBcK+aF2puc6nC78eZj5LexxoalT4= gopkg.in/h2non/bimg.v1 v1.0.18/go.mod h1:PgsZL7dLwUbsGm1NYps320GxGgvQNTnecMCZqxV11So= gopkg.in/h2non/gock.v1 v1.0.14 h1:fTeu9fcUvSnLNacYvYI54h+1/XEteDyHvrVCZEEEYNM= gopkg.in/h2non/gock.v1 v1.0.14/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE= gopkg.in/macaroon.v2 v2.1.0 h1:HZcsjBCzq9t0eBPMKqTN/uSN6JOm78ZJ2INbqcBQOUI= gopkg.in/macaroon.v2 v2.1.0/go.mod h1:OUb+TQP/OP0WOerC2Jp/3CwhIKyIa9kQjuc7H24e6/o= +gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= +gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From d71b72816d30f35b1fde9f2ca4826ac82c61d2b8 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 10 Mar 2020 18:10:10 +0000 Subject: [PATCH 32/86] p2p: use separate topics! (#886) Without it, setting a display name fails and /sync wedges forever. --- cmd/dendritejs/main.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index a8791de67..214c887ed 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -90,6 +90,10 @@ func main() { cfg.Database.RoomServer = "file:dendritejs_roomserver.db" cfg.Database.ServerKey = "file:dendritejs_serverkey.db" cfg.Database.SyncAPI = "file:dendritejs_syncapi.db" + cfg.Kafka.Topics.UserUpdates = "user_updates" + cfg.Kafka.Topics.OutputTypingEvent = "output_typing_event" + cfg.Kafka.Topics.OutputClientData = "output_client_data" + cfg.Kafka.Topics.OutputRoomEvent = "output_room_event" cfg.Matrix.TrustedIDServers = []string{ "matrix.org", "vector.im", } From 8bc5084d8d26fa02a2c8f3ecc346db62f31a24cb Mon Sep 17 00:00:00 2001 From: Kegsay Date: Wed, 11 Mar 2020 12:18:37 +0000 Subject: [PATCH 33/86] p2p: Use JSServer for comms rather than GoJsConn (#888) * p2p: Use JSServer for comms rather than GoJsConn This has several benefits: - it fixes a bug whereby you could not transmit >4k bytes to/from JS/Go land. - it more clearly exposes the interface point between Go and JS: a single global function call. - it presents a nicer API shape than the previous `net.Conn`. - it doesn't needlessly 'stream' data which is already sitting in-memory. This is currently only active for local CS API traffic, another PR will add Federation P2P support. * Typo --- cmd/dendritejs/jsServer.go | 104 +++++++++++++++++++++++++++++++++++++ cmd/dendritejs/main.go | 8 +-- go.mod | 2 +- go.sum | 2 + p2p.md | 17 +++--- 5 files changed, 118 insertions(+), 15 deletions(-) create mode 100644 cmd/dendritejs/jsServer.go diff --git a/cmd/dendritejs/jsServer.go b/cmd/dendritejs/jsServer.go new file mode 100644 index 000000000..a5ac574d8 --- /dev/null +++ b/cmd/dendritejs/jsServer.go @@ -0,0 +1,104 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build wasm + +package main + +import ( + "bufio" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "syscall/js" +) + +// JSServer exposes an HTTP-like server interface which allows JS to 'send' requests to it. +type JSServer struct { + // The router which will service requests + Mux *http.ServeMux +} + +// OnRequestFromJS is the function that JS will invoke when there is a new request. +// The JS function signature is: +// function(reqString: string): Promise<{result: string, error: string}> +// Usage is like: +// const res = await global._go_js_server.fetch(reqString); +// if (res.error) { +// // handle error: this is a 'network' error, not a non-2xx error. +// } +// const rawHttpResponse = res.result; +func (h *JSServer) OnRequestFromJS(this js.Value, args []js.Value) interface{} { + // we HAVE to spawn a new goroutine and return immediately or else Go will deadlock + // if this request blocks at all e.g for /sync calls + httpStr := args[0].String() + promise := js.Global().Get("Promise").New(js.FuncOf(func(pthis js.Value, pargs []js.Value) interface{} { + // The initial callback code for new Promise() is also called on the critical path, which is why + // we need to put this in an immediately invoked goroutine. + go func() { + resolve := pargs[0] + fmt.Println("Received request:") + fmt.Printf("%s\n", httpStr) + resStr, err := h.handle(httpStr) + errStr := "" + if err != nil { + errStr = err.Error() + } + fmt.Println("Sending response:") + fmt.Printf("%s\n", resStr) + resolve.Invoke(map[string]interface{}{ + "result": resStr, + "error": errStr, + }) + }() + return nil + })) + return promise +} + +// handle invokes the http.ServeMux for this request and returns the raw HTTP response. +func (h *JSServer) handle(httpStr string) (resStr string, err error) { + req, err := http.ReadRequest(bufio.NewReader(strings.NewReader(httpStr))) + if err != nil { + return + } + w := httptest.NewRecorder() + + h.Mux.ServeHTTP(w, req) + + res := w.Result() + var resBuffer strings.Builder + err = res.Write(&resBuffer) + return resBuffer.String(), err +} + +// ListenAndServe registers a variable in JS-land with the given namespace. This variable is +// a function which JS-land can call to 'send' HTTP requests. The function is attached to +// a global object called "_go_js_server". See OnRequestFromJS for more info. +func (h *JSServer) ListenAndServe(namespace string) { + globalName := "_go_js_server" + // register a hook in JS-land for it to invoke stuff + server := js.Global().Get(globalName) + if !server.Truthy() { + server = js.Global().Get("Object").New() + js.Global().Set(globalName, server) + } + + server.Set(namespace, js.FuncOf(h.OnRequestFromJS)) + + fmt.Printf("Listening for requests from JS on function %s.%s\n", globalName, namespace) + // Block forever to mimic http.ListenAndServe + select {} +} diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 214c887ed..458427a7b 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -156,10 +156,10 @@ func main() { // Expose the matrix APIs via fetch - for local traffic go func() { logrus.Info("Listening for service-worker fetch traffic") - - listener := go_http_js_libp2p.NewFetchListener() - s := &http.Server{} - go s.Serve(listener) + s := JSServer{ + Mux: http.DefaultServeMux, + } + s.ListenAndServe("fetch") }() // We want to block forever to let the fetch and libp2p handler serve the APIs diff --git a/go.mod b/go.mod index cda7bdf1b..92e0b7201 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/lib/pq v1.2.0 github.com/libp2p/go-libp2p-core v0.5.0 github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 - github.com/matrix-org/go-http-js-libp2p v0.0.0-20200306192008-b9e71eeaa437 + github.com/matrix-org/go-http-js-libp2p v0.0.0-20200310180544-7f3fad43b51c github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 github.com/matrix-org/gomatrixserverlib v0.0.0-20200306154041-df6bb9a3e424 diff --git a/go.sum b/go.sum index 70be0cbb3..c28254d46 100644 --- a/go.sum +++ b/go.sum @@ -122,6 +122,8 @@ github.com/matrix-org/go-http-js-libp2p v0.0.0-20200306190227-af44d7013315 h1:tE github.com/matrix-org/go-http-js-libp2p v0.0.0-20200306190227-af44d7013315/go.mod h1:/giSXVd8D6DZGSfTmhQrLEoZZwsfkC14kSqP9MiLqIY= github.com/matrix-org/go-http-js-libp2p v0.0.0-20200306192008-b9e71eeaa437 h1:zcGpWvVV6swXw9LBMRsdDHPOugQYSwesH2RByUfBx2I= github.com/matrix-org/go-http-js-libp2p v0.0.0-20200306192008-b9e71eeaa437/go.mod h1:/giSXVd8D6DZGSfTmhQrLEoZZwsfkC14kSqP9MiLqIY= +github.com/matrix-org/go-http-js-libp2p v0.0.0-20200310180544-7f3fad43b51c h1:jj/LIZKMO7GK6O0UarpRwse9L3ZyzozpyMtdPA7ddSk= +github.com/matrix-org/go-http-js-libp2p v0.0.0-20200310180544-7f3fad43b51c/go.mod h1:qK3LUW7RCLhFM7gC3pabj3EXT9A1DsCK33MHstUhhbk= github.com/matrix-org/go-sqlite3-js v0.0.0-20200226144546-ea6ed5b90074 h1:UWz6vfhmQVshBuE67X1BCsdMhEDtd+uOz8CJ48Fc0F4= github.com/matrix-org/go-sqlite3-js v0.0.0-20200226144546-ea6ed5b90074/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/go-sqlite3-js v0.0.0-20200304163011-cfb4884075db h1:ERuFJq4DI8fakfBZlvXHltHZ0ix3K5YsLG0tQfQn6TI= diff --git a/p2p.md b/p2p.md index 0c3dc83bb..141aaa1fc 100644 --- a/p2p.md +++ b/p2p.md @@ -16,7 +16,7 @@ $ cp main.wasm ../riot-web/src/vector/dendrite.wasm This is how peers discover each other and communicate. -By default, Dendrite uses the IPFS-hosted websocket star **Development** relay server at `/dns4/ws-star.discovery.libp2p.io/tcp/443/wss/p2p-websocket-star`. +By default, Dendrite uses the Matrix-hosted websocket star relay server at TODO `/dns4/ws-star.discovery.libp2p.io/tcp/443/wss/p2p-websocket-star`. This is currently hard-coded in `./cmd/dendritejs/main.go` - you can also use a local one if you run your own relay: ``` @@ -24,13 +24,13 @@ $ npm install --global libp2p-websocket-star-rendezvous $ rendezvous --port=9090 --host=127.0.0.1 ``` -Then use `/ip4/127.0.0.1/tcp/9090/ws/p2p-websocket-star/`. We'll probably run our own relay server at some point. +Then use `/ip4/127.0.0.1/tcp/9090/ws/p2p-websocket-star/`. ### Riot-web You need to check out these repos: -`` +``` $ git clone git@github.com:matrix-org/go-http-js-libp2p.git $ git clone git@github.com:matrix-org/go-sqlite3-js.git ``` @@ -39,6 +39,7 @@ Make sure to `yarn install` in both of these repos. Then: - `$ cp "$(go env GOROOT)/misc/wasm/wasm_exec.js" ./src/vector/` - Comment out the lines in `wasm_exec.js` which contains: + ``` if (!global.fs && global.require) { global.fs = require("fs"); @@ -56,17 +57,13 @@ NB: If you don't run the server with `yarn start` you need to make sure your ser TODO: Make a Docker image with all of this in it and a volume mount for `dendrite.wasm`. -## Running +### Running You need a Chrome and a Firefox running to test locally as service workers don't work in incognito tabs. - For Chrome, use `chrome://serviceworker-internals/` to unregister/see logs. - For Firefox, use `about:debugging#/runtime/this-firefox` to unregister. Use the console window to see logs. -Assuming you've `yarn start`ed Riot-Web, go to `http://localhost:8080` and wait a bit. Then refresh the page (this is required -because the fetch interceptor races with setting up dendrite. If you don't refresh, you won't be able to contact your HS). After -the refresh, click Register and use `http://localhost:8080` as your HS URL. - -TODO: Fix the race so we don't need multiple refreshes. +Assuming you've `yarn start`ed Riot-Web, go to `http://localhost:8080` and register with `http://localhost:8080` as your HS URL. You can join rooms by room alias e.g `/join #foo:bar`. @@ -74,7 +71,7 @@ You can join rooms by room alias e.g `/join #foo:bar`. - When registering you may be unable to find the server, it'll seem flakey. This happens because the SW, particularly in Firefox, gets killed after 30s of inactivity. When you are not registered, you aren't doing `/sync` calls to keep the SW alive, so if you - don't register for a while and idle on the page, the HS will disappear. To fix, unregister the SW, and then refresh the page *twice*. + don't register for a while and idle on the page, the HS will disappear. To fix, unregister the SW, and then refresh the page. - The libp2p layer has rate limits, so frequent Federation traffic may cause the connection to drop and messages to not be transferred. I guess in other words, don't send too much traffic? From 7353de7dbb146a16696cf2e9683753c3b08291f2 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 13 Mar 2020 10:15:01 +0000 Subject: [PATCH 34/86] Make P2P use JSServer --- cmd/dendritejs/main.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 458427a7b..25492b166 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -69,7 +69,7 @@ func createFederationClient(cfg *config.Dendrite, node *go_http_js_libp2p.P2pLoc } func createP2PNode(privKey ed25519.PrivateKey) (serverName string, node *go_http_js_libp2p.P2pLocalNode) { - hosted := "/ip4/127.0.0.1/tcp/9090/ws/p2p-websocket-star/" + hosted := "/dns4/rendezvous.matrix.org/tcp/8443/wss/p2p-websocket-star/" node = go_http_js_libp2p.NewP2pLocalNode("org.matrix.p2p.experiment", privKey.Seed(), []string{hosted}) serverName = node.Id fmt.Println("p2p assigned ServerName: ", serverName) @@ -145,11 +145,10 @@ func main() { if node != nil { go func() { logrus.Info("Listening on libp2p-js host ID ", node.Id) - - listener := go_http_js_libp2p.NewP2pListener(node) - defer listener.Close() - s := &http.Server{} - s.Serve(listener) + s := JSServer{ + Mux: http.DefaultServeMux, + } + s.ListenAndServe("p2p") }() } From dfd8b93d93a586f0c4400b3a317c978a1c0e04d9 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Sat, 14 Mar 2020 20:08:54 +0000 Subject: [PATCH 35/86] Federation: Add event_auth endpoint (#905) * Federation: Add event_auth endpoint Signed-off-by: Sudhanshu Jaiswal * Update metric names Signed-off-by: Sudhanshu Jaiswal * Rebase * Add comment from @APwhitehat Co-authored-by: Sudhanshu Jaiswal --- federationapi/routing/eventauth.go | 43 ++++++++++++++++++++++++++++++ federationapi/routing/routing.go | 14 ++++++++-- 2 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 federationapi/routing/eventauth.go diff --git a/federationapi/routing/eventauth.go b/federationapi/routing/eventauth.go new file mode 100644 index 000000000..003165c85 --- /dev/null +++ b/federationapi/routing/eventauth.go @@ -0,0 +1,43 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routing + +import ( + "context" + "net/http" + + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" +) + +// GetEventAuth returns event auth for the roomID and eventID +func GetEventAuth( + ctx context.Context, + request *gomatrixserverlib.FederationRequest, + query api.RoomserverQueryAPI, + roomID string, + eventID string, +) util.JSONResponse { + // TODO: Optimisation: we shouldn't be querying all the room state + // that is in state.StateEvents - we just ignore it. + state, err := getState(ctx, request, query, roomID, eventID) + if err != nil { + return *err + } + + return util.JSONResponse{ + Code: http.StatusOK, + JSON: gomatrixserverlib.RespEventAuth{AuthEvents: state.AuthEvents}, + } +} diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index 3b119301a..b5c8e53de 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -131,7 +131,7 @@ func Setup( )).Methods(http.MethodGet) v1fedmux.Handle("/state/{roomID}", common.MakeFedAPI( - "federation_get_event_auth", cfg.Matrix.ServerName, keys, + "federation_get_state", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) if err != nil { @@ -144,7 +144,7 @@ func Setup( )).Methods(http.MethodGet) v1fedmux.Handle("/state_ids/{roomID}", common.MakeFedAPI( - "federation_get_event_auth", cfg.Matrix.ServerName, keys, + "federation_get_state_ids", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) if err != nil { @@ -156,6 +156,16 @@ func Setup( }, )).Methods(http.MethodGet) + v1fedmux.Handle("/event_auth/{roomID}/{eventID}", common.MakeFedAPI( + "federation_get_event_auth", cfg.Matrix.ServerName, keys, + func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { + vars := mux.Vars(httpReq) + return GetEventAuth( + httpReq.Context(), request, query, vars["roomID"], vars["eventID"], + ) + }, + )).Methods(http.MethodGet) + v1fedmux.Handle("/query/directory", common.MakeFedAPI( "federation_query_room_alias", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { From 452f393dd7de5203c5cd3ebe5697af6ea0f4b009 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 16 Mar 2020 16:05:29 +0000 Subject: [PATCH 36/86] Update room version descriptors, add error handling (#906) * Update room version descriptors, add error handling * Fix database queries * Drop Get from version package * Fix database wrapping, add comments for version descriptions * Don't set default room_version value in SQL --- go.mod | 2 +- go.sum | 4 + roomserver/query/query.go | 7 +- roomserver/storage/interface.go | 2 +- roomserver/storage/postgres/rooms_table.go | 14 +- roomserver/storage/postgres/storage.go | 13 +- roomserver/storage/sqlite3/rooms_table.go | 14 +- roomserver/storage/sqlite3/storage.go | 13 +- roomserver/version/version.go | 147 +++++++++++---------- 9 files changed, 121 insertions(+), 95 deletions(-) diff --git a/go.mod b/go.mod index 92e0b7201..0728a03cc 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200310180544-7f3fad43b51c github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200306154041-df6bb9a3e424 + github.com/matrix-org/gomatrixserverlib v0.0.0-20200316144058-cc6847798a3f github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible diff --git a/go.sum b/go.sum index c28254d46..af0876489 100644 --- a/go.sum +++ b/go.sum @@ -138,6 +138,10 @@ github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:km github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= github.com/matrix-org/gomatrixserverlib v0.0.0-20200306154041-df6bb9a3e424 h1:H61lT6ckUFIDl9qb636qNomfo0B52lFt73ecioiqF10= github.com/matrix-org/gomatrixserverlib v0.0.0-20200306154041-df6bb9a3e424/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316100021-ac4a53d49690 h1:aQPPypOdoIsquJHhoalRYvKtDoiJfSyyJqOEn6R7yTY= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316100021-ac4a53d49690/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316144058-cc6847798a3f h1:JXSvzG4movqXT1KcdX+XZ9HM61m1FW4rIMFKUM9j/Dk= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316144058-cc6847798a3f/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 h1:W7l5CP4V7wPyPb4tYE11dbmeAOwtFQBTW0rf4OonOS8= diff --git a/roomserver/query/query.go b/roomserver/query/query.go index f138686b5..83f89777f 100644 --- a/roomserver/query/query.go +++ b/roomserver/query/query.go @@ -20,7 +20,6 @@ import ( "context" "encoding/json" "net/http" - "strconv" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/api" @@ -733,10 +732,10 @@ func (r *RoomserverQueryAPI) QueryRoomVersionCapabilities( request *api.QueryRoomVersionCapabilitiesRequest, response *api.QueryRoomVersionCapabilitiesResponse, ) error { - response.DefaultRoomVersion = strconv.Itoa(int(version.GetDefaultRoomVersion())) + response.DefaultRoomVersion = string(version.DefaultRoomVersion()) response.AvailableRoomVersions = make(map[string]string) - for v, desc := range version.GetSupportedRoomVersions() { - sv := strconv.Itoa(int(v)) + for v, desc := range version.SupportedRoomVersions() { + sv := string(v) if desc.Stable { response.AvailableRoomVersions[sv] = "stable" } else { diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go index f03d6feab..7f32b53f8 100644 --- a/roomserver/storage/interface.go +++ b/roomserver/storage/interface.go @@ -45,5 +45,5 @@ type Database interface { GetMembership(ctx context.Context, roomNID types.RoomNID, requestSenderUserID string) (membershipEventNID types.EventNID, stillInRoom bool, err error) GetMembershipEventNIDsForRoom(ctx context.Context, roomNID types.RoomNID, joinOnly bool) ([]types.EventNID, error) EventsFromIDs(ctx context.Context, eventIDs []string) ([]types.Event, error) - GetRoomVersionForRoom(ctx context.Context, roomNID types.RoomNID) (int64, error) + GetRoomVersionForRoom(ctx context.Context, roomNID types.RoomNID) (gomatrixserverlib.RoomVersion, error) } diff --git a/roomserver/storage/postgres/rooms_table.go b/roomserver/storage/postgres/rooms_table.go index edd15a338..ef8b8ecef 100644 --- a/roomserver/storage/postgres/rooms_table.go +++ b/roomserver/storage/postgres/rooms_table.go @@ -22,6 +22,7 @@ import ( "github.com/lib/pq" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/gomatrixserverlib" ) const roomsSchema = ` @@ -42,13 +43,13 @@ CREATE TABLE IF NOT EXISTS roomserver_rooms ( state_snapshot_nid BIGINT NOT NULL DEFAULT 0, -- The version of the room, which will assist in determining the state resolution -- algorithm, event ID format, etc. - room_version BIGINT NOT NULL DEFAULT 1 + room_version TEXT NOT NULL ); ` // Same as insertEventTypeNIDSQL const insertRoomNIDSQL = "" + - "INSERT INTO roomserver_rooms (room_id) VALUES ($1)" + + "INSERT INTO roomserver_rooms (room_id, room_version) VALUES ($1, $2)" + " ON CONFLICT ON CONSTRAINT roomserver_room_id_unique" + " DO NOTHING RETURNING (room_nid)" @@ -92,11 +93,12 @@ func (s *roomStatements) prepare(db *sql.DB) (err error) { } func (s *roomStatements) insertRoomNID( - ctx context.Context, txn *sql.Tx, roomID string, + ctx context.Context, txn *sql.Tx, + roomID string, roomVersion gomatrixserverlib.RoomVersion, ) (types.RoomNID, error) { var roomNID int64 stmt := common.TxStmt(txn, s.insertRoomNIDStmt) - err := stmt.QueryRowContext(ctx, roomID).Scan(&roomNID) + err := stmt.QueryRowContext(ctx, roomID, roomVersion).Scan(&roomNID) return types.RoomNID(roomNID), err } @@ -165,8 +167,8 @@ func (s *roomStatements) updateLatestEventNIDs( func (s *roomStatements) selectRoomVersionForRoomNID( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, -) (int64, error) { - var roomVersion int64 +) (gomatrixserverlib.RoomVersion, error) { + var roomVersion gomatrixserverlib.RoomVersion stmt := common.TxStmt(txn, s.selectRoomVersionForRoomNIDStmt) err := stmt.QueryRowContext(ctx, roomNID).Scan(&roomVersion) return roomVersion, err diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go index 77a792d68..ee1a1cd4d 100644 --- a/roomserver/storage/postgres/storage.go +++ b/roomserver/storage/postgres/storage.go @@ -68,7 +68,8 @@ func (d *Database) StoreEvent( } } - if roomNID, err = d.assignRoomNID(ctx, nil, event.RoomID()); err != nil { + // TODO: Room version here + if roomNID, err = d.assignRoomNID(ctx, nil, event.RoomID(), "1"); err != nil { return 0, types.StateAtEvent{}, err } @@ -121,13 +122,14 @@ func (d *Database) StoreEvent( } func (d *Database) assignRoomNID( - ctx context.Context, txn *sql.Tx, roomID string, + ctx context.Context, txn *sql.Tx, + roomID string, roomVersion gomatrixserverlib.RoomVersion, ) (types.RoomNID, error) { // Check if we already have a numeric ID in the database. roomNID, err := d.statements.selectRoomNID(ctx, txn, roomID) if err == sql.ErrNoRows { // We don't have a numeric ID so insert one into the database. - roomNID, err = d.statements.insertRoomNID(ctx, txn, roomID) + roomNID, err = d.statements.insertRoomNID(ctx, txn, roomID, roomVersion) if err == sql.ErrNoRows { // We raced with another insert so run the select again. roomNID, err = d.statements.selectRoomNID(ctx, txn, roomID) @@ -494,7 +496,8 @@ func (d *Database) MembershipUpdater( } }() - roomNID, err := d.assignRoomNID(ctx, txn, roomID) + // TODO: Room version here + roomNID, err := d.assignRoomNID(ctx, txn, roomID, "1") if err != nil { return nil, err } @@ -699,7 +702,7 @@ func (d *Database) EventsFromIDs(ctx context.Context, eventIDs []string) ([]type func (d *Database) GetRoomVersionForRoom( ctx context.Context, roomNID types.RoomNID, -) (int64, error) { +) (gomatrixserverlib.RoomVersion, error) { return d.statements.selectRoomVersionForRoomNID( ctx, nil, roomNID, ) diff --git a/roomserver/storage/sqlite3/rooms_table.go b/roomserver/storage/sqlite3/rooms_table.go index 99c03476f..b750f63ea 100644 --- a/roomserver/storage/sqlite3/rooms_table.go +++ b/roomserver/storage/sqlite3/rooms_table.go @@ -22,6 +22,7 @@ import ( "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/gomatrixserverlib" ) const roomsSchema = ` @@ -31,13 +32,13 @@ const roomsSchema = ` latest_event_nids TEXT NOT NULL DEFAULT '[]', last_event_sent_nid INTEGER NOT NULL DEFAULT 0, state_snapshot_nid INTEGER NOT NULL DEFAULT 0, - room_version INTEGER NOT NULL DEFAULT 1 + room_version TEXT NOT NULL ); ` // Same as insertEventTypeNIDSQL const insertRoomNIDSQL = ` - INSERT INTO roomserver_rooms (room_id) VALUES ($1) + INSERT INTO roomserver_rooms (room_id, room_version) VALUES ($1, $2) ON CONFLICT DO NOTHING; ` @@ -81,11 +82,12 @@ func (s *roomStatements) prepare(db *sql.DB) (err error) { } func (s *roomStatements) insertRoomNID( - ctx context.Context, txn *sql.Tx, roomID string, + ctx context.Context, txn *sql.Tx, + roomID string, roomVersion gomatrixserverlib.RoomVersion, ) (types.RoomNID, error) { var err error insertStmt := common.TxStmt(txn, s.insertRoomNIDStmt) - if _, err = insertStmt.ExecContext(ctx, roomID); err == nil { + if _, err = insertStmt.ExecContext(ctx, roomID, roomVersion); err == nil { return s.selectRoomNID(ctx, txn, roomID) } else { return types.RoomNID(0), err @@ -157,8 +159,8 @@ func (s *roomStatements) updateLatestEventNIDs( func (s *roomStatements) selectRoomVersionForRoomNID( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, -) (int64, error) { - var roomVersion int64 +) (gomatrixserverlib.RoomVersion, error) { + var roomVersion gomatrixserverlib.RoomVersion stmt := common.TxStmt(txn, s.selectRoomVersionForRoomNIDStmt) err := stmt.QueryRowContext(ctx, roomNID).Scan(&roomVersion) return roomVersion, err diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index 41d4c2a94..d9513611e 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -90,7 +90,8 @@ func (d *Database) StoreEvent( } } - if roomNID, err = d.assignRoomNID(ctx, txn, event.RoomID()); err != nil { + // TODO: Room version here + if roomNID, err = d.assignRoomNID(ctx, txn, event.RoomID(), "1"); err != nil { return err } @@ -150,13 +151,14 @@ func (d *Database) StoreEvent( } func (d *Database) assignRoomNID( - ctx context.Context, txn *sql.Tx, roomID string, + ctx context.Context, txn *sql.Tx, + roomID string, roomVersion gomatrixserverlib.RoomVersion, ) (roomNID types.RoomNID, err error) { // Check if we already have a numeric ID in the database. roomNID, err = d.statements.selectRoomNID(ctx, txn, roomID) if err == sql.ErrNoRows { // We don't have a numeric ID so insert one into the database. - roomNID, err = d.statements.insertRoomNID(ctx, txn, roomID) + roomNID, err = d.statements.insertRoomNID(ctx, txn, roomID, roomVersion) if err == nil { // Now get the numeric ID back out of the database roomNID, err = d.statements.selectRoomNID(ctx, txn, roomID) @@ -630,7 +632,8 @@ func (d *Database) MembershipUpdater( } }() - roomNID, err := d.assignRoomNID(ctx, txn, roomID) + // TODO: Room version here + roomNID, err := d.assignRoomNID(ctx, txn, roomID, "1") if err != nil { return nil, err } @@ -853,7 +856,7 @@ func (d *Database) EventsFromIDs(ctx context.Context, eventIDs []string) ([]type func (d *Database) GetRoomVersionForRoom( ctx context.Context, roomNID types.RoomNID, -) (int64, error) { +) (gomatrixserverlib.RoomVersion, error) { return d.statements.selectRoomVersionForRoomNID( ctx, nil, roomNID, ) diff --git a/roomserver/version/version.go b/roomserver/version/version.go index 0943e3843..b18df49f5 100644 --- a/roomserver/version/version.go +++ b/roomserver/version/version.go @@ -15,85 +15,64 @@ package version import ( - "errors" + "fmt" - "github.com/matrix-org/dendrite/roomserver/state" -) - -type RoomVersionID int -type EventFormatID int - -const ( - RoomVersionV1 RoomVersionID = iota + 1 - RoomVersionV2 - RoomVersionV3 - RoomVersionV4 - RoomVersionV5 -) - -const ( - EventFormatV1 EventFormatID = iota + 1 // original event ID formatting - EventFormatV2 // event ID is event hash - EventFormatV3 // event ID is URL-safe base64 event hash + "github.com/matrix-org/gomatrixserverlib" ) +// RoomVersionDescription contains information about a room version, +// namely whether it is marked as supported or stable in this server +// version. +// A version is supported if the server has some support for rooms +// that are this version. A version is marked as stable or unstable +// in order to hint whether the version should be used to clients +// calling the /capabilities endpoint. +// https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-capabilities type RoomVersionDescription struct { - Supported bool - Stable bool - StateResolution state.StateResolutionVersion - EventFormat EventFormatID - EnforceSigningKeyValidity bool + Supported bool + Stable bool } -var roomVersions = map[RoomVersionID]RoomVersionDescription{ - RoomVersionV1: RoomVersionDescription{ - Supported: true, - Stable: true, - StateResolution: state.StateResolutionAlgorithmV1, - EventFormat: EventFormatV1, - EnforceSigningKeyValidity: false, +var roomVersions = map[gomatrixserverlib.RoomVersion]RoomVersionDescription{ + gomatrixserverlib.RoomVersionV1: RoomVersionDescription{ + Supported: true, + Stable: true, }, - RoomVersionV2: RoomVersionDescription{ - Supported: false, - Stable: true, - StateResolution: state.StateResolutionAlgorithmV2, - EventFormat: EventFormatV1, - EnforceSigningKeyValidity: false, + gomatrixserverlib.RoomVersionV2: RoomVersionDescription{ + Supported: false, + Stable: false, }, - RoomVersionV3: RoomVersionDescription{ - Supported: false, - Stable: true, - StateResolution: state.StateResolutionAlgorithmV2, - EventFormat: EventFormatV2, - EnforceSigningKeyValidity: false, + gomatrixserverlib.RoomVersionV3: RoomVersionDescription{ + Supported: false, + Stable: false, }, - RoomVersionV4: RoomVersionDescription{ - Supported: false, - Stable: true, - StateResolution: state.StateResolutionAlgorithmV2, - EventFormat: EventFormatV3, - EnforceSigningKeyValidity: false, + gomatrixserverlib.RoomVersionV4: RoomVersionDescription{ + Supported: false, + Stable: false, }, - RoomVersionV5: RoomVersionDescription{ - Supported: false, - Stable: true, - StateResolution: state.StateResolutionAlgorithmV2, - EventFormat: EventFormatV3, - EnforceSigningKeyValidity: true, + gomatrixserverlib.RoomVersionV5: RoomVersionDescription{ + Supported: false, + Stable: false, }, } -func GetDefaultRoomVersion() RoomVersionID { - return RoomVersionV1 +// DefaultRoomVersion contains the room version that will, by +// default, be used to create new rooms on this server. +func DefaultRoomVersion() gomatrixserverlib.RoomVersion { + return gomatrixserverlib.RoomVersionV1 } -func GetRoomVersions() map[RoomVersionID]RoomVersionDescription { +// RoomVersions returns a map of all known room versions to this +// server. +func RoomVersions() map[gomatrixserverlib.RoomVersion]RoomVersionDescription { return roomVersions } -func GetSupportedRoomVersions() map[RoomVersionID]RoomVersionDescription { - versions := make(map[RoomVersionID]RoomVersionDescription) - for id, version := range GetRoomVersions() { +// SupportedRoomVersions returns a map of descriptions for room +// versions that are supported by this homeserver. +func SupportedRoomVersions() map[gomatrixserverlib.RoomVersion]RoomVersionDescription { + versions := make(map[gomatrixserverlib.RoomVersion]RoomVersionDescription) + for id, version := range RoomVersions() { if version.Supported { versions[id] = version } @@ -101,12 +80,46 @@ func GetSupportedRoomVersions() map[RoomVersionID]RoomVersionDescription { return versions } -func GetSupportedRoomVersion(version RoomVersionID) (desc RoomVersionDescription, err error) { +// RoomVersion returns information about a specific room version. +// An UnknownVersionError is returned if the version is not known +// to the server. +func RoomVersion(version gomatrixserverlib.RoomVersion) (RoomVersionDescription, error) { if version, ok := roomVersions[version]; ok { - desc = version + return version, nil } - if !desc.Supported { - err = errors.New("unsupported room version") - } - return + return RoomVersionDescription{}, UnknownVersionError{version} +} + +// SupportedRoomVersion returns information about a specific room +// version. An UnknownVersionError is returned if the version is not +// known to the server, or an UnsupportedVersionError is returned if +// the version is known but specifically marked as unsupported. +func SupportedRoomVersion(version gomatrixserverlib.RoomVersion) (RoomVersionDescription, error) { + result, err := RoomVersion(version) + if err != nil { + return RoomVersionDescription{}, err + } + if !result.Supported { + return RoomVersionDescription{}, UnsupportedVersionError{version} + } + return result, nil +} + +// UnknownVersionError is caused when the room version is not known. +type UnknownVersionError struct { + Version gomatrixserverlib.RoomVersion +} + +func (e UnknownVersionError) Error() string { + return fmt.Sprintf("room version '%s' is not known", e.Version) +} + +// UnsupportedVersionError is caused when the room version is specifically +// marked as unsupported. +type UnsupportedVersionError struct { + Version gomatrixserverlib.RoomVersion +} + +func (e UnsupportedVersionError) Error() string { + return fmt.Sprintf("room version '%s' is marked as unsupported", e.Version) } From acb505b71751f637531ba9113d4b4daae589bcbc Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 16 Mar 2020 17:29:52 +0000 Subject: [PATCH 37/86] Implement gomatrixserverlib.HeaderedEvent in roomserver query API (#912) * Implement gomatrixserverlib.HeaderedEvent, which should allow us to store room version headers along with the event across API boundaries and consumers/producers, and intercept unmarshalling to get the event structure right * Add federationsender to previous --- appservice/consumers/roomserver.go | 4 +- clientapi/consumers/roomserver.go | 4 +- clientapi/routing/getevent.go | 2 +- clientapi/routing/sendevent.go | 2 +- common/events.go | 2 +- federationapi/routing/backfill.go | 4 +- federationapi/routing/events.go | 2 +- federationapi/routing/join.go | 2 +- federationapi/routing/leave.go | 2 +- federationapi/routing/missingevents.go | 4 +- federationapi/routing/send.go | 6 +- federationapi/routing/state.go | 12 ++- federationapi/routing/threepid.go | 2 +- federationsender/consumers/roomserver.go | 5 +- go.mod | 2 +- go.sum | 2 + publicroomsapi/consumers/roomserver.go | 11 ++- roomserver/alias/alias.go | 2 +- roomserver/api/query.go | 14 +-- roomserver/query/query.go | 110 +++++++++++++++++++++-- syncapi/consumers/roomserver.go | 5 +- 21 files changed, 163 insertions(+), 36 deletions(-) diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go index 6d3ea808f..04eb3d992 100644 --- a/appservice/consumers/roomserver.go +++ b/appservice/consumers/roomserver.go @@ -143,7 +143,9 @@ func (s *OutputRoomEventConsumer) lookupMissingStateEvents( return nil, err } - result = append(result, eventResp.Events...) + for _, headeredEvent := range eventResp.Events { + result = append(result, headeredEvent.Event) + } return result, nil } diff --git a/clientapi/consumers/roomserver.go b/clientapi/consumers/roomserver.go index a65281514..e90d56693 100644 --- a/clientapi/consumers/roomserver.go +++ b/clientapi/consumers/roomserver.go @@ -138,7 +138,9 @@ func (s *OutputRoomEventConsumer) lookupStateEvents( return nil, err } - result = append(result, eventResp.Events...) + for _, headeredEvent := range eventResp.Events { + result = append(result, headeredEvent.Event) + } return result, nil } diff --git a/clientapi/routing/getevent.go b/clientapi/routing/getevent.go index 1c1a7add9..2d3152510 100644 --- a/clientapi/routing/getevent.go +++ b/clientapi/routing/getevent.go @@ -66,7 +66,7 @@ func GetEvent( } } - requestedEvent := eventsResp.Events[0] + requestedEvent := eventsResp.Events[0].Event r := getEventRequest{ req: req, diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go index da96c5c65..dbfea09c5 100644 --- a/clientapi/routing/sendevent.go +++ b/clientapi/routing/sendevent.go @@ -144,7 +144,7 @@ func generateSendEvent( // check to see if this user can perform this operation stateEvents := make([]*gomatrixserverlib.Event, len(queryRes.StateEvents)) for i := range queryRes.StateEvents { - stateEvents[i] = &queryRes.StateEvents[i] + stateEvents[i] = &queryRes.StateEvents[i].Event } provider := gomatrixserverlib.NewAuthEvents(stateEvents) if err = gomatrixserverlib.Allowed(*e, &provider); err != nil { diff --git a/common/events.go b/common/events.go index 3c060ee65..ff46e8e0b 100644 --- a/common/events.go +++ b/common/events.go @@ -89,7 +89,7 @@ func AddPrevEventsToEvent( authEvents := gomatrixserverlib.NewAuthEvents(nil) for i := range queryRes.StateEvents { - err = authEvents.AddEvent(&queryRes.StateEvents[i]) + err = authEvents.AddEvent(&queryRes.StateEvents[i].Event) if err != nil { return err } diff --git a/federationapi/routing/backfill.go b/federationapi/routing/backfill.go index b74bfe3ac..a4bc3c67d 100644 --- a/federationapi/routing/backfill.go +++ b/federationapi/routing/backfill.go @@ -84,10 +84,10 @@ func Backfill( // Filter any event that's not from the requested room out. evs := make([]gomatrixserverlib.Event, 0) - var ev gomatrixserverlib.Event + var ev gomatrixserverlib.HeaderedEvent for _, ev = range res.Events { if ev.RoomID() == roomID { - evs = append(evs, ev) + evs = append(evs, ev.Event) } } diff --git a/federationapi/routing/events.go b/federationapi/routing/events.go index c4248022b..a91528b3d 100644 --- a/federationapi/routing/events.go +++ b/federationapi/routing/events.go @@ -80,5 +80,5 @@ func getEvent( return nil, &util.JSONResponse{Code: http.StatusNotFound, JSON: nil} } - return &eventsResponse.Events[0], nil + return &eventsResponse.Events[0].Event, nil } diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go index 369e19aed..7d48c86dc 100644 --- a/federationapi/routing/join.go +++ b/federationapi/routing/join.go @@ -78,7 +78,7 @@ func MakeJoin( // Check that the join is allowed or not stateEvents := make([]*gomatrixserverlib.Event, len(queryRes.StateEvents)) for i := range queryRes.StateEvents { - stateEvents[i] = &queryRes.StateEvents[i] + stateEvents[i] = &queryRes.StateEvents[i].Event } provider := gomatrixserverlib.NewAuthEvents(stateEvents) if err = gomatrixserverlib.Allowed(*event, &provider); err != nil { diff --git a/federationapi/routing/leave.go b/federationapi/routing/leave.go index f27989ff7..3eceb6f25 100644 --- a/federationapi/routing/leave.go +++ b/federationapi/routing/leave.go @@ -76,7 +76,7 @@ func MakeLeave( // Check that the leave is allowed or not stateEvents := make([]*gomatrixserverlib.Event, len(queryRes.StateEvents)) for i := range queryRes.StateEvents { - stateEvents[i] = &queryRes.StateEvents[i] + stateEvents[i] = &queryRes.StateEvents[i].Event } provider := gomatrixserverlib.NewAuthEvents(stateEvents) if err = gomatrixserverlib.Allowed(*event, &provider); err != nil { diff --git a/federationapi/routing/missingevents.go b/federationapi/routing/missingevents.go index 661ec33a4..069bff3dd 100644 --- a/federationapi/routing/missingevents.go +++ b/federationapi/routing/missingevents.go @@ -68,8 +68,8 @@ func GetMissingEvents( // filterEvents returns only those events with matching roomID and having depth greater than minDepth func filterEvents( - events []gomatrixserverlib.Event, minDepth int64, roomID string, -) []gomatrixserverlib.Event { + events []gomatrixserverlib.HeaderedEvent, minDepth int64, roomID string, +) []gomatrixserverlib.HeaderedEvent { ref := events[:0] for _, ev := range events { if ev.Depth() >= minDepth && ev.RoomID() == roomID { diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index 079e121f9..ce9ad8b13 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -162,7 +162,11 @@ func (t *txnReq) processEvent(e gomatrixserverlib.Event) error { } // Check that the event is allowed by the state at the event. - if err := checkAllowedByState(e, stateResp.StateEvents); err != nil { + var events []gomatrixserverlib.Event + for _, headeredEvent := range stateResp.StateEvents { + events = append(events, headeredEvent.Event) + } + if err := checkAllowedByState(e, events); err != nil { return err } diff --git a/federationapi/routing/state.go b/federationapi/routing/state.go index 86cf1cf54..86a1e08d2 100644 --- a/federationapi/routing/state.go +++ b/federationapi/routing/state.go @@ -129,9 +129,17 @@ func getState( return nil, &util.JSONResponse{Code: http.StatusNotFound, JSON: nil} } + var stateEvents, authEvents []gomatrixserverlib.Event + for _, headeredEvent := range response.StateEvents { + stateEvents = append(stateEvents, headeredEvent.Event) + } + for _, headeredEvent := range response.AuthChainEvents { + authEvents = append(authEvents, headeredEvent.Event) + } + return &gomatrixserverlib.RespState{ - StateEvents: response.StateEvents, - AuthEvents: response.AuthChainEvents, + StateEvents: stateEvents, + AuthEvents: authEvents, }, nil } diff --git a/federationapi/routing/threepid.go b/federationapi/routing/threepid.go index 03f3c5bba..18ebc07e1 100644 --- a/federationapi/routing/threepid.go +++ b/federationapi/routing/threepid.go @@ -264,7 +264,7 @@ func buildMembershipEvent( authEvents := gomatrixserverlib.NewAuthEvents(nil) for i := range queryRes.StateEvents { - err = authEvents.AddEvent(&queryRes.StateEvents[i]) + err = authEvents.AddEvent(&queryRes.StateEvents[i].Event) if err != nil { return nil, err } diff --git a/federationsender/consumers/roomserver.go b/federationsender/consumers/roomserver.go index 4568f44dc..39d8d62db 100644 --- a/federationsender/consumers/roomserver.go +++ b/federationsender/consumers/roomserver.go @@ -325,7 +325,10 @@ func (s *OutputRoomEventConsumer) lookupStateEvents( return nil, err } - result = append(result, eventResp.Events...) + for _, headeredEvent := range eventResp.Events { + result = append(result, headeredEvent.Event) + } + missing = missingEventsFrom(result, addsStateEventIDs) if len(missing) != 0 { diff --git a/go.mod b/go.mod index 0728a03cc..9b5aa1414 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200310180544-7f3fad43b51c github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200316144058-cc6847798a3f + github.com/matrix-org/gomatrixserverlib v0.0.0-20200316163031-36bb5f40ae9b github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible diff --git a/go.sum b/go.sum index af0876489..95d39e5b0 100644 --- a/go.sum +++ b/go.sum @@ -142,6 +142,8 @@ github.com/matrix-org/gomatrixserverlib v0.0.0-20200316100021-ac4a53d49690 h1:aQ github.com/matrix-org/gomatrixserverlib v0.0.0-20200316100021-ac4a53d49690/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= github.com/matrix-org/gomatrixserverlib v0.0.0-20200316144058-cc6847798a3f h1:JXSvzG4movqXT1KcdX+XZ9HM61m1FW4rIMFKUM9j/Dk= github.com/matrix-org/gomatrixserverlib v0.0.0-20200316144058-cc6847798a3f/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316163031-36bb5f40ae9b h1:DdVa6Ledhi5dqwYMw8e9zG+kZmRlnWMgsYOxb3NaFOU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316163031-36bb5f40ae9b/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 h1:W7l5CP4V7wPyPb4tYE11dbmeAOwtFQBTW0rf4OonOS8= diff --git a/publicroomsapi/consumers/roomserver.go b/publicroomsapi/consumers/roomserver.go index 9a817735a..2bbd92b72 100644 --- a/publicroomsapi/consumers/roomserver.go +++ b/publicroomsapi/consumers/roomserver.go @@ -22,6 +22,7 @@ import ( "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/gomatrixserverlib" log "github.com/sirupsen/logrus" sarama "gopkg.in/Shopify/sarama.v1" ) @@ -98,5 +99,13 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { return err } - return s.db.UpdateRoomFromEvents(context.TODO(), addQueryRes.Events, remQueryRes.Events) + var addQueryEvents, remQueryEvents []gomatrixserverlib.Event + for _, headeredEvent := range addQueryRes.Events { + addQueryEvents = append(addQueryEvents, headeredEvent.Event) + } + for _, headeredEvent := range remQueryRes.Events { + remQueryEvents = append(remQueryEvents, headeredEvent.Event) + } + + return s.db.UpdateRoomFromEvents(context.TODO(), addQueryEvents, remQueryEvents) } diff --git a/roomserver/alias/alias.go b/roomserver/alias/alias.go index aeaf5ae94..59da35b3f 100644 --- a/roomserver/alias/alias.go +++ b/roomserver/alias/alias.go @@ -229,7 +229,7 @@ func (r *RoomserverAliasAPI) sendUpdatedAliasesEvent( // Add auth events authEvents := gomatrixserverlib.NewAuthEvents(nil) for i := range res.StateEvents { - err = authEvents.AddEvent(&res.StateEvents[i]) + err = authEvents.AddEvent(&res.StateEvents[i].Event) if err != nil { return err } diff --git a/roomserver/api/query.go b/roomserver/api/query.go index e1850e723..24cca03ec 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -50,7 +50,7 @@ type QueryLatestEventsAndStateResponse struct { // This list will be in an arbitrary order. // These are used to set the auth_events when sending an event. // These are used to check whether the event is allowed. - StateEvents []gomatrixserverlib.Event `json:"state_events"` + StateEvents []gomatrixserverlib.HeaderedEvent `json:"state_events"` // The depth of the latest events. // This is one greater than the maximum depth of the latest events. // This is used to set the depth when sending an event. @@ -79,7 +79,7 @@ type QueryStateAfterEventsResponse struct { PrevEventsExist bool `json:"prev_events_exist"` // The state events requested. // This list will be in an arbitrary order. - StateEvents []gomatrixserverlib.Event `json:"state_events"` + StateEvents []gomatrixserverlib.HeaderedEvent `json:"state_events"` } // QueryEventsByIDRequest is a request to QueryEventsByID @@ -99,7 +99,7 @@ type QueryEventsByIDResponse struct { // fails to read it from the database then it will fail // the entire request. // This list will be in an arbitrary order. - Events []gomatrixserverlib.Event `json:"events"` + Events []gomatrixserverlib.HeaderedEvent `json:"events"` } // QueryMembershipForUserRequest is a request to QueryMembership @@ -186,7 +186,7 @@ type QueryMissingEventsRequest struct { // QueryMissingEventsResponse is a response to QueryMissingEvents type QueryMissingEventsResponse struct { // Missing events, arbritrary order. - Events []gomatrixserverlib.Event `json:"events"` + Events []gomatrixserverlib.HeaderedEvent `json:"events"` } // QueryStateAndAuthChainRequest is a request to QueryStateAndAuthChain @@ -212,8 +212,8 @@ type QueryStateAndAuthChainResponse struct { PrevEventsExist bool `json:"prev_events_exist"` // The state and auth chain events that were requested. // The lists will be in an arbitrary order. - StateEvents []gomatrixserverlib.Event `json:"state_events"` - AuthChainEvents []gomatrixserverlib.Event `json:"auth_chain_events"` + StateEvents []gomatrixserverlib.HeaderedEvent `json:"state_events"` + AuthChainEvents []gomatrixserverlib.HeaderedEvent `json:"auth_chain_events"` } // QueryBackfillRequest is a request to QueryBackfill. @@ -229,7 +229,7 @@ type QueryBackfillRequest struct { // QueryBackfillResponse is a response to QueryBackfill. type QueryBackfillResponse struct { // Missing events, arbritrary order. - Events []gomatrixserverlib.Event `json:"events"` + Events []gomatrixserverlib.HeaderedEvent `json:"events"` } // QueryServersInRoomAtEventRequest is a request to QueryServersInRoomAtEvent diff --git a/roomserver/query/query.go b/roomserver/query/query.go index 83f89777f..7f69699f7 100644 --- a/roomserver/query/query.go +++ b/roomserver/query/query.go @@ -89,6 +89,10 @@ type RoomserverQueryAPIDatabase interface { EventStateKeys( context.Context, []types.EventStateKeyNID, ) (map[types.EventStateKeyNID]string, error) + // Look up the room version for a given room. + GetRoomVersionForRoom( + ctx context.Context, roomNID types.RoomNID, + ) (gomatrixserverlib.RoomVersion, error) } // RoomserverQueryAPI is an implementation of api.RoomserverQueryAPI @@ -116,6 +120,12 @@ func (r *RoomserverQueryAPI) QueryLatestEventsAndState( return nil } response.RoomExists = true + + roomVersion, err := r.DB.GetRoomVersionForRoom(ctx, roomNID) + if err != nil { + return err + } + var currentStateSnapshotNID types.StateSnapshotNID response.LatestEvents, currentStateSnapshotNID, response.Depth, err = r.DB.LatestEventIDs(ctx, roomNID) @@ -136,7 +146,15 @@ func (r *RoomserverQueryAPI) QueryLatestEventsAndState( return err } - response.StateEvents = stateEvents + for _, event := range stateEvents { + response.StateEvents = append(response.StateEvents, gomatrixserverlib.HeaderedEvent{ + EventHeader: gomatrixserverlib.EventHeader{ + RoomVersion: roomVersion, + }, + Event: event, + }) + } + return nil } @@ -161,6 +179,11 @@ func (r *RoomserverQueryAPI) QueryStateAfterEvents( } response.RoomExists = true + roomVersion, err := r.DB.GetRoomVersionForRoom(ctx, roomNID) + if err != nil { + return err + } + prevStates, err := r.DB.StateAtEventIDs(ctx, request.PrevEventIDs) if err != nil { switch err.(type) { @@ -185,7 +208,15 @@ func (r *RoomserverQueryAPI) QueryStateAfterEvents( return err } - response.StateEvents = stateEvents + for _, event := range stateEvents { + response.StateEvents = append(response.StateEvents, gomatrixserverlib.HeaderedEvent{ + EventHeader: gomatrixserverlib.EventHeader{ + RoomVersion: roomVersion, + }, + Event: event, + }) + } + return nil } @@ -212,7 +243,18 @@ func (r *RoomserverQueryAPI) QueryEventsByID( return err } - response.Events = events + for _, event := range events { + // TODO: Room version here + roomVersion := gomatrixserverlib.RoomVersionV1 + + response.Events = append(response.Events, gomatrixserverlib.HeaderedEvent{ + EventHeader: gomatrixserverlib.EventHeader{ + RoomVersion: roomVersion, + }, + Event: event, + }) + } + return nil } @@ -486,10 +528,18 @@ func (r *RoomserverQueryAPI) QueryMissingEvents( return err } - response.Events = make([]gomatrixserverlib.Event, 0, len(loadedEvents)-len(eventsToFilter)) + response.Events = make([]gomatrixserverlib.HeaderedEvent, 0, len(loadedEvents)-len(eventsToFilter)) for _, event := range loadedEvents { if !eventsToFilter[event.EventID()] { - response.Events = append(response.Events, event) + // TODO: Room version here + roomVersion := gomatrixserverlib.RoomVersionV1 + + response.Events = append(response.Events, gomatrixserverlib.HeaderedEvent{ + EventHeader: gomatrixserverlib.EventHeader{ + RoomVersion: roomVersion, + }, + Event: event, + }) } } @@ -525,7 +575,24 @@ func (r *RoomserverQueryAPI) QueryBackfill( } // Retrieve events from the list that was filled previously. - response.Events, err = r.loadEvents(ctx, resultNIDs) + var loadedEvents []gomatrixserverlib.Event + loadedEvents, err = r.loadEvents(ctx, resultNIDs) + if err != nil { + return err + } + + for _, event := range loadedEvents { + // TODO: Room version here + roomVersion := gomatrixserverlib.RoomVersionV1 + + response.Events = append(response.Events, gomatrixserverlib.HeaderedEvent{ + EventHeader: gomatrixserverlib.EventHeader{ + RoomVersion: roomVersion, + }, + Event: event, + }) + } + return err } @@ -635,8 +702,35 @@ func (r *RoomserverQueryAPI) QueryStateAndAuthChain( return err } - response.StateEvents = stateEvents - response.AuthChainEvents, err = getAuthChain(ctx, r.DB, request.AuthEventIDs) + authEvents, err := getAuthChain(ctx, r.DB, request.AuthEventIDs) + if err != nil { + return err + } + + for _, event := range stateEvents { + // TODO: Room version here + roomVersion := gomatrixserverlib.RoomVersionV1 + + response.StateEvents = append(response.StateEvents, gomatrixserverlib.HeaderedEvent{ + EventHeader: gomatrixserverlib.EventHeader{ + RoomVersion: roomVersion, + }, + Event: event, + }) + } + + for _, event := range authEvents { + // TODO: Room version here + roomVersion := gomatrixserverlib.RoomVersionV1 + + response.AuthChainEvents = append(response.AuthChainEvents, gomatrixserverlib.HeaderedEvent{ + EventHeader: gomatrixserverlib.EventHeader{ + RoomVersion: roomVersion, + }, + Event: event, + }) + } + return err } diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index 5dbef4b7d..976dd8e8f 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -229,7 +229,10 @@ func (s *OutputRoomEventConsumer) lookupStateEvents( return nil, err } - result = append(result, eventResp.Events...) + for _, headeredEvent := range eventResp.Events { + result = append(result, headeredEvent.Event) + } + missing = missingEventsFrom(result, addsStateEventIDs) if len(missing) != 0 { From 9f74a8798e26d9482f1964ff79a28acfe61d6dd1 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Mon, 16 Mar 2020 17:51:58 +0000 Subject: [PATCH 38/86] bugfix: Fix #908 by setting the correct state after the event (#913) * bugfix: Fix #908 by setting the correct state after the event Previously, this would only happen if the state already existed previously! * Structured logging --- federationapi/routing/send.go | 1 + roomserver/state/v1/state.go | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index ce9ad8b13..d3e060ac2 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -116,6 +116,7 @@ func (t *txnReq) processTransaction() (*gomatrixserverlib.RespSend, error) { results[e.EventID()] = gomatrixserverlib.PDUResult{ Error: err.Error(), } + util.GetLogger(t.context).WithError(err).WithField("event_id", e.EventID()).Warn("Failed to process incoming federation event, skipping it.") } else { results[e.EventID()] = gomatrixserverlib.PDUResult{} } diff --git a/roomserver/state/v1/state.go b/roomserver/state/v1/state.go index 5683745bf..3eb601925 100644 --- a/roomserver/state/v1/state.go +++ b/roomserver/state/v1/state.go @@ -366,11 +366,16 @@ func (v StateResolutionV1) loadStateAfterEventsForNumericTuples( // update that key in the result. // If the requested event wasn't a state event then the state after // it is the same as the state before it. + set := false for i := range result { if result[i].StateKeyTuple == prevState.StateKeyTuple { result[i] = prevState.StateEntry + set = true } } + if !set { // no previous state exists for this event: add new state + result = append(result, prevState.StateEntry) + } } return result, nil } From aebf347a79102597a359dcd59bb45593c8352780 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 17 Mar 2020 11:01:25 +0000 Subject: [PATCH 39/86] Implement gomatrixserverlib.HeaderedEvent in roomserver Kafka output (#914) * Use Event.Headered * Use HeaderedEvent in roomserver kafka output * Fix syncserver-integration-tests * Update producers to roomserver inputs * Update gomatrixserverlib * Update gomatrixserverlib * Update gomatrixserverlib * Update gomatrixserverlib * Update gomatrixserverlib * Update gomatrixserverlib --- appservice/consumers/roomserver.go | 4 +- clientapi/consumers/roomserver.go | 2 +- clientapi/producers/roomserver.go | 18 +- clientapi/routing/createroom.go | 3 +- cmd/create-room-events/main.go | 2 +- cmd/syncserver-integration-tests/main.go | 2 +- federationsender/consumers/roomserver.go | 6 +- go.mod | 3 +- go.sum | 234 +++++++++++++++++++++++ roomserver/alias/alias.go | 5 +- roomserver/api/input.go | 4 +- roomserver/api/output.go | 4 +- roomserver/input/events.go | 10 +- roomserver/input/latest_events.go | 5 +- roomserver/input/membership.go | 3 +- roomserver/query/query.go | 49 +---- syncapi/consumers/roomserver.go | 6 +- 17 files changed, 289 insertions(+), 71 deletions(-) diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go index 04eb3d992..3e47dee56 100644 --- a/appservice/consumers/roomserver.go +++ b/appservice/consumers/roomserver.go @@ -101,11 +101,11 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { "type": ev.Type(), }).Info("appservice received an event from roomserver") - missingEvents, err := s.lookupMissingStateEvents(output.NewRoomEvent.AddsStateEventIDs, ev) + missingEvents, err := s.lookupMissingStateEvents(output.NewRoomEvent.AddsStateEventIDs, ev.Event) if err != nil { return err } - events := append(missingEvents, ev) + events := append(missingEvents, ev.Event) // Send event to any relevant application services return s.filterRoomserverEvents(context.TODO(), events) diff --git a/clientapi/consumers/roomserver.go b/clientapi/consumers/roomserver.go index e90d56693..6d5bb09a6 100644 --- a/clientapi/consumers/roomserver.go +++ b/clientapi/consumers/roomserver.go @@ -91,7 +91,7 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { "type": ev.Type(), }).Info("received event from roomserver") - events, err := s.lookupStateEvents(output.NewRoomEvent.AddsStateEventIDs, ev) + events, err := s.lookupStateEvents(output.NewRoomEvent.AddsStateEventIDs, ev.Event) if err != nil { return err } diff --git a/clientapi/producers/roomserver.go b/clientapi/producers/roomserver.go index e50561a70..0fe2d556b 100644 --- a/clientapi/producers/roomserver.go +++ b/clientapi/producers/roomserver.go @@ -40,9 +40,11 @@ func (c *RoomserverProducer) SendEvents( ) (string, error) { ires := make([]api.InputRoomEvent, len(events)) for i, event := range events { + roomVersion := gomatrixserverlib.RoomVersionV1 + ires[i] = api.InputRoomEvent{ Kind: api.KindNew, - Event: event, + Event: event.Headered(roomVersion), AuthEventIDs: event.AuthEventIDs(), SendAsServer: string(sendAsServer), TransactionID: txnID, @@ -61,11 +63,14 @@ func (c *RoomserverProducer) SendEventWithState( return err } + // TODO: Room version here + roomVersion := gomatrixserverlib.RoomVersionV1 + ires := make([]api.InputRoomEvent, len(outliers)+1) for i, outlier := range outliers { ires[i] = api.InputRoomEvent{ Kind: api.KindOutlier, - Event: outlier, + Event: outlier.Headered(roomVersion), AuthEventIDs: outlier.AuthEventIDs(), } } @@ -77,7 +82,7 @@ func (c *RoomserverProducer) SendEventWithState( ires[len(outliers)] = api.InputRoomEvent{ Kind: api.KindNew, - Event: event, + Event: event.Headered(roomVersion), AuthEventIDs: event.AuthEventIDs(), HasState: true, StateEventIDs: stateEventIDs, @@ -104,8 +109,13 @@ func (c *RoomserverProducer) SendInputRoomEvents( func (c *RoomserverProducer) SendInvite( ctx context.Context, inviteEvent gomatrixserverlib.Event, ) error { + // TODO: Room version here + roomVersion := gomatrixserverlib.RoomVersionV1 + request := api.InputRoomEventsRequest{ - InputInviteEvents: []api.InputInviteEvent{{Event: inviteEvent}}, + InputInviteEvents: []api.InputInviteEvent{{ + Event: inviteEvent.Headered(roomVersion), + }}, } var response api.InputRoomEventsResponse return c.InputAPI.InputRoomEvents(ctx, &request, &response) diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index c9623acb9..3465d9854 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -180,7 +180,8 @@ func createRoom( } r.CreationContent["creator"] = userID - r.CreationContent["room_version"] = "1" // TODO: We set this to 1 before we support Room versioning + // TODO: Room version here + r.CreationContent["room_version"] = gomatrixserverlib.RoomVersionV1 // TODO: visibility/presets/raw initial state // TODO: Create room alias association diff --git a/cmd/create-room-events/main.go b/cmd/create-room-events/main.go index 8475914f0..ef442a0c4 100644 --- a/cmd/create-room-events/main.go +++ b/cmd/create-room-events/main.go @@ -125,7 +125,7 @@ func writeEvent(event gomatrixserverlib.Event) { if *format == "InputRoomEvent" { var ire api.InputRoomEvent ire.Kind = api.KindNew - ire.Event = event + ire.Event = event.Headered(gomatrixserverlib.RoomVersionV1) authEventIDs := []string{} for _, ref := range b.AuthEvents { authEventIDs = append(authEventIDs, ref.EventID) diff --git a/cmd/syncserver-integration-tests/main.go b/cmd/syncserver-integration-tests/main.go index 7f315cd33..d14e854c3 100644 --- a/cmd/syncserver-integration-tests/main.go +++ b/cmd/syncserver-integration-tests/main.go @@ -104,7 +104,7 @@ func clientEventJSONForOutputRoomEvent(outputRoomEvent string) string { panic("failed to unmarshal output room event: " + err.Error()) } clientEvs := gomatrixserverlib.ToClientEvents([]gomatrixserverlib.Event{ - out.NewRoomEvent.Event, + out.NewRoomEvent.Event.Event, }, gomatrixserverlib.FormatSync) b, err := json.Marshal(clientEvs[0]) if err != nil { diff --git a/federationsender/consumers/roomserver.go b/federationsender/consumers/roomserver.go index 39d8d62db..2d4b1b2ef 100644 --- a/federationsender/consumers/roomserver.go +++ b/federationsender/consumers/roomserver.go @@ -109,7 +109,7 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { // processMessage updates the list of currently joined hosts in the room // and then sends the event to the hosts that were joined before the event. func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent) error { - addsStateEvents, err := s.lookupStateEvents(ore.AddsStateEventIDs, ore.Event) + addsStateEvents, err := s.lookupStateEvents(ore.AddsStateEventIDs, ore.Event.Event) if err != nil { return err } @@ -155,7 +155,7 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent) err // Send the event. return s.queues.SendEvent( - &ore.Event, gomatrixserverlib.ServerName(ore.SendAsServer), joinedHostsAtEvent, + &ore.Event.Event, gomatrixserverlib.ServerName(ore.SendAsServer), joinedHostsAtEvent, ) } @@ -178,7 +178,7 @@ func (s *OutputRoomEventConsumer) joinedHostsAtEvent( ore.AddsStateEventIDs, ore.RemovesStateEventIDs, ore.StateBeforeAddsEventIDs, ore.StateBeforeRemovesEventIDs, ) - combinedAddsEvents, err := s.lookupStateEvents(combinedAdds, ore.Event) + combinedAddsEvents, err := s.lookupStateEvents(combinedAdds, ore.Event.Event) if err != nil { return nil, err } diff --git a/go.mod b/go.mod index 9b5aa1414..114a548e5 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,7 @@ module github.com/matrix-org/dendrite require ( github.com/btcsuite/btcutil v1.0.1 + github.com/golangci/golangci-lint v1.19.1 // indirect github.com/gorilla/mux v1.7.3 github.com/hashicorp/golang-lru v0.5.4 github.com/lib/pq v1.2.0 @@ -10,7 +11,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200310180544-7f3fad43b51c github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200316163031-36bb5f40ae9b + github.com/matrix-org/gomatrixserverlib v0.0.0-20200317103236-134415251e65 github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible diff --git a/go.sum b/go.sum index 95d39e5b0..a02e29542 100644 --- a/go.sum +++ b/go.sum @@ -1,14 +1,20 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/zstd v1.4.4 h1:+IawcoXhCBylN7ccwdwf8LOH2jKq7NavGpEPanrlTzE= github.com/DataDog/zstd v1.4.4/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/OpenPeeDeeP/depguard v1.0.1 h1:VlW4R6jmBIv3/u1JNlawEvJMM4J+dPORPaZasQee8Us= +github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmUx/1V+TNhjQvM= github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -24,37 +30,81 @@ github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVa github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q= github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/frankban/quicktest v1.0.0/go.mod h1:R98jIehRai+d1/3Hv2//jOVCTJhW1VBavT6B6CuGq2k= github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3BTYk= github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-critic/go-critic v0.3.5-0.20190904082202-d79a9f0c64db h1:GYXWx7Vr3+zv833u+8IoXbNnQY0AdXsxAgI0kX7xcwA= +github.com/go-critic/go-critic v0.3.5-0.20190904082202-d79a9f0c64db/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-lintpack/lintpack v0.5.2 h1:DI5mA3+eKdWeJ40nU4d6Wc26qmdG8RCi/btYq0TuRN0= +github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-toolsmith/astcast v1.0.0 h1:JojxlmI6STnFVG9yOImLeGREv8W2ocNUM+iOhR6jE7g= +github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4= +github.com/go-toolsmith/astcopy v1.0.0 h1:OMgl1b1MEpjFQ1m5ztEO06rz5CUd3oBv9RF7+DyvdG8= +github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ= +github.com/go-toolsmith/astequal v0.0.0-20180903214952-dcb477bfacd6/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY= +github.com/go-toolsmith/astequal v1.0.0 h1:4zxD8j3JRFNyLN46lodQuqz3xdKSrur7U/sr0SDS/gQ= +github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY= +github.com/go-toolsmith/astfmt v0.0.0-20180903215011-8f8ee99c3086/go.mod h1:mP93XdblcopXwlyN4X4uodxXQhldPGZbcEJIimQHrkg= +github.com/go-toolsmith/astfmt v1.0.0 h1:A0vDDXt+vsvLEdbMFJAUBI/uTbRw1ffOPnxsILnFL6k= +github.com/go-toolsmith/astfmt v1.0.0/go.mod h1:cnWmsOAuq4jJY6Ct5YWlVLmcmLMn1JUPuQIHCY7CJDw= +github.com/go-toolsmith/astinfo v0.0.0-20180906194353-9809ff7efb21/go.mod h1:dDStQCHtmZpYOmjRP/8gHHnCCch3Zz3oEgCdZVdtweU= +github.com/go-toolsmith/astp v0.0.0-20180903215135-0af7e3c24f30/go.mod h1:SV2ur98SGypH1UjcPpCatrV5hPazG6+IfNHbkDXBRrk= +github.com/go-toolsmith/astp v1.0.0 h1:alXE75TXgcmupDsMK1fRAy0YUzLzqPVvBKoyWV+KPXg= +github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdYrA8meBmLI= +github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8/go.mod h1:WoMrjiy4zvdS+Bg6z9jZH82QXwkcgCBX6nOfnmdaHks= +github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc= +github.com/go-toolsmith/strparse v1.0.0 h1:Vcw78DnpCAKlM20kSbAyO4mPfJn/lyYA4BJUDxe2Jb4= +github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8= +github.com/go-toolsmith/typep v1.0.0 h1:zKymWyA1TRYvqYrYDrfEMZULyrhcnGY3x7LDKU2XQaA= +github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -63,6 +113,37 @@ github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 h1:23T5iq8rbUYlhpt5DB4XJkc6BU31uODLD1o1gKvZmD0= +github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4= +github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a h1:w8hkcTqaFpzKqonE9uMCefW1WDie15eSP/4MssdenaM= +github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk= +github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6 h1:YYWNAGTKWhKpcLLt7aSj/odlKrSrelQwlovBpDuf19w= +github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0= +github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613 h1:9kfjN3AdxcbsZBf8NjltjWihK2QfBBBZuv91cMFfDHw= +github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8= +github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3 h1:pe9JHs3cHHDQgOFXJJdYkK6fLz2PWyYtP4hthoCMvs8= +github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o= +github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee h1:J2XAy40+7yz70uaOiMbNnluTg7gyQhtGqLQncQh+4J8= +github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU= +github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98 h1:0OkFarm1Zy2CjCiDKfK9XHgmc2wbDlRMD2hD8anAJHU= +github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU= +github.com/golangci/golangci-lint v1.19.1 h1:g9xL8KW7UZDCkVlgHYJMA6F4Sj/sRVa0FoCeXI+Z3iM= +github.com/golangci/golangci-lint v1.19.1/go.mod h1:2CEc4Fxx3vxDv7g8DyXkHCBF73AOzAymcJAprs2vCps= +github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc h1:gLLhTLMk2/SutryVJ6D4VZCU3CUqr8YloG7FPIBWFpI= +github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU= +github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217 h1:En/tZdwhAn0JNwLuXzP3k2RVtMqMmOEK7Yu/g3tmtJE= +github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg= +github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca h1:kNY3/svz5T29MYHubXix4aDDuE3RWHkPvopM/EDv/MA= +github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o= +github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770 h1:EL/O5HGrF7Jaq0yNhBLucz9hTuRzj2LdwGBOaENgxIk= +github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA= +github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21 h1:leSNB7iYzLYSSx3J/s5sVf4Drkc68W2wm4Ixh/mr0us= +github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI= +github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0 h1:HVfrLniijszjS1aiNg8JbBMO2+E1WIQ+j/gL4SQqGPg= +github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4= +github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 h1:zwtduBRr5SSWhqsYNgcuWO2kFlpdOZbP0+yRjmvPGys= +github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= @@ -70,13 +151,24 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3 h1:JVnpOZS+qxli+rgVl98ILOXVNbW+kb5wcxeGx8ShUIw= +github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/go-cid v0.0.5 h1:o0Ix8e/ql7Zb5UVUJEUfjsWCIY8t48++9lR8qi6oiJU= github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog= github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= @@ -84,15 +176,22 @@ github.com/jbenet/goprocess v0.1.3 h1:YKyIEECS/XvcfHtBzxtjBBbWK+MbvA6dG8ASiqwvr1 github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -100,6 +199,7 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= @@ -110,7 +210,12 @@ github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS github.com/libp2p/go-libp2p-core v0.5.0 h1:FBQ1fpq2Fo/ClyjojVJ5AKXlKhvNc/B6U0O+7AN1ffE= github.com/libp2p/go-libp2p-core v0.5.0/go.mod h1:49XGI+kc38oGVwqSBhDEwytaAxgZasHhFfQKibzTls0= github.com/libp2p/go-openssl v0.0.4/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= +github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/matoous/godox v0.0.0-20190910121045-032ad8106c86 h1:q6SrfsK4FojRnJ1j8+8OJzyq3g9Y1oSVyL6nYGJXXBk= +github.com/matoous/godox v0.0.0-20190910121045-032ad8106c86/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s= github.com/matrix-org/dendrite v0.0.0-20200220135450-0352f250b857/go.mod h1:DZ35IoR+ViBNVPe9umdlOSnjvKl7wfyRmZg4QfWGvTo= github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 h1:nMX2t7hbGF0NYDYySx0pCqEKGKAeZIiSqlWSspetlhY= github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg= @@ -144,6 +249,20 @@ github.com/matrix-org/gomatrixserverlib v0.0.0-20200316144058-cc6847798a3f h1:JX github.com/matrix-org/gomatrixserverlib v0.0.0-20200316144058-cc6847798a3f/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= github.com/matrix-org/gomatrixserverlib v0.0.0-20200316163031-36bb5f40ae9b h1:DdVa6Ledhi5dqwYMw8e9zG+kZmRlnWMgsYOxb3NaFOU= github.com/matrix-org/gomatrixserverlib v0.0.0-20200316163031-36bb5f40ae9b/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316173242-f66834855555 h1:7M7QwpGbc5fJ6Uxn62ECsagekKkV36hjvnbaDQGJhls= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316173242-f66834855555/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316191245-60f1fa78147c h1:hbu6d/7LOKwPpISyE62cUXapx6askaDtKnpb5dtqr1I= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316191245-60f1fa78147c/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316200447-0820dfcdff10 h1:O6EHrD7m9Ah9SjMoHuEHHWh4moKllCYo0HRVtSSGzYc= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316200447-0820dfcdff10/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316202926-7ce8b3298af6 h1:ZeRF0DpsnaxwYL7b6U1J3XMyUBpB8GWiNCqz3bWdivs= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316202926-7ce8b3298af6/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316203446-64795a12215c h1:s7XUmnqbJTJmMHuZ9Ooad6vKlBsSpIe/C3aJc6G6cy8= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316203446-64795a12215c/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316222111-b66d3b63dd85 h1:I7ocq675X1B+RYDUQPvMcYcpfHkdTcreil48r3qs2KE= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200316222111-b66d3b63dd85/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200317103236-134415251e65 h1:llq2yETQhD75ImQhCwHbnivSKOgzOP6QdUqH9sse7XM= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200317103236-134415251e65/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 h1:W7l5CP4V7wPyPb4tYE11dbmeAOwtFQBTW0rf4OonOS8= @@ -151,9 +270,14 @@ github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5/go.mod h1:lePuOiXL github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-sqlite3 v2.0.2+incompatible h1:qzw9c2GNT8UFrgWNDhCTqRqYUSmu/Dav/9Z58LGpk7U= github.com/mattn/go-sqlite3 v2.0.2+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= @@ -166,10 +290,16 @@ github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8Rv github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mozilla/tls-observatory v0.0.0-20190404164649-a3c1b6cfecfd/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc= github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= @@ -187,15 +317,22 @@ github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXS github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= +github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d h1:AREM5mwr4u1ORQBMvzfzBgpsctsbQikCVpvC+tX285E= +github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU= github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 h1:BvoENQQU+fZ9uukda/RzCAL/191HHwJA5b13R6diVlY= github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pierrec/lz4 v2.4.1+incompatible h1:mFe7ttWaflA46Mhqh+jUfjp2qTbPYxLB2/OyBppH9dg= github.com/pierrec/lz4 v2.4.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -204,6 +341,7 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U= github.com/prometheus/client_golang v1.4.1 h1:FFSuS004yOQEtDdTq+TAOLP5xUq63KqAFYyOi8zA+Y8= @@ -213,29 +351,63 @@ github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1: github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/securego/gosec v0.0.0-20190912120752-140048b2a218 h1:O0yPHYL49quNL4Oj2wVq+zbGMu4dAM6iLoOQtm49TrQ= +github.com/securego/gosec v0.0.0-20190912120752-140048b2a218/go.mod h1:q6oYAujd2qyeU4cJqIri4LBIgdHXGvxWHZ1E29HNFRE= +github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc= +github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= +github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= +github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/sourcegraph/go-diff v0.5.1 h1:gO6i5zugwzo1RVTvgvfwCOSVegNuvnNi6bAD1QCmkHs= +github.com/sourcegraph/go-diff v0.5.1/go.mod h1:j2dHj3m8aZgQO8lMTcTnBcXkRRRqi34cd2MNlA9u1mE= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= @@ -251,6 +423,9 @@ github.com/tidwall/pretty v1.0.1 h1:WE4RBSZ1x6McVVC8S/Md+Qse8YUv6HRObAx6ke00NY8= github.com/tidwall/pretty v1.0.1/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/sjson v1.0.3 h1:DeF+0LZqvIt4fKYw41aPB29ZGlvwVkHKktoXJ1YW9Y8= github.com/tidwall/sjson v1.0.3/go.mod h1:bURseu1nuBkFpIES5cz6zBtjmYeOQmEESshn7VpF15Y= +github.com/timakin/bodyclose v0.0.0-20190721030226-87058b9bfcec h1:AmoEvWAO3nDx1MEcMzPh+GzOOIA5Znpv6++c7bePPY0= +github.com/timakin/bodyclose v0.0.0-20190721030226-87058b9bfcec/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/uber-go/atomic v1.3.0 h1:ylWoWcs+jXihgo3Us1Sdsatf2R6+OlBGm8fexR3oFG4= github.com/uber-go/atomic v1.3.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= github.com/uber/jaeger-client-go v2.15.0+incompatible h1:NP3qsSqNxh8VYr956ur1N/1C1PjvOJnJykCzcD5QHbk= @@ -261,18 +436,37 @@ github.com/uber/jaeger-lib v1.5.0 h1:OHbgr8l656Ub3Fw5k9SWnBfIEwvoHQ+W2y+Aa9D1Uyo github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw= github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/ultraware/funlen v0.0.2 h1:Av96YVBwwNSe4MLR7iI/BIa3VyI7/djnto/pK3Uxbdo= +github.com/ultraware/funlen v0.0.2/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA= +github.com/ultraware/whitespace v0.0.3 h1:S5BCRRB5sttNy0bSOhbpw+0mb+cHiCmWfrvxpEzuUk0= +github.com/ultraware/whitespace v0.0.3/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s= +github.com/valyala/quicktemplate v1.2.0/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4= +github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.0 h1:vs7fgriifsPbGdK3bNuMWapNn3qnZhCRXc19NRdq010= go.uber.org/atomic v1.3.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613 h1:MQ/ZZiDsUapFFiMS+vzwXkCTeEKaum+Do5rINYJDmxc= golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d h1:2+ZP7EfsZV7Vvmx3TIqSlSzATMkTAKqM14YGFPoSKjI= @@ -281,48 +475,74 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190301231341-16b79f2e4e95 h1:fY7Dsw114eJN4boqzVSbpVHO6rTdhq6/GnXeu+PKnzU= golang.org/x/net v0.0.0-20190301231341-16b79f2e4e95/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190909003024-a7b16738d86b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0Is3p+EHBKNgquIzo1OI= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 h1:I6FyU15t786LL7oL/hn43zqTuEGr4PN7F4XJ1p4E3Y8= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190911201528-7ad0cfa0b7b5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 h1:ywK/j/KkyTHcdyYSZNXGjMwgmDSfjglYZ3vStQ/gSCU= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190719005602-e377ae9d6386/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= +golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911230505-6bfd74cf029c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190912215617-3720d1ec3678 h1:rM1Udd0CgtYI3KUIhu9ROz0QCqjW+n/ODp/hH7c60Xc= +golang.org/x/tools v0.0.0-20190912215617-3720d1ec3678/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= @@ -331,6 +551,7 @@ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoA google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= gopkg.in/Shopify/sarama.v1 v1.20.1 h1:Gi09A3fJXm0Jgt8kuKZ8YK+r60GfYn7MQuEmI3oq6hE= gopkg.in/Shopify/sarama.v1 v1.20.1/go.mod h1:AxnvoaevB2nBjNK17cG61A3LleFcWFwVBHBt+cot4Oc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -340,6 +561,7 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/h2non/bimg.v1 v1.0.18 h1:qn6/RpBHt+7WQqoBcK+aF2puc6nC78eZj5LexxoalT4= gopkg.in/h2non/bimg.v1 v1.0.18/go.mod h1:PgsZL7dLwUbsGm1NYps320GxGgvQNTnecMCZqxV11So= @@ -347,9 +569,11 @@ gopkg.in/h2non/gock.v1 v1.0.14 h1:fTeu9fcUvSnLNacYvYI54h+1/XEteDyHvrVCZEEEYNM= gopkg.in/h2non/gock.v1 v1.0.14/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE= gopkg.in/macaroon.v2 v2.1.0 h1:HZcsjBCzq9t0eBPMKqTN/uSN6JOm78ZJ2INbqcBQOUI= gopkg.in/macaroon.v2 v2.1.0/go.mod h1:OUb+TQP/OP0WOerC2Jp/3CwhIKyIa9kQjuc7H24e6/o= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -357,3 +581,13 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed h1:WX1yoOaKQfddO/mLzdV4wptyWgoH/6hwLs7QHTixo0I= +mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc= +mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b h1:DxJ5nJdkhDlLok9K6qO+5290kphDJbHOQO1DFFFTeBo= +mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4= +mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f h1:Cq7MalBHYACRd6EesksG1Q8EoIAKOsiZviGKbOLIej4= +mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f/go.mod h1:4G1h5nDURzA3bwVMZIVpwbkw+04kSxk3rAtzlimaUJw= +sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4 h1:JPJh2pk3+X4lXAkZIk2RuE/7/FoK9maXw+TNPJhVS/c= +sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= diff --git a/roomserver/alias/alias.go b/roomserver/alias/alias.go index 59da35b3f..f4f5c20ce 100644 --- a/roomserver/alias/alias.go +++ b/roomserver/alias/alias.go @@ -250,10 +250,13 @@ func (r *RoomserverAliasAPI) sendUpdatedAliasesEvent( return err } + // TODO: Room version here + roomVersion := gomatrixserverlib.RoomVersionV1 + // Create the request ire := roomserverAPI.InputRoomEvent{ Kind: roomserverAPI.KindNew, - Event: event, + Event: event.Headered(roomVersion), AuthEventIDs: event.AuthEventIDs(), SendAsServer: serverName, } diff --git a/roomserver/api/input.go b/roomserver/api/input.go index 9643a927c..f07cc0221 100644 --- a/roomserver/api/input.go +++ b/roomserver/api/input.go @@ -51,7 +51,7 @@ type InputRoomEvent struct { // This controls how the event is processed. Kind int `json:"kind"` // The event JSON for the event to add. - Event gomatrixserverlib.Event `json:"event"` + Event gomatrixserverlib.HeaderedEvent `json:"event"` // List of state event IDs that authenticate this event. // These are likely derived from the "auth_events" JSON key of the event. // But can be different because the "auth_events" key can be incomplete or wrong. @@ -85,7 +85,7 @@ type TransactionID struct { // the usual context a matrix room event would have. We usually do not have // access to the events needed to check the event auth rules for the invite. type InputInviteEvent struct { - Event gomatrixserverlib.Event `json:"event"` + Event gomatrixserverlib.HeaderedEvent `json:"event"` } // InputRoomEventsRequest is a request to InputRoomEvents diff --git a/roomserver/api/output.go b/roomserver/api/output.go index c09d5a1e5..4e7adff79 100644 --- a/roomserver/api/output.go +++ b/roomserver/api/output.go @@ -54,7 +54,7 @@ type OutputEvent struct { // prev_events. type OutputNewRoomEvent struct { // The Event. - Event gomatrixserverlib.Event `json:"event"` + Event gomatrixserverlib.HeaderedEvent `json:"event"` // The latest events in the room after this event. // This can be used to set the prev events for new events in the room. // This also can be used to get the full current state after this event. @@ -117,7 +117,7 @@ type OutputNewRoomEvent struct { // tracked separately from the room events themselves. type OutputNewInviteEvent struct { // The "m.room.member" invite event. - Event gomatrixserverlib.Event `json:"event"` + Event gomatrixserverlib.HeaderedEvent `json:"event"` } // An OutputRetireInviteEvent is written whenever an existing invite is no longer diff --git a/roomserver/input/events.go b/roomserver/input/events.go index a3b70753e..7fbc5d8a9 100644 --- a/roomserver/input/events.go +++ b/roomserver/input/events.go @@ -95,7 +95,7 @@ func processRoomEvent( event := input.Event // Check that the event passes authentication checks and work out the numeric IDs for the auth events. - authEventNIDs, err := checkAuthEvents(ctx, db, event, input.AuthEventIDs) + authEventNIDs, err := checkAuthEvents(ctx, db, event.Event, input.AuthEventIDs) if err != nil { return } @@ -112,7 +112,7 @@ func processRoomEvent( } // Store the event - roomNID, stateAtEvent, err := db.StoreEvent(ctx, event, input.TransactionID, authEventNIDs) + roomNID, stateAtEvent, err := db.StoreEvent(ctx, event.Event, input.TransactionID, authEventNIDs) if err != nil { return } @@ -127,7 +127,7 @@ func processRoomEvent( if stateAtEvent.BeforeStateSnapshotNID == 0 { // We haven't calculated a state for this event yet. // Lets calculate one. - err = calculateAndSetState(ctx, db, input, roomNID, &stateAtEvent, event) + err = calculateAndSetState(ctx, db, input, roomNID, &stateAtEvent, event.Event) if err != nil { return } @@ -140,7 +140,7 @@ func processRoomEvent( // Update the extremities of the event graph for the room return event.EventID(), updateLatestEvents( - ctx, db, ow, roomNID, stateAtEvent, event, input.SendAsServer, input.TransactionID, + ctx, db, ow, roomNID, stateAtEvent, event.Event, input.SendAsServer, input.TransactionID, ) } @@ -234,7 +234,7 @@ func processInviteEvent( return nil } - outputUpdates, err := updateToInviteMembership(updater, &input.Event, nil) + outputUpdates, err := updateToInviteMembership(updater, &input.Event.Event, nil) if err != nil { return err } diff --git a/roomserver/input/latest_events.go b/roomserver/input/latest_events.go index f9fd1d5d4..9a99ad76f 100644 --- a/roomserver/input/latest_events.go +++ b/roomserver/input/latest_events.go @@ -253,8 +253,11 @@ func (u *latestEventsUpdater) makeOutputNewRoomEvent() (*api.OutputEvent, error) latestEventIDs[i] = u.latest[i].EventID } + // TODO: Room version here + roomVersion := gomatrixserverlib.RoomVersionV1 + ore := api.OutputNewRoomEvent{ - Event: u.event, + Event: u.event.Headered(roomVersion), LastSentEventID: u.lastEventIDSent, LatestEventIDs: latestEventIDs, TransactionID: u.transactionID, diff --git a/roomserver/input/membership.go b/roomserver/input/membership.go index 841c5fec6..f2ac3b510 100644 --- a/roomserver/input/membership.go +++ b/roomserver/input/membership.go @@ -136,13 +136,14 @@ func updateToInviteMembership( return nil, err } if needsSending { + roomVersion := gomatrixserverlib.RoomVersionV1 // We notify the consumers using a special event even though we will // notify them about the change in current state as part of the normal // room event stream. This ensures that the consumers only have to // consider a single stream of events when determining whether a user // is invited, rather than having to combine multiple streams themselves. onie := api.OutputNewInviteEvent{ - Event: *add, + Event: (*add).Headered(roomVersion), } updates = append(updates, api.OutputEvent{ Type: api.OutputTypeNewInviteEvent, diff --git a/roomserver/query/query.go b/roomserver/query/query.go index 7f69699f7..52b678ac3 100644 --- a/roomserver/query/query.go +++ b/roomserver/query/query.go @@ -147,12 +147,7 @@ func (r *RoomserverQueryAPI) QueryLatestEventsAndState( } for _, event := range stateEvents { - response.StateEvents = append(response.StateEvents, gomatrixserverlib.HeaderedEvent{ - EventHeader: gomatrixserverlib.EventHeader{ - RoomVersion: roomVersion, - }, - Event: event, - }) + response.StateEvents = append(response.StateEvents, event.Headered(roomVersion)) } return nil @@ -209,12 +204,7 @@ func (r *RoomserverQueryAPI) QueryStateAfterEvents( } for _, event := range stateEvents { - response.StateEvents = append(response.StateEvents, gomatrixserverlib.HeaderedEvent{ - EventHeader: gomatrixserverlib.EventHeader{ - RoomVersion: roomVersion, - }, - Event: event, - }) + response.StateEvents = append(response.StateEvents, event.Headered(roomVersion)) } return nil @@ -247,12 +237,7 @@ func (r *RoomserverQueryAPI) QueryEventsByID( // TODO: Room version here roomVersion := gomatrixserverlib.RoomVersionV1 - response.Events = append(response.Events, gomatrixserverlib.HeaderedEvent{ - EventHeader: gomatrixserverlib.EventHeader{ - RoomVersion: roomVersion, - }, - Event: event, - }) + response.Events = append(response.Events, event.Headered(roomVersion)) } return nil @@ -534,12 +519,7 @@ func (r *RoomserverQueryAPI) QueryMissingEvents( // TODO: Room version here roomVersion := gomatrixserverlib.RoomVersionV1 - response.Events = append(response.Events, gomatrixserverlib.HeaderedEvent{ - EventHeader: gomatrixserverlib.EventHeader{ - RoomVersion: roomVersion, - }, - Event: event, - }) + response.Events = append(response.Events, event.Headered(roomVersion)) } } @@ -585,12 +565,7 @@ func (r *RoomserverQueryAPI) QueryBackfill( // TODO: Room version here roomVersion := gomatrixserverlib.RoomVersionV1 - response.Events = append(response.Events, gomatrixserverlib.HeaderedEvent{ - EventHeader: gomatrixserverlib.EventHeader{ - RoomVersion: roomVersion, - }, - Event: event, - }) + response.Events = append(response.Events, event.Headered(roomVersion)) } return err @@ -711,24 +686,14 @@ func (r *RoomserverQueryAPI) QueryStateAndAuthChain( // TODO: Room version here roomVersion := gomatrixserverlib.RoomVersionV1 - response.StateEvents = append(response.StateEvents, gomatrixserverlib.HeaderedEvent{ - EventHeader: gomatrixserverlib.EventHeader{ - RoomVersion: roomVersion, - }, - Event: event, - }) + response.StateEvents = append(response.StateEvents, event.Headered(roomVersion)) } for _, event := range authEvents { // TODO: Room version here roomVersion := gomatrixserverlib.RoomVersionV1 - response.AuthChainEvents = append(response.AuthChainEvents, gomatrixserverlib.HeaderedEvent{ - EventHeader: gomatrixserverlib.EventHeader{ - RoomVersion: roomVersion, - }, - Event: event, - }) + response.AuthChainEvents = append(response.AuthChainEvents, event.Headered(roomVersion)) } return err diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index 976dd8e8f..5aaff5627 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -98,7 +98,7 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { func (s *OutputRoomEventConsumer) onNewRoomEvent( ctx context.Context, msg api.OutputNewRoomEvent, ) error { - ev := msg.Event + ev := msg.Event.Event log.WithFields(log.Fields{ "event_id": ev.EventID(), "room_id": ev.RoomID(), @@ -153,7 +153,7 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent( func (s *OutputRoomEventConsumer) onNewInviteEvent( ctx context.Context, msg api.OutputNewInviteEvent, ) error { - pduPos, err := s.db.AddInviteEvent(ctx, msg.Event) + pduPos, err := s.db.AddInviteEvent(ctx, msg.Event.Event) if err != nil { // panic rather than continue with an inconsistent database log.WithFields(log.Fields{ @@ -163,7 +163,7 @@ func (s *OutputRoomEventConsumer) onNewInviteEvent( }).Panicf("roomserver output log: write invite failure") return nil } - s.notifier.OnNewEvent(&msg.Event, "", nil, types.PaginationToken{PDUPosition: pduPos}) + s.notifier.OnNewEvent(&msg.Event.Event, "", nil, types.PaginationToken{PDUPosition: pduPos}) return nil } From a66c701b29255a5cb7ec68b970948fb77f1f31af Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 17 Mar 2020 15:12:01 +0000 Subject: [PATCH 40/86] Add room version to room create request, persist in storage (#915) * Add room version into createRoomReq * Extract room version from m.room.create event when persisting * Reduce cyclomatic complexity * Update whitelist, gomatrixserverlib, tweaks to roomserver * Update sytest-whitelist again --- clientapi/jsonerror/jsonerror.go | 6 ++++ clientapi/routing/createroom.go | 40 +++++++++++++++-------- go.mod | 2 +- go.sum | 6 ++++ roomserver/storage/postgres/storage.go | 43 +++++++++++++++++++++++-- roomserver/storage/sqlite3/storage.go | 44 ++++++++++++++++++++++++-- sytest-whitelist | 12 ------- 7 files changed, 123 insertions(+), 30 deletions(-) diff --git a/clientapi/jsonerror/jsonerror.go b/clientapi/jsonerror/jsonerror.go index 8df1fead2..735de5bea 100644 --- a/clientapi/jsonerror/jsonerror.go +++ b/clientapi/jsonerror/jsonerror.go @@ -124,6 +124,12 @@ func GuestAccessForbidden(msg string) *MatrixError { return &MatrixError{"M_GUEST_ACCESS_FORBIDDEN", msg} } +// UnsupportedRoomVersion is an error which is returned when the client +// requests a room with a version that is unsupported. +func UnsupportedRoomVersion(msg string) *MatrixError { + return &MatrixError{"M_UNSUPPORTED_ROOM_VERSION", msg} +} + // LimitExceededError is a rate-limiting error. type LimitExceededError struct { MatrixError diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index 3465d9854..92b1ae887 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -23,6 +23,7 @@ import ( appserviceAPI "github.com/matrix-org/dendrite/appservice/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + roomserverVersion "github.com/matrix-org/dendrite/roomserver/version" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" @@ -38,15 +39,16 @@ import ( // https://matrix.org/docs/spec/client_server/r0.2.0.html#post-matrix-client-r0-createroom type createRoomRequest struct { - Invite []string `json:"invite"` - Name string `json:"name"` - Visibility string `json:"visibility"` - Topic string `json:"topic"` - Preset string `json:"preset"` - CreationContent map[string]interface{} `json:"creation_content"` - InitialState []fledglingEvent `json:"initial_state"` - RoomAliasName string `json:"room_alias_name"` - GuestCanJoin bool `json:"guest_can_join"` + Invite []string `json:"invite"` + Name string `json:"name"` + Visibility string `json:"visibility"` + Topic string `json:"topic"` + Preset string `json:"preset"` + CreationContent map[string]interface{} `json:"creation_content"` + InitialState []fledglingEvent `json:"initial_state"` + RoomAliasName string `json:"room_alias_name"` + GuestCanJoin bool `json:"guest_can_join"` + RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"` } const ( @@ -180,16 +182,28 @@ func createRoom( } r.CreationContent["creator"] = userID - // TODO: Room version here - r.CreationContent["room_version"] = gomatrixserverlib.RoomVersionV1 + roomVersion := roomserverVersion.DefaultRoomVersion() + if r.RoomVersion != "" { + candidateVersion := gomatrixserverlib.RoomVersion(r.RoomVersion) + _, roomVersionError := roomserverVersion.SupportedRoomVersion(candidateVersion) + if roomVersionError != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.UnsupportedRoomVersion(roomVersionError.Error()), + } + } + roomVersion = candidateVersion + } + r.CreationContent["room_version"] = roomVersion // TODO: visibility/presets/raw initial state // TODO: Create room alias association // Make sure this doesn't fall into an application service's namespace though! logger.WithFields(log.Fields{ - "userID": userID, - "roomID": roomID, + "userID": userID, + "roomID": roomID, + "roomVersion": r.CreationContent["room_version"], }).Info("Creating new room") profile, err := appserviceAPI.RetrieveUserProfile(req.Context(), userID, asAPI, accountDB) diff --git a/go.mod b/go.mod index 114a548e5..958706db4 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200310180544-7f3fad43b51c github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200317103236-134415251e65 + github.com/matrix-org/gomatrixserverlib v0.0.0-20200317140257-ddc7feaaf2fd github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible diff --git a/go.sum b/go.sum index a02e29542..517d0bf1c 100644 --- a/go.sum +++ b/go.sum @@ -30,6 +30,7 @@ github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVa github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= @@ -263,6 +264,10 @@ github.com/matrix-org/gomatrixserverlib v0.0.0-20200316222111-b66d3b63dd85 h1:I7 github.com/matrix-org/gomatrixserverlib v0.0.0-20200316222111-b66d3b63dd85/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= github.com/matrix-org/gomatrixserverlib v0.0.0-20200317103236-134415251e65 h1:llq2yETQhD75ImQhCwHbnivSKOgzOP6QdUqH9sse7XM= github.com/matrix-org/gomatrixserverlib v0.0.0-20200317103236-134415251e65/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200317114945-9a368ea4620d h1:0GYO2Jye1TNVzsn02IF5tqV80psDi0KIWC4+glH6+/Q= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200317114945-9a368ea4620d/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200317140257-ddc7feaaf2fd h1:n95A8YyiCZ8Nu2beqw4akCaPIRrZr/nesHYDZV8WkXI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200317140257-ddc7feaaf2fd/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 h1:W7l5CP4V7wPyPb4tYE11dbmeAOwtFQBTW0rf4OonOS8= @@ -453,6 +458,7 @@ go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.0 h1:vs7fgriifsPbGdK3bNuMWapNn3qnZhCRXc19NRdq010= go.uber.org/atomic v1.3.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go index ee1a1cd4d..b2b4159c9 100644 --- a/roomserver/storage/postgres/storage.go +++ b/roomserver/storage/postgres/storage.go @@ -18,6 +18,9 @@ package postgres import ( "context" "database/sql" + "encoding/json" + + roomserverVersion "github.com/matrix-org/dendrite/roomserver/version" // Import the postgres database driver. _ "github.com/lib/pq" @@ -68,8 +71,21 @@ func (d *Database) StoreEvent( } } - // TODO: Room version here - if roomNID, err = d.assignRoomNID(ctx, nil, event.RoomID(), "1"); err != nil { + // TODO: Here we should aim to have two different code paths for new rooms + // vs existing ones. + + // Get the default room version. If the client doesn't supply a room_version + // then we will use our configured default to create the room. + // https://matrix.org/docs/spec/client_server/r0.6.0#post-matrix-client-r0-createroom + // Note that the below logic depends on the m.room.create event being the + // first event that is persisted to the database when creating or joining a + // room. + var roomVersion gomatrixserverlib.RoomVersion + if roomVersion, err = extractRoomVersionFromCreateEvent(event); err != nil { + return 0, types.StateAtEvent{}, err + } + + if roomNID, err = d.assignRoomNID(ctx, nil, event.RoomID(), roomVersion); err != nil { return 0, types.StateAtEvent{}, err } @@ -121,6 +137,29 @@ func (d *Database) StoreEvent( }, nil } +func extractRoomVersionFromCreateEvent(event gomatrixserverlib.Event) ( + gomatrixserverlib.RoomVersion, error, +) { + var err error + var roomVersion gomatrixserverlib.RoomVersion + // Look for m.room.create events. + if event.Type() != gomatrixserverlib.MRoomCreate { + return gomatrixserverlib.RoomVersion(""), nil + } + roomVersion = roomserverVersion.DefaultRoomVersion() + var createContent gomatrixserverlib.CreateContent + // The m.room.create event contains an optional "room_version" key in + // the event content, so we need to unmarshal that first. + if err = json.Unmarshal(event.Content(), &createContent); err != nil { + return gomatrixserverlib.RoomVersion(""), err + } + // A room version was specified in the event content? + if createContent.RoomVersion != nil { + roomVersion = *createContent.RoomVersion + } + return roomVersion, err +} + func (d *Database) assignRoomNID( ctx context.Context, txn *sql.Tx, roomID string, roomVersion gomatrixserverlib.RoomVersion, diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index d9513611e..b912b1c0e 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -18,9 +18,12 @@ package sqlite3 import ( "context" "database/sql" + "encoding/json" "errors" "net/url" + roomserverVersion "github.com/matrix-org/dendrite/roomserver/version" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/types" @@ -67,6 +70,7 @@ func Open(dataSourceName string) (*Database, error) { } // StoreEvent implements input.EventDatabase +// nolint:gocyclo func (d *Database) StoreEvent( ctx context.Context, event gomatrixserverlib.Event, txnAndSessionID *api.TransactionID, authEventNIDs []types.EventNID, @@ -90,8 +94,21 @@ func (d *Database) StoreEvent( } } - // TODO: Room version here - if roomNID, err = d.assignRoomNID(ctx, txn, event.RoomID(), "1"); err != nil { + // TODO: Here we should aim to have two different code paths for new rooms + // vs existing ones. + + // Get the default room version. If the client doesn't supply a room_version + // then we will use our configured default to create the room. + // https://matrix.org/docs/spec/client_server/r0.6.0#post-matrix-client-r0-createroom + // Note that the below logic depends on the m.room.create event being the + // first event that is persisted to the database when creating or joining a + // room. + var roomVersion gomatrixserverlib.RoomVersion + if roomVersion, err = extractRoomVersionFromCreateEvent(event); err != nil { + return err + } + + if roomNID, err = d.assignRoomNID(ctx, txn, event.RoomID(), roomVersion); err != nil { return err } @@ -150,6 +167,29 @@ func (d *Database) StoreEvent( }, nil } +func extractRoomVersionFromCreateEvent(event gomatrixserverlib.Event) ( + gomatrixserverlib.RoomVersion, error, +) { + var err error + var roomVersion gomatrixserverlib.RoomVersion + // Look for m.room.create events. + if event.Type() != gomatrixserverlib.MRoomCreate { + return gomatrixserverlib.RoomVersion(""), nil + } + roomVersion = roomserverVersion.DefaultRoomVersion() + var createContent gomatrixserverlib.CreateContent + // The m.room.create event contains an optional "room_version" key in + // the event content, so we need to unmarshal that first. + if err = json.Unmarshal(event.Content(), &createContent); err != nil { + return gomatrixserverlib.RoomVersion(""), err + } + // A room version was specified in the event content? + if createContent.RoomVersion != nil { + roomVersion = *createContent.RoomVersion + } + return roomVersion, err +} + func (d *Database) assignRoomNID( ctx context.Context, txn *sql.Tx, roomID string, roomVersion gomatrixserverlib.RoomVersion, diff --git a/sytest-whitelist b/sytest-whitelist index eb91b3f0c..cac718288 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -104,12 +104,6 @@ Newly banned rooms appear in the leave section of incremental sync Newly banned rooms appear in the leave section of incremental sync local user can join room with version 1 User can invite local user to room with version 1 -local user can join room with version 2 -User can invite local user to room with version 2 -local user can join room with version 3 -User can invite local user to room with version 3 -local user can join room with version 4 -User can invite local user to room with version 4 Should reject keys claiming to belong to a different user Can add account data Can add account data to room @@ -140,8 +134,6 @@ Changing the actions of an unknown rule fails with 404 Enabling an unknown default rule fails with 404 Trying to get push rules with unknown rule_id fails with 404 Events come down the correct room -local user can join room with version 5 -User can invite local user to room with version 5 # SyTest currently only implements the v1 endpoints for /send_join and /send_leave, # whereas Dendrite only supports the v2 endpoints for those, so let's ignore this # test for now. @@ -198,10 +190,6 @@ Local non-members don't see posted message events Remote room members also see posted message events Lazy loading parameters in the filter are strictly boolean remote user can join room with version 1 -remote user can join room with version 2 -remote user can join room with version 3 -remote user can join room with version 4 -remote user can join room with version 5 Inbound federation can query room alias directory Outbound federation can query v2 /send_join Inbound federation can receive v2 /send_join From 4af8323df3af61e2f5e705de9ab0f6d0b84a978b Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 17 Mar 2020 16:45:40 +0000 Subject: [PATCH 41/86] bugfix: Fix a bug which caused prev_content not to be sent to clients (#919) I don't know how this ever passed QA... also fix a missing rows.Close() --- go.sum | 1 + syncapi/consumers/roomserver.go | 2 +- syncapi/storage/sqlite3/output_room_events_table.go | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.sum b/go.sum index 517d0bf1c..63423760c 100644 --- a/go.sum +++ b/go.sum @@ -497,6 +497,7 @@ golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190909003024-a7b16738d86b h1:XfVGCX+0T4WOStkaOsJRllbsiImhB2jgVBGc9L0lPGc= golang.org/x/net v0.0.0-20190909003024-a7b16738d86b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index 5aaff5627..332ac6205 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -253,7 +253,7 @@ func (s *OutputRoomEventConsumer) updateStateEvent(event gomatrixserverlib.Event } prevEvent, err := s.db.GetStateEvent( - context.TODO(), event.Type(), event.RoomID(), stateKey, + context.TODO(), event.RoomID(), event.Type(), stateKey, ) if err != nil { return event, err diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index be8937435..05a33c067 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -160,6 +160,7 @@ func (s *outputRoomEventsStatements) selectStateInRange( if err != nil { return nil, nil, err } + defer rows.Close() // nolint: errcheck // Fetch all the state change events for all rooms between the two positions then loop each event and: // - Keep a cache of the event by ID (99% of state change events are for the event itself) // - For each room ID, build up an array of event IDs which represents cumulative adds/removes From 1467cc10d8bd40b9ea85bc4f0aa4644b16cb37bb Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 17 Mar 2020 17:18:48 +0000 Subject: [PATCH 42/86] bugfix: Fix a bug which caused failures to join rooms over federation (#917) * bugfix: Fix a bug which caused failures to join rooms over federation The cause of this was the semantics of `/send_join`'s `auth_chain` response. Previously, we would only send back the auth chain *for the join event* and not the entire room state. However, we would then try to check that the room state is valid, and then be missing auth events. Now, we send back the entire auth chain for all room state in `/send_join`. The spec needs to be clarified that this is what the chain should be. * refactor: split out grabbing state to reduce cyclo complexity --- roomserver/query/query.go | 62 +++++++++++++++++++++++---------------- 1 file changed, 37 insertions(+), 25 deletions(-) diff --git a/roomserver/query/query.go b/roomserver/query/query.go index 52b678ac3..2de8e0d08 100644 --- a/roomserver/query/query.go +++ b/roomserver/query/query.go @@ -637,12 +637,6 @@ func (r *RoomserverQueryAPI) QueryStateAndAuthChain( request *api.QueryStateAndAuthChainRequest, response *api.QueryStateAndAuthChainResponse, ) error { - // TODO: get the correct room version - roomState, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, r.DB) - if err != nil { - return err - } - response.QueryStateAndAuthChainRequest = *request roomNID, err := r.DB.RoomNID(ctx, request.RoomID) if err != nil { @@ -653,31 +647,21 @@ func (r *RoomserverQueryAPI) QueryStateAndAuthChain( } response.RoomExists = true - prevStates, err := r.DB.StateAtEventIDs(ctx, request.PrevEventIDs) + stateEvents, err := r.loadStateAtEventIDs(ctx, request.PrevEventIDs) if err != nil { - switch err.(type) { - case types.MissingEventError: - return nil - default: - return err - } + return err } response.PrevEventsExist = true - // Look up the currrent state for the requested tuples. - stateEntries, err := roomState.LoadCombinedStateAfterEvents( - ctx, prevStates, - ) - if err != nil { - return err + // add the auth event IDs for the current state events too + var authEventIDs []string + authEventIDs = append(authEventIDs, request.AuthEventIDs...) + for _, se := range stateEvents { + authEventIDs = append(authEventIDs, se.AuthEventIDs()...) } + authEventIDs = util.UniqueStrings(authEventIDs) // de-dupe - stateEvents, err := r.loadStateEvents(ctx, stateEntries) - if err != nil { - return err - } - - authEvents, err := getAuthChain(ctx, r.DB, request.AuthEventIDs) + authEvents, err := getAuthChain(ctx, r.DB, authEventIDs) if err != nil { return err } @@ -699,6 +683,34 @@ func (r *RoomserverQueryAPI) QueryStateAndAuthChain( return err } +func (r *RoomserverQueryAPI) loadStateAtEventIDs(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.Event, error) { + // TODO: get the correct room version + roomState, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, r.DB) + if err != nil { + return nil, err + } + + prevStates, err := r.DB.StateAtEventIDs(ctx, eventIDs) + if err != nil { + switch err.(type) { + case types.MissingEventError: + return nil, nil + default: + return nil, err + } + } + + // Look up the currrent state for the requested tuples. + stateEntries, err := roomState.LoadCombinedStateAfterEvents( + ctx, prevStates, + ) + if err != nil { + return nil, err + } + + return r.loadStateEvents(ctx, stateEntries) +} + // getAuthChain fetches the auth chain for the given auth events. An auth chain // is the list of all events that are referenced in the auth_events section, and // all their auth_events, recursively. The returned set of events contain the From c2bd0b97b34c9040325bf31c5c4a2a06579239d9 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 17 Mar 2020 18:00:10 +0000 Subject: [PATCH 43/86] Get room versions from database (#918) * Retrieve room version where known in roomserver * Get room versions in alias code * Increase gocyclothreshold to 13, since we hit that number a lot * Remove gocyclo nolint from StoreEvent * Update interface to get room version from room ID instead of NID * Remove new API * Fixed this query for SQLite but not for Postgres --- .golangci.yml | 2 +- roomserver/alias/alias.go | 12 ++++++-- roomserver/alias/alias_test.go | 7 +++++ roomserver/input/events.go | 4 +++ roomserver/input/latest_events.go | 6 ++-- roomserver/query/query.go | 35 ++++++++++++---------- roomserver/storage/interface.go | 2 +- roomserver/storage/postgres/rooms_table.go | 16 +++++----- roomserver/storage/postgres/storage.go | 6 ++-- roomserver/storage/sqlite3/rooms_table.go | 16 +++++----- roomserver/storage/sqlite3/storage.go | 7 ++--- 11 files changed, 68 insertions(+), 45 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 0d0f51bd2..7fdd4d003 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -102,7 +102,7 @@ linters-settings: #local-prefixes: github.com/org/project gocyclo: # minimal code complexity to report, 30 by default (but we recommend 10-20) - min-complexity: 12 + min-complexity: 13 maligned: # print struct with more effective memory layout or not, false by default suggest-new: true diff --git a/roomserver/alias/alias.go b/roomserver/alias/alias.go index f4f5c20ce..dfb7e76a7 100644 --- a/roomserver/alias/alias.go +++ b/roomserver/alias/alias.go @@ -46,6 +46,10 @@ type RoomserverAliasAPIDatabase interface { // Remove a given room alias. // Returns an error if there was a problem talking to the database. RemoveRoomAlias(ctx context.Context, alias string) error + // Look up the room version for a given room. + GetRoomVersionForRoom( + ctx context.Context, roomID string, + ) (gomatrixserverlib.RoomVersion, error) } // RoomserverAliasAPI is an implementation of alias.RoomserverAliasAPI @@ -240,6 +244,11 @@ func (r *RoomserverAliasAPI) sendUpdatedAliasesEvent( } builder.AuthEvents = refs + roomVersion, err := r.DB.GetRoomVersionForRoom(ctx, roomID) + if err != nil { + return err + } + // Build the event eventID := fmt.Sprintf("$%s:%s", util.RandomString(16), r.Cfg.Matrix.ServerName) now := time.Now() @@ -250,9 +259,6 @@ func (r *RoomserverAliasAPI) sendUpdatedAliasesEvent( return err } - // TODO: Room version here - roomVersion := gomatrixserverlib.RoomVersionV1 - // Create the request ire := roomserverAPI.InputRoomEvent{ Kind: roomserverAPI.KindNew, diff --git a/roomserver/alias/alias_test.go b/roomserver/alias/alias_test.go index 6ddb63a73..0aefa19d9 100644 --- a/roomserver/alias/alias_test.go +++ b/roomserver/alias/alias_test.go @@ -22,6 +22,7 @@ import ( appserviceAPI "github.com/matrix-org/dendrite/appservice/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/gomatrixserverlib" ) type MockRoomserverAliasAPIDatabase struct { @@ -49,6 +50,12 @@ func (db *MockRoomserverAliasAPIDatabase) GetCreatorIDForAlias( return "", nil } +func (db *MockRoomserverAliasAPIDatabase) GetRoomVersionForRoom( + ctx context.Context, roomID string, +) (gomatrixserverlib.RoomVersion, error) { + return gomatrixserverlib.RoomVersionV1, nil +} + // This method needs to change depending on test case func (db *MockRoomserverAliasAPIDatabase) GetRoomIDForAlias( ctx context.Context, diff --git a/roomserver/input/events.go b/roomserver/input/events.go index 7fbc5d8a9..034b06c11 100644 --- a/roomserver/input/events.go +++ b/roomserver/input/events.go @@ -72,6 +72,10 @@ type RoomEventDatabase interface { ctx context.Context, transactionID string, sessionID int64, userID string, ) (string, error) + // Look up the room version for a given room. + GetRoomVersionForRoom( + ctx context.Context, roomID string, + ) (gomatrixserverlib.RoomVersion, error) } // OutputRoomEventWriter has the APIs needed to write an event to the output logs. diff --git a/roomserver/input/latest_events.go b/roomserver/input/latest_events.go index 9a99ad76f..4d75daae9 100644 --- a/roomserver/input/latest_events.go +++ b/roomserver/input/latest_events.go @@ -253,8 +253,10 @@ func (u *latestEventsUpdater) makeOutputNewRoomEvent() (*api.OutputEvent, error) latestEventIDs[i] = u.latest[i].EventID } - // TODO: Room version here - roomVersion := gomatrixserverlib.RoomVersionV1 + roomVersion, err := u.db.GetRoomVersionForRoom(u.ctx, u.event.RoomID()) + if err != nil { + return nil, err + } ore := api.OutputNewRoomEvent{ Event: u.event.Headered(roomVersion), diff --git a/roomserver/query/query.go b/roomserver/query/query.go index 2de8e0d08..3ab12d60a 100644 --- a/roomserver/query/query.go +++ b/roomserver/query/query.go @@ -91,7 +91,7 @@ type RoomserverQueryAPIDatabase interface { ) (map[types.EventStateKeyNID]string, error) // Look up the room version for a given room. GetRoomVersionForRoom( - ctx context.Context, roomNID types.RoomNID, + ctx context.Context, roomID string, ) (gomatrixserverlib.RoomVersion, error) } @@ -121,7 +121,7 @@ func (r *RoomserverQueryAPI) QueryLatestEventsAndState( } response.RoomExists = true - roomVersion, err := r.DB.GetRoomVersionForRoom(ctx, roomNID) + roomVersion, err := r.DB.GetRoomVersionForRoom(ctx, request.RoomID) if err != nil { return err } @@ -174,7 +174,7 @@ func (r *RoomserverQueryAPI) QueryStateAfterEvents( } response.RoomExists = true - roomVersion, err := r.DB.GetRoomVersionForRoom(ctx, roomNID) + roomVersion, err := r.DB.GetRoomVersionForRoom(ctx, request.RoomID) if err != nil { return err } @@ -234,8 +234,10 @@ func (r *RoomserverQueryAPI) QueryEventsByID( } for _, event := range events { - // TODO: Room version here - roomVersion := gomatrixserverlib.RoomVersionV1 + roomVersion, verr := r.DB.GetRoomVersionForRoom(ctx, event.RoomID()) + if verr != nil { + return verr + } response.Events = append(response.Events, event.Headered(roomVersion)) } @@ -516,8 +518,10 @@ func (r *RoomserverQueryAPI) QueryMissingEvents( response.Events = make([]gomatrixserverlib.HeaderedEvent, 0, len(loadedEvents)-len(eventsToFilter)) for _, event := range loadedEvents { if !eventsToFilter[event.EventID()] { - // TODO: Room version here - roomVersion := gomatrixserverlib.RoomVersionV1 + roomVersion, verr := r.DB.GetRoomVersionForRoom(ctx, event.RoomID()) + if verr != nil { + return verr + } response.Events = append(response.Events, event.Headered(roomVersion)) } @@ -562,8 +566,10 @@ func (r *RoomserverQueryAPI) QueryBackfill( } for _, event := range loadedEvents { - // TODO: Room version here - roomVersion := gomatrixserverlib.RoomVersionV1 + roomVersion, verr := r.DB.GetRoomVersionForRoom(ctx, event.RoomID()) + if verr != nil { + return verr + } response.Events = append(response.Events, event.Headered(roomVersion)) } @@ -647,6 +653,11 @@ func (r *RoomserverQueryAPI) QueryStateAndAuthChain( } response.RoomExists = true + roomVersion, err := r.DB.GetRoomVersionForRoom(ctx, request.RoomID) + if err != nil { + return err + } + stateEvents, err := r.loadStateAtEventIDs(ctx, request.PrevEventIDs) if err != nil { return err @@ -667,16 +678,10 @@ func (r *RoomserverQueryAPI) QueryStateAndAuthChain( } for _, event := range stateEvents { - // TODO: Room version here - roomVersion := gomatrixserverlib.RoomVersionV1 - response.StateEvents = append(response.StateEvents, event.Headered(roomVersion)) } for _, event := range authEvents { - // TODO: Room version here - roomVersion := gomatrixserverlib.RoomVersionV1 - response.AuthChainEvents = append(response.AuthChainEvents, event.Headered(roomVersion)) } diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go index 7f32b53f8..20db7ef7f 100644 --- a/roomserver/storage/interface.go +++ b/roomserver/storage/interface.go @@ -45,5 +45,5 @@ type Database interface { GetMembership(ctx context.Context, roomNID types.RoomNID, requestSenderUserID string) (membershipEventNID types.EventNID, stillInRoom bool, err error) GetMembershipEventNIDsForRoom(ctx context.Context, roomNID types.RoomNID, joinOnly bool) ([]types.EventNID, error) EventsFromIDs(ctx context.Context, eventIDs []string) ([]types.Event, error) - GetRoomVersionForRoom(ctx context.Context, roomNID types.RoomNID) (gomatrixserverlib.RoomVersion, error) + GetRoomVersionForRoom(ctx context.Context, roomID string) (gomatrixserverlib.RoomVersion, error) } diff --git a/roomserver/storage/postgres/rooms_table.go b/roomserver/storage/postgres/rooms_table.go index ef8b8ecef..6bb96f1de 100644 --- a/roomserver/storage/postgres/rooms_table.go +++ b/roomserver/storage/postgres/rooms_table.go @@ -65,8 +65,8 @@ const selectLatestEventNIDsForUpdateSQL = "" + const updateLatestEventNIDsSQL = "" + "UPDATE roomserver_rooms SET latest_event_nids = $2, last_event_sent_nid = $3, state_snapshot_nid = $4 WHERE room_nid = $1" -const selectRoomVersionForRoomNIDSQL = "" + - "SELECT room_version FROM roomserver_rooms WHERE room_nid = $1" +const selectRoomVersionForRoomIDSQL = "" + + "SELECT room_version FROM roomserver_rooms WHERE room_id = $1" type roomStatements struct { insertRoomNIDStmt *sql.Stmt @@ -74,7 +74,7 @@ type roomStatements struct { selectLatestEventNIDsStmt *sql.Stmt selectLatestEventNIDsForUpdateStmt *sql.Stmt updateLatestEventNIDsStmt *sql.Stmt - selectRoomVersionForRoomNIDStmt *sql.Stmt + selectRoomVersionForRoomIDStmt *sql.Stmt } func (s *roomStatements) prepare(db *sql.DB) (err error) { @@ -88,7 +88,7 @@ func (s *roomStatements) prepare(db *sql.DB) (err error) { {&s.selectLatestEventNIDsStmt, selectLatestEventNIDsSQL}, {&s.selectLatestEventNIDsForUpdateStmt, selectLatestEventNIDsForUpdateSQL}, {&s.updateLatestEventNIDsStmt, updateLatestEventNIDsSQL}, - {&s.selectRoomVersionForRoomNIDStmt, selectRoomVersionForRoomNIDSQL}, + {&s.selectRoomVersionForRoomIDStmt, selectRoomVersionForRoomIDSQL}, }.prepare(db) } @@ -165,11 +165,11 @@ func (s *roomStatements) updateLatestEventNIDs( return err } -func (s *roomStatements) selectRoomVersionForRoomNID( - ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, +func (s *roomStatements) selectRoomVersionForRoomID( + ctx context.Context, txn *sql.Tx, roomID string, ) (gomatrixserverlib.RoomVersion, error) { var roomVersion gomatrixserverlib.RoomVersion - stmt := common.TxStmt(txn, s.selectRoomVersionForRoomNIDStmt) - err := stmt.QueryRowContext(ctx, roomNID).Scan(&roomVersion) + stmt := common.TxStmt(txn, s.selectRoomVersionForRoomIDStmt) + err := stmt.QueryRowContext(ctx, roomID).Scan(&roomVersion) return roomVersion, err } diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go index b2b4159c9..af6afe5c1 100644 --- a/roomserver/storage/postgres/storage.go +++ b/roomserver/storage/postgres/storage.go @@ -740,10 +740,10 @@ func (d *Database) EventsFromIDs(ctx context.Context, eventIDs []string) ([]type } func (d *Database) GetRoomVersionForRoom( - ctx context.Context, roomNID types.RoomNID, + ctx context.Context, roomID string, ) (gomatrixserverlib.RoomVersion, error) { - return d.statements.selectRoomVersionForRoomNID( - ctx, nil, roomNID, + return d.statements.selectRoomVersionForRoomID( + ctx, nil, roomID, ) } diff --git a/roomserver/storage/sqlite3/rooms_table.go b/roomserver/storage/sqlite3/rooms_table.go index b750f63ea..49fa07ea8 100644 --- a/roomserver/storage/sqlite3/rooms_table.go +++ b/roomserver/storage/sqlite3/rooms_table.go @@ -54,8 +54,8 @@ const selectLatestEventNIDsForUpdateSQL = "" + const updateLatestEventNIDsSQL = "" + "UPDATE roomserver_rooms SET latest_event_nids = $1, last_event_sent_nid = $2, state_snapshot_nid = $3 WHERE room_nid = $4" -const selectRoomVersionForRoomNIDSQL = "" + - "SELECT room_version FROM roomserver_rooms WHERE room_nid = $1" +const selectRoomVersionForRoomIDSQL = "" + + "SELECT room_version FROM roomserver_rooms WHERE room_id = $1" type roomStatements struct { insertRoomNIDStmt *sql.Stmt @@ -63,7 +63,7 @@ type roomStatements struct { selectLatestEventNIDsStmt *sql.Stmt selectLatestEventNIDsForUpdateStmt *sql.Stmt updateLatestEventNIDsStmt *sql.Stmt - selectRoomVersionForRoomNIDStmt *sql.Stmt + selectRoomVersionForRoomIDStmt *sql.Stmt } func (s *roomStatements) prepare(db *sql.DB) (err error) { @@ -77,7 +77,7 @@ func (s *roomStatements) prepare(db *sql.DB) (err error) { {&s.selectLatestEventNIDsStmt, selectLatestEventNIDsSQL}, {&s.selectLatestEventNIDsForUpdateStmt, selectLatestEventNIDsForUpdateSQL}, {&s.updateLatestEventNIDsStmt, updateLatestEventNIDsSQL}, - {&s.selectRoomVersionForRoomNIDStmt, selectRoomVersionForRoomNIDSQL}, + {&s.selectRoomVersionForRoomIDStmt, selectRoomVersionForRoomIDSQL}, }.prepare(db) } @@ -157,11 +157,11 @@ func (s *roomStatements) updateLatestEventNIDs( return err } -func (s *roomStatements) selectRoomVersionForRoomNID( - ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, +func (s *roomStatements) selectRoomVersionForRoomID( + ctx context.Context, txn *sql.Tx, roomID string, ) (gomatrixserverlib.RoomVersion, error) { var roomVersion gomatrixserverlib.RoomVersion - stmt := common.TxStmt(txn, s.selectRoomVersionForRoomNIDStmt) - err := stmt.QueryRowContext(ctx, roomNID).Scan(&roomVersion) + stmt := common.TxStmt(txn, s.selectRoomVersionForRoomIDStmt) + err := stmt.QueryRowContext(ctx, roomID).Scan(&roomVersion) return roomVersion, err } diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index b912b1c0e..ea926ca66 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -70,7 +70,6 @@ func Open(dataSourceName string) (*Database, error) { } // StoreEvent implements input.EventDatabase -// nolint:gocyclo func (d *Database) StoreEvent( ctx context.Context, event gomatrixserverlib.Event, txnAndSessionID *api.TransactionID, authEventNIDs []types.EventNID, @@ -895,10 +894,10 @@ func (d *Database) EventsFromIDs(ctx context.Context, eventIDs []string) ([]type } func (d *Database) GetRoomVersionForRoom( - ctx context.Context, roomNID types.RoomNID, + ctx context.Context, roomID string, ) (gomatrixserverlib.RoomVersion, error) { - return d.statements.selectRoomVersionForRoomNID( - ctx, nil, roomNID, + return d.statements.selectRoomVersionForRoomID( + ctx, nil, roomID, ) } From c019ad708669a4c269072bea933f0a520a78ca7c Mon Sep 17 00:00:00 2001 From: Prateek Sachan <42961174+prateek2211@users.noreply.github.com> Date: Wed, 18 Mar 2020 15:47:18 +0530 Subject: [PATCH 44/86] Log errors from rows.Close (#920) * Log errors from rows.Close * fixed imports * Added contextual messages * fixed review changes --- .../accounts/postgres/account_data_table.go | 4 +++- .../accounts/postgres/membership_table.go | 4 +++- .../accounts/sqlite3/membership_table.go | 2 +- .../storage/accounts/sqlite3/threepid_table.go | 2 +- .../storage/devices/postgres/devices_table.go | 2 +- common/keydb/postgres/server_key_table.go | 4 +++- common/keydb/sqlite3/server_key_table.go | 2 +- common/log.go | 18 ++++++++++++++++++ common/partition_offset_table.go | 2 +- .../storage/postgres/joined_hosts_table.go | 2 +- .../storage/sqlite3/joined_hosts_table.go | 2 +- go.mod | 3 +++ go.sum | 2 ++ mediaapi/storage/postgres/thumbnail_table.go | 4 +++- mediaapi/storage/sqlite3/thumbnail_table.go | 4 +++- .../storage/postgres/public_rooms_table.go | 4 +++- .../storage/postgres/event_json_table.go | 4 +++- .../storage/postgres/event_state_keys_table.go | 4 ++-- .../storage/postgres/event_types_table.go | 4 +++- roomserver/storage/postgres/events_table.go | 12 ++++++------ roomserver/storage/postgres/invite_table.go | 4 ++-- .../storage/postgres/membership_table.go | 4 ++-- .../storage/postgres/room_aliases_table.go | 4 +++- .../storage/postgres/state_block_table.go | 6 ++++-- roomserver/storage/sqlite3/event_json_table.go | 2 +- .../storage/sqlite3/event_state_keys_table.go | 4 ++-- .../storage/sqlite3/event_types_table.go | 2 +- roomserver/storage/sqlite3/events_table.go | 12 ++++++------ roomserver/storage/sqlite3/invite_table.go | 2 +- roomserver/storage/sqlite3/membership_table.go | 4 ++-- .../storage/sqlite3/room_aliases_table.go | 2 +- .../storage/sqlite3/state_block_table.go | 4 ++-- .../storage/sqlite3/state_snapshot_table.go | 2 +- syncapi/storage/postgres/account_data_table.go | 2 +- .../postgres/backward_extremities_table.go | 4 +++- .../postgres/current_room_state_table.go | 8 ++++---- syncapi/storage/postgres/invites_table.go | 2 +- .../postgres/output_room_events_table.go | 8 ++++---- .../output_room_events_topology_table.go | 6 ++++-- syncapi/storage/sqlite3/account_data_table.go | 4 +++- .../sqlite3/current_room_state_table.go | 8 ++++---- syncapi/storage/sqlite3/invites_table.go | 2 +- .../sqlite3/output_room_events_table.go | 6 +++--- 43 files changed, 118 insertions(+), 69 deletions(-) diff --git a/clientapi/auth/storage/accounts/postgres/account_data_table.go b/clientapi/auth/storage/accounts/postgres/account_data_table.go index 4573999b4..9198a7440 100644 --- a/clientapi/auth/storage/accounts/postgres/account_data_table.go +++ b/clientapi/auth/storage/accounts/postgres/account_data_table.go @@ -18,6 +18,8 @@ import ( "context" "database/sql" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/gomatrixserverlib" ) @@ -90,7 +92,7 @@ func (s *accountDataStatements) selectAccountData( if err != nil { return } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectAccountData: rows.close() failed") global = []gomatrixserverlib.ClientEvent{} rooms = make(map[string][]gomatrixserverlib.ClientEvent) diff --git a/clientapi/auth/storage/accounts/postgres/membership_table.go b/clientapi/auth/storage/accounts/postgres/membership_table.go index 426c2d6ac..27570b67d 100644 --- a/clientapi/auth/storage/accounts/postgres/membership_table.go +++ b/clientapi/auth/storage/accounts/postgres/membership_table.go @@ -18,6 +18,8 @@ import ( "context" "database/sql" + "github.com/matrix-org/dendrite/common" + "github.com/lib/pq" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" ) @@ -118,7 +120,7 @@ func (s *membershipStatements) selectMembershipsByLocalpart( memberships = []authtypes.Membership{} - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectMembershipsByLocalpart: rows.close() failed") for rows.Next() { var m authtypes.Membership m.Localpart = localpart diff --git a/clientapi/auth/storage/accounts/sqlite3/membership_table.go b/clientapi/auth/storage/accounts/sqlite3/membership_table.go index 38f21b7f3..b4bff6334 100644 --- a/clientapi/auth/storage/accounts/sqlite3/membership_table.go +++ b/clientapi/auth/storage/accounts/sqlite3/membership_table.go @@ -118,7 +118,7 @@ func (s *membershipStatements) selectMembershipsByLocalpart( memberships = []authtypes.Membership{} - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectMembershipsByLocalpart: rows.close() failed") for rows.Next() { var m authtypes.Membership m.Localpart = localpart diff --git a/clientapi/auth/storage/accounts/sqlite3/threepid_table.go b/clientapi/auth/storage/accounts/sqlite3/threepid_table.go index 53f6408d1..29ee4c3d0 100644 --- a/clientapi/auth/storage/accounts/sqlite3/threepid_table.go +++ b/clientapi/auth/storage/accounts/sqlite3/threepid_table.go @@ -97,7 +97,7 @@ func (s *threepidStatements) selectThreePIDsForLocalpart( if err != nil { return } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectThreePIDsForLocalpart: rows.close() failed") threepids = []authtypes.ThreePID{} for rows.Next() { diff --git a/clientapi/auth/storage/devices/postgres/devices_table.go b/clientapi/auth/storage/devices/postgres/devices_table.go index 6cae3743e..ee5591706 100644 --- a/clientapi/auth/storage/devices/postgres/devices_table.go +++ b/clientapi/auth/storage/devices/postgres/devices_table.go @@ -226,7 +226,7 @@ func (s *devicesStatements) selectDevicesByLocalpart( if err != nil { return devices, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectDevicesByLocalpart: rows.close() failed") for rows.Next() { var dev authtypes.Device diff --git a/common/keydb/postgres/server_key_table.go b/common/keydb/postgres/server_key_table.go index 6b13cc3c2..0434eb8b1 100644 --- a/common/keydb/postgres/server_key_table.go +++ b/common/keydb/postgres/server_key_table.go @@ -19,6 +19,8 @@ import ( "context" "database/sql" + "github.com/matrix-org/dendrite/common" + "github.com/lib/pq" "github.com/matrix-org/gomatrixserverlib" ) @@ -91,7 +93,7 @@ func (s *serverKeyStatements) bulkSelectServerKeys( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectServerKeys: rows.close() failed") results := map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult{} for rows.Next() { var serverName string diff --git a/common/keydb/sqlite3/server_key_table.go b/common/keydb/sqlite3/server_key_table.go index 3caf4ea4c..ba1cc0606 100644 --- a/common/keydb/sqlite3/server_key_table.go +++ b/common/keydb/sqlite3/server_key_table.go @@ -124,7 +124,7 @@ func (s *serverKeyStatements) bulkSelectServerKeys( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectServerKeys: rows.close() failed") results := map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult{} for rows.Next() { var serverName string diff --git a/common/log.go b/common/log.go index f9ed84edb..11339ada4 100644 --- a/common/log.go +++ b/common/log.go @@ -15,13 +15,17 @@ package common import ( + "context" "fmt" + "io" "os" "path" "path/filepath" "runtime" "strings" + "github.com/matrix-org/util" + "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dugong" "github.com/sirupsen/logrus" @@ -156,3 +160,17 @@ func setupFileHook(hook config.LogrusHook, level logrus.Level, componentName str ), }) } + +//CloseAndLogIfError Closes io.Closer and logs the error if any +func CloseAndLogIfError(ctx context.Context, closer io.Closer, message string) { + if closer == nil { + return + } + err := closer.Close() + if ctx == nil { + ctx = context.TODO() + } + if err != nil { + util.GetLogger(ctx).WithError(err).Error(message) + } +} diff --git a/common/partition_offset_table.go b/common/partition_offset_table.go index 6bc066a69..aa799f8a0 100644 --- a/common/partition_offset_table.go +++ b/common/partition_offset_table.go @@ -90,7 +90,7 @@ func (s *PartitionOffsetStatements) selectPartitionOffsets( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer CloseAndLogIfError(ctx, rows, "selectPartitionOffsets: rows.close() failed") var results []PartitionOffset for rows.Next() { var offset PartitionOffset diff --git a/federationsender/storage/postgres/joined_hosts_table.go b/federationsender/storage/postgres/joined_hosts_table.go index e5c30a010..b3c45abda 100644 --- a/federationsender/storage/postgres/joined_hosts_table.go +++ b/federationsender/storage/postgres/joined_hosts_table.go @@ -118,7 +118,7 @@ func joinedHostsFromStmt( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "joinedHostsFromStmt: rows.close() failed") var result []types.JoinedHost for rows.Next() { diff --git a/federationsender/storage/sqlite3/joined_hosts_table.go b/federationsender/storage/sqlite3/joined_hosts_table.go index 1437a062b..466ae4991 100644 --- a/federationsender/storage/sqlite3/joined_hosts_table.go +++ b/federationsender/storage/sqlite3/joined_hosts_table.go @@ -121,7 +121,7 @@ func joinedHostsFromStmt( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "joinedHostsFromStmt: rows.close() failed") var result []types.JoinedHost for rows.Next() { diff --git a/go.mod b/go.mod index 958706db4..82474944f 100644 --- a/go.mod +++ b/go.mod @@ -15,10 +15,13 @@ require ( github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 github.com/opentracing/opentracing-go v1.1.0 github.com/pkg/errors v0.8.1 github.com/prometheus/client_golang v1.4.1 + github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect github.com/sirupsen/logrus v1.4.2 github.com/tidwall/gjson v1.6.0 // indirect github.com/tidwall/pretty v1.0.1 // indirect diff --git a/go.sum b/go.sum index 63423760c..3c7cf281e 100644 --- a/go.sum +++ b/go.sum @@ -372,6 +372,8 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ= +github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= diff --git a/mediaapi/storage/postgres/thumbnail_table.go b/mediaapi/storage/postgres/thumbnail_table.go index 127b86bb9..08bddc36f 100644 --- a/mediaapi/storage/postgres/thumbnail_table.go +++ b/mediaapi/storage/postgres/thumbnail_table.go @@ -20,6 +20,8 @@ import ( "database/sql" "time" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/mediaapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -144,7 +146,7 @@ func (s *thumbnailStatements) selectThumbnails( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectThumbnails: rows.close() failed") var thumbnails []*types.ThumbnailMetadata for rows.Next() { diff --git a/mediaapi/storage/sqlite3/thumbnail_table.go b/mediaapi/storage/sqlite3/thumbnail_table.go index 95332c9d4..280fafe8d 100644 --- a/mediaapi/storage/sqlite3/thumbnail_table.go +++ b/mediaapi/storage/sqlite3/thumbnail_table.go @@ -20,6 +20,8 @@ import ( "database/sql" "time" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/mediaapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -134,7 +136,7 @@ func (s *thumbnailStatements) selectThumbnails( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectThumbnails: rows.close() failed") var thumbnails []*types.ThumbnailMetadata for rows.Next() { diff --git a/publicroomsapi/storage/postgres/public_rooms_table.go b/publicroomsapi/storage/postgres/public_rooms_table.go index edf9ad2ab..7e606e939 100644 --- a/publicroomsapi/storage/postgres/public_rooms_table.go +++ b/publicroomsapi/storage/postgres/public_rooms_table.go @@ -21,6 +21,8 @@ import ( "errors" "fmt" + "github.com/matrix-org/dendrite/common" + "github.com/lib/pq" "github.com/matrix-org/dendrite/publicroomsapi/types" ) @@ -203,7 +205,7 @@ func (s *publicRoomsStatements) selectPublicRooms( if err != nil { return []types.PublicRoom{}, nil } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectPublicRooms: rows.close() failed") rooms := []types.PublicRoom{} for rows.Next() { diff --git a/roomserver/storage/postgres/event_json_table.go b/roomserver/storage/postgres/event_json_table.go index 0b7ef6aa7..616eaf318 100644 --- a/roomserver/storage/postgres/event_json_table.go +++ b/roomserver/storage/postgres/event_json_table.go @@ -19,6 +19,8 @@ import ( "context" "database/sql" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/roomserver/types" ) @@ -86,7 +88,7 @@ func (s *eventJSONStatements) bulkSelectEventJSON( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventJSON: rows.close() failed") // We know that we will only get as many results as event NIDs // because of the unique constraint on event NIDs. diff --git a/roomserver/storage/postgres/event_state_keys_table.go b/roomserver/storage/postgres/event_state_keys_table.go index cbc29a69d..4c3496d91 100644 --- a/roomserver/storage/postgres/event_state_keys_table.go +++ b/roomserver/storage/postgres/event_state_keys_table.go @@ -114,7 +114,7 @@ func (s *eventStateKeyStatements) bulkSelectEventStateKeyNID( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventStateKeyNID: rows.close() failed") result := make(map[string]types.EventStateKeyNID, len(eventStateKeys)) for rows.Next() { @@ -139,7 +139,7 @@ func (s *eventStateKeyStatements) bulkSelectEventStateKey( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventStateKey: rows.close() failed") result := make(map[types.EventStateKeyNID]string, len(eventStateKeyNIDs)) for rows.Next() { diff --git a/roomserver/storage/postgres/event_types_table.go b/roomserver/storage/postgres/event_types_table.go index faa887545..6537a5457 100644 --- a/roomserver/storage/postgres/event_types_table.go +++ b/roomserver/storage/postgres/event_types_table.go @@ -19,6 +19,8 @@ import ( "context" "database/sql" + "github.com/matrix-org/dendrite/common" + "github.com/lib/pq" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -132,7 +134,7 @@ func (s *eventTypeStatements) bulkSelectEventTypeNID( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventTypeNID: rows.close() failed") result := make(map[string]types.EventTypeNID, len(eventTypes)) for rows.Next() { diff --git a/roomserver/storage/postgres/events_table.go b/roomserver/storage/postgres/events_table.go index d9b269bc8..0caa8199a 100644 --- a/roomserver/storage/postgres/events_table.go +++ b/roomserver/storage/postgres/events_table.go @@ -192,7 +192,7 @@ func (s *eventStatements) bulkSelectStateEventByID( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectStateEventByID: rows.close() failed") // We know that we will only get as many results as event IDs // because of the unique constraint on event IDs. // So we can allocate an array of the correct size now. @@ -235,7 +235,7 @@ func (s *eventStatements) bulkSelectStateAtEventByID( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectStateAtEventByID: rows.close() failed") results := make([]types.StateAtEvent, len(eventIDs)) i := 0 for ; rows.Next(); i++ { @@ -302,7 +302,7 @@ func (s *eventStatements) bulkSelectStateAtEventAndReference( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectStateAtEventAndReference: rows.close() failed") results := make([]types.StateAtEventAndReference, len(eventNIDs)) i := 0 for ; rows.Next(); i++ { @@ -343,7 +343,7 @@ func (s *eventStatements) bulkSelectEventReference( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventReference: rows.close() failed") results := make([]gomatrixserverlib.EventReference, len(eventNIDs)) i := 0 for ; rows.Next(); i++ { @@ -367,7 +367,7 @@ func (s *eventStatements) bulkSelectEventID(ctx context.Context, eventNIDs []typ if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventID: rows.close() failed") results := make(map[types.EventNID]string, len(eventNIDs)) i := 0 for ; rows.Next(); i++ { @@ -394,7 +394,7 @@ func (s *eventStatements) bulkSelectEventNID(ctx context.Context, eventIDs []str if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventNID: rows.close() failed") results := make(map[string]types.EventNID, len(eventIDs)) for rows.Next() { var eventID string diff --git a/roomserver/storage/postgres/invite_table.go b/roomserver/storage/postgres/invite_table.go index 603fed31b..f764b1561 100644 --- a/roomserver/storage/postgres/invite_table.go +++ b/roomserver/storage/postgres/invite_table.go @@ -120,7 +120,7 @@ func (s *inviteStatements) updateInviteRetired( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "updateInviteRetired: rows.close() failed") var eventIDs []string for rows.Next() { @@ -144,7 +144,7 @@ func (s *inviteStatements) selectInviteActiveForUserInRoom( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectInviteActiveForUserInRoom: rows.close() failed") var result []types.EventStateKeyNID for rows.Next() { var senderUserNID int64 diff --git a/roomserver/storage/postgres/membership_table.go b/roomserver/storage/postgres/membership_table.go index 70032fd1e..9c8a4c259 100644 --- a/roomserver/storage/postgres/membership_table.go +++ b/roomserver/storage/postgres/membership_table.go @@ -151,7 +151,7 @@ func (s *membershipStatements) selectMembershipsFromRoom( if err != nil { return } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectMembershipsFromRoom: rows.close() failed") for rows.Next() { var eNID types.EventNID @@ -172,7 +172,7 @@ func (s *membershipStatements) selectMembershipsFromRoomAndMembership( if err != nil { return } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectMembershipsFromRoomAndMembership: rows.close() failed") for rows.Next() { var eNID types.EventNID diff --git a/roomserver/storage/postgres/room_aliases_table.go b/roomserver/storage/postgres/room_aliases_table.go index 6de898c41..c37f383c9 100644 --- a/roomserver/storage/postgres/room_aliases_table.go +++ b/roomserver/storage/postgres/room_aliases_table.go @@ -18,6 +18,8 @@ package postgres import ( "context" "database/sql" + + "github.com/matrix-org/dendrite/common" ) const roomAliasesSchema = ` @@ -95,7 +97,7 @@ func (s *roomAliasesStatements) selectAliasesFromRoomID( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectAliasesFromRoomID: rows.close() failed") var aliases []string for rows.Next() { diff --git a/roomserver/storage/postgres/state_block_table.go b/roomserver/storage/postgres/state_block_table.go index e6f4f7fe9..b9246b763 100644 --- a/roomserver/storage/postgres/state_block_table.go +++ b/roomserver/storage/postgres/state_block_table.go @@ -21,6 +21,8 @@ import ( "fmt" "sort" + "github.com/matrix-org/dendrite/common" + "github.com/lib/pq" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/util" @@ -138,7 +140,7 @@ func (s *stateBlockStatements) bulkSelectStateBlockEntries( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectStateBlockEntries: rows.close() failed") results := make([]types.StateEntryList, len(stateBlockNIDs)) // current is a pointer to the StateEntryList to append the state entries to. @@ -197,7 +199,7 @@ func (s *stateBlockStatements) bulkSelectFilteredStateBlockEntries( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectFilteredStateBlockEntries: rows.close() failed") var results []types.StateEntryList var current types.StateEntryList diff --git a/roomserver/storage/sqlite3/event_json_table.go b/roomserver/storage/sqlite3/event_json_table.go index f6c83906a..fc661c1da 100644 --- a/roomserver/storage/sqlite3/event_json_table.go +++ b/roomserver/storage/sqlite3/event_json_table.go @@ -88,7 +88,7 @@ func (s *eventJSONStatements) bulkSelectEventJSON( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventJSON: rows.close() failed") // We know that we will only get as many results as event NIDs // because of the unique constraint on event NIDs. diff --git a/roomserver/storage/sqlite3/event_state_keys_table.go b/roomserver/storage/sqlite3/event_state_keys_table.go index b8bc6c02d..fa8fc57eb 100644 --- a/roomserver/storage/sqlite3/event_state_keys_table.go +++ b/roomserver/storage/sqlite3/event_state_keys_table.go @@ -116,7 +116,7 @@ func (s *eventStateKeyStatements) bulkSelectEventStateKeyNID( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventStateKeyNID: rows.close() failed") result := make(map[string]types.EventStateKeyNID, len(eventStateKeys)) for rows.Next() { var stateKey string @@ -142,7 +142,7 @@ func (s *eventStateKeyStatements) bulkSelectEventStateKey( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventStateKey: rows.close() failed") result := make(map[types.EventStateKeyNID]string, len(eventStateKeyNIDs)) for rows.Next() { var stateKey string diff --git a/roomserver/storage/sqlite3/event_types_table.go b/roomserver/storage/sqlite3/event_types_table.go index edc06d4c6..777f8be79 100644 --- a/roomserver/storage/sqlite3/event_types_table.go +++ b/roomserver/storage/sqlite3/event_types_table.go @@ -138,7 +138,7 @@ func (s *eventTypeStatements) bulkSelectEventTypeNID( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventTypeNID: rows.close() failed") result := make(map[string]types.EventTypeNID, len(eventTypes)) for rows.Next() { diff --git a/roomserver/storage/sqlite3/events_table.go b/roomserver/storage/sqlite3/events_table.go index 4fa095913..1e4ed448f 100644 --- a/roomserver/storage/sqlite3/events_table.go +++ b/roomserver/storage/sqlite3/events_table.go @@ -194,7 +194,7 @@ func (s *eventStatements) bulkSelectStateEventByID( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectStateEventByID: rows.close() failed") // We know that we will only get as many results as event IDs // because of the unique constraint on event IDs. // So we can allocate an array of the correct size now. @@ -247,7 +247,7 @@ func (s *eventStatements) bulkSelectStateAtEventByID( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectStateAtEventByID: rows.close() failed") results := make([]types.StateAtEvent, len(eventIDs)) i := 0 for ; rows.Next(); i++ { @@ -323,7 +323,7 @@ func (s *eventStatements) bulkSelectStateAtEventAndReference( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectStateAtEventAndReference: rows.close() failed") results := make([]types.StateAtEventAndReference, len(eventNIDs)) i := 0 for ; rows.Next(); i++ { @@ -374,7 +374,7 @@ func (s *eventStatements) bulkSelectEventReference( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventReference: rows.close() failed") results := make([]gomatrixserverlib.EventReference, len(eventNIDs)) i := 0 for ; rows.Next(); i++ { @@ -408,7 +408,7 @@ func (s *eventStatements) bulkSelectEventID(ctx context.Context, txn *sql.Tx, ev if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventID: rows.close() failed") results := make(map[types.EventNID]string, len(eventNIDs)) i := 0 for ; rows.Next(); i++ { @@ -445,7 +445,7 @@ func (s *eventStatements) bulkSelectEventNID(ctx context.Context, txn *sql.Tx, e if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventNID: rows.close() failed") results := make(map[string]types.EventNID, len(eventIDs)) for rows.Next() { var eventID string diff --git a/roomserver/storage/sqlite3/invite_table.go b/roomserver/storage/sqlite3/invite_table.go index 641f80156..0ab3e6f36 100644 --- a/roomserver/storage/sqlite3/invite_table.go +++ b/roomserver/storage/sqlite3/invite_table.go @@ -137,7 +137,7 @@ func (s *inviteStatements) selectInviteActiveForUserInRoom( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectInviteActiveForUserInRoom: rows.close() failed") var result []types.EventStateKeyNID for rows.Next() { var senderUserNID int64 diff --git a/roomserver/storage/sqlite3/membership_table.go b/roomserver/storage/sqlite3/membership_table.go index 978776738..7ae28e4b8 100644 --- a/roomserver/storage/sqlite3/membership_table.go +++ b/roomserver/storage/sqlite3/membership_table.go @@ -134,7 +134,7 @@ func (s *membershipStatements) selectMembershipsFromRoom( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectMembershipsFromRoom: rows.close() failed") for rows.Next() { var eNID types.EventNID @@ -154,7 +154,7 @@ func (s *membershipStatements) selectMembershipsFromRoomAndMembership( if err != nil { return } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectMembershipsFromRoomAndMembership: rows.close() failed") for rows.Next() { var eNID types.EventNID diff --git a/roomserver/storage/sqlite3/room_aliases_table.go b/roomserver/storage/sqlite3/room_aliases_table.go index 71238b0e4..d29833918 100644 --- a/roomserver/storage/sqlite3/room_aliases_table.go +++ b/roomserver/storage/sqlite3/room_aliases_table.go @@ -103,7 +103,7 @@ func (s *roomAliasesStatements) selectAliasesFromRoomID( return } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectAliasesFromRoomID: rows.close() failed") for rows.Next() { var alias string diff --git a/roomserver/storage/sqlite3/state_block_table.go b/roomserver/storage/sqlite3/state_block_table.go index c94ca2385..cc7c75733 100644 --- a/roomserver/storage/sqlite3/state_block_table.go +++ b/roomserver/storage/sqlite3/state_block_table.go @@ -137,7 +137,7 @@ func (s *stateBlockStatements) bulkSelectStateBlockEntries( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectStateBlockEntries: rows.close() failed") results := make([]types.StateEntryList, len(stateBlockNIDs)) // current is a pointer to the StateEntryList to append the state entries to. @@ -207,7 +207,7 @@ func (s *stateBlockStatements) bulkSelectFilteredStateBlockEntries( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectFilteredStateBlockEntries: rows.close() failed") var results []types.StateEntryList var current types.StateEntryList diff --git a/roomserver/storage/sqlite3/state_snapshot_table.go b/roomserver/storage/sqlite3/state_snapshot_table.go index 0e1786eae..f367a779b 100644 --- a/roomserver/storage/sqlite3/state_snapshot_table.go +++ b/roomserver/storage/sqlite3/state_snapshot_table.go @@ -99,7 +99,7 @@ func (s *stateSnapshotStatements) bulkSelectStateBlockNIDs( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "bulkSelectStateBlockNIDs: rows.close() failed") results := make([]types.StateBlockNIDList, len(stateNIDs)) i := 0 for ; rows.Next(); i++ { diff --git a/syncapi/storage/postgres/account_data_table.go b/syncapi/storage/postgres/account_data_table.go index d1811aa66..d1e3b527f 100644 --- a/syncapi/storage/postgres/account_data_table.go +++ b/syncapi/storage/postgres/account_data_table.go @@ -118,7 +118,7 @@ func (s *accountDataStatements) selectAccountDataInRange( if err != nil { return } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectAccountDataInRange: rows.close() failed") for rows.Next() { var dataType string diff --git a/syncapi/storage/postgres/backward_extremities_table.go b/syncapi/storage/postgres/backward_extremities_table.go index d63c546e3..8286ca434 100644 --- a/syncapi/storage/postgres/backward_extremities_table.go +++ b/syncapi/storage/postgres/backward_extremities_table.go @@ -17,6 +17,8 @@ package postgres import ( "context" "database/sql" + + "github.com/matrix-org/dendrite/common" ) const backwardExtremitiesSchema = ` @@ -91,7 +93,7 @@ func (s *backwardExtremitiesStatements) selectBackwardExtremitiesForRoom( if err != nil { return } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectBackwardExtremitiesForRoom: rows.close() failed") for rows.Next() { var eID string diff --git a/syncapi/storage/postgres/current_room_state_table.go b/syncapi/storage/postgres/current_room_state_table.go index 6f5c1e803..48cc2b5fd 100644 --- a/syncapi/storage/postgres/current_room_state_table.go +++ b/syncapi/storage/postgres/current_room_state_table.go @@ -140,7 +140,7 @@ func (s *currentRoomStateStatements) selectJoinedUsers( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectJoinedUsers: rows.close() failed") result := make(map[string][]string) for rows.Next() { @@ -168,7 +168,7 @@ func (s *currentRoomStateStatements) selectRoomIDsWithMembership( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectRoomIDsWithMembership: rows.close() failed") var result []string for rows.Next() { @@ -198,7 +198,7 @@ func (s *currentRoomStateStatements) selectCurrentState( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectCurrentState: rows.close() failed") return rowsToEvents(rows) } @@ -248,7 +248,7 @@ func (s *currentRoomStateStatements) selectEventsWithEventIDs( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectEventsWithEventIDs: rows.close() failed") return rowsToStreamEvents(rows) } diff --git a/syncapi/storage/postgres/invites_table.go b/syncapi/storage/postgres/invites_table.go index 2cb8fb199..6a7f9ef28 100644 --- a/syncapi/storage/postgres/invites_table.go +++ b/syncapi/storage/postgres/invites_table.go @@ -115,7 +115,7 @@ func (s *inviteEventsStatements) selectInviteEventsInRange( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectInviteEventsInRange: rows.close() failed") result := map[string]gomatrixserverlib.Event{} for rows.Next() { var ( diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go index 2db46c5db..e7cf19354 100644 --- a/syncapi/storage/postgres/output_room_events_table.go +++ b/syncapi/storage/postgres/output_room_events_table.go @@ -169,7 +169,7 @@ func (s *outputRoomEventsStatements) selectStateInRange( if err != nil { return nil, nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectStateInRange: rows.close() failed") // Fetch all the state change events for all rooms between the two positions then loop each event and: // - Keep a cache of the event by ID (99% of state change events are for the event itself) // - For each room ID, build up an array of event IDs which represents cumulative adds/removes @@ -303,7 +303,7 @@ func (s *outputRoomEventsStatements) selectRecentEvents( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectRecentEvents: rows.close() failed") events, err := rowsToStreamEvents(rows) if err != nil { return nil, err @@ -330,7 +330,7 @@ func (s *outputRoomEventsStatements) selectEarlyEvents( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectEarlyEvents: rows.close() failed") events, err := rowsToStreamEvents(rows) if err != nil { return nil, err @@ -354,7 +354,7 @@ func (s *outputRoomEventsStatements) selectEvents( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectEvents: rows.close() failed") return rowsToStreamEvents(rows) } diff --git a/syncapi/storage/postgres/output_room_events_topology_table.go b/syncapi/storage/postgres/output_room_events_topology_table.go index 78a381da9..b114fd43d 100644 --- a/syncapi/storage/postgres/output_room_events_topology_table.go +++ b/syncapi/storage/postgres/output_room_events_topology_table.go @@ -18,6 +18,8 @@ import ( "context" "database/sql" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -134,7 +136,7 @@ func (s *outputRoomEventsTopologyStatements) selectEventIDsInRange( } else if err != nil { return } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectEventIDsInRange: rows.close() failed") // Return the IDs. var eventID string @@ -177,7 +179,7 @@ func (s *outputRoomEventsTopologyStatements) selectEventIDsFromPosition( } else if err != nil { return } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectEventIDsFromPosition: rows.close() failed") // Return the IDs. var eventID string for rows.Next() { diff --git a/syncapi/storage/sqlite3/account_data_table.go b/syncapi/storage/sqlite3/account_data_table.go index 71105d0c3..3dbf961b4 100644 --- a/syncapi/storage/sqlite3/account_data_table.go +++ b/syncapi/storage/sqlite3/account_data_table.go @@ -19,6 +19,8 @@ import ( "context" "database/sql" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -102,7 +104,7 @@ func (s *accountDataStatements) selectAccountDataInRange( if err != nil { return } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectAccountDataInRange: rows.close() failed") var entries int diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go index ed76177be..87007f2b0 100644 --- a/syncapi/storage/sqlite3/current_room_state_table.go +++ b/syncapi/storage/sqlite3/current_room_state_table.go @@ -126,7 +126,7 @@ func (s *currentRoomStateStatements) selectJoinedUsers( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectJoinedUsers: rows.close() failed") result := make(map[string][]string) for rows.Next() { @@ -154,7 +154,7 @@ func (s *currentRoomStateStatements) selectRoomIDsWithMembership( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectRoomIDsWithMembership: rows.close() failed") var result []string for rows.Next() { @@ -184,7 +184,7 @@ func (s *currentRoomStateStatements) selectCurrentState( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectCurrentState: rows.close() failed") return rowsToEvents(rows) } @@ -238,7 +238,7 @@ func (s *currentRoomStateStatements) selectEventsWithEventIDs( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectEventsWithEventIDs: rows.close() failed") return rowsToStreamEvents(rows) } diff --git a/syncapi/storage/sqlite3/invites_table.go b/syncapi/storage/sqlite3/invites_table.go index baf8871bd..df6f1c9a5 100644 --- a/syncapi/storage/sqlite3/invites_table.go +++ b/syncapi/storage/sqlite3/invites_table.go @@ -113,7 +113,7 @@ func (s *inviteEventsStatements) selectInviteEventsInRange( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectInviteEventsInRange: rows.close() failed") result := map[string]gomatrixserverlib.Event{} for rows.Next() { var ( diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index 05a33c067..5ae28ca3f 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -316,7 +316,7 @@ func (s *outputRoomEventsStatements) selectRecentEvents( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectRecentEvents: rows.close() failed") events, err := rowsToStreamEvents(rows) if err != nil { return nil, err @@ -343,7 +343,7 @@ func (s *outputRoomEventsStatements) selectEarlyEvents( if err != nil { return nil, err } - defer rows.Close() // nolint: errcheck + defer common.CloseAndLogIfError(ctx, rows, "selectEarlyEvents: rows.close() failed") events, err := rowsToStreamEvents(rows) if err != nil { return nil, err @@ -372,7 +372,7 @@ func (s *outputRoomEventsStatements) selectEvents( if streamEvents, err := rowsToStreamEvents(rows); err == nil { returnEvents = append(returnEvents, streamEvents...) } - rows.Close() // nolint: errcheck + common.CloseAndLogIfError(ctx, rows, "selectEvents: rows.close() failed") } return returnEvents, nil } From ec38783192dd551718902719c8446c6cc3942dfa Mon Sep 17 00:00:00 2001 From: Abhinav Krishna C K Date: Wed, 18 Mar 2020 18:18:51 +0530 Subject: [PATCH 45/86] maintenance: Fix matrix-org#896 use %w format verb to wrap errors (#916) * maintenance: Fix matrix-org#896 use %w format verb to wrap errors * In Go version 1.13 a new formatting verb introduced for fmt.Errorf %w https://blog.golang.org/go1.13-errors * update %s to %w to wrap errors. * Update all instances of error type to use %w Signed-off-by: Abhinav Krishna C K Co-authored-by: Kegsay --- clientapi/routing/createroom.go | 2 +- cmd/dendritejs/keyfetcher.go | 6 +++--- common/consumers.go | 2 +- mediaapi/fileutils/fileutils.go | 20 ++++++++++---------- syncapi/routing/messages.go | 6 +++--- syncapi/sync/notifier_test.go | 12 ++++++------ 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index 92b1ae887..964200760 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -381,7 +381,7 @@ func buildEvent( eventID := fmt.Sprintf("$%s:%s", util.RandomString(16), cfg.Matrix.ServerName) event, err := builder.Build(eventID, evTime, cfg.Matrix.ServerName, cfg.Matrix.KeyID, cfg.Matrix.PrivateKey) if err != nil { - return nil, fmt.Errorf("cannot build event %s : Builder failed to build. %s", builder.Type, err) + return nil, fmt.Errorf("cannot build event %s : Builder failed to build. %w", builder.Type, err) } return &event, nil } diff --git a/cmd/dendritejs/keyfetcher.go b/cmd/dendritejs/keyfetcher.go index 2b0a54fd9..ee4905d4f 100644 --- a/cmd/dendritejs/keyfetcher.go +++ b/cmd/dendritejs/keyfetcher.go @@ -53,15 +53,15 @@ func (f *libp2pKeyFetcher) FetchKeys( peerIDStr := string(req.ServerName) peerID, err := peer.Decode(peerIDStr) if err != nil { - return nil, fmt.Errorf("Failed to decode peer ID from server name '%s': %s", peerIDStr, err) + return nil, fmt.Errorf("Failed to decode peer ID from server name '%s': %w", peerIDStr, err) } pubKey, err := peerID.ExtractPublicKey() if err != nil { - return nil, fmt.Errorf("Failed to extract public key from peer ID: %s", err) + return nil, fmt.Errorf("Failed to extract public key from peer ID: %w", err) } pubKeyBytes, err := pubKey.Raw() if err != nil { - return nil, fmt.Errorf("Failed to extract raw bytes from public key: %s", err) + return nil, fmt.Errorf("Failed to extract raw bytes from public key: %w", err) } util.GetLogger(ctx).Info("libp2pKeyFetcher.FetchKeys: Using public key %v for server name %s", pubKeyBytes, req.ServerName) diff --git a/common/consumers.go b/common/consumers.go index 2e58a6241..f33993494 100644 --- a/common/consumers.go +++ b/common/consumers.go @@ -112,7 +112,7 @@ func (c *ContinualConsumer) consumePartition(pc sarama.PartitionConsumer) { msgErr := c.ProcessMessage(message) // Advance our position in the stream so that we will start at the right position after a restart. if err := c.PartitionStore.SetPartitionOffset(context.TODO(), c.Topic, message.Partition, message.Offset); err != nil { - panic(fmt.Errorf("the ContinualConsumer failed to SetPartitionOffset: %s", err)) + panic(fmt.Errorf("the ContinualConsumer failed to SetPartitionOffset: %w", err)) } // Shutdown if we were told to do so. if msgErr == ErrShutdown { diff --git a/mediaapi/fileutils/fileutils.go b/mediaapi/fileutils/fileutils.go index 36b2c5b89..1de242a00 100644 --- a/mediaapi/fileutils/fileutils.go +++ b/mediaapi/fileutils/fileutils.go @@ -49,7 +49,7 @@ func GetPathFromBase64Hash(base64Hash types.Base64Hash, absBasePath config.Path) "file", )) if err != nil { - return "", fmt.Errorf("Unable to construct filePath: %q", err) + return "", fmt.Errorf("Unable to construct filePath: %w", err) } // check if the absolute absBasePath is a prefix of the absolute filePath @@ -73,7 +73,7 @@ func MoveFileWithHashCheck(tmpDir types.Path, mediaMetadata *types.MediaMetadata duplicate := false finalPath, err := GetPathFromBase64Hash(mediaMetadata.Base64Hash, absBasePath) if err != nil { - return "", duplicate, fmt.Errorf("failed to get file path from metadata: %q", err) + return "", duplicate, fmt.Errorf("failed to get file path from metadata: %w", err) } var stat os.FileInfo @@ -91,7 +91,7 @@ func MoveFileWithHashCheck(tmpDir types.Path, mediaMetadata *types.MediaMetadata types.Path(finalPath), ) if err != nil { - return "", duplicate, fmt.Errorf("failed to move file to final destination (%v): %q", finalPath, err) + return "", duplicate, fmt.Errorf("failed to move file to final destination (%v): %w", finalPath, err) } return types.Path(finalPath), duplicate, nil } @@ -143,11 +143,11 @@ func moveFile(src types.Path, dst types.Path) error { err := os.MkdirAll(dstDir, 0770) if err != nil { - return fmt.Errorf("Failed to make directory: %q", err) + return fmt.Errorf("Failed to make directory: %w", err) } err = os.Rename(string(src), string(dst)) if err != nil { - return fmt.Errorf("Failed to move directory: %q", err) + return fmt.Errorf("Failed to move directory: %w", err) } return nil } @@ -155,11 +155,11 @@ func moveFile(src types.Path, dst types.Path) error { func createTempFileWriter(absBasePath config.Path) (*bufio.Writer, *os.File, types.Path, error) { tmpDir, err := createTempDir(absBasePath) if err != nil { - return nil, nil, "", fmt.Errorf("Failed to create temp dir: %q", err) + return nil, nil, "", fmt.Errorf("Failed to create temp dir: %w", err) } writer, tmpFile, err := createFileWriter(tmpDir) if err != nil { - return nil, nil, "", fmt.Errorf("Failed to create file writer: %q", err) + return nil, nil, "", fmt.Errorf("Failed to create file writer: %w", err) } return writer, tmpFile, tmpDir, nil } @@ -168,11 +168,11 @@ func createTempFileWriter(absBasePath config.Path) (*bufio.Writer, *os.File, typ func createTempDir(baseDirectory config.Path) (types.Path, error) { baseTmpDir := filepath.Join(string(baseDirectory), "tmp") if err := os.MkdirAll(baseTmpDir, 0770); err != nil { - return "", fmt.Errorf("Failed to create base temp dir: %v", err) + return "", fmt.Errorf("Failed to create base temp dir: %w", err) } tmpDir, err := ioutil.TempDir(baseTmpDir, "") if err != nil { - return "", fmt.Errorf("Failed to create temp dir: %v", err) + return "", fmt.Errorf("Failed to create temp dir: %w", err) } return types.Path(tmpDir), nil } @@ -184,7 +184,7 @@ func createFileWriter(directory types.Path) (*bufio.Writer, *os.File, error) { filePath := filepath.Join(string(directory), "content") file, err := os.Create(filePath) if err != nil { - return nil, nil, fmt.Errorf("Failed to create file: %v", err) + return nil, nil, fmt.Errorf("Failed to create file: %w", err) } return bufio.NewWriter(file), file, nil diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index 83bf75b2e..e0901a780 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -177,7 +177,7 @@ func (r *messagesReq) retrieveEvents() ( r.ctx, r.from, r.to, r.roomID, r.limit, r.backwardOrdering, ) if err != nil { - err = fmt.Errorf("GetEventsInRange: %s", err) + err = fmt.Errorf("GetEventsInRange: %w", err) return } @@ -228,14 +228,14 @@ func (r *messagesReq) retrieveEvents() ( r.ctx, events[0].EventID(), ) if err != nil { - err = fmt.Errorf("EventPositionInTopology: for start event %s: %s", events[0].EventID(), err) + err = fmt.Errorf("EventPositionInTopology: for start event %s: %w", events[0].EventID(), err) return } endPos, err := r.db.EventPositionInTopology( r.ctx, events[len(events)-1].EventID(), ) if err != nil { - err = fmt.Errorf("EventPositionInTopology: for end event %s: %s", events[len(events)-1].EventID(), err) + err = fmt.Errorf("EventPositionInTopology: for end event %s: %w", events[len(events)-1].EventID(), err) return } // Generate pagination tokens to send to the client using the positions diff --git a/syncapi/sync/notifier_test.go b/syncapi/sync/notifier_test.go index 02da0f7e6..350f85594 100644 --- a/syncapi/sync/notifier_test.go +++ b/syncapi/sync/notifier_test.go @@ -135,7 +135,7 @@ func TestNewEventAndJoinedToRoom(t *testing.T) { go func() { pos, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionBefore)) if err != nil { - t.Errorf("TestNewEventAndJoinedToRoom error: %s", err) + t.Errorf("TestNewEventAndJoinedToRoom error: %w", err) } if pos != syncPositionAfter { t.Errorf("TestNewEventAndJoinedToRoom want %v, got %v", syncPositionAfter, pos) @@ -163,7 +163,7 @@ func TestNewInviteEventForUser(t *testing.T) { go func() { pos, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionBefore)) if err != nil { - t.Errorf("TestNewInviteEventForUser error: %s", err) + t.Errorf("TestNewInviteEventForUser error: %w", err) } if pos != syncPositionAfter { t.Errorf("TestNewInviteEventForUser want %v, got %v", syncPositionAfter, pos) @@ -191,7 +191,7 @@ func TestEDUWakeup(t *testing.T) { go func() { pos, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionAfter)) if err != nil { - t.Errorf("TestNewInviteEventForUser error: %s", err) + t.Errorf("TestNewInviteEventForUser error: %w", err) } if pos != syncPositionNewEDU { t.Errorf("TestNewInviteEventForUser want %v, got %v", syncPositionNewEDU, pos) @@ -219,7 +219,7 @@ func TestMultipleRequestWakeup(t *testing.T) { poll := func() { pos, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionBefore)) if err != nil { - t.Errorf("TestMultipleRequestWakeup error: %s", err) + t.Errorf("TestMultipleRequestWakeup error: %w", err) } if pos != syncPositionAfter { t.Errorf("TestMultipleRequestWakeup want %v, got %v", syncPositionAfter, pos) @@ -259,7 +259,7 @@ func TestNewEventAndWasPreviouslyJoinedToRoom(t *testing.T) { go func() { pos, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionBefore)) if err != nil { - t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom error: %s", err) + t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom error: %w", err) } if pos != syncPositionAfter { t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom want %v, got %v", syncPositionAfter, pos) @@ -278,7 +278,7 @@ func TestNewEventAndWasPreviouslyJoinedToRoom(t *testing.T) { go func() { pos, err := waitForEvents(n, newTestSyncRequest(alice, syncPositionAfter)) if err != nil { - t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom error: %s", err) + t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom error: %w", err) } if pos != syncPositionAfter2 { t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom want %v, got %v", syncPositionAfter2, pos) From dc06c69887f7061411868b3a7a0ee8221a7053e1 Mon Sep 17 00:00:00 2001 From: Prateek Sachan <42961174+prateek2211@users.noreply.github.com> Date: Thu, 19 Mar 2020 15:55:36 +0530 Subject: [PATCH 46/86] Implement /joined_rooms (#911) * Implemented /joined_rooms * Removed account endpoint added by mistake * trigger ci --- clientapi/auth/storage/accounts/interface.go | 1 + .../accounts/postgres/membership_table.go | 27 ++++++++++++++++ .../auth/storage/accounts/postgres/storage.go | 10 ++++++ .../accounts/sqlite3/membership_table.go | 26 ++++++++++++++++ .../auth/storage/accounts/sqlite3/storage.go | 10 ++++++ clientapi/routing/memberships.go | 31 +++++++++++++++++-- clientapi/routing/routing.go | 6 ++++ go.mod | 1 + sytest-whitelist | 2 ++ 9 files changed, 112 insertions(+), 2 deletions(-) diff --git a/clientapi/auth/storage/accounts/interface.go b/clientapi/auth/storage/accounts/interface.go index 9f6e3e1ea..a5052b047 100644 --- a/clientapi/auth/storage/accounts/interface.go +++ b/clientapi/auth/storage/accounts/interface.go @@ -33,6 +33,7 @@ type Database interface { CreateGuestAccount(ctx context.Context) (*authtypes.Account, error) UpdateMemberships(ctx context.Context, eventsToAdd []gomatrixserverlib.Event, idsToRemove []string) error GetMembershipInRoomByLocalpart(ctx context.Context, localpart, roomID string) (authtypes.Membership, error) + GetRoomIDsByLocalPart(ctx context.Context, localpart string) ([]string, error) GetMembershipsByLocalpart(ctx context.Context, localpart string) (memberships []authtypes.Membership, err error) SaveAccountData(ctx context.Context, localpart, roomID, dataType, content string) error GetAccountData(ctx context.Context, localpart string) (global []gomatrixserverlib.ClientEvent, rooms map[string][]gomatrixserverlib.ClientEvent, err error) diff --git a/clientapi/auth/storage/accounts/postgres/membership_table.go b/clientapi/auth/storage/accounts/postgres/membership_table.go index 27570b67d..04e9095e9 100644 --- a/clientapi/auth/storage/accounts/postgres/membership_table.go +++ b/clientapi/auth/storage/accounts/postgres/membership_table.go @@ -53,6 +53,9 @@ const selectMembershipsByLocalpartSQL = "" + const selectMembershipInRoomByLocalpartSQL = "" + "SELECT event_id FROM account_memberships WHERE localpart = $1 AND room_id = $2" +const selectRoomIDsByLocalPartSQL = "" + + "SELECT room_id FROM account_memberships WHERE localpart = $1" + const deleteMembershipsByEventIDsSQL = "" + "DELETE FROM account_memberships WHERE event_id = ANY($1)" @@ -61,6 +64,7 @@ type membershipStatements struct { insertMembershipStmt *sql.Stmt selectMembershipInRoomByLocalpartStmt *sql.Stmt selectMembershipsByLocalpartStmt *sql.Stmt + selectRoomIDsByLocalPartStmt *sql.Stmt } func (s *membershipStatements) prepare(db *sql.DB) (err error) { @@ -80,6 +84,9 @@ func (s *membershipStatements) prepare(db *sql.DB) (err error) { if s.selectMembershipsByLocalpartStmt, err = db.Prepare(selectMembershipsByLocalpartSQL); err != nil { return } + if s.selectRoomIDsByLocalPartStmt, err = db.Prepare(selectRoomIDsByLocalPartSQL); err != nil { + return + } return } @@ -131,3 +138,23 @@ func (s *membershipStatements) selectMembershipsByLocalpart( } return memberships, rows.Err() } + +func (s *membershipStatements) selectRoomIDsByLocalPart( + ctx context.Context, localPart string, +) ([]string, error) { + stmt := s.selectRoomIDsByLocalPartStmt + rows, err := stmt.QueryContext(ctx, localPart) + if err != nil { + return nil, err + } + roomIDs := []string{} + defer rows.Close() // nolint: errcheck + for rows.Next() { + var roomID string + if err = rows.Scan(&roomID); err != nil { + return nil, err + } + roomIDs = append(roomIDs, roomID) + } + return roomIDs, rows.Err() +} diff --git a/clientapi/auth/storage/accounts/postgres/storage.go b/clientapi/auth/storage/accounts/postgres/storage.go index 8115dca43..4a0a2060b 100644 --- a/clientapi/auth/storage/accounts/postgres/storage.go +++ b/clientapi/auth/storage/accounts/postgres/storage.go @@ -234,6 +234,16 @@ func (d *Database) GetMembershipInRoomByLocalpart( return d.memberships.selectMembershipInRoomByLocalpart(ctx, localpart, roomID) } +// GetRoomIDsByLocalPart returns an array containing the room ids of all +// the rooms a user matching a given localpart is a member of +// If no membership match the given localpart, returns an empty array +// If there was an issue during the retrieval, returns the SQL error +func (d *Database) GetRoomIDsByLocalPart( + ctx context.Context, localpart string, +) ([]string, error) { + return d.memberships.selectRoomIDsByLocalPart(ctx, localpart) +} + // GetMembershipsByLocalpart returns an array containing the memberships for all // the rooms a user matching a given localpart is a member of // If no membership match the given localpart, returns an empty array diff --git a/clientapi/auth/storage/accounts/sqlite3/membership_table.go b/clientapi/auth/storage/accounts/sqlite3/membership_table.go index b4bff6334..bd9838b6b 100644 --- a/clientapi/auth/storage/accounts/sqlite3/membership_table.go +++ b/clientapi/auth/storage/accounts/sqlite3/membership_table.go @@ -51,6 +51,9 @@ const selectMembershipsByLocalpartSQL = "" + const selectMembershipInRoomByLocalpartSQL = "" + "SELECT event_id FROM account_memberships WHERE localpart = $1 AND room_id = $2" +const selectRoomIDsByLocalPartSQL = "" + + "SELECT room_id FROM account_memberships WHERE localpart = $1" + const deleteMembershipsByEventIDsSQL = "" + "DELETE FROM account_memberships WHERE event_id IN ($1)" @@ -58,6 +61,7 @@ type membershipStatements struct { insertMembershipStmt *sql.Stmt selectMembershipInRoomByLocalpartStmt *sql.Stmt selectMembershipsByLocalpartStmt *sql.Stmt + selectRoomIDsByLocalPartStmt *sql.Stmt } func (s *membershipStatements) prepare(db *sql.DB) (err error) { @@ -74,6 +78,9 @@ func (s *membershipStatements) prepare(db *sql.DB) (err error) { if s.selectMembershipsByLocalpartStmt, err = db.Prepare(selectMembershipsByLocalpartSQL); err != nil { return } + if s.selectRoomIDsByLocalPartStmt, err = db.Prepare(selectRoomIDsByLocalPartSQL); err != nil { + return + } return } @@ -130,3 +137,22 @@ func (s *membershipStatements) selectMembershipsByLocalpart( return } +func (s *membershipStatements) selectRoomIDsByLocalPart( + ctx context.Context, localPart string, +) ([]string, error) { + stmt := s.selectRoomIDsByLocalPartStmt + rows, err := stmt.QueryContext(ctx, localPart) + if err != nil { + return nil, err + } + roomIDs := []string{} + defer rows.Close() // nolint: errcheck + for rows.Next() { + var roomID string + if err = rows.Scan(&roomID); err != nil { + return nil, err + } + roomIDs = append(roomIDs, roomID) + } + return roomIDs, rows.Err() +} diff --git a/clientapi/auth/storage/accounts/sqlite3/storage.go b/clientapi/auth/storage/accounts/sqlite3/storage.go index 9124640c6..bfb7b4ea1 100644 --- a/clientapi/auth/storage/accounts/sqlite3/storage.go +++ b/clientapi/auth/storage/accounts/sqlite3/storage.go @@ -253,6 +253,16 @@ func (d *Database) GetMembershipsByLocalpart( return d.memberships.selectMembershipsByLocalpart(ctx, localpart) } +// GetRoomIDsByLocalPart returns an array containing the room ids of all +// the rooms a user matching a given localpart is a member of +// If no membership match the given localpart, returns an empty array +// If there was an issue during the retrieval, returns the SQL error +func (d *Database) GetRoomIDsByLocalPart( + ctx context.Context, localpart string, +) ([]string, error) { + return d.memberships.selectRoomIDsByLocalPart(ctx, localpart) +} + // newMembership saves a new membership in the database. // If the event isn't a valid m.room.member event with type `join`, does nothing. // If an error occurred, returns the SQL error diff --git a/clientapi/routing/memberships.go b/clientapi/routing/memberships.go index a6899eeb8..0b846e5e3 100644 --- a/clientapi/routing/memberships.go +++ b/clientapi/routing/memberships.go @@ -17,6 +17,8 @@ package routing import ( "net/http" + "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/common/config" @@ -25,10 +27,14 @@ import ( "github.com/matrix-org/util" ) -type response struct { +type getMembershipResponse struct { Chunk []gomatrixserverlib.ClientEvent `json:"chunk"` } +type getJoinedRoomsResponse struct { + JoinedRooms []string `json:"joined_rooms"` +} + // GetMemberships implements GET /rooms/{roomId}/members func GetMemberships( req *http.Request, device *authtypes.Device, roomID string, joinedOnly bool, @@ -55,6 +61,27 @@ func GetMemberships( return util.JSONResponse{ Code: http.StatusOK, - JSON: response{queryRes.JoinEvents}, + JSON: getMembershipResponse{queryRes.JoinEvents}, + } +} + +func GetJoinedRooms( + req *http.Request, + device *authtypes.Device, + accountsDB accounts.Database, +) util.JSONResponse { + localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") + return jsonerror.InternalServerError() + } + joinedRooms, err := accountsDB.GetRoomIDsByLocalPart(req.Context(), localpart) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("accountsDB.GetRoomIDsByLocalPart failed") + return jsonerror.InternalServerError() + } + return util.JSONResponse{ + Code: http.StatusOK, + JSON: getJoinedRoomsResponse{joinedRooms}, } } diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index f0841b796..47b7b2675 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -105,6 +105,12 @@ func Setup( ) }), ).Methods(http.MethodPost, http.MethodOptions) + r0mux.Handle("/joined_rooms", + common.MakeAuthAPI("joined_rooms", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + return GetJoinedRooms(req, device, accountDB) + }), + ).Methods(http.MethodGet, http.MethodOptions) + r0mux.Handle("/rooms/{roomID}/{membership:(?:join|kick|ban|unban|leave|invite)}", common.MakeAuthAPI("membership", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { vars, err := common.URLDecodeMapValues(mux.Vars(req)) diff --git a/go.mod b/go.mod index 82474944f..25debd748 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( github.com/tidwall/pretty v1.0.1 // indirect github.com/uber/jaeger-client-go v2.22.1+incompatible github.com/uber/jaeger-lib v2.2.0+incompatible + go.uber.org/atomic v1.6.0 // indirect golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d gopkg.in/Shopify/sarama.v1 v1.20.1 gopkg.in/h2non/bimg.v1 v1.0.18 diff --git a/sytest-whitelist b/sytest-whitelist index cac718288..03e11b83b 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -218,3 +218,5 @@ Push rules come down in an initial /sync Regular users can add and delete aliases in the default room configuration Regular users can add and delete aliases when m.room.aliases is restricted GET /r0/capabilities is not public +GET /joined_rooms lists newly-created room +/joined_rooms returns only joined rooms From bfbf96eec9152f61cb3e54154f1ed82148d82a8a Mon Sep 17 00:00:00 2001 From: Kegsay Date: Thu, 19 Mar 2020 11:04:08 +0000 Subject: [PATCH 47/86] p2p: Implement published rooms (#923) * Create and glue ExternalPublicRoomsProvider into the public rooms component This is how we will link p2p stuff to dendrite proper. * Use gmsl structs rather than our own * Implement federated public rooms - Make thirdparty endpoint r0 so riot-web loads the public room list * Typo * Missing callsites --- clientapi/routing/routing.go | 2 +- cmd/dendrite-monolith-server/main.go | 2 +- cmd/dendrite-public-rooms-api-server/main.go | 2 +- cmd/dendritejs/main.go | 3 +- cmd/dendritejs/publicrooms.go | 46 ++++ go.mod | 4 +- go.sum | 2 + publicroomsapi/directory/public_rooms.go | 205 ++++++++++++++---- publicroomsapi/publicroomsapi.go | 6 +- publicroomsapi/routing/routing.go | 17 +- publicroomsapi/storage/interface.go | 3 +- .../storage/postgres/public_rooms_table.go | 12 +- publicroomsapi/storage/postgres/storage.go | 3 +- .../storage/sqlite3/public_rooms_table.go | 16 +- publicroomsapi/storage/sqlite3/storage.go | 3 +- publicroomsapi/types/types.go | 18 +- 16 files changed, 263 insertions(+), 81 deletions(-) create mode 100644 cmd/dendritejs/publicrooms.go diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 47b7b2675..22ff12b02 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -396,7 +396,7 @@ func Setup( }), ).Methods(http.MethodGet, http.MethodOptions) - unstableMux.Handle("/thirdparty/protocols", + r0mux.Handle("/thirdparty/protocols", common.MakeExternalAPI("thirdparty_protocols", func(req *http.Request) util.JSONResponse { // TODO: Return the third party protcols return util.JSONResponse{ diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index b3de9adde..27c3054b8 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -69,7 +69,7 @@ func main() { ) federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fedSenderAPI) mediaapi.SetupMediaAPIComponent(base, deviceDB) - publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, query) + publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, query, federation, nil) syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query, federation, cfg) httpHandler := common.WrapHandlerInCORS(base.APIMux) diff --git a/cmd/dendrite-public-rooms-api-server/main.go b/cmd/dendrite-public-rooms-api-server/main.go index f8bd8b06a..6b7eac7d5 100644 --- a/cmd/dendrite-public-rooms-api-server/main.go +++ b/cmd/dendrite-public-rooms-api-server/main.go @@ -28,7 +28,7 @@ func main() { _, _, query := base.CreateHTTPRoomserverAPIs() - publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, query) + publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, query, nil, nil) base.SetupAndServeHTTP(string(base.Cfg.Bind.PublicRoomsAPI), string(base.Cfg.Listen.PublicRoomsAPI)) diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 25492b166..7c8526715 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -119,6 +119,7 @@ func main() { }, KeyDatabase: keyDB, } + p2pPublicRoomProvider := NewLibP2PPublicRoomsProvider(node) alias, input, query := roomserver.SetupRoomServerComponent(base) typingInputAPI := typingserver.SetupTypingServerComponent(base, cache.NewTypingCache()) @@ -134,7 +135,7 @@ func main() { ) federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fedSenderAPI) mediaapi.SetupMediaAPIComponent(base, deviceDB) - publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, query) + publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, query, federation, p2pPublicRoomProvider) syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query, federation, cfg) httpHandler := common.WrapHandlerInCORS(base.APIMux) diff --git a/cmd/dendritejs/publicrooms.go b/cmd/dendritejs/publicrooms.go new file mode 100644 index 000000000..17822e7ad --- /dev/null +++ b/cmd/dendritejs/publicrooms.go @@ -0,0 +1,46 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build wasm + +package main + +import ( + "github.com/matrix-org/go-http-js-libp2p/go_http_js_libp2p" +) + +type libp2pPublicRoomsProvider struct { + node *go_http_js_libp2p.P2pLocalNode + providers []go_http_js_libp2p.PeerInfo +} + +func NewLibP2PPublicRoomsProvider(node *go_http_js_libp2p.P2pLocalNode) *libp2pPublicRoomsProvider { + p := &libp2pPublicRoomsProvider{ + node: node, + } + node.RegisterFoundProviders(p.foundProviders) + return p +} + +func (p *libp2pPublicRoomsProvider) foundProviders(peerInfos []go_http_js_libp2p.PeerInfo) { + p.providers = peerInfos +} + +func (p *libp2pPublicRoomsProvider) Homeservers() []string { + result := make([]string, len(p.providers)) + for i := range p.providers { + result[i] = p.providers[i].Id + } + return result +} diff --git a/go.mod b/go.mod index 25debd748..556a6b8f1 100644 --- a/go.mod +++ b/go.mod @@ -8,15 +8,13 @@ require ( github.com/lib/pq v1.2.0 github.com/libp2p/go-libp2p-core v0.5.0 github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 - github.com/matrix-org/go-http-js-libp2p v0.0.0-20200310180544-7f3fad43b51c + github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 github.com/matrix-org/gomatrixserverlib v0.0.0-20200317140257-ddc7feaaf2fd github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.1 // indirect github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 github.com/opentracing/opentracing-go v1.1.0 github.com/pkg/errors v0.8.1 diff --git a/go.sum b/go.sum index 3c7cf281e..2c51915bb 100644 --- a/go.sum +++ b/go.sum @@ -230,6 +230,8 @@ github.com/matrix-org/go-http-js-libp2p v0.0.0-20200306192008-b9e71eeaa437 h1:zc github.com/matrix-org/go-http-js-libp2p v0.0.0-20200306192008-b9e71eeaa437/go.mod h1:/giSXVd8D6DZGSfTmhQrLEoZZwsfkC14kSqP9MiLqIY= github.com/matrix-org/go-http-js-libp2p v0.0.0-20200310180544-7f3fad43b51c h1:jj/LIZKMO7GK6O0UarpRwse9L3ZyzozpyMtdPA7ddSk= github.com/matrix-org/go-http-js-libp2p v0.0.0-20200310180544-7f3fad43b51c/go.mod h1:qK3LUW7RCLhFM7gC3pabj3EXT9A1DsCK33MHstUhhbk= +github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f h1:5TOte9uk/epk8L+Pbp6qwaV8YsKYXKjyECPHUhJTWQc= +github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f/go.mod h1:qK3LUW7RCLhFM7gC3pabj3EXT9A1DsCK33MHstUhhbk= github.com/matrix-org/go-sqlite3-js v0.0.0-20200226144546-ea6ed5b90074 h1:UWz6vfhmQVshBuE67X1BCsdMhEDtd+uOz8CJ48Fc0F4= github.com/matrix-org/go-sqlite3-js v0.0.0-20200226144546-ea6ed5b90074/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/go-sqlite3-js v0.0.0-20200304163011-cfb4884075db h1:ERuFJq4DI8fakfBZlvXHltHZ0ix3K5YsLG0tQfQn6TI= diff --git a/publicroomsapi/directory/public_rooms.go b/publicroomsapi/directory/public_rooms.go index fd327942e..7bd6740eb 100644 --- a/publicroomsapi/directory/public_rooms.go +++ b/publicroomsapi/directory/public_rooms.go @@ -15,17 +15,21 @@ package directory import ( + "context" "net/http" "strconv" + "sync" + "time" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/publicroomsapi/types" + "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) -type publicRoomReq struct { +type PublicRoomReq struct { Since string `json:"since,omitempty"` Limit int16 `json:"limit,omitempty"` Filter filter `json:"filter,omitempty"` @@ -35,65 +39,182 @@ type filter struct { SearchTerms string `json:"generic_search_term,omitempty"` } -type publicRoomRes struct { - Chunk []types.PublicRoom `json:"chunk"` - NextBatch string `json:"next_batch,omitempty"` - PrevBatch string `json:"prev_batch,omitempty"` - Estimate int64 `json:"total_room_count_estimate,omitempty"` -} - // GetPostPublicRooms implements GET and POST /publicRooms func GetPostPublicRooms( req *http.Request, publicRoomDatabase storage.Database, ) util.JSONResponse { - var limit int16 - var offset int64 - var request publicRoomReq - var response publicRoomRes - + var request PublicRoomReq if fillErr := fillPublicRoomsReq(req, &request); fillErr != nil { return *fillErr } - - limit = request.Limit - offset, err := strconv.ParseInt(request.Since, 10, 64) - // ParseInt returns 0 and an error when trying to parse an empty string - // In that case, we want to assign 0 so we ignore the error - if err != nil && len(request.Since) > 0 { - util.GetLogger(req.Context()).WithError(err).Error("strconv.ParseInt failed") + response, err := publicRooms(req.Context(), request, publicRoomDatabase) + if err != nil { return jsonerror.InternalServerError() } - - if response.Estimate, err = publicRoomDatabase.CountPublicRooms(req.Context()); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("publicRoomDatabase.CountPublicRooms failed") - return jsonerror.InternalServerError() - } - - if offset > 0 { - response.PrevBatch = strconv.Itoa(int(offset) - 1) - } - nextIndex := int(offset) + int(limit) - if response.Estimate > int64(nextIndex) { - response.NextBatch = strconv.Itoa(nextIndex) - } - - if response.Chunk, err = publicRoomDatabase.GetPublicRooms( - req.Context(), offset, limit, request.Filter.SearchTerms, - ); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("publicRoomDatabase.GetPublicRooms failed") - return jsonerror.InternalServerError() - } - return util.JSONResponse{ Code: http.StatusOK, JSON: response, } } +// GetPostPublicRoomsWithExternal is the same as GetPostPublicRooms but also mixes in public rooms from the provider supplied. +func GetPostPublicRoomsWithExternal( + req *http.Request, publicRoomDatabase storage.Database, fedClient *gomatrixserverlib.FederationClient, + extRoomsProvider types.ExternalPublicRoomsProvider, +) util.JSONResponse { + var request PublicRoomReq + if fillErr := fillPublicRoomsReq(req, &request); fillErr != nil { + return *fillErr + } + response, err := publicRooms(req.Context(), request, publicRoomDatabase) + if err != nil { + return jsonerror.InternalServerError() + } + + if request.Since != "" { + // TODO: handle pagination tokens sensibly rather than ignoring them. + // ignore paginated requests since we don't handle them yet over federation. + // Only the initial request will contain federated rooms. + return util.JSONResponse{ + Code: http.StatusOK, + JSON: response, + } + } + + // If we have already hit the limit on the number of rooms, bail. + var limit int + if request.Limit > 0 { + limit = int(request.Limit) - len(response.Chunk) + if limit <= 0 { + return util.JSONResponse{ + Code: http.StatusOK, + JSON: response, + } + } + } + + // downcasting `limit` is safe as we know it isn't bigger than request.Limit which is int16 + fedRooms := bulkFetchPublicRoomsFromServers(req.Context(), fedClient, extRoomsProvider.Homeservers(), int16(limit)) + response.Chunk = append(response.Chunk, fedRooms...) + return util.JSONResponse{ + Code: http.StatusOK, + JSON: response, + } +} + +// bulkFetchPublicRoomsFromServers fetches public rooms from the list of homeservers. +// Returns a list of public rooms up to the limit specified. +func bulkFetchPublicRoomsFromServers( + ctx context.Context, fedClient *gomatrixserverlib.FederationClient, homeservers []string, limit int16, +) (publicRooms []gomatrixserverlib.PublicRoom) { + // follow pipeline semantics, see https://blog.golang.org/pipelines for more info. + // goroutines send rooms to this channel + roomCh := make(chan gomatrixserverlib.PublicRoom, int(limit)) + // signalling channel to tell goroutines to stop sending rooms and quit + done := make(chan bool) + // signalling to say when we can close the room channel + var wg sync.WaitGroup + wg.Add(len(homeservers)) + // concurrently query for public rooms + for _, hs := range homeservers { + go func(homeserverDomain string) { + defer wg.Done() + util.GetLogger(ctx).WithField("hs", homeserverDomain).Info("Querying HS for public rooms") + fres, err := fedClient.GetPublicRooms(ctx, gomatrixserverlib.ServerName(homeserverDomain), int(limit), "", false, "") + if err != nil { + util.GetLogger(ctx).WithError(err).WithField("hs", homeserverDomain).Warn( + "bulkFetchPublicRoomsFromServers: failed to query hs", + ) + return + } + for _, room := range fres.Chunk { + // atomically send a room or stop + select { + case roomCh <- room: + case <-done: + util.GetLogger(ctx).WithError(err).WithField("hs", homeserverDomain).Info("Interrupted whilst sending rooms") + return + } + } + }(hs) + } + + // Close the room channel when the goroutines have quit so we don't leak, but don't let it stop the in-flight request. + // This also allows the request to fail fast if all HSes experience errors as it will cause the room channel to be + // closed. + go func() { + wg.Wait() + util.GetLogger(ctx).Info("Cleaning up resources") + close(roomCh) + }() + + // fan-in results with timeout. We stop when we reach the limit. +FanIn: + for len(publicRooms) < int(limit) || limit == 0 { + // add a room or timeout + select { + case room, ok := <-roomCh: + if !ok { + util.GetLogger(ctx).Info("All homeservers have been queried, returning results.") + break FanIn + } + publicRooms = append(publicRooms, room) + case <-time.After(15 * time.Second): // we've waited long enough, let's tell the client what we got. + util.GetLogger(ctx).Info("Waited 15s for federated public rooms, returning early") + break FanIn + case <-ctx.Done(): // the client hung up on us, let's stop. + util.GetLogger(ctx).Info("Client hung up, returning early") + break FanIn + } + } + // tell goroutines to stop + close(done) + + return publicRooms +} + +func publicRooms(ctx context.Context, request PublicRoomReq, publicRoomDatabase storage.Database) (*gomatrixserverlib.RespPublicRooms, error) { + var response gomatrixserverlib.RespPublicRooms + var limit int16 + var offset int64 + limit = request.Limit + offset, err := strconv.ParseInt(request.Since, 10, 64) + // ParseInt returns 0 and an error when trying to parse an empty string + // In that case, we want to assign 0 so we ignore the error + if err != nil && len(request.Since) > 0 { + util.GetLogger(ctx).WithError(err).Error("strconv.ParseInt failed") + return nil, err + } + + est, err := publicRoomDatabase.CountPublicRooms(ctx) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("publicRoomDatabase.CountPublicRooms failed") + return nil, err + } + response.TotalRoomCountEstimate = int(est) + + if offset > 0 { + response.PrevBatch = strconv.Itoa(int(offset) - 1) + } + nextIndex := int(offset) + int(limit) + if response.TotalRoomCountEstimate > nextIndex { + response.NextBatch = strconv.Itoa(nextIndex) + } + + if response.Chunk, err = publicRoomDatabase.GetPublicRooms( + ctx, offset, limit, request.Filter.SearchTerms, + ); err != nil { + util.GetLogger(ctx).WithError(err).Error("publicRoomDatabase.GetPublicRooms failed") + return nil, err + } + + return &response, nil +} + // fillPublicRoomsReq fills the Limit, Since and Filter attributes of a GET or POST request // on /publicRooms by parsing the incoming HTTP request // Filter is only filled for POST requests -func fillPublicRoomsReq(httpReq *http.Request, request *publicRoomReq) *util.JSONResponse { +func fillPublicRoomsReq(httpReq *http.Request, request *PublicRoomReq) *util.JSONResponse { if httpReq.Method == http.MethodGet { limit, err := strconv.Atoi(httpReq.FormValue("limit")) // Atoi returns 0 and an error when trying to parse an empty string diff --git a/publicroomsapi/publicroomsapi.go b/publicroomsapi/publicroomsapi.go index 1e2a3f9bb..399c0cc57 100644 --- a/publicroomsapi/publicroomsapi.go +++ b/publicroomsapi/publicroomsapi.go @@ -20,7 +20,9 @@ import ( "github.com/matrix-org/dendrite/publicroomsapi/consumers" "github.com/matrix-org/dendrite/publicroomsapi/routing" "github.com/matrix-org/dendrite/publicroomsapi/storage" + "github.com/matrix-org/dendrite/publicroomsapi/types" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" ) @@ -30,6 +32,8 @@ func SetupPublicRoomsAPIComponent( base *basecomponent.BaseDendrite, deviceDB devices.Database, rsQueryAPI roomserverAPI.RoomserverQueryAPI, + fedClient *gomatrixserverlib.FederationClient, + extRoomsProvider types.ExternalPublicRoomsProvider, ) { publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI)) if err != nil { @@ -43,5 +47,5 @@ func SetupPublicRoomsAPIComponent( logrus.WithError(err).Panic("failed to start public rooms server consumer") } - routing.Setup(base.APIMux, deviceDB, publicRoomsDB) + routing.Setup(base.APIMux, deviceDB, publicRoomsDB, fedClient, extRoomsProvider) } diff --git a/publicroomsapi/routing/routing.go b/publicroomsapi/routing/routing.go index 1953e04fc..321b61b89 100644 --- a/publicroomsapi/routing/routing.go +++ b/publicroomsapi/routing/routing.go @@ -24,6 +24,8 @@ import ( "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/publicroomsapi/directory" "github.com/matrix-org/dendrite/publicroomsapi/storage" + "github.com/matrix-org/dendrite/publicroomsapi/types" + "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -34,7 +36,10 @@ const pathPrefixR0 = "/_matrix/client/r0" // Due to Setup being used to call many other functions, a gocyclo nolint is // applied: // nolint: gocyclo -func Setup(apiMux *mux.Router, deviceDB devices.Database, publicRoomsDB storage.Database) { +func Setup( + apiMux *mux.Router, deviceDB devices.Database, publicRoomsDB storage.Database, + fedClient *gomatrixserverlib.FederationClient, extRoomsProvider types.ExternalPublicRoomsProvider, +) { r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter() authData := auth.Data{ @@ -64,7 +69,17 @@ func Setup(apiMux *mux.Router, deviceDB devices.Database, publicRoomsDB storage. ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/publicRooms", common.MakeExternalAPI("public_rooms", func(req *http.Request) util.JSONResponse { + if extRoomsProvider != nil { + return directory.GetPostPublicRoomsWithExternal(req, publicRoomsDB, fedClient, extRoomsProvider) + } return directory.GetPostPublicRooms(req, publicRoomsDB) }), ).Methods(http.MethodGet, http.MethodPost, http.MethodOptions) + + // Federation - TODO: should this live here or in federation API? It's sure easier if it's here so here it is. + apiMux.Handle("/_matrix/federation/v1/publicRooms", + common.MakeExternalAPI("federation_public_rooms", func(req *http.Request) util.JSONResponse { + return directory.GetPostPublicRooms(req, publicRoomsDB) + }), + ).Methods(http.MethodGet) } diff --git a/publicroomsapi/storage/interface.go b/publicroomsapi/storage/interface.go index 3f7e65892..0feca0e20 100644 --- a/publicroomsapi/storage/interface.go +++ b/publicroomsapi/storage/interface.go @@ -18,7 +18,6 @@ import ( "context" "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/publicroomsapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -27,7 +26,7 @@ type Database interface { GetRoomVisibility(ctx context.Context, roomID string) (bool, error) SetRoomVisibility(ctx context.Context, visible bool, roomID string) error CountPublicRooms(ctx context.Context) (int64, error) - GetPublicRooms(ctx context.Context, offset int64, limit int16, filter string) ([]types.PublicRoom, error) + GetPublicRooms(ctx context.Context, offset int64, limit int16, filter string) ([]gomatrixserverlib.PublicRoom, error) UpdateRoomFromEvents(ctx context.Context, eventsToAdd []gomatrixserverlib.Event, eventsToRemove []gomatrixserverlib.Event) error UpdateRoomFromEvent(ctx context.Context, event gomatrixserverlib.Event) error } diff --git a/publicroomsapi/storage/postgres/public_rooms_table.go b/publicroomsapi/storage/postgres/public_rooms_table.go index 7e606e939..7e31afd2a 100644 --- a/publicroomsapi/storage/postgres/public_rooms_table.go +++ b/publicroomsapi/storage/postgres/public_rooms_table.go @@ -22,9 +22,9 @@ import ( "fmt" "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/gomatrixserverlib" "github.com/lib/pq" - "github.com/matrix-org/dendrite/publicroomsapi/types" ) var editableAttributes = []string{ @@ -177,7 +177,7 @@ func (s *publicRoomsStatements) countPublicRooms(ctx context.Context) (nb int64, func (s *publicRoomsStatements) selectPublicRooms( ctx context.Context, offset int64, limit int16, filter string, -) ([]types.PublicRoom, error) { +) ([]gomatrixserverlib.PublicRoom, error) { var rows *sql.Rows var err error @@ -203,17 +203,17 @@ func (s *publicRoomsStatements) selectPublicRooms( } if err != nil { - return []types.PublicRoom{}, nil + return []gomatrixserverlib.PublicRoom{}, nil } defer common.CloseAndLogIfError(ctx, rows, "selectPublicRooms: rows.close() failed") - rooms := []types.PublicRoom{} + rooms := []gomatrixserverlib.PublicRoom{} for rows.Next() { - var r types.PublicRoom + var r gomatrixserverlib.PublicRoom var aliases pq.StringArray err = rows.Scan( - &r.RoomID, &r.NumJoinedMembers, &aliases, &r.CanonicalAlias, + &r.RoomID, &r.JoinedMembersCount, &aliases, &r.CanonicalAlias, &r.Name, &r.Topic, &r.WorldReadable, &r.GuestCanJoin, &r.AvatarURL, ) if err != nil { diff --git a/publicroomsapi/storage/postgres/storage.go b/publicroomsapi/storage/postgres/storage.go index 5365c766f..5a4bc8f9a 100644 --- a/publicroomsapi/storage/postgres/storage.go +++ b/publicroomsapi/storage/postgres/storage.go @@ -21,7 +21,6 @@ import ( "encoding/json" "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/publicroomsapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -85,7 +84,7 @@ func (d *PublicRoomsServerDatabase) CountPublicRooms(ctx context.Context) (int64 // Returns an error if the retrieval failed. func (d *PublicRoomsServerDatabase) GetPublicRooms( ctx context.Context, offset int64, limit int16, filter string, -) ([]types.PublicRoom, error) { +) ([]gomatrixserverlib.PublicRoom, error) { return d.statements.selectPublicRooms(ctx, offset, limit, filter) } diff --git a/publicroomsapi/storage/sqlite3/public_rooms_table.go b/publicroomsapi/storage/sqlite3/public_rooms_table.go index beba0d694..44679837f 100644 --- a/publicroomsapi/storage/sqlite3/public_rooms_table.go +++ b/publicroomsapi/storage/sqlite3/public_rooms_table.go @@ -22,7 +22,8 @@ import ( "errors" "fmt" - "github.com/matrix-org/dendrite/publicroomsapi/types" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/gomatrixserverlib" ) var editableAttributes = []string{ @@ -66,7 +67,7 @@ const selectPublicRoomsWithLimitSQL = "" + "SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" + " FROM publicroomsapi_public_rooms WHERE visibility = true" + " ORDER BY joined_members DESC" + - " LIMIT $2 OFFSET $1" + " LIMIT $1 OFFSET $2" const selectPublicRoomsWithFilterSQL = "" + "SELECT room_id, joined_members, aliases, canonical_alias, name, topic, world_readable, guest_can_join, avatar_url" + @@ -164,7 +165,7 @@ func (s *publicRoomsStatements) countPublicRooms(ctx context.Context) (nb int64, func (s *publicRoomsStatements) selectPublicRooms( ctx context.Context, offset int64, limit int16, filter string, -) ([]types.PublicRoom, error) { +) ([]gomatrixserverlib.PublicRoom, error) { var rows *sql.Rows var err error @@ -190,16 +191,17 @@ func (s *publicRoomsStatements) selectPublicRooms( } if err != nil { - return []types.PublicRoom{}, nil + return []gomatrixserverlib.PublicRoom{}, nil } + defer common.CloseAndLogIfError(ctx, rows, "selectPublicRooms failed to close rows") - rooms := []types.PublicRoom{} + rooms := []gomatrixserverlib.PublicRoom{} for rows.Next() { - var r types.PublicRoom + var r gomatrixserverlib.PublicRoom var aliasesJSON string err = rows.Scan( - &r.RoomID, &r.NumJoinedMembers, &aliasesJSON, &r.CanonicalAlias, + &r.RoomID, &r.JoinedMembersCount, &aliasesJSON, &r.CanonicalAlias, &r.Name, &r.Topic, &r.WorldReadable, &r.GuestCanJoin, &r.AvatarURL, ) if err != nil { diff --git a/publicroomsapi/storage/sqlite3/storage.go b/publicroomsapi/storage/sqlite3/storage.go index f8ba71a89..80c04cab7 100644 --- a/publicroomsapi/storage/sqlite3/storage.go +++ b/publicroomsapi/storage/sqlite3/storage.go @@ -23,7 +23,6 @@ import ( _ "github.com/mattn/go-sqlite3" "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/publicroomsapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -87,7 +86,7 @@ func (d *PublicRoomsServerDatabase) CountPublicRooms(ctx context.Context) (int64 // Returns an error if the retrieval failed. func (d *PublicRoomsServerDatabase) GetPublicRooms( ctx context.Context, offset int64, limit int16, filter string, -) ([]types.PublicRoom, error) { +) ([]gomatrixserverlib.PublicRoom, error) { return d.statements.selectPublicRooms(ctx, offset, limit, filter) } diff --git a/publicroomsapi/types/types.go b/publicroomsapi/types/types.go index c284bcca4..11cb0d204 100644 --- a/publicroomsapi/types/types.go +++ b/publicroomsapi/types/types.go @@ -14,15 +14,11 @@ package types -// PublicRoom represents a local public room -type PublicRoom struct { - RoomID string `json:"room_id"` - Aliases []string `json:"aliases,omitempty"` - CanonicalAlias string `json:"canonical_alias,omitempty"` - Name string `json:"name,omitempty"` - Topic string `json:"topic,omitempty"` - AvatarURL string `json:"avatar_url,omitempty"` - NumJoinedMembers int64 `json:"num_joined_members"` - WorldReadable bool `json:"world_readable"` - GuestCanJoin bool `json:"guest_can_join"` +// ExternalPublicRoomsProvider provides a list of homeservers who should be queried +// periodically for a list of public rooms on their server. +type ExternalPublicRoomsProvider interface { + // The list of homeserver domains to query. These servers will receive a request + // via this API: https://matrix.org/docs/spec/server_server/latest#public-room-directory + // This will be called -on demand- by clients, so cache appropriately! + Homeservers() []string } From ad5849d2224245e1277b8bc7d3e2104ed6061a99 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 19 Mar 2020 12:07:01 +0000 Subject: [PATCH 48/86] HeaderedEvents in sync API (#922) * Use HeaderedEvent in syncapi * Update notifier test * Fix persisting headered event * Clean up unused API function * Fix overshadowed err from linter * Write headered JSON to invites table too * Rename event_json to headered_event_json in syncapi database schemae * Fix invites_table queries * Update QueryRoomVersionCapabilitiesResponse comment * Fix syncapi SQLite --- go.mod | 2 +- go.sum | 6 ++ roomserver/api/query.go | 14 +++-- roomserver/query/query.go | 9 ++- roomserver/storage/postgres/storage.go | 2 +- roomserver/storage/sqlite3/storage.go | 2 +- syncapi/consumers/roomserver.go | 29 +++++----- syncapi/routing/messages.go | 29 ++++++---- syncapi/routing/state.go | 6 +- syncapi/storage/interface.go | 12 ++-- .../postgres/current_room_state_table.go | 40 +++++++------ syncapi/storage/postgres/invites_table.go | 25 ++++++--- .../postgres/output_room_events_table.go | 39 +++++++------ .../output_room_events_topology_table.go | 2 +- syncapi/storage/postgres/syncserver.go | 44 +++++++-------- .../sqlite3/current_room_state_table.go | 40 +++++++------ syncapi/storage/sqlite3/invites_table.go | 25 ++++++--- .../sqlite3/output_room_events_table.go | 36 +++++++----- .../output_room_events_topology_table.go | 2 +- syncapi/storage/sqlite3/syncserver.go | 56 ++++++++++--------- syncapi/sync/notifier.go | 2 +- syncapi/sync/notifier_test.go | 25 ++++++--- syncapi/types/types.go | 2 +- 23 files changed, 260 insertions(+), 189 deletions(-) diff --git a/go.mod b/go.mod index 556a6b8f1..3ceaea528 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200317140257-ddc7feaaf2fd + github.com/matrix-org/gomatrixserverlib v0.0.0-20200318180716-bc4ff56961e2 github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible diff --git a/go.sum b/go.sum index 2c51915bb..0f4542b28 100644 --- a/go.sum +++ b/go.sum @@ -270,6 +270,12 @@ github.com/matrix-org/gomatrixserverlib v0.0.0-20200317114945-9a368ea4620d h1:0G github.com/matrix-org/gomatrixserverlib v0.0.0-20200317114945-9a368ea4620d/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= github.com/matrix-org/gomatrixserverlib v0.0.0-20200317140257-ddc7feaaf2fd h1:n95A8YyiCZ8Nu2beqw4akCaPIRrZr/nesHYDZV8WkXI= github.com/matrix-org/gomatrixserverlib v0.0.0-20200317140257-ddc7feaaf2fd/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200318141718-859b9f256ffd h1:381JBgdNYOQwWelV/LOPqDoUkdSBUCkVZY8Of+n5jAM= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200318141718-859b9f256ffd/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200318145320-bc896516d72a h1:7+e7ManmyiGNqo06sQIRIoUtoV92XWzSbc0o3e4aTGY= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200318145320-bc896516d72a/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200318150006-bc27294f9203 h1:7HkL6bF7/M2cYteNFVtvGW5qjD4wHIiR0HsdCm2Rqao= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200318150006-bc27294f9203/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 h1:W7l5CP4V7wPyPb4tYE11dbmeAOwtFQBTW0rf4OonOS8= diff --git a/roomserver/api/query.go b/roomserver/api/query.go index 24cca03ec..9003a95a3 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -43,6 +43,8 @@ type QueryLatestEventsAndStateResponse struct { // Does the room exist? // If the room doesn't exist this will be false and LatestEvents will be empty. RoomExists bool `json:"room_exists"` + // The room version of the room. + RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"` // The latest events in the room. // These are used to set the prev_events when sending an event. LatestEvents []gomatrixserverlib.EventReference `json:"latest_events"` @@ -74,6 +76,8 @@ type QueryStateAfterEventsResponse struct { // Does the room exist on this roomserver? // If the room doesn't exist this will be false and StateEvents will be empty. RoomExists bool `json:"room_exists"` + // The room version of the room. + RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"` // Do all the previous events exist on this roomserver? // If some of previous events do not exist this will be false and StateEvents will be empty. PrevEventsExist bool `json:"prev_events_exist"` @@ -207,6 +211,8 @@ type QueryStateAndAuthChainResponse struct { // Does the room exist on this roomserver? // If the room doesn't exist this will be false and StateEvents will be empty. RoomExists bool `json:"room_exists"` + // The room version of the room. + RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"` // Do all the previous events exist on this roomserver? // If some of previous events do not exist this will be false and StateEvents will be empty. PrevEventsExist bool `json:"prev_events_exist"` @@ -249,10 +255,10 @@ type QueryServersInRoomAtEventResponse struct { // QueryRoomVersionCapabilities asks for the default room version type QueryRoomVersionCapabilitiesRequest struct{} -// QueryRoomVersionCapabilitiesResponse is a response to QueryServersInRoomAtEventResponse +// QueryRoomVersionCapabilitiesResponse is a response to QueryRoomVersionCapabilitiesRequest type QueryRoomVersionCapabilitiesResponse struct { - DefaultRoomVersion string `json:"default"` - AvailableRoomVersions map[string]string `json:"available"` + DefaultRoomVersion gomatrixserverlib.RoomVersion `json:"default"` + AvailableRoomVersions map[gomatrixserverlib.RoomVersion]string `json:"available"` } // RoomserverQueryAPI is used to query information from the room server. @@ -536,7 +542,7 @@ func (h *httpRoomserverQueryAPI) QueryServersInRoomAtEvent( return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } -// QueryServersInRoomAtEvent implements RoomServerQueryAPI +// QueryRoomVersionCapabilities implements RoomServerQueryAPI func (h *httpRoomserverQueryAPI) QueryRoomVersionCapabilities( ctx context.Context, request *QueryRoomVersionCapabilitiesRequest, diff --git a/roomserver/query/query.go b/roomserver/query/query.go index 3ab12d60a..e0f385883 100644 --- a/roomserver/query/query.go +++ b/roomserver/query/query.go @@ -808,14 +808,13 @@ func (r *RoomserverQueryAPI) QueryRoomVersionCapabilities( request *api.QueryRoomVersionCapabilitiesRequest, response *api.QueryRoomVersionCapabilitiesResponse, ) error { - response.DefaultRoomVersion = string(version.DefaultRoomVersion()) - response.AvailableRoomVersions = make(map[string]string) + response.DefaultRoomVersion = version.DefaultRoomVersion() + response.AvailableRoomVersions = make(map[gomatrixserverlib.RoomVersion]string) for v, desc := range version.SupportedRoomVersions() { - sv := string(v) if desc.Stable { - response.AvailableRoomVersions[sv] = "stable" + response.AvailableRoomVersions[v] = "stable" } else { - response.AvailableRoomVersions[sv] = "unstable" + response.AvailableRoomVersions[v] = "unstable" } } return nil diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go index af6afe5c1..9bb6de9d7 100644 --- a/roomserver/storage/postgres/storage.go +++ b/roomserver/storage/postgres/storage.go @@ -155,7 +155,7 @@ func extractRoomVersionFromCreateEvent(event gomatrixserverlib.Event) ( } // A room version was specified in the event content? if createContent.RoomVersion != nil { - roomVersion = *createContent.RoomVersion + roomVersion = gomatrixserverlib.RoomVersion(*createContent.RoomVersion) } return roomVersion, err } diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index ea926ca66..ae09a88ad 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -184,7 +184,7 @@ func extractRoomVersionFromCreateEvent(event gomatrixserverlib.Event) ( } // A room version was specified in the event content? if createContent.RoomVersion != nil { - roomVersion = *createContent.RoomVersion + roomVersion = gomatrixserverlib.RoomVersion(*createContent.RoomVersion) } return roomVersion, err } diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index 332ac6205..f1e68c262 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -98,10 +98,11 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { func (s *OutputRoomEventConsumer) onNewRoomEvent( ctx context.Context, msg api.OutputNewRoomEvent, ) error { - ev := msg.Event.Event + ev := msg.Event log.WithFields(log.Fields{ - "event_id": ev.EventID(), - "room_id": ev.RoomID(), + "event_id": ev.EventID(), + "room_id": ev.RoomID(), + "room_version": ev.RoomVersion, }).Info("received event from roomserver") addsStateEvents, err := s.lookupStateEvents(msg.AddsStateEventIDs, ev) @@ -153,7 +154,7 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent( func (s *OutputRoomEventConsumer) onNewInviteEvent( ctx context.Context, msg api.OutputNewInviteEvent, ) error { - pduPos, err := s.db.AddInviteEvent(ctx, msg.Event.Event) + pduPos, err := s.db.AddInviteEvent(ctx, msg.Event) if err != nil { // panic rather than continue with an inconsistent database log.WithFields(log.Fields{ @@ -163,7 +164,7 @@ func (s *OutputRoomEventConsumer) onNewInviteEvent( }).Panicf("roomserver output log: write invite failure") return nil } - s.notifier.OnNewEvent(&msg.Event.Event, "", nil, types.PaginationToken{PDUPosition: pduPos}) + s.notifier.OnNewEvent(&msg.Event, "", nil, types.PaginationToken{PDUPosition: pduPos}) return nil } @@ -186,8 +187,8 @@ func (s *OutputRoomEventConsumer) onRetireInviteEvent( // lookupStateEvents looks up the state events that are added by a new event. func (s *OutputRoomEventConsumer) lookupStateEvents( - addsStateEventIDs []string, event gomatrixserverlib.Event, -) ([]gomatrixserverlib.Event, error) { + addsStateEventIDs []string, event gomatrixserverlib.HeaderedEvent, +) ([]gomatrixserverlib.HeaderedEvent, error) { // Fast path if there aren't any new state events. if len(addsStateEventIDs) == 0 { return nil, nil @@ -195,7 +196,7 @@ func (s *OutputRoomEventConsumer) lookupStateEvents( // Fast path if the only state event added is the event itself. if len(addsStateEventIDs) == 1 && addsStateEventIDs[0] == event.EventID() { - return []gomatrixserverlib.Event{event}, nil + return []gomatrixserverlib.HeaderedEvent{event}, nil } // Check if this is re-adding a state events that we previously processed @@ -229,10 +230,7 @@ func (s *OutputRoomEventConsumer) lookupStateEvents( return nil, err } - for _, headeredEvent := range eventResp.Events { - result = append(result, headeredEvent.Event) - } - + result = append(result, eventResp.Events...) missing = missingEventsFrom(result, addsStateEventIDs) if len(missing) != 0 { @@ -244,7 +242,7 @@ func (s *OutputRoomEventConsumer) lookupStateEvents( return result, nil } -func (s *OutputRoomEventConsumer) updateStateEvent(event gomatrixserverlib.Event) (gomatrixserverlib.Event, error) { +func (s *OutputRoomEventConsumer) updateStateEvent(event gomatrixserverlib.HeaderedEvent) (gomatrixserverlib.HeaderedEvent, error) { var stateKey string if event.StateKey() == nil { stateKey = "" @@ -269,10 +267,11 @@ func (s *OutputRoomEventConsumer) updateStateEvent(event gomatrixserverlib.Event PrevSender: prevEvent.Sender(), } - return event.SetUnsigned(prev) + event.Event, err = event.SetUnsigned(prev) + return event, err } -func missingEventsFrom(events []gomatrixserverlib.Event, required []string) []string { +func missingEventsFrom(events []gomatrixserverlib.HeaderedEvent, required []string) []string { have := map[string]bool{} for _, event := range events { have[event.EventID()] = true diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index e0901a780..cd21c3bd8 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -181,7 +181,7 @@ func (r *messagesReq) retrieveEvents() ( return } - var events []gomatrixserverlib.Event + var events []gomatrixserverlib.HeaderedEvent // There can be two reasons for streamEvents to be empty: either we've // reached the oldest event in the room (or the most recent one, depending @@ -217,7 +217,7 @@ func (r *messagesReq) retrieveEvents() ( } // Convert all of the events into client events. - clientEvents = gomatrixserverlib.ToClientEvents(events, gomatrixserverlib.FormatAll) + clientEvents = gomatrixserverlib.HeaderedToClientEvents(events, gomatrixserverlib.FormatAll) // Get the position of the first and the last event in the room's topology. // This position is currently determined by the event's depth, so we could // also use it instead of retrieving from the database. However, if we ever @@ -273,7 +273,7 @@ func (r *messagesReq) retrieveEvents() ( // Returns an error if there was an issue talking with the database or // backfilling. func (r *messagesReq) handleEmptyEventsSlice() ( - events []gomatrixserverlib.Event, err error, + events []gomatrixserverlib.HeaderedEvent, err error, ) { backwardExtremities, err := r.db.BackwardExtremitiesForRoom(r.ctx, r.roomID) @@ -287,7 +287,7 @@ func (r *messagesReq) handleEmptyEventsSlice() ( } else { // If not, it means the slice was empty because we reached the room's // creation, so return an empty slice. - events = []gomatrixserverlib.Event{} + events = []gomatrixserverlib.HeaderedEvent{} } return @@ -299,7 +299,7 @@ func (r *messagesReq) handleEmptyEventsSlice() ( // through backfilling if needed. // Returns an error if there was an issue while backfilling. func (r *messagesReq) handleNonEmptyEventsSlice(streamEvents []types.StreamEvent) ( - events []gomatrixserverlib.Event, err error, + events []gomatrixserverlib.HeaderedEvent, err error, ) { // Check if we have enough events. isSetLargeEnough := true @@ -326,7 +326,7 @@ func (r *messagesReq) handleNonEmptyEventsSlice(streamEvents []types.StreamEvent // Backfill is needed if we've reached a backward extremity and need more // events. It's only needed if the direction is backward. if len(backwardExtremities) > 0 && !isSetLargeEnough && r.backwardOrdering { - var pdus []gomatrixserverlib.Event + var pdus []gomatrixserverlib.HeaderedEvent // Only ask the remote server for enough events to reach the limit. pdus, err = r.backfill(backwardExtremities, r.limit-len(streamEvents)) if err != nil { @@ -400,7 +400,7 @@ func (r *messagesReq) containsBackwardExtremity(events []types.StreamEvent) (boo // event, or if there is no remote homeserver to contact. // Returns an error if there was an issue with retrieving the list of servers in // the room or sending the request. -func (r *messagesReq) backfill(fromEventIDs []string, limit int) ([]gomatrixserverlib.Event, error) { +func (r *messagesReq) backfill(fromEventIDs []string, limit int) ([]gomatrixserverlib.HeaderedEvent, error) { // Query the list of servers in the room when one of the backward extremities // was sent. var serversResponse api.QueryServersInRoomAtEventResponse @@ -428,7 +428,7 @@ func (r *messagesReq) backfill(fromEventIDs []string, limit int) ([]gomatrixserv } } - pdus := make([]gomatrixserverlib.Event, 0) + pdus := make([]gomatrixserverlib.HeaderedEvent, 0) // If the roomserver responded with at least one server that isn't us, // send it a request for backfill. @@ -440,13 +440,20 @@ func (r *messagesReq) backfill(fromEventIDs []string, limit int) ([]gomatrixserv return nil, err } - pdus = txn.PDUs + for _, p := range txn.PDUs { + pdus = append(pdus, p.Headered(gomatrixserverlib.RoomVersionV1)) + } // Store the events in the database, while marking them as unfit to show // up in responses to sync requests. for _, pdu := range pdus { + headered := pdu.Headered(gomatrixserverlib.RoomVersionV1) if _, err = r.db.WriteEvent( - r.ctx, &pdu, []gomatrixserverlib.Event{}, []string{}, []string{}, + r.ctx, + &headered, + []gomatrixserverlib.HeaderedEvent{}, + []string{}, + []string{}, nil, true, ); err != nil { return nil, err @@ -486,7 +493,7 @@ func setToDefault( // timestamp of two Matrix events. // Returns true if the first event happened before the second one, false // otherwise. -func sortEvents(e1 *gomatrixserverlib.Event, e2 *gomatrixserverlib.Event) bool { +func sortEvents(e1 *gomatrixserverlib.HeaderedEvent, e2 *gomatrixserverlib.HeaderedEvent) bool { t := e1.OriginServerTS().Time() return e2.OriginServerTS().Time().After(t) } diff --git a/syncapi/routing/state.go b/syncapi/routing/state.go index 5688086cd..87b6396ac 100644 --- a/syncapi/routing/state.go +++ b/syncapi/routing/state.go @@ -56,7 +56,9 @@ func OnIncomingStateRequest(req *http.Request, db storage.Database, roomID strin // Fill the prev_content and replaces_state keys if necessary for _, event := range stateEvents { stateEvent := stateEventInStateResp{ - ClientEvent: gomatrixserverlib.ToClientEvent(event, gomatrixserverlib.FormatAll), + ClientEvent: gomatrixserverlib.HeaderedToClientEvents( + []gomatrixserverlib.HeaderedEvent{event}, gomatrixserverlib.FormatAll, + )[0], } var prevEventRef types.PrevEventRef if len(event.Unsigned()) > 0 { @@ -113,7 +115,7 @@ func OnIncomingStateTypeRequest(req *http.Request, db storage.Database, roomID s } stateEvent := stateEventInStateResp{ - ClientEvent: gomatrixserverlib.ToClientEvent(*event, gomatrixserverlib.FormatAll), + ClientEvent: gomatrixserverlib.HeaderedToClientEvent(*event, gomatrixserverlib.FormatAll), } return util.JSONResponse{ diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index 0abeac34d..b6dc19696 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -29,16 +29,16 @@ import ( type Database interface { common.PartitionStorer AllJoinedUsersInRooms(ctx context.Context) (map[string][]string, error) - Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.Event, error) - WriteEvent(context.Context, *gomatrixserverlib.Event, []gomatrixserverlib.Event, []string, []string, *api.TransactionID, bool) (types.StreamPosition, error) - GetStateEvent(ctx context.Context, roomID, evType, stateKey string) (*gomatrixserverlib.Event, error) - GetStateEventsForRoom(ctx context.Context, roomID string, stateFilterPart *gomatrixserverlib.StateFilter) (stateEvents []gomatrixserverlib.Event, err error) + Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.HeaderedEvent, error) + WriteEvent(context.Context, *gomatrixserverlib.HeaderedEvent, []gomatrixserverlib.HeaderedEvent, []string, []string, *api.TransactionID, bool) (types.StreamPosition, error) + GetStateEvent(ctx context.Context, roomID, evType, stateKey string) (*gomatrixserverlib.HeaderedEvent, error) + GetStateEventsForRoom(ctx context.Context, roomID string, stateFilterPart *gomatrixserverlib.StateFilter) (stateEvents []gomatrixserverlib.HeaderedEvent, err error) SyncPosition(ctx context.Context) (types.PaginationToken, error) IncrementalSync(ctx context.Context, device authtypes.Device, fromPos, toPos types.PaginationToken, numRecentEventsPerRoom int, wantFullState bool) (*types.Response, error) CompleteSync(ctx context.Context, userID string, numRecentEventsPerRoom int) (*types.Response, error) GetAccountDataInRange(ctx context.Context, userID string, oldPos, newPos types.StreamPosition, accountDataFilterPart *gomatrixserverlib.EventFilter) (map[string][]string, error) UpsertAccountData(ctx context.Context, userID, roomID, dataType string) (types.StreamPosition, error) - AddInviteEvent(ctx context.Context, inviteEvent gomatrixserverlib.Event) (types.StreamPosition, error) + AddInviteEvent(ctx context.Context, inviteEvent gomatrixserverlib.HeaderedEvent) (types.StreamPosition, error) RetireInviteEvent(ctx context.Context, inviteEventID string) error SetTypingTimeoutCallback(fn cache.TimeoutCallbackFn) AddTypingUser(userID, roomID string, expireTime *time.Time) types.StreamPosition @@ -48,6 +48,6 @@ type Database interface { EventsAtTopologicalPosition(ctx context.Context, roomID string, pos types.StreamPosition) ([]types.StreamEvent, error) BackwardExtremitiesForRoom(ctx context.Context, roomID string) (backwardExtremities []string, err error) MaxTopologicalPosition(ctx context.Context, roomID string) (types.StreamPosition, error) - StreamEventsToEvents(device *authtypes.Device, in []types.StreamEvent) []gomatrixserverlib.Event + StreamEventsToEvents(device *authtypes.Device, in []types.StreamEvent) []gomatrixserverlib.HeaderedEvent SyncStreamPosition(ctx context.Context) (types.StreamPosition, error) } diff --git a/syncapi/storage/postgres/current_room_state_table.go b/syncapi/storage/postgres/current_room_state_table.go index 48cc2b5fd..ab8f07b21 100644 --- a/syncapi/storage/postgres/current_room_state_table.go +++ b/syncapi/storage/postgres/current_room_state_table.go @@ -42,7 +42,7 @@ CREATE TABLE IF NOT EXISTS syncapi_current_room_state ( -- The state_key value for this state event e.g '' state_key TEXT NOT NULL, -- The JSON for the event. Stored as TEXT because this should be valid UTF-8. - event_json TEXT NOT NULL, + headered_event_json TEXT NOT NULL, -- The 'content.membership' value if this event is an m.room.member event. For other -- events, this will be NULL. membership TEXT, @@ -59,10 +59,10 @@ CREATE INDEX IF NOT EXISTS syncapi_membership_idx ON syncapi_current_room_state( ` const upsertRoomStateSQL = "" + - "INSERT INTO syncapi_current_room_state (room_id, event_id, type, sender, contains_url, state_key, event_json, membership, added_at)" + + "INSERT INTO syncapi_current_room_state (room_id, event_id, type, sender, contains_url, state_key, headered_event_json, membership, added_at)" + " VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)" + " ON CONFLICT ON CONSTRAINT syncapi_room_state_unique" + - " DO UPDATE SET event_id = $2, sender=$4, contains_url=$5, event_json = $7, membership = $8, added_at = $9" + " DO UPDATE SET event_id = $2, sender=$4, contains_url=$5, headered_event_json = $7, membership = $8, added_at = $9" const deleteRoomStateByEventIDSQL = "" + "DELETE FROM syncapi_current_room_state WHERE event_id = $1" @@ -71,7 +71,7 @@ const selectRoomIDsWithMembershipSQL = "" + "SELECT room_id FROM syncapi_current_room_state WHERE type = 'm.room.member' AND state_key = $1 AND membership = $2" const selectCurrentStateSQL = "" + - "SELECT event_json FROM syncapi_current_room_state WHERE room_id = $1" + + "SELECT headered_event_json FROM syncapi_current_room_state WHERE room_id = $1" + " AND ( $2::text[] IS NULL OR sender = ANY($2) )" + " AND ( $3::text[] IS NULL OR NOT(sender = ANY($3)) )" + " AND ( $4::text[] IS NULL OR type LIKE ANY($4) )" + @@ -83,14 +83,14 @@ const selectJoinedUsersSQL = "" + "SELECT room_id, state_key FROM syncapi_current_room_state WHERE type = 'm.room.member' AND membership = 'join'" const selectStateEventSQL = "" + - "SELECT event_json FROM syncapi_current_room_state WHERE room_id = $1 AND type = $2 AND state_key = $3" + "SELECT headered_event_json FROM syncapi_current_room_state WHERE room_id = $1 AND type = $2 AND state_key = $3" const selectEventsWithEventIDsSQL = "" + // TODO: The session_id and transaction_id blanks are here because otherwise // the rowsToStreamEvents expects there to be exactly five columns. We need to // figure out if these really need to be in the DB, and if so, we need a // better permanent fix for this. - neilalexander, 2 Jan 2020 - "SELECT added_at, event_json, 0 AS session_id, false AS exclude_from_sync, '' AS transaction_id" + + "SELECT added_at, headered_event_json, 0 AS session_id, false AS exclude_from_sync, '' AS transaction_id" + " FROM syncapi_current_room_state WHERE event_id = ANY($1)" type currentRoomStateStatements struct { @@ -185,7 +185,7 @@ func (s *currentRoomStateStatements) selectRoomIDsWithMembership( func (s *currentRoomStateStatements) selectCurrentState( ctx context.Context, txn *sql.Tx, roomID string, stateFilter *gomatrixserverlib.StateFilter, -) ([]gomatrixserverlib.Event, error) { +) ([]gomatrixserverlib.HeaderedEvent, error) { stmt := common.TxStmt(txn, s.selectCurrentStateStmt) rows, err := stmt.QueryContext(ctx, roomID, pq.StringArray(stateFilter.Senders), @@ -213,7 +213,7 @@ func (s *currentRoomStateStatements) deleteRoomStateByEventID( func (s *currentRoomStateStatements) upsertRoomState( ctx context.Context, txn *sql.Tx, - event gomatrixserverlib.Event, membership *string, addedAt types.StreamPosition, + event gomatrixserverlib.HeaderedEvent, membership *string, addedAt types.StreamPosition, ) error { // Parse content as JSON and search for an "url" key containsURL := false @@ -223,9 +223,14 @@ func (s *currentRoomStateStatements) upsertRoomState( _, containsURL = content["url"] } + headeredJSON, err := json.Marshal(event) + if err != nil { + return err + } + // upsert state event stmt := common.TxStmt(txn, s.upsertRoomStateStmt) - _, err := stmt.ExecContext( + _, err = stmt.ExecContext( ctx, event.RoomID(), event.EventID(), @@ -233,7 +238,7 @@ func (s *currentRoomStateStatements) upsertRoomState( event.Sender(), containsURL, *event.StateKey(), - event.JSON(), + headeredJSON, membership, addedAt, ) @@ -252,16 +257,16 @@ func (s *currentRoomStateStatements) selectEventsWithEventIDs( return rowsToStreamEvents(rows) } -func rowsToEvents(rows *sql.Rows) ([]gomatrixserverlib.Event, error) { - result := []gomatrixserverlib.Event{} +func rowsToEvents(rows *sql.Rows) ([]gomatrixserverlib.HeaderedEvent, error) { + result := []gomatrixserverlib.HeaderedEvent{} for rows.Next() { var eventBytes []byte if err := rows.Scan(&eventBytes); err != nil { return nil, err } // TODO: Handle redacted events - ev, err := gomatrixserverlib.NewEventFromTrustedJSON(eventBytes, false) - if err != nil { + var ev gomatrixserverlib.HeaderedEvent + if err := json.Unmarshal(eventBytes, &ev); err != nil { return nil, err } result = append(result, ev) @@ -271,7 +276,7 @@ func rowsToEvents(rows *sql.Rows) ([]gomatrixserverlib.Event, error) { func (s *currentRoomStateStatements) selectStateEvent( ctx context.Context, roomID, evType, stateKey string, -) (*gomatrixserverlib.Event, error) { +) (*gomatrixserverlib.HeaderedEvent, error) { stmt := s.selectStateEventStmt var res []byte err := stmt.QueryRowContext(ctx, roomID, evType, stateKey).Scan(&res) @@ -281,6 +286,9 @@ func (s *currentRoomStateStatements) selectStateEvent( if err != nil { return nil, err } - ev, err := gomatrixserverlib.NewEventFromTrustedJSON(res, false) + var ev gomatrixserverlib.HeaderedEvent + if err = json.Unmarshal(res, &ev); err != nil { + return nil, err + } return &ev, err } diff --git a/syncapi/storage/postgres/invites_table.go b/syncapi/storage/postgres/invites_table.go index 6a7f9ef28..ca0c64fb9 100644 --- a/syncapi/storage/postgres/invites_table.go +++ b/syncapi/storage/postgres/invites_table.go @@ -18,6 +18,7 @@ package postgres import ( "context" "database/sql" + "encoding/json" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/syncapi/types" @@ -30,7 +31,7 @@ CREATE TABLE IF NOT EXISTS syncapi_invite_events ( event_id TEXT NOT NULL, room_id TEXT NOT NULL, target_user_id TEXT NOT NULL, - event_json TEXT NOT NULL + headered_event_json TEXT NOT NULL ); -- For looking up the invites for a given user. @@ -44,14 +45,14 @@ CREATE INDEX IF NOT EXISTS syncapi_invites_event_id_idx const insertInviteEventSQL = "" + "INSERT INTO syncapi_invite_events (" + - " room_id, event_id, target_user_id, event_json" + + " room_id, event_id, target_user_id, headered_event_json" + ") VALUES ($1, $2, $3, $4) RETURNING id" const deleteInviteEventSQL = "" + "DELETE FROM syncapi_invite_events WHERE event_id = $1" const selectInviteEventsInRangeSQL = "" + - "SELECT room_id, event_json FROM syncapi_invite_events" + + "SELECT room_id, headered_event_json FROM syncapi_invite_events" + " WHERE target_user_id = $1 AND id > $2 AND id <= $3" + " ORDER BY id DESC" @@ -86,14 +87,20 @@ func (s *inviteEventsStatements) prepare(db *sql.DB) (err error) { } func (s *inviteEventsStatements) insertInviteEvent( - ctx context.Context, inviteEvent gomatrixserverlib.Event, + ctx context.Context, inviteEvent gomatrixserverlib.HeaderedEvent, ) (streamPos types.StreamPosition, err error) { + var headeredJSON []byte + headeredJSON, err = json.Marshal(inviteEvent) + if err != nil { + return + } + err = s.insertInviteEventStmt.QueryRowContext( ctx, inviteEvent.RoomID(), inviteEvent.EventID(), *inviteEvent.StateKey(), - inviteEvent.JSON(), + headeredJSON, ).Scan(&streamPos) return } @@ -109,14 +116,14 @@ func (s *inviteEventsStatements) deleteInviteEvent( // active invites for the target user ID in the supplied range. func (s *inviteEventsStatements) selectInviteEventsInRange( ctx context.Context, txn *sql.Tx, targetUserID string, startPos, endPos types.StreamPosition, -) (map[string]gomatrixserverlib.Event, error) { +) (map[string]gomatrixserverlib.HeaderedEvent, error) { stmt := common.TxStmt(txn, s.selectInviteEventsInRangeStmt) rows, err := stmt.QueryContext(ctx, targetUserID, startPos, endPos) if err != nil { return nil, err } defer common.CloseAndLogIfError(ctx, rows, "selectInviteEventsInRange: rows.close() failed") - result := map[string]gomatrixserverlib.Event{} + result := map[string]gomatrixserverlib.HeaderedEvent{} for rows.Next() { var ( roomID string @@ -126,8 +133,8 @@ func (s *inviteEventsStatements) selectInviteEventsInRange( return nil, err } - event, err := gomatrixserverlib.NewEventFromTrustedJSON(eventJSON, false) - if err != nil { + var event gomatrixserverlib.HeaderedEvent + if err := json.Unmarshal(eventJSON, &event); err != nil { return nil, err } diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go index e7cf19354..5f9a1d0c6 100644 --- a/syncapi/storage/postgres/output_room_events_table.go +++ b/syncapi/storage/postgres/output_room_events_table.go @@ -44,8 +44,9 @@ CREATE TABLE IF NOT EXISTS syncapi_output_room_events ( event_id TEXT NOT NULL CONSTRAINT syncapi_event_id_idx UNIQUE, -- The 'room_id' key for the event. room_id TEXT NOT NULL, - -- The JSON for the event. Stored as TEXT because this should be valid UTF-8. - event_json TEXT NOT NULL, + -- The headered JSON for the event, containing potentially additional metadata such as + -- the room version. Stored as TEXT because this should be valid UTF-8. + headered_event_json TEXT NOT NULL, -- The event type e.g 'm.room.member'. type TEXT NOT NULL, -- The 'sender' property of the event. @@ -70,26 +71,26 @@ CREATE TABLE IF NOT EXISTS syncapi_output_room_events ( const insertEventSQL = "" + "INSERT INTO syncapi_output_room_events (" + - "room_id, event_id, event_json, type, sender, contains_url, add_state_ids, remove_state_ids, session_id, transaction_id, exclude_from_sync" + + "room_id, event_id, headered_event_json, type, sender, contains_url, add_state_ids, remove_state_ids, session_id, transaction_id, exclude_from_sync" + ") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) " + "ON CONFLICT ON CONSTRAINT syncapi_event_id_idx DO UPDATE SET exclude_from_sync = $11 " + "RETURNING id" const selectEventsSQL = "" + - "SELECT id, event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events WHERE event_id = ANY($1)" + "SELECT id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events WHERE event_id = ANY($1)" const selectRecentEventsSQL = "" + - "SELECT id, event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + + "SELECT id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + " WHERE room_id = $1 AND id > $2 AND id <= $3" + " ORDER BY id DESC LIMIT $4" const selectRecentEventsForSyncSQL = "" + - "SELECT id, event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + + "SELECT id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + " WHERE room_id = $1 AND id > $2 AND id <= $3 AND exclude_from_sync = FALSE" + " ORDER BY id DESC LIMIT $4" const selectEarlyEventsSQL = "" + - "SELECT id, event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + + "SELECT id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + " WHERE room_id = $1 AND id > $2 AND id <= $3" + " ORDER BY id ASC LIMIT $4" @@ -98,7 +99,7 @@ const selectMaxEventIDSQL = "" + // In order for us to apply the state updates correctly, rows need to be ordered in the order they were received (id). const selectStateInRangeSQL = "" + - "SELECT id, event_json, exclude_from_sync, add_state_ids, remove_state_ids" + + "SELECT id, headered_event_json, exclude_from_sync, add_state_ids, remove_state_ids" + " FROM syncapi_output_room_events" + " WHERE (id > $1 AND id <= $2) AND (add_state_ids IS NOT NULL OR remove_state_ids IS NOT NULL)" + " AND ( $3::text[] IS NULL OR sender = ANY($3) )" + @@ -203,8 +204,8 @@ func (s *outputRoomEventsStatements) selectStateInRange( } // TODO: Handle redacted events - ev, err := gomatrixserverlib.NewEventFromTrustedJSON(eventBytes, false) - if err != nil { + var ev gomatrixserverlib.HeaderedEvent + if err := json.Unmarshal(eventBytes, &ev); err != nil { return nil, nil, err } needSet := stateNeeded[ev.RoomID()] @@ -220,7 +221,7 @@ func (s *outputRoomEventsStatements) selectStateInRange( stateNeeded[ev.RoomID()] = needSet eventIDToEvent[ev.EventID()] = types.StreamEvent{ - Event: ev, + HeaderedEvent: ev, StreamPosition: streamPos, ExcludeFromSync: excludeFromSync, } @@ -248,7 +249,7 @@ func (s *outputRoomEventsStatements) selectMaxEventID( // of the inserted event. func (s *outputRoomEventsStatements) insertEvent( ctx context.Context, txn *sql.Tx, - event *gomatrixserverlib.Event, addState, removeState []string, + event *gomatrixserverlib.HeaderedEvent, addState, removeState []string, transactionID *api.TransactionID, excludeFromSync bool, ) (streamPos types.StreamPosition, err error) { var txnID *string @@ -266,12 +267,18 @@ func (s *outputRoomEventsStatements) insertEvent( _, containsURL = content["url"] } + var headeredJSON []byte + headeredJSON, err = json.Marshal(event) + if err != nil { + return + } + stmt := common.TxStmt(txn, s.insertEventStmt) err = stmt.QueryRowContext( ctx, event.RoomID(), event.EventID(), - event.JSON(), + headeredJSON, event.Type(), event.Sender(), containsURL, @@ -373,8 +380,8 @@ func rowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) { return nil, err } // TODO: Handle redacted events - ev, err := gomatrixserverlib.NewEventFromTrustedJSON(eventBytes, false) - if err != nil { + var ev gomatrixserverlib.HeaderedEvent + if err := json.Unmarshal(eventBytes, &ev); err != nil { return nil, err } @@ -386,7 +393,7 @@ func rowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) { } result = append(result, types.StreamEvent{ - Event: ev, + HeaderedEvent: ev, StreamPosition: streamPos, TransactionID: transactionID, ExcludeFromSync: excludeFromSync, diff --git a/syncapi/storage/postgres/output_room_events_topology_table.go b/syncapi/storage/postgres/output_room_events_topology_table.go index b114fd43d..280d4ec39 100644 --- a/syncapi/storage/postgres/output_room_events_topology_table.go +++ b/syncapi/storage/postgres/output_room_events_topology_table.go @@ -104,7 +104,7 @@ func (s *outputRoomEventsTopologyStatements) prepare(db *sql.DB) (err error) { // insertEventInTopology inserts the given event in the room's topology, based // on the event's depth. func (s *outputRoomEventsTopologyStatements) insertEventInTopology( - ctx context.Context, event *gomatrixserverlib.Event, + ctx context.Context, event *gomatrixserverlib.HeaderedEvent, ) (err error) { _, err = s.insertEventInTopologyStmt.ExecContext( ctx, event.EventID(), event.Depth(), event.RoomID(), diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go index abb5b4a4c..ccf1c5656 100644 --- a/syncapi/storage/postgres/syncserver.go +++ b/syncapi/storage/postgres/syncserver.go @@ -37,7 +37,7 @@ import ( type stateDelta struct { roomID string - stateEvents []gomatrixserverlib.Event + stateEvents []gomatrixserverlib.HeaderedEvent membership string // The PDU stream position of the latest membership event for this user, if applicable. // Can be 0 if there is no membership event in this delta. @@ -100,7 +100,7 @@ func (d *SyncServerDatasource) AllJoinedUsersInRooms(ctx context.Context) (map[s // If an event is not found in the database then it will be omitted from the list. // Returns an error if there was a problem talking with the database. // Does not include any transaction IDs in the returned events. -func (d *SyncServerDatasource) Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.Event, error) { +func (d *SyncServerDatasource) Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.HeaderedEvent, error) { streamEvents, err := d.events.selectEvents(ctx, nil, eventIDs) if err != nil { return nil, err @@ -111,7 +111,7 @@ func (d *SyncServerDatasource) Events(ctx context.Context, eventIDs []string) ([ return d.StreamEventsToEvents(nil, streamEvents), nil } -func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, ev *gomatrixserverlib.Event) error { +func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, ev *gomatrixserverlib.HeaderedEvent) error { // If the event is already known as a backward extremity, don't consider // it as such anymore now that we have it. isBackwardExtremity, err := d.backwardExtremities.isBackwardExtremity(ctx, ev.RoomID(), ev.EventID()) @@ -155,8 +155,8 @@ func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, ev // Returns an error if there was a problem inserting this event. func (d *SyncServerDatasource) WriteEvent( ctx context.Context, - ev *gomatrixserverlib.Event, - addStateEvents []gomatrixserverlib.Event, + ev *gomatrixserverlib.HeaderedEvent, + addStateEvents []gomatrixserverlib.HeaderedEvent, addStateEventIDs, removeStateEventIDs []string, transactionID *api.TransactionID, excludeFromSync bool, ) (pduPosition types.StreamPosition, returnErr error) { @@ -192,7 +192,7 @@ func (d *SyncServerDatasource) WriteEvent( func (d *SyncServerDatasource) updateRoomState( ctx context.Context, txn *sql.Tx, removedEventIDs []string, - addedEvents []gomatrixserverlib.Event, + addedEvents []gomatrixserverlib.HeaderedEvent, pduPosition types.StreamPosition, ) error { // remove first, then add, as we do not ever delete state, but do replace state which is a remove followed by an add. @@ -228,7 +228,7 @@ func (d *SyncServerDatasource) updateRoomState( // If there was an issue during the retrieval, returns an error func (d *SyncServerDatasource) GetStateEvent( ctx context.Context, roomID, evType, stateKey string, -) (*gomatrixserverlib.Event, error) { +) (*gomatrixserverlib.HeaderedEvent, error) { return d.roomstate.selectStateEvent(ctx, roomID, evType, stateKey) } @@ -237,7 +237,7 @@ func (d *SyncServerDatasource) GetStateEvent( // Returns an error if there was an issue with the retrieval. func (d *SyncServerDatasource) GetStateEventsForRoom( ctx context.Context, roomID string, stateFilter *gomatrixserverlib.StateFilter, -) (stateEvents []gomatrixserverlib.Event, err error) { +) (stateEvents []gomatrixserverlib.HeaderedEvent, err error) { err = common.WithTransaction(d.db, func(txn *sql.Tx) error { stateEvents, err = d.roomstate.selectCurrentState(ctx, txn, roomID, stateFilter) return err @@ -599,7 +599,7 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( // Build up a /sync response. Add joined rooms. for _, roomID := range joinedRoomIDs { - var stateEvents []gomatrixserverlib.Event + var stateEvents []gomatrixserverlib.HeaderedEvent stateEvents, err = d.roomstate.selectCurrentState(ctx, txn, roomID, &stateFilter) if err != nil { return @@ -633,9 +633,9 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( jr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition( types.PaginationTokenTypeTopology, backwardTopologyPos, 0, ).String() - jr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync) + jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = true - jr.State.Events = gomatrixserverlib.ToClientEvents(stateEvents, gomatrixserverlib.FormatSync) + jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(stateEvents, gomatrixserverlib.FormatSync) res.Rooms.Join[roomID] = *jr } @@ -707,7 +707,7 @@ func (d *SyncServerDatasource) UpsertAccountData( // If the invite was successfully stored this returns the stream ID it was stored at. // Returns an error if there was a problem communicating with the database. func (d *SyncServerDatasource) AddInviteEvent( - ctx context.Context, inviteEvent gomatrixserverlib.Event, + ctx context.Context, inviteEvent gomatrixserverlib.HeaderedEvent, ) (types.StreamPosition, error) { return d.invites.insertInviteEvent(ctx, inviteEvent) } @@ -758,7 +758,7 @@ func (d *SyncServerDatasource) addInvitesToResponse( for roomID, inviteEvent := range invites { ir := types.NewInviteResponse() ir.InviteState.Events = gomatrixserverlib.ToClientEvents( - []gomatrixserverlib.Event{inviteEvent}, gomatrixserverlib.FormatSync, + []gomatrixserverlib.Event{inviteEvent.Event}, gomatrixserverlib.FormatSync, ) // TODO: add the invite state from the invite event. res.Rooms.Invite[roomID] = *ir @@ -821,9 +821,9 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( jr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition( types.PaginationTokenTypeTopology, backwardTopologyPos, 0, ).String() - jr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync) + jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true - jr.State.Events = gomatrixserverlib.ToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) + jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) res.Rooms.Join[delta.roomID] = *jr case gomatrixserverlib.Leave: fallthrough // transitions to leave are the same as ban @@ -834,9 +834,9 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( lr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition( types.PaginationTokenTypeTopology, backwardTopologyPos, 0, ).String() - lr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync) + lr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) lr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true - lr.State.Events = gomatrixserverlib.ToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) + lr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) res.Rooms.Leave[delta.roomID] = *lr } @@ -1074,7 +1074,7 @@ func (d *SyncServerDatasource) currentStateStreamEventsForRoom( } s := make([]types.StreamEvent, len(allState)) for i := 0; i < len(s); i++ { - s[i] = types.StreamEvent{Event: allState[i], StreamPosition: 0} + s[i] = types.StreamEvent{HeaderedEvent: allState[i], StreamPosition: 0} } return s, nil } @@ -1082,10 +1082,10 @@ func (d *SyncServerDatasource) currentStateStreamEventsForRoom( // StreamEventsToEvents converts streamEvent to Event. If device is non-nil and // matches the streamevent.transactionID device then the transaction ID gets // added to the unsigned section of the output event. -func (d *SyncServerDatasource) StreamEventsToEvents(device *authtypes.Device, in []types.StreamEvent) []gomatrixserverlib.Event { - out := make([]gomatrixserverlib.Event, len(in)) +func (d *SyncServerDatasource) StreamEventsToEvents(device *authtypes.Device, in []types.StreamEvent) []gomatrixserverlib.HeaderedEvent { + out := make([]gomatrixserverlib.HeaderedEvent, len(in)) for i := 0; i < len(in); i++ { - out[i] = in[i].Event + out[i] = in[i].HeaderedEvent if device != nil && in[i].TransactionID != nil { if device.UserID == in[i].Sender() && device.SessionID == in[i].TransactionID.SessionID { err := out[i].SetUnsignedField( @@ -1105,7 +1105,7 @@ func (d *SyncServerDatasource) StreamEventsToEvents(device *authtypes.Device, in // There may be some overlap where events in stateEvents are already in recentEvents, so filter // them out so we don't include them twice in the /sync response. They should be in recentEvents // only, so clients get to the correct state once they have rolled forward. -func removeDuplicates(stateEvents, recentEvents []gomatrixserverlib.Event) []gomatrixserverlib.Event { +func removeDuplicates(stateEvents, recentEvents []gomatrixserverlib.HeaderedEvent) []gomatrixserverlib.HeaderedEvent { for _, recentEv := range recentEvents { if recentEv.StateKey() == nil { continue // not a state event diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go index 87007f2b0..9fafdbede 100644 --- a/syncapi/storage/sqlite3/current_room_state_table.go +++ b/syncapi/storage/sqlite3/current_room_state_table.go @@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS syncapi_current_room_state ( sender TEXT NOT NULL, contains_url BOOL NOT NULL DEFAULT false, state_key TEXT NOT NULL, - event_json TEXT NOT NULL, + headered_event_json TEXT NOT NULL, membership TEXT, added_at BIGINT, UNIQUE (room_id, type, state_key) @@ -47,10 +47,10 @@ CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_id_idx ON syncapi_current_room_s ` const upsertRoomStateSQL = "" + - "INSERT INTO syncapi_current_room_state (room_id, event_id, type, sender, contains_url, state_key, event_json, membership, added_at)" + + "INSERT INTO syncapi_current_room_state (room_id, event_id, type, sender, contains_url, state_key, headered_event_json, membership, added_at)" + " VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)" + " ON CONFLICT (event_id, room_id, type, sender, contains_url)" + - " DO UPDATE SET event_id = $2, sender=$4, contains_url=$5, event_json = $7, membership = $8, added_at = $9" + " DO UPDATE SET event_id = $2, sender=$4, contains_url=$5, headered_event_json = $7, membership = $8, added_at = $9" const deleteRoomStateByEventIDSQL = "" + "DELETE FROM syncapi_current_room_state WHERE event_id = $1" @@ -59,7 +59,7 @@ const selectRoomIDsWithMembershipSQL = "" + "SELECT room_id FROM syncapi_current_room_state WHERE type = 'm.room.member' AND state_key = $1 AND membership = $2" const selectCurrentStateSQL = "" + - "SELECT event_json FROM syncapi_current_room_state WHERE room_id = $1" + + "SELECT headered_event_json FROM syncapi_current_room_state WHERE room_id = $1" + " AND ( $2 IS NULL OR sender IN ($2) )" + " AND ( $3 IS NULL OR NOT(sender IN ($3)) )" + " AND ( $4 IS NULL OR type IN ($4) )" + @@ -71,14 +71,14 @@ const selectJoinedUsersSQL = "" + "SELECT room_id, state_key FROM syncapi_current_room_state WHERE type = 'm.room.member' AND membership = 'join'" const selectStateEventSQL = "" + - "SELECT event_json FROM syncapi_current_room_state WHERE room_id = $1 AND type = $2 AND state_key = $3" + "SELECT headered_event_json FROM syncapi_current_room_state WHERE room_id = $1 AND type = $2 AND state_key = $3" const selectEventsWithEventIDsSQL = "" + // TODO: The session_id and transaction_id blanks are here because otherwise // the rowsToStreamEvents expects there to be exactly five columns. We need to // figure out if these really need to be in the DB, and if so, we need a // better permanent fix for this. - neilalexander, 2 Jan 2020 - "SELECT added_at, event_json, 0 AS session_id, false AS exclude_from_sync, '' AS transaction_id" + + "SELECT added_at, headered_event_json, 0 AS session_id, false AS exclude_from_sync, '' AS transaction_id" + " FROM syncapi_current_room_state WHERE event_id IN ($1)" type currentRoomStateStatements struct { @@ -171,7 +171,7 @@ func (s *currentRoomStateStatements) selectRoomIDsWithMembership( func (s *currentRoomStateStatements) selectCurrentState( ctx context.Context, txn *sql.Tx, roomID string, stateFilterPart *gomatrixserverlib.StateFilter, -) ([]gomatrixserverlib.Event, error) { +) ([]gomatrixserverlib.HeaderedEvent, error) { stmt := common.TxStmt(txn, s.selectCurrentStateStmt) rows, err := stmt.QueryContext(ctx, roomID, nil, // FIXME: pq.StringArray(stateFilterPart.Senders), @@ -199,7 +199,7 @@ func (s *currentRoomStateStatements) deleteRoomStateByEventID( func (s *currentRoomStateStatements) upsertRoomState( ctx context.Context, txn *sql.Tx, - event gomatrixserverlib.Event, membership *string, addedAt types.StreamPosition, + event gomatrixserverlib.HeaderedEvent, membership *string, addedAt types.StreamPosition, ) error { // Parse content as JSON and search for an "url" key containsURL := false @@ -209,9 +209,14 @@ func (s *currentRoomStateStatements) upsertRoomState( _, containsURL = content["url"] } + headeredJSON, err := json.Marshal(event) + if err != nil { + return err + } + // upsert state event stmt := common.TxStmt(txn, s.upsertRoomStateStmt) - _, err := stmt.ExecContext( + _, err = stmt.ExecContext( ctx, event.RoomID(), event.EventID(), @@ -219,7 +224,7 @@ func (s *currentRoomStateStatements) upsertRoomState( event.Sender(), containsURL, *event.StateKey(), - event.JSON(), + headeredJSON, membership, addedAt, ) @@ -242,16 +247,16 @@ func (s *currentRoomStateStatements) selectEventsWithEventIDs( return rowsToStreamEvents(rows) } -func rowsToEvents(rows *sql.Rows) ([]gomatrixserverlib.Event, error) { - result := []gomatrixserverlib.Event{} +func rowsToEvents(rows *sql.Rows) ([]gomatrixserverlib.HeaderedEvent, error) { + result := []gomatrixserverlib.HeaderedEvent{} for rows.Next() { var eventBytes []byte if err := rows.Scan(&eventBytes); err != nil { return nil, err } // TODO: Handle redacted events - ev, err := gomatrixserverlib.NewEventFromTrustedJSON(eventBytes, false) - if err != nil { + var ev gomatrixserverlib.HeaderedEvent + if err := json.Unmarshal(eventBytes, &ev); err != nil { return nil, err } result = append(result, ev) @@ -261,7 +266,7 @@ func rowsToEvents(rows *sql.Rows) ([]gomatrixserverlib.Event, error) { func (s *currentRoomStateStatements) selectStateEvent( ctx context.Context, roomID, evType, stateKey string, -) (*gomatrixserverlib.Event, error) { +) (*gomatrixserverlib.HeaderedEvent, error) { stmt := s.selectStateEventStmt var res []byte err := stmt.QueryRowContext(ctx, roomID, evType, stateKey).Scan(&res) @@ -271,6 +276,9 @@ func (s *currentRoomStateStatements) selectStateEvent( if err != nil { return nil, err } - ev, err := gomatrixserverlib.NewEventFromTrustedJSON(res, false) + var ev gomatrixserverlib.HeaderedEvent + if err = json.Unmarshal(res, &ev); err != nil { + return nil, err + } return &ev, err } diff --git a/syncapi/storage/sqlite3/invites_table.go b/syncapi/storage/sqlite3/invites_table.go index df6f1c9a5..22efeaeb0 100644 --- a/syncapi/storage/sqlite3/invites_table.go +++ b/syncapi/storage/sqlite3/invites_table.go @@ -18,6 +18,7 @@ package sqlite3 import ( "context" "database/sql" + "encoding/json" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/syncapi/types" @@ -30,7 +31,7 @@ CREATE TABLE IF NOT EXISTS syncapi_invite_events ( event_id TEXT NOT NULL, room_id TEXT NOT NULL, target_user_id TEXT NOT NULL, - event_json TEXT NOT NULL + headered_event_json TEXT NOT NULL ); CREATE INDEX IF NOT EXISTS syncapi_invites_target_user_id_idx ON syncapi_invite_events (target_user_id, id); @@ -39,14 +40,14 @@ CREATE INDEX IF NOT EXISTS syncapi_invites_event_id_idx ON syncapi_invite_events const insertInviteEventSQL = "" + "INSERT INTO syncapi_invite_events" + - " (id, room_id, event_id, target_user_id, event_json)" + + " (id, room_id, event_id, target_user_id, headered_event_json)" + " VALUES ($1, $2, $3, $4, $5)" const deleteInviteEventSQL = "" + "DELETE FROM syncapi_invite_events WHERE event_id = $1" const selectInviteEventsInRangeSQL = "" + - "SELECT room_id, event_json FROM syncapi_invite_events" + + "SELECT room_id, headered_event_json FROM syncapi_invite_events" + " WHERE target_user_id = $1 AND id > $2 AND id <= $3" + " ORDER BY id DESC" @@ -83,15 +84,21 @@ func (s *inviteEventsStatements) prepare(db *sql.DB, streamID *streamIDStatement } func (s *inviteEventsStatements) insertInviteEvent( - ctx context.Context, txn *sql.Tx, inviteEvent gomatrixserverlib.Event, streamPos types.StreamPosition, + ctx context.Context, txn *sql.Tx, inviteEvent gomatrixserverlib.HeaderedEvent, streamPos types.StreamPosition, ) (err error) { + var headeredJSON []byte + headeredJSON, err = json.Marshal(inviteEvent) + if err != nil { + return + } + _, err = txn.Stmt(s.insertInviteEventStmt).ExecContext( ctx, streamPos, inviteEvent.RoomID(), inviteEvent.EventID(), *inviteEvent.StateKey(), - inviteEvent.JSON(), + headeredJSON, ) return } @@ -107,14 +114,14 @@ func (s *inviteEventsStatements) deleteInviteEvent( // active invites for the target user ID in the supplied range. func (s *inviteEventsStatements) selectInviteEventsInRange( ctx context.Context, txn *sql.Tx, targetUserID string, startPos, endPos types.StreamPosition, -) (map[string]gomatrixserverlib.Event, error) { +) (map[string]gomatrixserverlib.HeaderedEvent, error) { stmt := common.TxStmt(txn, s.selectInviteEventsInRangeStmt) rows, err := stmt.QueryContext(ctx, targetUserID, startPos, endPos) if err != nil { return nil, err } defer common.CloseAndLogIfError(ctx, rows, "selectInviteEventsInRange: rows.close() failed") - result := map[string]gomatrixserverlib.Event{} + result := map[string]gomatrixserverlib.HeaderedEvent{} for rows.Next() { var ( roomID string @@ -124,8 +131,8 @@ func (s *inviteEventsStatements) selectInviteEventsInRange( return nil, err } - event, err := gomatrixserverlib.NewEventFromTrustedJSON(eventJSON, false) - if err != nil { + var event gomatrixserverlib.HeaderedEvent + if err := json.Unmarshal(eventJSON, &event); err != nil { return nil, err } diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index 5ae28ca3f..08299f64b 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS syncapi_output_room_events ( id INTEGER PRIMARY KEY AUTOINCREMENT, event_id TEXT NOT NULL UNIQUE, room_id TEXT NOT NULL, - event_json TEXT NOT NULL, + headered_event_json TEXT NOT NULL, type TEXT NOT NULL, sender TEXT NOT NULL, contains_url BOOL NOT NULL, @@ -49,25 +49,25 @@ CREATE TABLE IF NOT EXISTS syncapi_output_room_events ( const insertEventSQL = "" + "INSERT INTO syncapi_output_room_events (" + - "id, room_id, event_id, event_json, type, sender, contains_url, add_state_ids, remove_state_ids, session_id, transaction_id, exclude_from_sync" + + "id, room_id, event_id, headered_event_json, type, sender, contains_url, add_state_ids, remove_state_ids, session_id, transaction_id, exclude_from_sync" + ") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) " + "ON CONFLICT (event_id) DO UPDATE SET exclude_from_sync = $13" const selectEventsSQL = "" + - "SELECT id, event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events WHERE event_id = $1" + "SELECT id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events WHERE event_id = $1" const selectRecentEventsSQL = "" + - "SELECT id, event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + + "SELECT id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + " WHERE room_id = $1 AND id > $2 AND id <= $3" + " ORDER BY id DESC LIMIT $4" const selectRecentEventsForSyncSQL = "" + - "SELECT id, event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + + "SELECT id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + " WHERE room_id = $1 AND id > $2 AND id <= $3 AND exclude_from_sync = FALSE" + " ORDER BY id DESC LIMIT $4" const selectEarlyEventsSQL = "" + - "SELECT id, event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + + "SELECT id, headered_event_json, session_id, exclude_from_sync, transaction_id FROM syncapi_output_room_events" + " WHERE room_id = $1 AND id > $2 AND id <= $3" + " ORDER BY id ASC LIMIT $4" @@ -86,7 +86,7 @@ const selectMaxEventIDSQL = "" + $8 = stateFilterPart.Limit, */ const selectStateInRangeSQL = "" + - "SELECT id, event_json, exclude_from_sync, add_state_ids, remove_state_ids" + + "SELECT id, headered_event_json, exclude_from_sync, add_state_ids, remove_state_ids" + " FROM syncapi_output_room_events" + " WHERE (id > $1 AND id <= $2)" + // old/new pos " AND (add_state_ids IS NOT NULL OR remove_state_ids IS NOT NULL)" + @@ -200,8 +200,8 @@ func (s *outputRoomEventsStatements) selectStateInRange( } // TODO: Handle redacted events - ev, err := gomatrixserverlib.NewEventFromTrustedJSON(eventBytes, false) - if err != nil { + var ev gomatrixserverlib.HeaderedEvent + if err := json.Unmarshal(eventBytes, &ev); err != nil { return nil, nil, err } needSet := stateNeeded[ev.RoomID()] @@ -217,7 +217,7 @@ func (s *outputRoomEventsStatements) selectStateInRange( stateNeeded[ev.RoomID()] = needSet eventIDToEvent[ev.EventID()] = types.StreamEvent{ - Event: ev, + HeaderedEvent: ev, StreamPosition: streamPos, ExcludeFromSync: excludeFromSync, } @@ -245,7 +245,7 @@ func (s *outputRoomEventsStatements) selectMaxEventID( // of the inserted event. func (s *outputRoomEventsStatements) insertEvent( ctx context.Context, txn *sql.Tx, - event *gomatrixserverlib.Event, addState, removeState []string, + event *gomatrixserverlib.HeaderedEvent, addState, removeState []string, transactionID *api.TransactionID, excludeFromSync bool, ) (streamPos types.StreamPosition, err error) { var txnID *string @@ -263,6 +263,12 @@ func (s *outputRoomEventsStatements) insertEvent( _, containsURL = content["url"] } + var headeredJSON []byte + headeredJSON, err = json.Marshal(event) + if err != nil { + return + } + streamPos, err = s.streamIDStatements.nextStreamID(ctx, txn) if err != nil { return @@ -283,7 +289,7 @@ func (s *outputRoomEventsStatements) insertEvent( streamPos, event.RoomID(), event.EventID(), - event.JSON(), + headeredJSON, event.Type(), event.Sender(), containsURL, @@ -392,8 +398,8 @@ func rowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) { return nil, err } // TODO: Handle redacted events - ev, err := gomatrixserverlib.NewEventFromTrustedJSON(eventBytes, false) - if err != nil { + var ev gomatrixserverlib.HeaderedEvent + if err := json.Unmarshal(eventBytes, &ev); err != nil { return nil, err } @@ -405,7 +411,7 @@ func rowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) { } result = append(result, types.StreamEvent{ - Event: ev, + HeaderedEvent: ev, StreamPosition: streamPos, TransactionID: transactionID, ExcludeFromSync: excludeFromSync, diff --git a/syncapi/storage/sqlite3/output_room_events_topology_table.go b/syncapi/storage/sqlite3/output_room_events_topology_table.go index f7075bd6f..a2944c2f9 100644 --- a/syncapi/storage/sqlite3/output_room_events_topology_table.go +++ b/syncapi/storage/sqlite3/output_room_events_topology_table.go @@ -101,7 +101,7 @@ func (s *outputRoomEventsTopologyStatements) prepare(db *sql.DB) (err error) { // insertEventInTopology inserts the given event in the room's topology, based // on the event's depth. func (s *outputRoomEventsTopologyStatements) insertEventInTopology( - ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.Event, + ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.HeaderedEvent, ) (err error) { stmt := common.TxStmt(txn, s.insertEventInTopologyStmt) _, err = stmt.ExecContext( diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index 0e84c8c86..221d96728 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -40,7 +40,7 @@ import ( type stateDelta struct { roomID string - stateEvents []gomatrixserverlib.Event + stateEvents []gomatrixserverlib.HeaderedEvent membership string // The PDU stream position of the latest membership event for this user, if applicable. // Can be 0 if there is no membership event in this delta. @@ -126,7 +126,7 @@ func (d *SyncServerDatasource) AllJoinedUsersInRooms(ctx context.Context) (map[s // If an event is not found in the database then it will be omitted from the list. // Returns an error if there was a problem talking with the database. // Does not include any transaction IDs in the returned events. -func (d *SyncServerDatasource) Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.Event, error) { +func (d *SyncServerDatasource) Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.HeaderedEvent, error) { streamEvents, err := d.events.selectEvents(ctx, nil, eventIDs) if err != nil { return nil, err @@ -137,7 +137,7 @@ func (d *SyncServerDatasource) Events(ctx context.Context, eventIDs []string) ([ return d.StreamEventsToEvents(nil, streamEvents), nil } -func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, txn *sql.Tx, ev *gomatrixserverlib.Event) error { +func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, txn *sql.Tx, ev *gomatrixserverlib.HeaderedEvent) error { // If the event is already known as a backward extremity, don't consider // it as such anymore now that we have it. isBackwardExtremity, err := d.backwardExtremities.isBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID()) @@ -181,8 +181,8 @@ func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, tx // Returns an error if there was a problem inserting this event. func (d *SyncServerDatasource) WriteEvent( ctx context.Context, - ev *gomatrixserverlib.Event, - addStateEvents []gomatrixserverlib.Event, + ev *gomatrixserverlib.HeaderedEvent, + addStateEvents []gomatrixserverlib.HeaderedEvent, addStateEventIDs, removeStateEventIDs []string, transactionID *api.TransactionID, excludeFromSync bool, ) (pduPosition types.StreamPosition, returnErr error) { @@ -218,7 +218,7 @@ func (d *SyncServerDatasource) WriteEvent( func (d *SyncServerDatasource) updateRoomState( ctx context.Context, txn *sql.Tx, removedEventIDs []string, - addedEvents []gomatrixserverlib.Event, + addedEvents []gomatrixserverlib.HeaderedEvent, pduPosition types.StreamPosition, ) error { // remove first, then add, as we do not ever delete state, but do replace state which is a remove followed by an add. @@ -254,7 +254,7 @@ func (d *SyncServerDatasource) updateRoomState( // If there was an issue during the retrieval, returns an error func (d *SyncServerDatasource) GetStateEvent( ctx context.Context, roomID, evType, stateKey string, -) (*gomatrixserverlib.Event, error) { +) (*gomatrixserverlib.HeaderedEvent, error) { return d.roomstate.selectStateEvent(ctx, roomID, evType, stateKey) } @@ -263,7 +263,7 @@ func (d *SyncServerDatasource) GetStateEvent( // Returns an error if there was an issue with the retrieval. func (d *SyncServerDatasource) GetStateEventsForRoom( ctx context.Context, roomID string, stateFilterPart *gomatrixserverlib.StateFilter, -) (stateEvents []gomatrixserverlib.Event, err error) { +) (stateEvents []gomatrixserverlib.HeaderedEvent, err error) { err = common.WithTransaction(d.db, func(txn *sql.Tx) error { stateEvents, err = d.roomstate.selectCurrentState(ctx, txn, roomID, stateFilterPart) return err @@ -633,8 +633,7 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( // Build up a /sync response. Add joined rooms. for _, roomID := range joinedRoomIDs { - - var stateEvents []gomatrixserverlib.Event + var stateEvents []gomatrixserverlib.HeaderedEvent stateEvents, err = d.roomstate.selectCurrentState(ctx, txn, roomID, &stateFilterPart) if err != nil { return @@ -665,14 +664,13 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( // We don't include a device here as we don't need to send down // transaction IDs for complete syncs recentEvents := d.StreamEventsToEvents(nil, recentStreamEvents) - stateEvents = removeDuplicates(stateEvents, recentEvents) jr := types.NewJoinResponse() jr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition( types.PaginationTokenTypeTopology, backwardTopologyPos, 0, ).String() - jr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync) + jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = true - jr.State.Events = gomatrixserverlib.ToClientEvents(stateEvents, gomatrixserverlib.FormatSync) + jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(stateEvents, gomatrixserverlib.FormatSync) res.Rooms.Join[roomID] = *jr } @@ -748,7 +746,7 @@ func (d *SyncServerDatasource) UpsertAccountData( // If the invite was successfully stored this returns the stream ID it was stored at. // Returns an error if there was a problem communicating with the database. func (d *SyncServerDatasource) AddInviteEvent( - ctx context.Context, inviteEvent gomatrixserverlib.Event, + ctx context.Context, inviteEvent gomatrixserverlib.HeaderedEvent, ) (streamPos types.StreamPosition, err error) { err = common.WithTransaction(d.db, func(txn *sql.Tx) error { streamPos, err = d.streamID.nextStreamID(ctx, txn) @@ -806,7 +804,7 @@ func (d *SyncServerDatasource) addInvitesToResponse( for roomID, inviteEvent := range invites { ir := types.NewInviteResponse() ir.InviteState.Events = gomatrixserverlib.ToClientEvents( - []gomatrixserverlib.Event{inviteEvent}, gomatrixserverlib.FormatSync, + []gomatrixserverlib.Event{inviteEvent.Event}, gomatrixserverlib.FormatSync, ) // TODO: add the invite state from the invite event. res.Rooms.Invite[roomID] = *ir @@ -858,8 +856,12 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( if err != nil { return err } - recentEvents := d.StreamEventsToEvents(device, recentStreamEvents) - delta.stateEvents = removeDuplicates(delta.stateEvents, recentEvents) // roll back + headeredRecentEvents := d.StreamEventsToEvents(device, recentStreamEvents) + var recentEvents []gomatrixserverlib.Event + for _, event := range d.StreamEventsToEvents(nil, recentStreamEvents) { + recentEvents = append(recentEvents, event.Event) + } + delta.stateEvents = removeDuplicates(delta.stateEvents, headeredRecentEvents) backwardTopologyPos := d.getBackwardTopologyPos(ctx, txn, recentStreamEvents) switch delta.membership { @@ -871,7 +873,7 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( ).String() jr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true - jr.State.Events = gomatrixserverlib.ToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) + jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) res.Rooms.Join[delta.roomID] = *jr case gomatrixserverlib.Leave: fallthrough // transitions to leave are the same as ban @@ -884,7 +886,7 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( ).String() lr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync) lr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true - lr.State.Events = gomatrixserverlib.ToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) + lr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) res.Rooms.Leave[delta.roomID] = *lr } @@ -1013,7 +1015,7 @@ func (d *SyncServerDatasource) getStateDeltas( // dupe join events will result in the entire room state coming down to the client again. This is added in // the 'state' part of the response though, so is transparent modulo bandwidth concerns as it is not added to // the timeline. - if membership := getMembershipFromEvent(&ev.Event, userID); membership != "" { + if membership := getMembershipFromEvent(&ev.HeaderedEvent, userID); membership != "" { if membership == gomatrixserverlib.Join { // send full room state down instead of a delta var s []types.StreamEvent @@ -1094,7 +1096,7 @@ func (d *SyncServerDatasource) getStateDeltasForFullStateSync( for roomID, stateStreamEvents := range state { for _, ev := range stateStreamEvents { - if membership := getMembershipFromEvent(&ev.Event, userID); membership != "" { + if membership := getMembershipFromEvent(&ev.HeaderedEvent, userID); membership != "" { if membership != gomatrixserverlib.Join { // We've already added full state for all joined rooms above. deltas = append(deltas, stateDelta{ membership: membership, @@ -1122,7 +1124,7 @@ func (d *SyncServerDatasource) currentStateStreamEventsForRoom( } s := make([]types.StreamEvent, len(allState)) for i := 0; i < len(s); i++ { - s[i] = types.StreamEvent{Event: allState[i], StreamPosition: 0} + s[i] = types.StreamEvent{HeaderedEvent: allState[i], StreamPosition: 0} } return s, nil } @@ -1130,10 +1132,10 @@ func (d *SyncServerDatasource) currentStateStreamEventsForRoom( // StreamEventsToEvents converts streamEvent to Event. If device is non-nil and // matches the streamevent.transactionID device then the transaction ID gets // added to the unsigned section of the output event. -func (d *SyncServerDatasource) StreamEventsToEvents(device *authtypes.Device, in []types.StreamEvent) []gomatrixserverlib.Event { - out := make([]gomatrixserverlib.Event, len(in)) +func (d *SyncServerDatasource) StreamEventsToEvents(device *authtypes.Device, in []types.StreamEvent) []gomatrixserverlib.HeaderedEvent { + out := make([]gomatrixserverlib.HeaderedEvent, len(in)) for i := 0; i < len(in); i++ { - out[i] = in[i].Event + out[i] = in[i].HeaderedEvent if device != nil && in[i].TransactionID != nil { if device.UserID == in[i].Sender() && device.SessionID == in[i].TransactionID.SessionID { err := out[i].SetUnsignedField( @@ -1153,7 +1155,7 @@ func (d *SyncServerDatasource) StreamEventsToEvents(device *authtypes.Device, in // There may be some overlap where events in stateEvents are already in recentEvents, so filter // them out so we don't include them twice in the /sync response. They should be in recentEvents // only, so clients get to the correct state once they have rolled forward. -func removeDuplicates(stateEvents, recentEvents []gomatrixserverlib.Event) []gomatrixserverlib.Event { +func removeDuplicates(stateEvents, recentEvents []gomatrixserverlib.HeaderedEvent) []gomatrixserverlib.HeaderedEvent { for _, recentEv := range recentEvents { if recentEv.StateKey() == nil { continue // not a state event @@ -1177,7 +1179,7 @@ func removeDuplicates(stateEvents, recentEvents []gomatrixserverlib.Event) []gom // getMembershipFromEvent returns the value of content.membership iff the event is a state event // with type 'm.room.member' and state_key of userID. Otherwise, an empty string is returned. -func getMembershipFromEvent(ev *gomatrixserverlib.Event, userID string) string { +func getMembershipFromEvent(ev *gomatrixserverlib.HeaderedEvent, userID string) string { if ev.Type() == "m.room.member" && ev.StateKeyEquals(userID) { membership, err := ev.Membership() if err != nil { diff --git a/syncapi/sync/notifier.go b/syncapi/sync/notifier.go index aaee49d34..0d8050112 100644 --- a/syncapi/sync/notifier.go +++ b/syncapi/sync/notifier.go @@ -67,7 +67,7 @@ func NewNotifier(pos types.PaginationToken) *Notifier { // Typically a consumer supplies a posUpdate with the latest sync position for the // event type it handles, leaving other fields as 0. func (n *Notifier) OnNewEvent( - ev *gomatrixserverlib.Event, roomID string, userIDs []string, + ev *gomatrixserverlib.HeaderedEvent, roomID string, userIDs []string, posUpdate types.PaginationToken, ) { // update the current position then notify relevant /sync streams. diff --git a/syncapi/sync/notifier_test.go b/syncapi/sync/notifier_test.go index 350f85594..350d757c6 100644 --- a/syncapi/sync/notifier_test.go +++ b/syncapi/sync/notifier_test.go @@ -16,6 +16,7 @@ package sync import ( "context" + "encoding/json" "fmt" "sync" "testing" @@ -29,9 +30,9 @@ import ( ) var ( - randomMessageEvent gomatrixserverlib.Event - aliceInviteBobEvent gomatrixserverlib.Event - bobLeaveEvent gomatrixserverlib.Event + randomMessageEvent gomatrixserverlib.HeaderedEvent + aliceInviteBobEvent gomatrixserverlib.HeaderedEvent + bobLeaveEvent gomatrixserverlib.HeaderedEvent syncPositionVeryOld types.PaginationToken syncPositionBefore types.PaginationToken syncPositionAfter types.PaginationToken @@ -67,7 +68,8 @@ func init() { syncPositionAfter2.PDUPosition = 13 var err error - randomMessageEvent, err = gomatrixserverlib.NewEventFromTrustedJSON([]byte(`{ + err = json.Unmarshal([]byte(`{ + "_room_version": "1", "type": "m.room.message", "content": { "body": "Hello World", @@ -75,13 +77,15 @@ func init() { }, "sender": "@noone:localhost", "room_id": "`+roomID+`", + "origin": "localhost", "origin_server_ts": 12345, "event_id": "$randomMessageEvent:localhost" - }`), false) + }`), &randomMessageEvent) if err != nil { panic(err) } - aliceInviteBobEvent, err = gomatrixserverlib.NewEventFromTrustedJSON([]byte(`{ + err = json.Unmarshal([]byte(`{ + "_room_version": "1", "type": "m.room.member", "state_key": "`+bob+`", "content": { @@ -89,13 +93,15 @@ func init() { }, "sender": "`+alice+`", "room_id": "`+roomID+`", + "origin": "localhost", "origin_server_ts": 12345, "event_id": "$aliceInviteBobEvent:localhost" - }`), false) + }`), &aliceInviteBobEvent) if err != nil { panic(err) } - bobLeaveEvent, err = gomatrixserverlib.NewEventFromTrustedJSON([]byte(`{ + err = json.Unmarshal([]byte(`{ + "_room_version": "1", "type": "m.room.member", "state_key": "`+bob+`", "content": { @@ -103,9 +109,10 @@ func init() { }, "sender": "`+bob+`", "room_id": "`+roomID+`", + "origin": "localhost", "origin_server_ts": 12345, "event_id": "$bobLeaveEvent:localhost" - }`), false) + }`), &bobLeaveEvent) if err != nil { panic(err) } diff --git a/syncapi/types/types.go b/syncapi/types/types.go index c25a38cdf..718906ecd 100644 --- a/syncapi/types/types.go +++ b/syncapi/types/types.go @@ -40,7 +40,7 @@ type StreamPosition int64 // Same as gomatrixserverlib.Event but also has the PDU stream position for this event. type StreamEvent struct { - gomatrixserverlib.Event + gomatrixserverlib.HeaderedEvent StreamPosition StreamPosition TransactionID *api.TransactionID ExcludeFromSync bool From 944d454cb03445e7c173d1b44e4254f425d09b09 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 19 Mar 2020 13:15:35 +0000 Subject: [PATCH 49/86] Fix SQLite problems after sync API changes --- syncapi/storage/sqlite3/syncserver.go | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index 221d96728..a06fc91f5 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -664,6 +664,7 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( // We don't include a device here as we don't need to send down // transaction IDs for complete syncs recentEvents := d.StreamEventsToEvents(nil, recentStreamEvents) + stateEvents = removeDuplicates(stateEvents, recentEvents) jr := types.NewJoinResponse() jr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition( types.PaginationTokenTypeTopology, backwardTopologyPos, 0, @@ -803,8 +804,8 @@ func (d *SyncServerDatasource) addInvitesToResponse( } for roomID, inviteEvent := range invites { ir := types.NewInviteResponse() - ir.InviteState.Events = gomatrixserverlib.ToClientEvents( - []gomatrixserverlib.Event{inviteEvent.Event}, gomatrixserverlib.FormatSync, + ir.InviteState.Events = gomatrixserverlib.HeaderedToClientEvents( + []gomatrixserverlib.HeaderedEvent{inviteEvent}, gomatrixserverlib.FormatSync, ) // TODO: add the invite state from the invite event. res.Rooms.Invite[roomID] = *ir @@ -856,12 +857,8 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( if err != nil { return err } - headeredRecentEvents := d.StreamEventsToEvents(device, recentStreamEvents) - var recentEvents []gomatrixserverlib.Event - for _, event := range d.StreamEventsToEvents(nil, recentStreamEvents) { - recentEvents = append(recentEvents, event.Event) - } - delta.stateEvents = removeDuplicates(delta.stateEvents, headeredRecentEvents) + recentEvents := d.StreamEventsToEvents(device, recentStreamEvents) + delta.stateEvents = removeDuplicates(delta.stateEvents, recentEvents) backwardTopologyPos := d.getBackwardTopologyPos(ctx, txn, recentStreamEvents) switch delta.membership { @@ -871,7 +868,7 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( jr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition( types.PaginationTokenTypeTopology, backwardTopologyPos, 0, ).String() - jr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync) + jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) res.Rooms.Join[delta.roomID] = *jr @@ -884,7 +881,7 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( lr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition( types.PaginationTokenTypeTopology, backwardTopologyPos, 0, ).String() - lr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync) + lr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) lr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true lr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) res.Rooms.Leave[delta.roomID] = *lr From f2030286de4838f26cffbb2fb3f48f850dc7335a Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 19 Mar 2020 18:33:04 +0000 Subject: [PATCH 50/86] Room server changes for room versions (#930) * Rearrange state package a bit, add some code to look up the right state resolution algorithm * Remove shared * Add GetRoomVersionForRoomNID * Try to use room version to get correct state resolution algorithm * Fix room joins over federation * nolint resolveConflictsV2 because all attempts to break it up so far just result in it being awfully less obvious how it works * Rename Prepare to NewStateResolution * Update comments * Re-add missing tests --- roomserver/input/events.go | 7 +- roomserver/input/latest_events.go | 6 +- roomserver/query/query.go | 50 +- roomserver/state/database/database.go | 3 + roomserver/state/shared/shared.go | 1 + roomserver/state/state.go | 1102 +++++++++++++++++++- roomserver/state/{v1 => }/state_test.go | 2 +- roomserver/state/v1/state.go | 932 ----------------- roomserver/storage/postgres/rooms_table.go | 14 + roomserver/storage/postgres/storage.go | 8 + roomserver/storage/sqlite3/rooms_table.go | 14 + roomserver/storage/sqlite3/storage.go | 8 + 12 files changed, 1122 insertions(+), 1025 deletions(-) create mode 100644 roomserver/state/shared/shared.go rename roomserver/state/{v1 => }/state_test.go (99%) delete mode 100644 roomserver/state/v1/state.go diff --git a/roomserver/input/events.go b/roomserver/input/events.go index 034b06c11..ab2bbe1cd 100644 --- a/roomserver/input/events.go +++ b/roomserver/input/events.go @@ -156,11 +156,8 @@ func calculateAndSetState( stateAtEvent *types.StateAtEvent, event gomatrixserverlib.Event, ) error { - // TODO: get the correct room version - roomState, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, db) - if err != nil { - return err - } + var err error + roomState := state.NewStateResolution(db) if input.HasState { // We've been told what the state at the event is so we don't need to calculate it. diff --git a/roomserver/input/latest_events.go b/roomserver/input/latest_events.go index 4d75daae9..525a6f518 100644 --- a/roomserver/input/latest_events.go +++ b/roomserver/input/latest_events.go @@ -178,11 +178,7 @@ func (u *latestEventsUpdater) doUpdateLatestEvents() error { func (u *latestEventsUpdater) latestState() error { var err error - // TODO: get the correct room version - roomState, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, u.db) - if err != nil { - return err - } + roomState := state.NewStateResolution(u.db) latestStateAtEvents := make([]types.StateAtEvent, len(u.latest)) for i := range u.latest { diff --git a/roomserver/query/query.go b/roomserver/query/query.go index e0f385883..2638919ad 100644 --- a/roomserver/query/query.go +++ b/roomserver/query/query.go @@ -106,11 +106,14 @@ func (r *RoomserverQueryAPI) QueryLatestEventsAndState( request *api.QueryLatestEventsAndStateRequest, response *api.QueryLatestEventsAndStateResponse, ) error { - // TODO: get the correct room version - roomState, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, r.DB) + roomVersion, err := r.DB.GetRoomVersionForRoom(ctx, request.RoomID) if err != nil { - return err + response.RoomExists = false + return nil } + + roomState := state.NewStateResolution(r.DB) + response.QueryLatestEventsAndStateRequest = *request roomNID, err := r.DB.RoomNID(ctx, request.RoomID) if err != nil { @@ -121,11 +124,6 @@ func (r *RoomserverQueryAPI) QueryLatestEventsAndState( } response.RoomExists = true - roomVersion, err := r.DB.GetRoomVersionForRoom(ctx, request.RoomID) - if err != nil { - return err - } - var currentStateSnapshotNID types.StateSnapshotNID response.LatestEvents, currentStateSnapshotNID, response.Depth, err = r.DB.LatestEventIDs(ctx, roomNID) @@ -159,11 +157,14 @@ func (r *RoomserverQueryAPI) QueryStateAfterEvents( request *api.QueryStateAfterEventsRequest, response *api.QueryStateAfterEventsResponse, ) error { - // TODO: get the correct room version - roomState, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, r.DB) + roomVersion, err := r.DB.GetRoomVersionForRoom(ctx, request.RoomID) if err != nil { - return err + response.RoomExists = false + return nil } + + roomState := state.NewStateResolution(r.DB) + response.QueryStateAfterEventsRequest = *request roomNID, err := r.DB.RoomNID(ctx, request.RoomID) if err != nil { @@ -174,11 +175,6 @@ func (r *RoomserverQueryAPI) QueryStateAfterEvents( } response.RoomExists = true - roomVersion, err := r.DB.GetRoomVersionForRoom(ctx, request.RoomID) - if err != nil { - return err - } - prevStates, err := r.DB.StateAtEventIDs(ctx, request.PrevEventIDs) if err != nil { switch err.(type) { @@ -192,7 +188,7 @@ func (r *RoomserverQueryAPI) QueryStateAfterEvents( // Look up the currrent state for the requested tuples. stateEntries, err := roomState.LoadStateAfterEventsForStringTuples( - ctx, prevStates, request.StateToFetch, + ctx, roomNID, prevStates, request.StateToFetch, ) if err != nil { return err @@ -358,11 +354,7 @@ func (r *RoomserverQueryAPI) QueryMembershipsForRoom( func (r *RoomserverQueryAPI) getMembershipsBeforeEventNID( ctx context.Context, eventNID types.EventNID, joinedOnly bool, ) ([]types.Event, error) { - // TODO: get the correct room version - roomState, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, r.DB) - if err != nil { - return []types.Event{}, err - } + roomState := state.NewStateResolution(r.DB) events := []types.Event{} // Lookup the event NID eIDs, err := r.DB.EventIDs(ctx, []types.EventNID{eventNID}) @@ -464,12 +456,7 @@ func (r *RoomserverQueryAPI) QueryServerAllowedToSeeEvent( func (r *RoomserverQueryAPI) checkServerAllowedToSeeEvent( ctx context.Context, eventID string, serverName gomatrixserverlib.ServerName, ) (bool, error) { - // TODO: get the correct room version - roomState, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, r.DB) - if err != nil { - return false, err - } - + roomState := state.NewStateResolution(r.DB) stateEntries, err := roomState.LoadStateAtEvent(ctx, eventID) if err != nil { return false, err @@ -689,12 +676,7 @@ func (r *RoomserverQueryAPI) QueryStateAndAuthChain( } func (r *RoomserverQueryAPI) loadStateAtEventIDs(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.Event, error) { - // TODO: get the correct room version - roomState, err := state.GetStateResolutionAlgorithm(state.StateResolutionAlgorithmV1, r.DB) - if err != nil { - return nil, err - } - + roomState := state.NewStateResolution(r.DB) prevStates, err := r.DB.StateAtEventIDs(ctx, eventIDs) if err != nil { switch err.(type) { diff --git a/roomserver/state/database/database.go b/roomserver/state/database/database.go index ede6c5ec3..80f1b14f4 100644 --- a/roomserver/state/database/database.go +++ b/roomserver/state/database/database.go @@ -20,6 +20,7 @@ import ( "context" "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/gomatrixserverlib" ) // A RoomStateDatabase has the storage APIs needed to load state from the database @@ -61,4 +62,6 @@ type RoomStateDatabase interface { Events(ctx context.Context, eventNIDs []types.EventNID) ([]types.Event, error) // Look up snapshot NID for an event ID string SnapshotNIDFromEventID(ctx context.Context, eventID string) (types.StateSnapshotNID, error) + // Look up a room version from the room NID. + GetRoomVersionForRoomNID(ctx context.Context, roomNID types.RoomNID) (gomatrixserverlib.RoomVersion, error) } diff --git a/roomserver/state/shared/shared.go b/roomserver/state/shared/shared.go new file mode 100644 index 000000000..a29b5e403 --- /dev/null +++ b/roomserver/state/shared/shared.go @@ -0,0 +1 @@ +package shared diff --git a/roomserver/state/state.go b/roomserver/state/state.go index 687a120e3..b8e3e18a1 100644 --- a/roomserver/state/state.go +++ b/roomserver/state/state.go @@ -19,63 +19,1069 @@ package state import ( "context" "errors" + "fmt" + "sort" + "time" "github.com/matrix-org/dendrite/roomserver/state/database" - v1 "github.com/matrix-org/dendrite/roomserver/state/v1" + "github.com/matrix-org/util" + "github.com/prometheus/client_golang/prometheus" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" ) -type StateResolutionVersion int +type StateResolution struct { + db database.RoomStateDatabase +} -const ( - StateResolutionAlgorithmV1 StateResolutionVersion = iota + 1 - StateResolutionAlgorithmV2 -) - -func GetStateResolutionAlgorithm( - version StateResolutionVersion, db database.RoomStateDatabase, -) (StateResolutionImpl, error) { - switch version { - case StateResolutionAlgorithmV1: - return v1.Prepare(db), nil - default: - return nil, errors.New("unsupported room version") +func NewStateResolution(db database.RoomStateDatabase) StateResolution { + return StateResolution{ + db: db, } } -type StateResolutionImpl interface { - LoadStateAtSnapshot( - ctx context.Context, stateNID types.StateSnapshotNID, - ) ([]types.StateEntry, error) - LoadStateAtEvent( - ctx context.Context, eventID string, - ) ([]types.StateEntry, error) - LoadCombinedStateAfterEvents( - ctx context.Context, prevStates []types.StateAtEvent, - ) ([]types.StateEntry, error) - DifferenceBetweeenStateSnapshots( - ctx context.Context, oldStateNID, newStateNID types.StateSnapshotNID, - ) (removed, added []types.StateEntry, err error) - LoadStateAtSnapshotForStringTuples( - ctx context.Context, - stateNID types.StateSnapshotNID, - stateKeyTuples []gomatrixserverlib.StateKeyTuple, - ) ([]types.StateEntry, error) - LoadStateAfterEventsForStringTuples( - ctx context.Context, - prevStates []types.StateAtEvent, - stateKeyTuples []gomatrixserverlib.StateKeyTuple, - ) ([]types.StateEntry, error) - CalculateAndStoreStateBeforeEvent( - ctx context.Context, - event gomatrixserverlib.Event, - roomNID types.RoomNID, - ) (types.StateSnapshotNID, error) - CalculateAndStoreStateAfterEvents( - ctx context.Context, - roomNID types.RoomNID, - prevStates []types.StateAtEvent, - ) (types.StateSnapshotNID, error) +// LoadStateAtSnapshot loads the full state of a room at a particular snapshot. +// This is typically the state before an event or the current state of a room. +// Returns a sorted list of state entries or an error if there was a problem talking to the database. +func (v StateResolution) LoadStateAtSnapshot( + ctx context.Context, stateNID types.StateSnapshotNID, +) ([]types.StateEntry, error) { + stateBlockNIDLists, err := v.db.StateBlockNIDs(ctx, []types.StateSnapshotNID{stateNID}) + if err != nil { + return nil, err + } + // We've asked for exactly one snapshot from the db so we should have exactly one entry in the result. + stateBlockNIDList := stateBlockNIDLists[0] + + stateEntryLists, err := v.db.StateEntries(ctx, stateBlockNIDList.StateBlockNIDs) + if err != nil { + return nil, err + } + stateEntriesMap := stateEntryListMap(stateEntryLists) + + // Combine all the state entries for this snapshot. + // The order of state block NIDs in the list tells us the order to combine them in. + var fullState []types.StateEntry + for _, stateBlockNID := range stateBlockNIDList.StateBlockNIDs { + entries, ok := stateEntriesMap.lookup(stateBlockNID) + if !ok { + // This should only get hit if the database is corrupt. + // It should be impossible for an event to reference a NID that doesn't exist + panic(fmt.Errorf("Corrupt DB: Missing state block numeric ID %d", stateBlockNID)) + } + fullState = append(fullState, entries...) + } + + // Stable sort so that the most recent entry for each state key stays + // remains later in the list than the older entries for the same state key. + sort.Stable(stateEntryByStateKeySorter(fullState)) + // Unique returns the last entry and hence the most recent entry for each state key. + fullState = fullState[:util.Unique(stateEntryByStateKeySorter(fullState))] + return fullState, nil +} + +// LoadStateAtEvent loads the full state of a room before a particular event. +func (v StateResolution) LoadStateAtEvent( + ctx context.Context, eventID string, +) ([]types.StateEntry, error) { + snapshotNID, err := v.db.SnapshotNIDFromEventID(ctx, eventID) + if err != nil { + return nil, err + } + + stateEntries, err := v.LoadStateAtSnapshot(ctx, snapshotNID) + if err != nil { + return nil, err + } + + return stateEntries, nil +} + +// LoadCombinedStateAfterEvents loads a snapshot of the state after each of the events +// and combines those snapshots together into a single list. At this point it is +// possible to run into duplicate (type, state key) tuples. +func (v StateResolution) LoadCombinedStateAfterEvents( + ctx context.Context, prevStates []types.StateAtEvent, +) ([]types.StateEntry, error) { + stateNIDs := make([]types.StateSnapshotNID, len(prevStates)) + for i, state := range prevStates { + stateNIDs[i] = state.BeforeStateSnapshotNID + } + // Fetch the state snapshots for the state before the each prev event from the database. + // Deduplicate the IDs before passing them to the database. + // There could be duplicates because the events could be state events where + // the snapshot of the room state before them was the same. + stateBlockNIDLists, err := v.db.StateBlockNIDs(ctx, uniqueStateSnapshotNIDs(stateNIDs)) + if err != nil { + return nil, err + } + + var stateBlockNIDs []types.StateBlockNID + for _, list := range stateBlockNIDLists { + stateBlockNIDs = append(stateBlockNIDs, list.StateBlockNIDs...) + } + // Fetch the state entries that will be combined to create the snapshots. + // Deduplicate the IDs before passing them to the database. + // There could be duplicates because a block of state entries could be reused by + // multiple snapshots. + stateEntryLists, err := v.db.StateEntries(ctx, uniqueStateBlockNIDs(stateBlockNIDs)) + if err != nil { + return nil, err + } + stateBlockNIDsMap := stateBlockNIDListMap(stateBlockNIDLists) + stateEntriesMap := stateEntryListMap(stateEntryLists) + + // Combine the entries from all the snapshots of state after each prev event into a single list. + var combined []types.StateEntry + for _, prevState := range prevStates { + // Grab the list of state data NIDs for this snapshot. + stateBlockNIDs, ok := stateBlockNIDsMap.lookup(prevState.BeforeStateSnapshotNID) + if !ok { + // This should only get hit if the database is corrupt. + // It should be impossible for an event to reference a NID that doesn't exist + panic(fmt.Errorf("Corrupt DB: Missing state snapshot numeric ID %d", prevState.BeforeStateSnapshotNID)) + } + + // Combine all the state entries for this snapshot. + // The order of state block NIDs in the list tells us the order to combine them in. + var fullState []types.StateEntry + for _, stateBlockNID := range stateBlockNIDs { + entries, ok := stateEntriesMap.lookup(stateBlockNID) + if !ok { + // This should only get hit if the database is corrupt. + // It should be impossible for an event to reference a NID that doesn't exist + panic(fmt.Errorf("Corrupt DB: Missing state block numeric ID %d", stateBlockNID)) + } + fullState = append(fullState, entries...) + } + if prevState.IsStateEvent() { + // If the prev event was a state event then add an entry for the event itself + // so that we get the state after the event rather than the state before. + fullState = append(fullState, prevState.StateEntry) + } + + // Stable sort so that the most recent entry for each state key stays + // remains later in the list than the older entries for the same state key. + sort.Stable(stateEntryByStateKeySorter(fullState)) + // Unique returns the last entry and hence the most recent entry for each state key. + fullState = fullState[:util.Unique(stateEntryByStateKeySorter(fullState))] + // Add the full state for this StateSnapshotNID. + combined = append(combined, fullState...) + } + return combined, nil +} + +// DifferenceBetweeenStateSnapshots works out which state entries have been added and removed between two snapshots. +func (v StateResolution) DifferenceBetweeenStateSnapshots( + ctx context.Context, oldStateNID, newStateNID types.StateSnapshotNID, +) (removed, added []types.StateEntry, err error) { + if oldStateNID == newStateNID { + // If the snapshot NIDs are the same then nothing has changed + return nil, nil, nil + } + + var oldEntries []types.StateEntry + var newEntries []types.StateEntry + if oldStateNID != 0 { + oldEntries, err = v.LoadStateAtSnapshot(ctx, oldStateNID) + if err != nil { + return nil, nil, err + } + } + if newStateNID != 0 { + newEntries, err = v.LoadStateAtSnapshot(ctx, newStateNID) + if err != nil { + return nil, nil, err + } + } + + var oldI int + var newI int + for { + switch { + case oldI == len(oldEntries): + // We've reached the end of the old entries. + // The rest of the new list must have been newly added. + added = append(added, newEntries[newI:]...) + return + case newI == len(newEntries): + // We've reached the end of the new entries. + // The rest of the old list must be have been removed. + removed = append(removed, oldEntries[oldI:]...) + return + case oldEntries[oldI] == newEntries[newI]: + // The entry is in both lists so skip over it. + oldI++ + newI++ + case oldEntries[oldI].LessThan(newEntries[newI]): + // The lists are sorted so the old entry being less than the new entry means that it only appears in the old list. + removed = append(removed, oldEntries[oldI]) + oldI++ + default: + // Reaching the default case implies that the new entry is less than the old entry. + // Since the lists are sorted this means that it only appears in the new list. + added = append(added, newEntries[newI]) + newI++ + } + } +} + +// LoadStateAtSnapshotForStringTuples loads the state for a list of event type and state key pairs at a snapshot. +// This is used when we only want to load a subset of the room state at a snapshot. +// If there is no entry for a given event type and state key pair then it will be discarded. +// This is typically the state before an event or the current state of a room. +// Returns a sorted list of state entries or an error if there was a problem talking to the database. +func (v StateResolution) LoadStateAtSnapshotForStringTuples( + ctx context.Context, + stateNID types.StateSnapshotNID, + stateKeyTuples []gomatrixserverlib.StateKeyTuple, +) ([]types.StateEntry, error) { + numericTuples, err := v.stringTuplesToNumericTuples(ctx, stateKeyTuples) + if err != nil { + return nil, err + } + return v.loadStateAtSnapshotForNumericTuples(ctx, stateNID, numericTuples) +} + +// stringTuplesToNumericTuples converts the string state key tuples into numeric IDs +// If there isn't a numeric ID for either the event type or the event state key then the tuple is discarded. +// Returns an error if there was a problem talking to the database. +func (v StateResolution) stringTuplesToNumericTuples( + ctx context.Context, + stringTuples []gomatrixserverlib.StateKeyTuple, +) ([]types.StateKeyTuple, error) { + eventTypes := make([]string, len(stringTuples)) + stateKeys := make([]string, len(stringTuples)) + for i := range stringTuples { + eventTypes[i] = stringTuples[i].EventType + stateKeys[i] = stringTuples[i].StateKey + } + eventTypes = util.UniqueStrings(eventTypes) + eventTypeMap, err := v.db.EventTypeNIDs(ctx, eventTypes) + if err != nil { + return nil, err + } + stateKeys = util.UniqueStrings(stateKeys) + stateKeyMap, err := v.db.EventStateKeyNIDs(ctx, stateKeys) + if err != nil { + return nil, err + } + + var result []types.StateKeyTuple + for _, stringTuple := range stringTuples { + var numericTuple types.StateKeyTuple + var ok1, ok2 bool + numericTuple.EventTypeNID, ok1 = eventTypeMap[stringTuple.EventType] + numericTuple.EventStateKeyNID, ok2 = stateKeyMap[stringTuple.StateKey] + // Discard the tuple if there wasn't a numeric ID for either the event type or the state key. + if ok1 && ok2 { + result = append(result, numericTuple) + } + } + + return result, nil +} + +// loadStateAtSnapshotForNumericTuples loads the state for a list of event type and state key pairs at a snapshot. +// This is used when we only want to load a subset of the room state at a snapshot. +// If there is no entry for a given event type and state key pair then it will be discarded. +// This is typically the state before an event or the current state of a room. +// Returns a sorted list of state entries or an error if there was a problem talking to the database. +func (v StateResolution) loadStateAtSnapshotForNumericTuples( + ctx context.Context, + stateNID types.StateSnapshotNID, + stateKeyTuples []types.StateKeyTuple, +) ([]types.StateEntry, error) { + stateBlockNIDLists, err := v.db.StateBlockNIDs(ctx, []types.StateSnapshotNID{stateNID}) + if err != nil { + return nil, err + } + // We've asked for exactly one snapshot from the db so we should have exactly one entry in the result. + stateBlockNIDList := stateBlockNIDLists[0] + + stateEntryLists, err := v.db.StateEntriesForTuples( + ctx, stateBlockNIDList.StateBlockNIDs, stateKeyTuples, + ) + if err != nil { + return nil, err + } + stateEntriesMap := stateEntryListMap(stateEntryLists) + + // Combine all the state entries for this snapshot. + // The order of state block NIDs in the list tells us the order to combine them in. + var fullState []types.StateEntry + for _, stateBlockNID := range stateBlockNIDList.StateBlockNIDs { + entries, ok := stateEntriesMap.lookup(stateBlockNID) + if !ok { + // If the block is missing from the map it means that none of its entries matched a requested tuple. + // This can happen if the block doesn't contain an update for one of the requested tuples. + // If none of the requested tuples are in the block then it can be safely skipped. + continue + } + fullState = append(fullState, entries...) + } + + // Stable sort so that the most recent entry for each state key stays + // remains later in the list than the older entries for the same state key. + sort.Stable(stateEntryByStateKeySorter(fullState)) + // Unique returns the last entry and hence the most recent entry for each state key. + fullState = fullState[:util.Unique(stateEntryByStateKeySorter(fullState))] + return fullState, nil +} + +// LoadStateAfterEventsForStringTuples loads the state for a list of event type +// and state key pairs after list of events. +// This is used when we only want to load a subset of the room state after a list of events. +// If there is no entry for a given event type and state key pair then it will be discarded. +// This is typically the state before an event. +// Returns a sorted list of state entries or an error if there was a problem talking to the database. +func (v StateResolution) LoadStateAfterEventsForStringTuples( + ctx context.Context, roomNID types.RoomNID, + prevStates []types.StateAtEvent, + stateKeyTuples []gomatrixserverlib.StateKeyTuple, +) ([]types.StateEntry, error) { + numericTuples, err := v.stringTuplesToNumericTuples(ctx, stateKeyTuples) + if err != nil { + return nil, err + } + return v.loadStateAfterEventsForNumericTuples(ctx, roomNID, prevStates, numericTuples) +} + +func (v StateResolution) loadStateAfterEventsForNumericTuples( + ctx context.Context, roomNID types.RoomNID, + prevStates []types.StateAtEvent, + stateKeyTuples []types.StateKeyTuple, +) ([]types.StateEntry, error) { + roomVersion, err := v.db.GetRoomVersionForRoomNID(ctx, roomNID) + if err != nil { + return nil, err + } + + if len(prevStates) == 1 { + // Fast path for a single event. + prevState := prevStates[0] + var result []types.StateEntry + result, err = v.loadStateAtSnapshotForNumericTuples( + ctx, prevState.BeforeStateSnapshotNID, stateKeyTuples, + ) + if err != nil { + return nil, err + } + if prevState.IsStateEvent() { + // The result is current the state before the requested event. + // We want the state after the requested event. + // If the requested event was a state event then we need to + // update that key in the result. + // If the requested event wasn't a state event then the state after + // it is the same as the state before it. + set := false + for i := range result { + if result[i].StateKeyTuple == prevState.StateKeyTuple { + result[i] = prevState.StateEntry + set = true + } + } + if !set { // no previous state exists for this event: add new state + result = append(result, prevState.StateEntry) + } + } + return result, nil + } + + // Slow path for more that one event. + // Load the entire state so that we can do conflict resolution if we need to. + // TODO: The are some optimistations we could do here: + // 1) We only need to do conflict resolution if there is a conflict in the + // requested tuples so we might try loading just those tuples and then + // checking for conflicts. + // 2) When there is a conflict we still only need to load the state + // needed to do conflict resolution which would save us having to load + // the full state. + + // TODO: Add metrics for this as it could take a long time for big rooms + // with large conflicts. + fullState, _, _, err := v.calculateStateAfterManyEvents(ctx, roomVersion, prevStates) + if err != nil { + return nil, err + } + + // Sort the full state so we can use it as a map. + sort.Sort(stateEntrySorter(fullState)) + + // Filter the full state down to the required tuples. + var result []types.StateEntry + for _, tuple := range stateKeyTuples { + eventNID, ok := stateEntryMap(fullState).lookup(tuple) + if ok { + result = append(result, types.StateEntry{ + StateKeyTuple: tuple, + EventNID: eventNID, + }) + } + } + sort.Sort(stateEntrySorter(result)) + return result, nil +} + +var calculateStateDurations = prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Namespace: "dendrite", + Subsystem: "roomserver", + Name: "calculate_state_duration_microseconds", + Help: "How long it takes to calculate the state after a list of events", + }, + // Takes two labels: + // algorithm: + // The algorithm used to calculate the state or the step it failed on if it failed. + // Labels starting with "_" are used to indicate when the algorithm fails halfway. + // outcome: + // Whether the state was successfully calculated. + // + // The possible values for algorithm are: + // empty_state -> The list of events was empty so the state is empty. + // no_change -> The state hasn't changed. + // single_delta -> There was a single event added to the state in a way that can be encoded as a single delta + // full_state_no_conflicts -> We created a new copy of the full room state, but didn't enounter any conflicts + // while doing so. + // full_state_with_conflicts -> We created a new copy of the full room state and had to resolve conflicts to do so. + // _load_state_block_nids -> Failed loading the state block nids for a single previous state. + // _load_combined_state -> Failed to load the combined state. + // _resolve_conflicts -> Failed to resolve conflicts. + []string{"algorithm", "outcome"}, +) + +var calculateStatePrevEventLength = prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Namespace: "dendrite", + Subsystem: "roomserver", + Name: "calculate_state_prev_event_length", + Help: "The length of the list of events to calculate the state after", + }, + []string{"algorithm", "outcome"}, +) + +var calculateStateFullStateLength = prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Namespace: "dendrite", + Subsystem: "roomserver", + Name: "calculate_state_full_state_length", + Help: "The length of the full room state.", + }, + []string{"algorithm", "outcome"}, +) + +var calculateStateConflictLength = prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Namespace: "dendrite", + Subsystem: "roomserver", + Name: "calculate_state_conflict_state_length", + Help: "The length of the conflicted room state.", + }, + []string{"algorithm", "outcome"}, +) + +type calculateStateMetrics struct { + algorithm string + startTime time.Time + prevEventLength int + fullStateLength int + conflictLength int +} + +func (c *calculateStateMetrics) stop(stateNID types.StateSnapshotNID, err error) (types.StateSnapshotNID, error) { + var outcome string + if err == nil { + outcome = "success" + } else { + outcome = "failure" + } + endTime := time.Now() + calculateStateDurations.WithLabelValues(c.algorithm, outcome).Observe( + float64(endTime.Sub(c.startTime).Nanoseconds()) / 1000., + ) + calculateStatePrevEventLength.WithLabelValues(c.algorithm, outcome).Observe( + float64(c.prevEventLength), + ) + calculateStateFullStateLength.WithLabelValues(c.algorithm, outcome).Observe( + float64(c.fullStateLength), + ) + calculateStateConflictLength.WithLabelValues(c.algorithm, outcome).Observe( + float64(c.conflictLength), + ) + return stateNID, err +} + +func init() { + prometheus.MustRegister( + calculateStateDurations, calculateStatePrevEventLength, + calculateStateFullStateLength, calculateStateConflictLength, + ) +} + +// CalculateAndStoreStateBeforeEvent calculates a snapshot of the state of a room before an event. +// Stores the snapshot of the state in the database. +// Returns a numeric ID for the snapshot of the state before the event. +func (v StateResolution) CalculateAndStoreStateBeforeEvent( + ctx context.Context, + event gomatrixserverlib.Event, + roomNID types.RoomNID, +) (types.StateSnapshotNID, error) { + // Load the state at the prev events. + prevEventRefs := event.PrevEvents() + prevEventIDs := make([]string, len(prevEventRefs)) + for i := range prevEventRefs { + prevEventIDs[i] = prevEventRefs[i].EventID + } + + prevStates, err := v.db.StateAtEventIDs(ctx, prevEventIDs) + if err != nil { + return 0, err + } + + // The state before this event will be the state after the events that came before it. + return v.CalculateAndStoreStateAfterEvents(ctx, roomNID, prevStates) +} + +// CalculateAndStoreStateAfterEvents finds the room state after the given events. +// Stores the resulting state in the database and returns a numeric ID for that snapshot. +func (v StateResolution) CalculateAndStoreStateAfterEvents( + ctx context.Context, + roomNID types.RoomNID, + prevStates []types.StateAtEvent, +) (types.StateSnapshotNID, error) { + metrics := calculateStateMetrics{startTime: time.Now(), prevEventLength: len(prevStates)} + + if len(prevStates) == 0 { + // 2) There weren't any prev_events for this event so the state is + // empty. + metrics.algorithm = "empty_state" + return metrics.stop(v.db.AddState(ctx, roomNID, nil, nil)) + } + + if len(prevStates) == 1 { + prevState := prevStates[0] + if prevState.EventStateKeyNID == 0 { + // 3) None of the previous events were state events and they all + // have the same state, so this event has exactly the same state + // as the previous events. + // This should be the common case. + metrics.algorithm = "no_change" + return metrics.stop(prevState.BeforeStateSnapshotNID, nil) + } + // The previous event was a state event so we need to store a copy + // of the previous state updated with that event. + stateBlockNIDLists, err := v.db.StateBlockNIDs( + ctx, []types.StateSnapshotNID{prevState.BeforeStateSnapshotNID}, + ) + if err != nil { + metrics.algorithm = "_load_state_blocks" + return metrics.stop(0, err) + } + stateBlockNIDs := stateBlockNIDLists[0].StateBlockNIDs + if len(stateBlockNIDs) < maxStateBlockNIDs { + // 4) The number of state data blocks is small enough that we can just + // add the state event as a block of size one to the end of the blocks. + metrics.algorithm = "single_delta" + return metrics.stop(v.db.AddState( + ctx, roomNID, stateBlockNIDs, []types.StateEntry{prevState.StateEntry}, + )) + } + // If there are too many deltas then we need to calculate the full state + // So fall through to calculateAndStoreStateAfterManyEvents + } + + return v.calculateAndStoreStateAfterManyEvents(ctx, roomNID, prevStates, metrics) +} + +// maxStateBlockNIDs is the maximum number of state data blocks to use to encode a snapshot of room state. +// Increasing this number means that we can encode more of the state changes as simple deltas which means that +// we need fewer entries in the state data table. However making this number bigger will increase the size of +// the rows in the state table itself and will require more index lookups when retrieving a snapshot. +// TODO: Tune this to get the right balance between size and lookup performance. +const maxStateBlockNIDs = 64 + +// calculateAndStoreStateAfterManyEvents finds the room state after the given events. +// This handles the slow path of calculateAndStoreStateAfterEvents for when there is more than one event. +// Stores the resulting state and returns a numeric ID for the snapshot. +func (v StateResolution) calculateAndStoreStateAfterManyEvents( + ctx context.Context, + roomNID types.RoomNID, + prevStates []types.StateAtEvent, + metrics calculateStateMetrics, +) (types.StateSnapshotNID, error) { + roomVersion, err := v.db.GetRoomVersionForRoomNID(ctx, roomNID) + if err != nil { + return metrics.stop(0, err) + } + + state, algorithm, conflictLength, err := + v.calculateStateAfterManyEvents(ctx, roomVersion, prevStates) + metrics.algorithm = algorithm + if err != nil { + return metrics.stop(0, err) + } + + // TODO: Check if we can encode the new state as a delta against the + // previous state. + metrics.conflictLength = conflictLength + metrics.fullStateLength = len(state) + return metrics.stop(v.db.AddState(ctx, roomNID, nil, state)) +} + +func (v StateResolution) calculateStateAfterManyEvents( + ctx context.Context, roomVersion gomatrixserverlib.RoomVersion, + prevStates []types.StateAtEvent, +) (state []types.StateEntry, algorithm string, conflictLength int, err error) { + var combined []types.StateEntry + // Conflict resolution. + // First stage: load the state after each of the prev events. + combined, err = v.LoadCombinedStateAfterEvents(ctx, prevStates) + if err != nil { + algorithm = "_load_combined_state" + return + } + + // Collect all the entries with the same type and key together. + // We don't care about the order here because the conflict resolution + // algorithm doesn't depend on the order of the prev events. + // Remove duplicate entires. + combined = combined[:util.SortAndUnique(stateEntrySorter(combined))] + + // Find the conflicts + conflicts := findDuplicateStateKeys(combined) + + if len(conflicts) > 0 { + conflictLength = len(conflicts) + + // 5) There are conflicting state events, for each conflict workout + // what the appropriate state event is. + + // Work out which entries aren't conflicted. + var notConflicted []types.StateEntry + for _, entry := range combined { + if _, ok := stateEntryMap(conflicts).lookup(entry.StateKeyTuple); !ok { + notConflicted = append(notConflicted, entry) + } + } + + var resolved []types.StateEntry + resolved, err = v.resolveConflicts(ctx, roomVersion, notConflicted, conflicts) + if err != nil { + algorithm = "_resolve_conflicts" + return + } + algorithm = "full_state_with_conflicts" + state = resolved + } else { + algorithm = "full_state_no_conflicts" + // 6) There weren't any conflicts + state = combined + } + return +} + +func (v StateResolution) resolveConflicts( + ctx context.Context, version gomatrixserverlib.RoomVersion, + notConflicted, conflicted []types.StateEntry, +) ([]types.StateEntry, error) { + stateResAlgo, err := version.StateResAlgorithm() + if err != nil { + return nil, err + } + switch stateResAlgo { + case gomatrixserverlib.StateResV1: + return v.resolveConflictsV1(ctx, notConflicted, conflicted) + case gomatrixserverlib.StateResV2: + return v.resolveConflictsV2(ctx, notConflicted, conflicted) + } + return nil, errors.New("unsupported state resolution algorithm") +} + +// resolveConflicts resolves a list of conflicted state entries. It takes two lists. +// The first is a list of all state entries that are not conflicted. +// The second is a list of all state entries that are conflicted +// A state entry is conflicted when there is more than one numeric event ID for the same state key tuple. +// Returns a list that combines the entries without conflicts with the result of state resolution for the entries with conflicts. +// The returned list is sorted by state key tuple. +// Returns an error if there was a problem talking to the database. +func (v StateResolution) resolveConflictsV1( + ctx context.Context, + notConflicted, conflicted []types.StateEntry, +) ([]types.StateEntry, error) { + + // Load the conflicted events + conflictedEvents, eventIDMap, err := v.loadStateEvents(ctx, conflicted) + if err != nil { + return nil, err + } + + // Work out which auth events we need to load. + needed := gomatrixserverlib.StateNeededForAuth(conflictedEvents) + + // Find the numeric IDs for the necessary state keys. + var neededStateKeys []string + neededStateKeys = append(neededStateKeys, needed.Member...) + neededStateKeys = append(neededStateKeys, needed.ThirdPartyInvite...) + stateKeyNIDMap, err := v.db.EventStateKeyNIDs(ctx, neededStateKeys) + if err != nil { + return nil, err + } + + // Load the necessary auth events. + tuplesNeeded := v.stateKeyTuplesNeeded(stateKeyNIDMap, needed) + var authEntries []types.StateEntry + for _, tuple := range tuplesNeeded { + if eventNID, ok := stateEntryMap(notConflicted).lookup(tuple); ok { + authEntries = append(authEntries, types.StateEntry{ + StateKeyTuple: tuple, + EventNID: eventNID, + }) + } + } + authEvents, _, err := v.loadStateEvents(ctx, authEntries) + if err != nil { + return nil, err + } + + // Resolve the conflicts. + resolvedEvents := gomatrixserverlib.ResolveStateConflicts(conflictedEvents, authEvents) + + // Map from the full events back to numeric state entries. + for _, resolvedEvent := range resolvedEvents { + entry, ok := eventIDMap[resolvedEvent.EventID()] + if !ok { + panic(fmt.Errorf("Missing state entry for event ID %q", resolvedEvent.EventID())) + } + notConflicted = append(notConflicted, entry) + } + + // Sort the result so it can be searched. + sort.Sort(stateEntrySorter(notConflicted)) + return notConflicted, nil +} + +// resolveConflicts resolves a list of conflicted state entries. It takes two lists. +// The first is a list of all state entries that are not conflicted. +// The second is a list of all state entries that are conflicted +// A state entry is conflicted when there is more than one numeric event ID for the same state key tuple. +// Returns a list that combines the entries without conflicts with the result of state resolution for the entries with conflicts. +// The returned list is sorted by state key tuple. +// Returns an error if there was a problem talking to the database. +// nolint:gocyclo +func (v StateResolution) resolveConflictsV2( + ctx context.Context, + notConflicted, conflicted []types.StateEntry, +) ([]types.StateEntry, error) { + eventIDMap := make(map[string]types.StateEntry) + + // Load the conflicted events + conflictedEvents, conflictedEventMap, err := v.loadStateEvents(ctx, conflicted) + if err != nil { + return nil, err + } + for k, v := range conflictedEventMap { + eventIDMap[k] = v + } + + // Load the non-conflicted events + nonConflictedEvents, nonConflictedEventMap, err := v.loadStateEvents(ctx, notConflicted) + if err != nil { + return nil, err + } + for k, v := range nonConflictedEventMap { + eventIDMap[k] = v + } + + // For each conflicted event, we will add a new set of auth events. Auth + // events may be duplicated across these sets but that's OK. + authSets := make(map[string][]gomatrixserverlib.Event) + var authEvents []gomatrixserverlib.Event + var authDifference []gomatrixserverlib.Event + + // For each conflicted event, let's try and get the needed auth events. + for _, conflictedEvent := range conflictedEvents { + // Work out which auth events we need to load. + key := conflictedEvent.EventID() + needed := gomatrixserverlib.StateNeededForAuth([]gomatrixserverlib.Event{conflictedEvent}) + + // Find the numeric IDs for the necessary state keys. + var neededStateKeys []string + neededStateKeys = append(neededStateKeys, needed.Member...) + neededStateKeys = append(neededStateKeys, needed.ThirdPartyInvite...) + stateKeyNIDMap, err := v.db.EventStateKeyNIDs(ctx, neededStateKeys) + if err != nil { + return nil, err + } + + // Load the necessary auth events. + tuplesNeeded := v.stateKeyTuplesNeeded(stateKeyNIDMap, needed) + var authEntries []types.StateEntry + for _, tuple := range tuplesNeeded { + if eventNID, ok := stateEntryMap(notConflicted).lookup(tuple); ok { + authEntries = append(authEntries, types.StateEntry{ + StateKeyTuple: tuple, + EventNID: eventNID, + }) + } + } + + // Store the newly found auth events in the auth set for this event. + authSets[key], _, err = v.loadStateEvents(ctx, authEntries) + if err != nil { + return nil, err + } + authEvents = append(authEvents, authSets[key]...) + } + + // This function helps us to work out whether an event exists in one of the + // auth sets. + isInAuthList := func(k string, event gomatrixserverlib.Event) bool { + for _, e := range authSets[k] { + if e.EventID() == event.EventID() { + return true + } + } + return false + } + + // This function works out if an event exists in all of the auth sets. + isInAllAuthLists := func(event gomatrixserverlib.Event) bool { + found := true + for k := range authSets { + found = found && isInAuthList(k, event) + } + return found + } + + // Look through all of the auth events that we've been given and work out if + // there are any events which don't appear in all of the auth sets. If they + // don't then we add them to the auth difference. + for _, event := range authEvents { + if !isInAllAuthLists(event) { + authDifference = append(authDifference, event) + } + } + + // Resolve the conflicts. + resolvedEvents := gomatrixserverlib.ResolveStateConflictsV2( + conflictedEvents, + nonConflictedEvents, + authEvents, + authDifference, + ) + + // Map from the full events back to numeric state entries. + for _, resolvedEvent := range resolvedEvents { + entry, ok := eventIDMap[resolvedEvent.EventID()] + if !ok { + panic(fmt.Errorf("Missing state entry for event ID %q", resolvedEvent.EventID())) + } + notConflicted = append(notConflicted, entry) + } + + // Sort the result so it can be searched. + sort.Sort(stateEntrySorter(notConflicted)) + return notConflicted, nil +} + +// stateKeyTuplesNeeded works out which numeric state key tuples we need to authenticate some events. +func (v StateResolution) stateKeyTuplesNeeded(stateKeyNIDMap map[string]types.EventStateKeyNID, stateNeeded gomatrixserverlib.StateNeeded) []types.StateKeyTuple { + var keyTuples []types.StateKeyTuple + if stateNeeded.Create { + keyTuples = append(keyTuples, types.StateKeyTuple{ + EventTypeNID: types.MRoomCreateNID, + EventStateKeyNID: types.EmptyStateKeyNID, + }) + } + if stateNeeded.PowerLevels { + keyTuples = append(keyTuples, types.StateKeyTuple{ + EventTypeNID: types.MRoomPowerLevelsNID, + EventStateKeyNID: types.EmptyStateKeyNID, + }) + } + if stateNeeded.JoinRules { + keyTuples = append(keyTuples, types.StateKeyTuple{ + EventTypeNID: types.MRoomJoinRulesNID, + EventStateKeyNID: types.EmptyStateKeyNID, + }) + } + for _, member := range stateNeeded.Member { + stateKeyNID, ok := stateKeyNIDMap[member] + if ok { + keyTuples = append(keyTuples, types.StateKeyTuple{ + EventTypeNID: types.MRoomMemberNID, + EventStateKeyNID: stateKeyNID, + }) + } + } + for _, token := range stateNeeded.ThirdPartyInvite { + stateKeyNID, ok := stateKeyNIDMap[token] + if ok { + keyTuples = append(keyTuples, types.StateKeyTuple{ + EventTypeNID: types.MRoomThirdPartyInviteNID, + EventStateKeyNID: stateKeyNID, + }) + } + } + return keyTuples +} + +// loadStateEvents loads the matrix events for a list of state entries. +// Returns a list of state events in no particular order and a map from string event ID back to state entry. +// The map can be used to recover which numeric state entry a given event is for. +// Returns an error if there was a problem talking to the database. +func (v StateResolution) loadStateEvents( + ctx context.Context, entries []types.StateEntry, +) ([]gomatrixserverlib.Event, map[string]types.StateEntry, error) { + eventNIDs := make([]types.EventNID, len(entries)) + for i := range entries { + eventNIDs[i] = entries[i].EventNID + } + events, err := v.db.Events(ctx, eventNIDs) + if err != nil { + return nil, nil, err + } + eventIDMap := map[string]types.StateEntry{} + result := make([]gomatrixserverlib.Event, len(entries)) + for i := range entries { + event, ok := eventMap(events).lookup(entries[i].EventNID) + if !ok { + panic(fmt.Errorf("Corrupt DB: Missing event numeric ID %d", entries[i].EventNID)) + } + result[i] = event.Event + eventIDMap[event.Event.EventID()] = entries[i] + } + return result, eventIDMap, nil +} + +// findDuplicateStateKeys finds the state entries where the state key tuple appears more than once in a sorted list. +// Returns a sorted list of those state entries. +func findDuplicateStateKeys(a []types.StateEntry) []types.StateEntry { + var result []types.StateEntry + // j is the starting index of a block of entries with the same state key tuple. + j := 0 + for i := 1; i < len(a); i++ { + // Check if the state key tuple matches the start of the block + if a[j].StateKeyTuple != a[i].StateKeyTuple { + // If the state key tuple is different then we've reached the end of a block of duplicates. + // Check if the size of the block is bigger than one. + // If the size is one then there was only a single entry with that state key tuple so we don't add it to the result + if j+1 != i { + // Add the block to the result. + result = append(result, a[j:i]...) + } + // Start a new block for the next state key tuple. + j = i + } + } + // Check if the last block with the same state key tuple had more than one event in it. + if j+1 != len(a) { + result = append(result, a[j:]...) + } + return result +} + +type stateEntrySorter []types.StateEntry + +func (s stateEntrySorter) Len() int { return len(s) } +func (s stateEntrySorter) Less(i, j int) bool { return s[i].LessThan(s[j]) } +func (s stateEntrySorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +type stateBlockNIDListMap []types.StateBlockNIDList + +func (m stateBlockNIDListMap) lookup(stateNID types.StateSnapshotNID) (stateBlockNIDs []types.StateBlockNID, ok bool) { + list := []types.StateBlockNIDList(m) + i := sort.Search(len(list), func(i int) bool { + return list[i].StateSnapshotNID >= stateNID + }) + if i < len(list) && list[i].StateSnapshotNID == stateNID { + ok = true + stateBlockNIDs = list[i].StateBlockNIDs + } + return +} + +type stateEntryListMap []types.StateEntryList + +func (m stateEntryListMap) lookup(stateBlockNID types.StateBlockNID) (stateEntries []types.StateEntry, ok bool) { + list := []types.StateEntryList(m) + i := sort.Search(len(list), func(i int) bool { + return list[i].StateBlockNID >= stateBlockNID + }) + if i < len(list) && list[i].StateBlockNID == stateBlockNID { + ok = true + stateEntries = list[i].StateEntries + } + return +} + +type stateEntryByStateKeySorter []types.StateEntry + +func (s stateEntryByStateKeySorter) Len() int { return len(s) } +func (s stateEntryByStateKeySorter) Less(i, j int) bool { + return s[i].StateKeyTuple.LessThan(s[j].StateKeyTuple) +} +func (s stateEntryByStateKeySorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +type stateNIDSorter []types.StateSnapshotNID + +func (s stateNIDSorter) Len() int { return len(s) } +func (s stateNIDSorter) Less(i, j int) bool { return s[i] < s[j] } +func (s stateNIDSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +func uniqueStateSnapshotNIDs(nids []types.StateSnapshotNID) []types.StateSnapshotNID { + return nids[:util.SortAndUnique(stateNIDSorter(nids))] +} + +type stateBlockNIDSorter []types.StateBlockNID + +func (s stateBlockNIDSorter) Len() int { return len(s) } +func (s stateBlockNIDSorter) Less(i, j int) bool { return s[i] < s[j] } +func (s stateBlockNIDSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +func uniqueStateBlockNIDs(nids []types.StateBlockNID) []types.StateBlockNID { + return nids[:util.SortAndUnique(stateBlockNIDSorter(nids))] +} + +// Map from event type, state key tuple to numeric event ID. +// Implemented using binary search on a sorted array. +type stateEntryMap []types.StateEntry + +// lookup an entry in the event map. +func (m stateEntryMap) lookup(stateKey types.StateKeyTuple) (eventNID types.EventNID, ok bool) { + // Since the list is sorted we can implement this using binary search. + // This is faster than using a hash map. + // We don't have to worry about pathological cases because the keys are fixed + // size and are controlled by us. + list := []types.StateEntry(m) + i := sort.Search(len(list), func(i int) bool { + return !list[i].StateKeyTuple.LessThan(stateKey) + }) + if i < len(list) && list[i].StateKeyTuple == stateKey { + ok = true + eventNID = list[i].EventNID + } + return +} + +// Map from numeric event ID to event. +// Implemented using binary search on a sorted array. +type eventMap []types.Event + +// lookup an entry in the event map. +func (m eventMap) lookup(eventNID types.EventNID) (event *types.Event, ok bool) { + // Since the list is sorted we can implement this using binary search. + // This is faster than using a hash map. + // We don't have to worry about pathological cases because the keys are fixed + // size are controlled by us. + list := []types.Event(m) + i := sort.Search(len(list), func(i int) bool { + return list[i].EventNID >= eventNID + }) + if i < len(list) && list[i].EventNID == eventNID { + ok = true + event = &list[i] + } + return } diff --git a/roomserver/state/v1/state_test.go b/roomserver/state/state_test.go similarity index 99% rename from roomserver/state/v1/state_test.go rename to roomserver/state/state_test.go index 4dc7e52ec..c57056786 100644 --- a/roomserver/state/v1/state_test.go +++ b/roomserver/state/state_test.go @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package v1 +package state import ( "testing" diff --git a/roomserver/state/v1/state.go b/roomserver/state/v1/state.go deleted file mode 100644 index 3eb601925..000000000 --- a/roomserver/state/v1/state.go +++ /dev/null @@ -1,932 +0,0 @@ -// Copyright 2017 Vector Creations Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package state provides functions for reading state from the database. -// The functions for writing state to the database are the input package. -package v1 - -import ( - "context" - "fmt" - "sort" - "time" - - "github.com/matrix-org/dendrite/roomserver/state/database" - "github.com/matrix-org/dendrite/roomserver/types" - "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" - "github.com/prometheus/client_golang/prometheus" -) - -type StateResolutionV1 struct { - db database.RoomStateDatabase -} - -func Prepare(db database.RoomStateDatabase) StateResolutionV1 { - return StateResolutionV1{ - db: db, - } -} - -// LoadStateAtSnapshot loads the full state of a room at a particular snapshot. -// This is typically the state before an event or the current state of a room. -// Returns a sorted list of state entries or an error if there was a problem talking to the database. -func (v StateResolutionV1) LoadStateAtSnapshot( - ctx context.Context, stateNID types.StateSnapshotNID, -) ([]types.StateEntry, error) { - stateBlockNIDLists, err := v.db.StateBlockNIDs(ctx, []types.StateSnapshotNID{stateNID}) - if err != nil { - return nil, err - } - // We've asked for exactly one snapshot from the db so we should have exactly one entry in the result. - stateBlockNIDList := stateBlockNIDLists[0] - - stateEntryLists, err := v.db.StateEntries(ctx, stateBlockNIDList.StateBlockNIDs) - if err != nil { - return nil, err - } - stateEntriesMap := stateEntryListMap(stateEntryLists) - - // Combine all the state entries for this snapshot. - // The order of state block NIDs in the list tells us the order to combine them in. - var fullState []types.StateEntry - for _, stateBlockNID := range stateBlockNIDList.StateBlockNIDs { - entries, ok := stateEntriesMap.lookup(stateBlockNID) - if !ok { - // This should only get hit if the database is corrupt. - // It should be impossible for an event to reference a NID that doesn't exist - panic(fmt.Errorf("Corrupt DB: Missing state block numeric ID %d", stateBlockNID)) - } - fullState = append(fullState, entries...) - } - - // Stable sort so that the most recent entry for each state key stays - // remains later in the list than the older entries for the same state key. - sort.Stable(stateEntryByStateKeySorter(fullState)) - // Unique returns the last entry and hence the most recent entry for each state key. - fullState = fullState[:util.Unique(stateEntryByStateKeySorter(fullState))] - return fullState, nil -} - -// LoadStateAtEvent loads the full state of a room at a particular event. -func (v StateResolutionV1) LoadStateAtEvent( - ctx context.Context, eventID string, -) ([]types.StateEntry, error) { - snapshotNID, err := v.db.SnapshotNIDFromEventID(ctx, eventID) - if err != nil { - return nil, err - } - - stateEntries, err := v.LoadStateAtSnapshot(ctx, snapshotNID) - if err != nil { - return nil, err - } - - return stateEntries, nil -} - -// LoadCombinedStateAfterEvents loads a snapshot of the state after each of the events -// and combines those snapshots together into a single list. -func (v StateResolutionV1) LoadCombinedStateAfterEvents( - ctx context.Context, prevStates []types.StateAtEvent, -) ([]types.StateEntry, error) { - stateNIDs := make([]types.StateSnapshotNID, len(prevStates)) - for i, state := range prevStates { - stateNIDs[i] = state.BeforeStateSnapshotNID - } - // Fetch the state snapshots for the state before the each prev event from the database. - // Deduplicate the IDs before passing them to the database. - // There could be duplicates because the events could be state events where - // the snapshot of the room state before them was the same. - stateBlockNIDLists, err := v.db.StateBlockNIDs(ctx, uniqueStateSnapshotNIDs(stateNIDs)) - if err != nil { - return nil, err - } - - var stateBlockNIDs []types.StateBlockNID - for _, list := range stateBlockNIDLists { - stateBlockNIDs = append(stateBlockNIDs, list.StateBlockNIDs...) - } - // Fetch the state entries that will be combined to create the snapshots. - // Deduplicate the IDs before passing them to the database. - // There could be duplicates because a block of state entries could be reused by - // multiple snapshots. - stateEntryLists, err := v.db.StateEntries(ctx, uniqueStateBlockNIDs(stateBlockNIDs)) - if err != nil { - return nil, err - } - stateBlockNIDsMap := stateBlockNIDListMap(stateBlockNIDLists) - stateEntriesMap := stateEntryListMap(stateEntryLists) - - // Combine the entries from all the snapshots of state after each prev event into a single list. - var combined []types.StateEntry - for _, prevState := range prevStates { - // Grab the list of state data NIDs for this snapshot. - stateBlockNIDs, ok := stateBlockNIDsMap.lookup(prevState.BeforeStateSnapshotNID) - if !ok { - // This should only get hit if the database is corrupt. - // It should be impossible for an event to reference a NID that doesn't exist - panic(fmt.Errorf("Corrupt DB: Missing state snapshot numeric ID %d", prevState.BeforeStateSnapshotNID)) - } - - // Combine all the state entries for this snapshot. - // The order of state block NIDs in the list tells us the order to combine them in. - var fullState []types.StateEntry - for _, stateBlockNID := range stateBlockNIDs { - entries, ok := stateEntriesMap.lookup(stateBlockNID) - if !ok { - // This should only get hit if the database is corrupt. - // It should be impossible for an event to reference a NID that doesn't exist - panic(fmt.Errorf("Corrupt DB: Missing state block numeric ID %d", stateBlockNID)) - } - fullState = append(fullState, entries...) - } - if prevState.IsStateEvent() { - // If the prev event was a state event then add an entry for the event itself - // so that we get the state after the event rather than the state before. - fullState = append(fullState, prevState.StateEntry) - } - - // Stable sort so that the most recent entry for each state key stays - // remains later in the list than the older entries for the same state key. - sort.Stable(stateEntryByStateKeySorter(fullState)) - // Unique returns the last entry and hence the most recent entry for each state key. - fullState = fullState[:util.Unique(stateEntryByStateKeySorter(fullState))] - // Add the full state for this StateSnapshotNID. - combined = append(combined, fullState...) - } - return combined, nil -} - -// DifferenceBetweeenStateSnapshots works out which state entries have been added and removed between two snapshots. -func (v StateResolutionV1) DifferenceBetweeenStateSnapshots( - ctx context.Context, oldStateNID, newStateNID types.StateSnapshotNID, -) (removed, added []types.StateEntry, err error) { - if oldStateNID == newStateNID { - // If the snapshot NIDs are the same then nothing has changed - return nil, nil, nil - } - - var oldEntries []types.StateEntry - var newEntries []types.StateEntry - if oldStateNID != 0 { - oldEntries, err = v.LoadStateAtSnapshot(ctx, oldStateNID) - if err != nil { - return nil, nil, err - } - } - if newStateNID != 0 { - newEntries, err = v.LoadStateAtSnapshot(ctx, newStateNID) - if err != nil { - return nil, nil, err - } - } - - var oldI int - var newI int - for { - switch { - case oldI == len(oldEntries): - // We've reached the end of the old entries. - // The rest of the new list must have been newly added. - added = append(added, newEntries[newI:]...) - return - case newI == len(newEntries): - // We've reached the end of the new entries. - // The rest of the old list must be have been removed. - removed = append(removed, oldEntries[oldI:]...) - return - case oldEntries[oldI] == newEntries[newI]: - // The entry is in both lists so skip over it. - oldI++ - newI++ - case oldEntries[oldI].LessThan(newEntries[newI]): - // The lists are sorted so the old entry being less than the new entry means that it only appears in the old list. - removed = append(removed, oldEntries[oldI]) - oldI++ - default: - // Reaching the default case implies that the new entry is less than the old entry. - // Since the lists are sorted this means that it only appears in the new list. - added = append(added, newEntries[newI]) - newI++ - } - } -} - -// LoadStateAtSnapshotForStringTuples loads the state for a list of event type and state key pairs at a snapshot. -// This is used when we only want to load a subset of the room state at a snapshot. -// If there is no entry for a given event type and state key pair then it will be discarded. -// This is typically the state before an event or the current state of a room. -// Returns a sorted list of state entries or an error if there was a problem talking to the database. -func (v StateResolutionV1) LoadStateAtSnapshotForStringTuples( - ctx context.Context, - stateNID types.StateSnapshotNID, - stateKeyTuples []gomatrixserverlib.StateKeyTuple, -) ([]types.StateEntry, error) { - numericTuples, err := v.stringTuplesToNumericTuples(ctx, stateKeyTuples) - if err != nil { - return nil, err - } - return v.loadStateAtSnapshotForNumericTuples(ctx, stateNID, numericTuples) -} - -// stringTuplesToNumericTuples converts the string state key tuples into numeric IDs -// If there isn't a numeric ID for either the event type or the event state key then the tuple is discarded. -// Returns an error if there was a problem talking to the database. -func (v StateResolutionV1) stringTuplesToNumericTuples( - ctx context.Context, - stringTuples []gomatrixserverlib.StateKeyTuple, -) ([]types.StateKeyTuple, error) { - eventTypes := make([]string, len(stringTuples)) - stateKeys := make([]string, len(stringTuples)) - for i := range stringTuples { - eventTypes[i] = stringTuples[i].EventType - stateKeys[i] = stringTuples[i].StateKey - } - eventTypes = util.UniqueStrings(eventTypes) - eventTypeMap, err := v.db.EventTypeNIDs(ctx, eventTypes) - if err != nil { - return nil, err - } - stateKeys = util.UniqueStrings(stateKeys) - stateKeyMap, err := v.db.EventStateKeyNIDs(ctx, stateKeys) - if err != nil { - return nil, err - } - - var result []types.StateKeyTuple - for _, stringTuple := range stringTuples { - var numericTuple types.StateKeyTuple - var ok1, ok2 bool - numericTuple.EventTypeNID, ok1 = eventTypeMap[stringTuple.EventType] - numericTuple.EventStateKeyNID, ok2 = stateKeyMap[stringTuple.StateKey] - // Discard the tuple if there wasn't a numeric ID for either the event type or the state key. - if ok1 && ok2 { - result = append(result, numericTuple) - } - } - - return result, nil -} - -// loadStateAtSnapshotForNumericTuples loads the state for a list of event type and state key pairs at a snapshot. -// This is used when we only want to load a subset of the room state at a snapshot. -// If there is no entry for a given event type and state key pair then it will be discarded. -// This is typically the state before an event or the current state of a room. -// Returns a sorted list of state entries or an error if there was a problem talking to the database. -func (v StateResolutionV1) loadStateAtSnapshotForNumericTuples( - ctx context.Context, - stateNID types.StateSnapshotNID, - stateKeyTuples []types.StateKeyTuple, -) ([]types.StateEntry, error) { - stateBlockNIDLists, err := v.db.StateBlockNIDs(ctx, []types.StateSnapshotNID{stateNID}) - if err != nil { - return nil, err - } - // We've asked for exactly one snapshot from the db so we should have exactly one entry in the result. - stateBlockNIDList := stateBlockNIDLists[0] - - stateEntryLists, err := v.db.StateEntriesForTuples( - ctx, stateBlockNIDList.StateBlockNIDs, stateKeyTuples, - ) - if err != nil { - return nil, err - } - stateEntriesMap := stateEntryListMap(stateEntryLists) - - // Combine all the state entries for this snapshot. - // The order of state block NIDs in the list tells us the order to combine them in. - var fullState []types.StateEntry - for _, stateBlockNID := range stateBlockNIDList.StateBlockNIDs { - entries, ok := stateEntriesMap.lookup(stateBlockNID) - if !ok { - // If the block is missing from the map it means that none of its entries matched a requested tuple. - // This can happen if the block doesn't contain an update for one of the requested tuples. - // If none of the requested tuples are in the block then it can be safely skipped. - continue - } - fullState = append(fullState, entries...) - } - - // Stable sort so that the most recent entry for each state key stays - // remains later in the list than the older entries for the same state key. - sort.Stable(stateEntryByStateKeySorter(fullState)) - // Unique returns the last entry and hence the most recent entry for each state key. - fullState = fullState[:util.Unique(stateEntryByStateKeySorter(fullState))] - return fullState, nil -} - -// LoadStateAfterEventsForStringTuples loads the state for a list of event type -// and state key pairs after list of events. -// This is used when we only want to load a subset of the room state after a list of events. -// If there is no entry for a given event type and state key pair then it will be discarded. -// This is typically the state before an event. -// Returns a sorted list of state entries or an error if there was a problem talking to the database. -func (v StateResolutionV1) LoadStateAfterEventsForStringTuples( - ctx context.Context, - prevStates []types.StateAtEvent, - stateKeyTuples []gomatrixserverlib.StateKeyTuple, -) ([]types.StateEntry, error) { - numericTuples, err := v.stringTuplesToNumericTuples(ctx, stateKeyTuples) - if err != nil { - return nil, err - } - return v.loadStateAfterEventsForNumericTuples(ctx, prevStates, numericTuples) -} - -func (v StateResolutionV1) loadStateAfterEventsForNumericTuples( - ctx context.Context, - prevStates []types.StateAtEvent, - stateKeyTuples []types.StateKeyTuple, -) ([]types.StateEntry, error) { - if len(prevStates) == 1 { - // Fast path for a single event. - prevState := prevStates[0] - result, err := v.loadStateAtSnapshotForNumericTuples( - ctx, prevState.BeforeStateSnapshotNID, stateKeyTuples, - ) - if err != nil { - return nil, err - } - if prevState.IsStateEvent() { - // The result is current the state before the requested event. - // We want the state after the requested event. - // If the requested event was a state event then we need to - // update that key in the result. - // If the requested event wasn't a state event then the state after - // it is the same as the state before it. - set := false - for i := range result { - if result[i].StateKeyTuple == prevState.StateKeyTuple { - result[i] = prevState.StateEntry - set = true - } - } - if !set { // no previous state exists for this event: add new state - result = append(result, prevState.StateEntry) - } - } - return result, nil - } - - // Slow path for more that one event. - // Load the entire state so that we can do conflict resolution if we need to. - // TODO: The are some optimistations we could do here: - // 1) We only need to do conflict resolution if there is a conflict in the - // requested tuples so we might try loading just those tuples and then - // checking for conflicts. - // 2) When there is a conflict we still only need to load the state - // needed to do conflict resolution which would save us having to load - // the full state. - - // TODO: Add metrics for this as it could take a long time for big rooms - // with large conflicts. - fullState, _, _, err := v.calculateStateAfterManyEvents(ctx, prevStates) - if err != nil { - return nil, err - } - - // Sort the full state so we can use it as a map. - sort.Sort(stateEntrySorter(fullState)) - - // Filter the full state down to the required tuples. - var result []types.StateEntry - for _, tuple := range stateKeyTuples { - eventNID, ok := stateEntryMap(fullState).lookup(tuple) - if ok { - result = append(result, types.StateEntry{ - StateKeyTuple: tuple, - EventNID: eventNID, - }) - } - } - sort.Sort(stateEntrySorter(result)) - return result, nil -} - -var calculateStateDurations = prometheus.NewSummaryVec( - prometheus.SummaryOpts{ - Namespace: "dendrite", - Subsystem: "roomserver", - Name: "calculate_state_duration_microseconds", - Help: "How long it takes to calculate the state after a list of events", - }, - // Takes two labels: - // algorithm: - // The algorithm used to calculate the state or the step it failed on if it failed. - // Labels starting with "_" are used to indicate when the algorithm fails halfway. - // outcome: - // Whether the state was successfully calculated. - // - // The possible values for algorithm are: - // empty_state -> The list of events was empty so the state is empty. - // no_change -> The state hasn't changed. - // single_delta -> There was a single event added to the state in a way that can be encoded as a single delta - // full_state_no_conflicts -> We created a new copy of the full room state, but didn't enounter any conflicts - // while doing so. - // full_state_with_conflicts -> We created a new copy of the full room state and had to resolve conflicts to do so. - // _load_state_block_nids -> Failed loading the state block nids for a single previous state. - // _load_combined_state -> Failed to load the combined state. - // _resolve_conflicts -> Failed to resolve conflicts. - []string{"algorithm", "outcome"}, -) - -var calculateStatePrevEventLength = prometheus.NewSummaryVec( - prometheus.SummaryOpts{ - Namespace: "dendrite", - Subsystem: "roomserver", - Name: "calculate_state_prev_event_length", - Help: "The length of the list of events to calculate the state after", - }, - []string{"algorithm", "outcome"}, -) - -var calculateStateFullStateLength = prometheus.NewSummaryVec( - prometheus.SummaryOpts{ - Namespace: "dendrite", - Subsystem: "roomserver", - Name: "calculate_state_full_state_length", - Help: "The length of the full room state.", - }, - []string{"algorithm", "outcome"}, -) - -var calculateStateConflictLength = prometheus.NewSummaryVec( - prometheus.SummaryOpts{ - Namespace: "dendrite", - Subsystem: "roomserver", - Name: "calculate_state_conflict_state_length", - Help: "The length of the conflicted room state.", - }, - []string{"algorithm", "outcome"}, -) - -type calculateStateMetrics struct { - algorithm string - startTime time.Time - prevEventLength int - fullStateLength int - conflictLength int -} - -func (c *calculateStateMetrics) stop(stateNID types.StateSnapshotNID, err error) (types.StateSnapshotNID, error) { - var outcome string - if err == nil { - outcome = "success" - } else { - outcome = "failure" - } - endTime := time.Now() - calculateStateDurations.WithLabelValues(c.algorithm, outcome).Observe( - float64(endTime.Sub(c.startTime).Nanoseconds()) / 1000., - ) - calculateStatePrevEventLength.WithLabelValues(c.algorithm, outcome).Observe( - float64(c.prevEventLength), - ) - calculateStateFullStateLength.WithLabelValues(c.algorithm, outcome).Observe( - float64(c.fullStateLength), - ) - calculateStateConflictLength.WithLabelValues(c.algorithm, outcome).Observe( - float64(c.conflictLength), - ) - return stateNID, err -} - -func init() { - prometheus.MustRegister( - calculateStateDurations, calculateStatePrevEventLength, - calculateStateFullStateLength, calculateStateConflictLength, - ) -} - -// CalculateAndStoreStateBeforeEvent calculates a snapshot of the state of a room before an event. -// Stores the snapshot of the state in the database. -// Returns a numeric ID for the snapshot of the state before the event. -func (v StateResolutionV1) CalculateAndStoreStateBeforeEvent( - ctx context.Context, - event gomatrixserverlib.Event, - roomNID types.RoomNID, -) (types.StateSnapshotNID, error) { - // Load the state at the prev events. - prevEventRefs := event.PrevEvents() - prevEventIDs := make([]string, len(prevEventRefs)) - for i := range prevEventRefs { - prevEventIDs[i] = prevEventRefs[i].EventID - } - - prevStates, err := v.db.StateAtEventIDs(ctx, prevEventIDs) - if err != nil { - return 0, err - } - - // The state before this event will be the state after the events that came before it. - return v.CalculateAndStoreStateAfterEvents(ctx, roomNID, prevStates) -} - -// CalculateAndStoreStateAfterEvents finds the room state after the given events. -// Stores the resulting state in the database and returns a numeric ID for that snapshot. -func (v StateResolutionV1) CalculateAndStoreStateAfterEvents( - ctx context.Context, - roomNID types.RoomNID, - prevStates []types.StateAtEvent, -) (types.StateSnapshotNID, error) { - metrics := calculateStateMetrics{startTime: time.Now(), prevEventLength: len(prevStates)} - - if len(prevStates) == 0 { - // 2) There weren't any prev_events for this event so the state is - // empty. - metrics.algorithm = "empty_state" - return metrics.stop(v.db.AddState(ctx, roomNID, nil, nil)) - } - - if len(prevStates) == 1 { - prevState := prevStates[0] - if prevState.EventStateKeyNID == 0 { - // 3) None of the previous events were state events and they all - // have the same state, so this event has exactly the same state - // as the previous events. - // This should be the common case. - metrics.algorithm = "no_change" - return metrics.stop(prevState.BeforeStateSnapshotNID, nil) - } - // The previous event was a state event so we need to store a copy - // of the previous state updated with that event. - stateBlockNIDLists, err := v.db.StateBlockNIDs( - ctx, []types.StateSnapshotNID{prevState.BeforeStateSnapshotNID}, - ) - if err != nil { - metrics.algorithm = "_load_state_blocks" - return metrics.stop(0, err) - } - stateBlockNIDs := stateBlockNIDLists[0].StateBlockNIDs - if len(stateBlockNIDs) < maxStateBlockNIDs { - // 4) The number of state data blocks is small enough that we can just - // add the state event as a block of size one to the end of the blocks. - metrics.algorithm = "single_delta" - return metrics.stop(v.db.AddState( - ctx, roomNID, stateBlockNIDs, []types.StateEntry{prevState.StateEntry}, - )) - } - // If there are too many deltas then we need to calculate the full state - // So fall through to calculateAndStoreStateAfterManyEvents - } - - return v.calculateAndStoreStateAfterManyEvents(ctx, roomNID, prevStates, metrics) -} - -// maxStateBlockNIDs is the maximum number of state data blocks to use to encode a snapshot of room state. -// Increasing this number means that we can encode more of the state changes as simple deltas which means that -// we need fewer entries in the state data table. However making this number bigger will increase the size of -// the rows in the state table itself and will require more index lookups when retrieving a snapshot. -// TODO: Tune this to get the right balance between size and lookup performance. -const maxStateBlockNIDs = 64 - -// calculateAndStoreStateAfterManyEvents finds the room state after the given events. -// This handles the slow path of calculateAndStoreStateAfterEvents for when there is more than one event. -// Stores the resulting state and returns a numeric ID for the snapshot. -func (v StateResolutionV1) calculateAndStoreStateAfterManyEvents( - ctx context.Context, - roomNID types.RoomNID, - prevStates []types.StateAtEvent, - metrics calculateStateMetrics, -) (types.StateSnapshotNID, error) { - - state, algorithm, conflictLength, err := - v.calculateStateAfterManyEvents(ctx, prevStates) - metrics.algorithm = algorithm - if err != nil { - return metrics.stop(0, err) - } - - // TODO: Check if we can encode the new state as a delta against the - // previous state. - metrics.conflictLength = conflictLength - metrics.fullStateLength = len(state) - return metrics.stop(v.db.AddState(ctx, roomNID, nil, state)) -} - -func (v StateResolutionV1) calculateStateAfterManyEvents( - ctx context.Context, prevStates []types.StateAtEvent, -) (state []types.StateEntry, algorithm string, conflictLength int, err error) { - var combined []types.StateEntry - // Conflict resolution. - // First stage: load the state after each of the prev events. - combined, err = v.LoadCombinedStateAfterEvents(ctx, prevStates) - if err != nil { - algorithm = "_load_combined_state" - return - } - - // Collect all the entries with the same type and key together. - // We don't care about the order here because the conflict resolution - // algorithm doesn't depend on the order of the prev events. - // Remove duplicate entires. - combined = combined[:util.SortAndUnique(stateEntrySorter(combined))] - - // Find the conflicts - conflicts := findDuplicateStateKeys(combined) - - if len(conflicts) > 0 { - conflictLength = len(conflicts) - - // 5) There are conflicting state events, for each conflict workout - // what the appropriate state event is. - - // Work out which entries aren't conflicted. - var notConflicted []types.StateEntry - for _, entry := range combined { - if _, ok := stateEntryMap(conflicts).lookup(entry.StateKeyTuple); !ok { - notConflicted = append(notConflicted, entry) - } - } - - var resolved []types.StateEntry - resolved, err = v.resolveConflicts(ctx, notConflicted, conflicts) - if err != nil { - algorithm = "_resolve_conflicts" - return - } - algorithm = "full_state_with_conflicts" - state = resolved - } else { - algorithm = "full_state_no_conflicts" - // 6) There weren't any conflicts - state = combined - } - return -} - -// resolveConflicts resolves a list of conflicted state entries. It takes two lists. -// The first is a list of all state entries that are not conflicted. -// The second is a list of all state entries that are conflicted -// A state entry is conflicted when there is more than one numeric event ID for the same state key tuple. -// Returns a list that combines the entries without conflicts with the result of state resolution for the entries with conflicts. -// The returned list is sorted by state key tuple. -// Returns an error if there was a problem talking to the database. -func (v StateResolutionV1) resolveConflicts( - ctx context.Context, - notConflicted, conflicted []types.StateEntry, -) ([]types.StateEntry, error) { - - // Load the conflicted events - conflictedEvents, eventIDMap, err := v.loadStateEvents(ctx, conflicted) - if err != nil { - return nil, err - } - - // Work out which auth events we need to load. - needed := gomatrixserverlib.StateNeededForAuth(conflictedEvents) - - // Find the numeric IDs for the necessary state keys. - var neededStateKeys []string - neededStateKeys = append(neededStateKeys, needed.Member...) - neededStateKeys = append(neededStateKeys, needed.ThirdPartyInvite...) - stateKeyNIDMap, err := v.db.EventStateKeyNIDs(ctx, neededStateKeys) - if err != nil { - return nil, err - } - - // Load the necessary auth events. - tuplesNeeded := v.stateKeyTuplesNeeded(stateKeyNIDMap, needed) - var authEntries []types.StateEntry - for _, tuple := range tuplesNeeded { - if eventNID, ok := stateEntryMap(notConflicted).lookup(tuple); ok { - authEntries = append(authEntries, types.StateEntry{ - StateKeyTuple: tuple, - EventNID: eventNID, - }) - } - } - authEvents, _, err := v.loadStateEvents(ctx, authEntries) - if err != nil { - return nil, err - } - - // Resolve the conflicts. - resolvedEvents := gomatrixserverlib.ResolveStateConflicts(conflictedEvents, authEvents) - - // Map from the full events back to numeric state entries. - for _, resolvedEvent := range resolvedEvents { - entry, ok := eventIDMap[resolvedEvent.EventID()] - if !ok { - panic(fmt.Errorf("Missing state entry for event ID %q", resolvedEvent.EventID())) - } - notConflicted = append(notConflicted, entry) - } - - // Sort the result so it can be searched. - sort.Sort(stateEntrySorter(notConflicted)) - return notConflicted, nil -} - -// stateKeyTuplesNeeded works out which numeric state key tuples we need to authenticate some events. -func (v StateResolutionV1) stateKeyTuplesNeeded(stateKeyNIDMap map[string]types.EventStateKeyNID, stateNeeded gomatrixserverlib.StateNeeded) []types.StateKeyTuple { - var keyTuples []types.StateKeyTuple - if stateNeeded.Create { - keyTuples = append(keyTuples, types.StateKeyTuple{ - EventTypeNID: types.MRoomCreateNID, - EventStateKeyNID: types.EmptyStateKeyNID, - }) - } - if stateNeeded.PowerLevels { - keyTuples = append(keyTuples, types.StateKeyTuple{ - EventTypeNID: types.MRoomPowerLevelsNID, - EventStateKeyNID: types.EmptyStateKeyNID, - }) - } - if stateNeeded.JoinRules { - keyTuples = append(keyTuples, types.StateKeyTuple{ - EventTypeNID: types.MRoomJoinRulesNID, - EventStateKeyNID: types.EmptyStateKeyNID, - }) - } - for _, member := range stateNeeded.Member { - stateKeyNID, ok := stateKeyNIDMap[member] - if ok { - keyTuples = append(keyTuples, types.StateKeyTuple{ - EventTypeNID: types.MRoomMemberNID, - EventStateKeyNID: stateKeyNID, - }) - } - } - for _, token := range stateNeeded.ThirdPartyInvite { - stateKeyNID, ok := stateKeyNIDMap[token] - if ok { - keyTuples = append(keyTuples, types.StateKeyTuple{ - EventTypeNID: types.MRoomThirdPartyInviteNID, - EventStateKeyNID: stateKeyNID, - }) - } - } - return keyTuples -} - -// loadStateEvents loads the matrix events for a list of state entries. -// Returns a list of state events in no particular order and a map from string event ID back to state entry. -// The map can be used to recover which numeric state entry a given event is for. -// Returns an error if there was a problem talking to the database. -func (v StateResolutionV1) loadStateEvents( - ctx context.Context, entries []types.StateEntry, -) ([]gomatrixserverlib.Event, map[string]types.StateEntry, error) { - eventNIDs := make([]types.EventNID, len(entries)) - for i := range entries { - eventNIDs[i] = entries[i].EventNID - } - events, err := v.db.Events(ctx, eventNIDs) - if err != nil { - return nil, nil, err - } - eventIDMap := map[string]types.StateEntry{} - result := make([]gomatrixserverlib.Event, len(entries)) - for i := range entries { - event, ok := eventMap(events).lookup(entries[i].EventNID) - if !ok { - panic(fmt.Errorf("Corrupt DB: Missing event numeric ID %d", entries[i].EventNID)) - } - result[i] = event.Event - eventIDMap[event.Event.EventID()] = entries[i] - } - return result, eventIDMap, nil -} - -// findDuplicateStateKeys finds the state entries where the state key tuple appears more than once in a sorted list. -// Returns a sorted list of those state entries. -func findDuplicateStateKeys(a []types.StateEntry) []types.StateEntry { - var result []types.StateEntry - // j is the starting index of a block of entries with the same state key tuple. - j := 0 - for i := 1; i < len(a); i++ { - // Check if the state key tuple matches the start of the block - if a[j].StateKeyTuple != a[i].StateKeyTuple { - // If the state key tuple is different then we've reached the end of a block of duplicates. - // Check if the size of the block is bigger than one. - // If the size is one then there was only a single entry with that state key tuple so we don't add it to the result - if j+1 != i { - // Add the block to the result. - result = append(result, a[j:i]...) - } - // Start a new block for the next state key tuple. - j = i - } - } - // Check if the last block with the same state key tuple had more than one event in it. - if j+1 != len(a) { - result = append(result, a[j:]...) - } - return result -} - -type stateEntrySorter []types.StateEntry - -func (s stateEntrySorter) Len() int { return len(s) } -func (s stateEntrySorter) Less(i, j int) bool { return s[i].LessThan(s[j]) } -func (s stateEntrySorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } - -type stateBlockNIDListMap []types.StateBlockNIDList - -func (m stateBlockNIDListMap) lookup(stateNID types.StateSnapshotNID) (stateBlockNIDs []types.StateBlockNID, ok bool) { - list := []types.StateBlockNIDList(m) - i := sort.Search(len(list), func(i int) bool { - return list[i].StateSnapshotNID >= stateNID - }) - if i < len(list) && list[i].StateSnapshotNID == stateNID { - ok = true - stateBlockNIDs = list[i].StateBlockNIDs - } - return -} - -type stateEntryListMap []types.StateEntryList - -func (m stateEntryListMap) lookup(stateBlockNID types.StateBlockNID) (stateEntries []types.StateEntry, ok bool) { - list := []types.StateEntryList(m) - i := sort.Search(len(list), func(i int) bool { - return list[i].StateBlockNID >= stateBlockNID - }) - if i < len(list) && list[i].StateBlockNID == stateBlockNID { - ok = true - stateEntries = list[i].StateEntries - } - return -} - -type stateEntryByStateKeySorter []types.StateEntry - -func (s stateEntryByStateKeySorter) Len() int { return len(s) } -func (s stateEntryByStateKeySorter) Less(i, j int) bool { - return s[i].StateKeyTuple.LessThan(s[j].StateKeyTuple) -} -func (s stateEntryByStateKeySorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } - -type stateNIDSorter []types.StateSnapshotNID - -func (s stateNIDSorter) Len() int { return len(s) } -func (s stateNIDSorter) Less(i, j int) bool { return s[i] < s[j] } -func (s stateNIDSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } - -func uniqueStateSnapshotNIDs(nids []types.StateSnapshotNID) []types.StateSnapshotNID { - return nids[:util.SortAndUnique(stateNIDSorter(nids))] -} - -type stateBlockNIDSorter []types.StateBlockNID - -func (s stateBlockNIDSorter) Len() int { return len(s) } -func (s stateBlockNIDSorter) Less(i, j int) bool { return s[i] < s[j] } -func (s stateBlockNIDSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } - -func uniqueStateBlockNIDs(nids []types.StateBlockNID) []types.StateBlockNID { - return nids[:util.SortAndUnique(stateBlockNIDSorter(nids))] -} - -// Map from event type, state key tuple to numeric event ID. -// Implemented using binary search on a sorted array. -type stateEntryMap []types.StateEntry - -// lookup an entry in the event map. -func (m stateEntryMap) lookup(stateKey types.StateKeyTuple) (eventNID types.EventNID, ok bool) { - // Since the list is sorted we can implement this using binary search. - // This is faster than using a hash map. - // We don't have to worry about pathological cases because the keys are fixed - // size and are controlled by us. - list := []types.StateEntry(m) - i := sort.Search(len(list), func(i int) bool { - return !list[i].StateKeyTuple.LessThan(stateKey) - }) - if i < len(list) && list[i].StateKeyTuple == stateKey { - ok = true - eventNID = list[i].EventNID - } - return -} - -// Map from numeric event ID to event. -// Implemented using binary search on a sorted array. -type eventMap []types.Event - -// lookup an entry in the event map. -func (m eventMap) lookup(eventNID types.EventNID) (event *types.Event, ok bool) { - // Since the list is sorted we can implement this using binary search. - // This is faster than using a hash map. - // We don't have to worry about pathological cases because the keys are fixed - // size are controlled by us. - list := []types.Event(m) - i := sort.Search(len(list), func(i int) bool { - return list[i].EventNID >= eventNID - }) - if i < len(list) && list[i].EventNID == eventNID { - ok = true - event = &list[i] - } - return -} diff --git a/roomserver/storage/postgres/rooms_table.go b/roomserver/storage/postgres/rooms_table.go index 6bb96f1de..0fd9d5b53 100644 --- a/roomserver/storage/postgres/rooms_table.go +++ b/roomserver/storage/postgres/rooms_table.go @@ -68,6 +68,9 @@ const updateLatestEventNIDsSQL = "" + const selectRoomVersionForRoomIDSQL = "" + "SELECT room_version FROM roomserver_rooms WHERE room_id = $1" +const selectRoomVersionForRoomNIDSQL = "" + + "SELECT room_version FROM roomserver_rooms WHERE room_nid = $1" + type roomStatements struct { insertRoomNIDStmt *sql.Stmt selectRoomNIDStmt *sql.Stmt @@ -75,6 +78,7 @@ type roomStatements struct { selectLatestEventNIDsForUpdateStmt *sql.Stmt updateLatestEventNIDsStmt *sql.Stmt selectRoomVersionForRoomIDStmt *sql.Stmt + selectRoomVersionForRoomNIDStmt *sql.Stmt } func (s *roomStatements) prepare(db *sql.DB) (err error) { @@ -89,6 +93,7 @@ func (s *roomStatements) prepare(db *sql.DB) (err error) { {&s.selectLatestEventNIDsForUpdateStmt, selectLatestEventNIDsForUpdateSQL}, {&s.updateLatestEventNIDsStmt, updateLatestEventNIDsSQL}, {&s.selectRoomVersionForRoomIDStmt, selectRoomVersionForRoomIDSQL}, + {&s.selectRoomVersionForRoomNIDStmt, selectRoomVersionForRoomNIDSQL}, }.prepare(db) } @@ -173,3 +178,12 @@ func (s *roomStatements) selectRoomVersionForRoomID( err := stmt.QueryRowContext(ctx, roomID).Scan(&roomVersion) return roomVersion, err } + +func (s *roomStatements) selectRoomVersionForRoomNID( + ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, +) (gomatrixserverlib.RoomVersion, error) { + var roomVersion gomatrixserverlib.RoomVersion + stmt := common.TxStmt(txn, s.selectRoomVersionForRoomNIDStmt) + err := stmt.QueryRowContext(ctx, roomNID).Scan(&roomVersion) + return roomVersion, err +} diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go index 9bb6de9d7..084b83ce6 100644 --- a/roomserver/storage/postgres/storage.go +++ b/roomserver/storage/postgres/storage.go @@ -747,6 +747,14 @@ func (d *Database) GetRoomVersionForRoom( ) } +func (d *Database) GetRoomVersionForRoomNID( + ctx context.Context, roomNID types.RoomNID, +) (gomatrixserverlib.RoomVersion, error) { + return d.statements.selectRoomVersionForRoomNID( + ctx, nil, roomNID, + ) +} + type transaction struct { ctx context.Context txn *sql.Tx diff --git a/roomserver/storage/sqlite3/rooms_table.go b/roomserver/storage/sqlite3/rooms_table.go index 49fa07ea8..512b98137 100644 --- a/roomserver/storage/sqlite3/rooms_table.go +++ b/roomserver/storage/sqlite3/rooms_table.go @@ -57,6 +57,9 @@ const updateLatestEventNIDsSQL = "" + const selectRoomVersionForRoomIDSQL = "" + "SELECT room_version FROM roomserver_rooms WHERE room_id = $1" +const selectRoomVersionForRoomNIDSQL = "" + + "SELECT room_version FROM roomserver_rooms WHERE room_nid = $1" + type roomStatements struct { insertRoomNIDStmt *sql.Stmt selectRoomNIDStmt *sql.Stmt @@ -64,6 +67,7 @@ type roomStatements struct { selectLatestEventNIDsForUpdateStmt *sql.Stmt updateLatestEventNIDsStmt *sql.Stmt selectRoomVersionForRoomIDStmt *sql.Stmt + selectRoomVersionForRoomNIDStmt *sql.Stmt } func (s *roomStatements) prepare(db *sql.DB) (err error) { @@ -78,6 +82,7 @@ func (s *roomStatements) prepare(db *sql.DB) (err error) { {&s.selectLatestEventNIDsForUpdateStmt, selectLatestEventNIDsForUpdateSQL}, {&s.updateLatestEventNIDsStmt, updateLatestEventNIDsSQL}, {&s.selectRoomVersionForRoomIDStmt, selectRoomVersionForRoomIDSQL}, + {&s.selectRoomVersionForRoomNIDStmt, selectRoomVersionForRoomNIDSQL}, }.prepare(db) } @@ -165,3 +170,12 @@ func (s *roomStatements) selectRoomVersionForRoomID( err := stmt.QueryRowContext(ctx, roomID).Scan(&roomVersion) return roomVersion, err } + +func (s *roomStatements) selectRoomVersionForRoomNID( + ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, +) (gomatrixserverlib.RoomVersion, error) { + var roomVersion gomatrixserverlib.RoomVersion + stmt := common.TxStmt(txn, s.selectRoomVersionForRoomNIDStmt) + err := stmt.QueryRowContext(ctx, roomNID).Scan(&roomVersion) + return roomVersion, err +} diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index ae09a88ad..28d608ca5 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -901,6 +901,14 @@ func (d *Database) GetRoomVersionForRoom( ) } +func (d *Database) GetRoomVersionForRoomNID( + ctx context.Context, roomNID types.RoomNID, +) (gomatrixserverlib.RoomVersion, error) { + return d.statements.selectRoomVersionForRoomNID( + ctx, nil, roomNID, + ) +} + type transaction struct { ctx context.Context txn *sql.Tx From c5b7a17272815a382f020b592312a08bff81860a Mon Sep 17 00:00:00 2001 From: Prateek Sachan <42961174+prateek2211@users.noreply.github.com> Date: Sat, 21 Mar 2020 03:10:24 +0530 Subject: [PATCH 51/86] Added special case for m.room.member event in appservice component (#874) * Added special case for m.room.member event in appservice component * fix review changes Co-authored-by: Kegsay --- appservice/consumers/roomserver.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go index 3e47dee56..9180d9ef0 100644 --- a/appservice/consumers/roomserver.go +++ b/appservice/consumers/roomserver.go @@ -193,6 +193,12 @@ func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Cont return true } + if event.Type() == gomatrixserverlib.MRoomMember && event.StateKey() != nil { + if appservice.IsInterestedInUserID(*event.StateKey()) { + return true + } + } + // Check all known room aliases of the room the event came from queryReq := api.GetAliasesForRoomIDRequest{RoomID: event.RoomID()} var queryRes api.GetAliasesForRoomIDResponse From 5a1a1ded1b5d8387f89e3d586729d175c1f1a1d0 Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Mon, 23 Mar 2020 19:16:17 +0800 Subject: [PATCH 52/86] Fix dendrite config file location in docker guide (#934) --- docker/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/README.md b/docker/README.md index ff88c0818..ee4f0f96f 100644 --- a/docker/README.md +++ b/docker/README.md @@ -11,10 +11,10 @@ and start working on dendrite. ### Configuration -Copy the `dendrite-docker.yaml` file to the root of the project and rename it to -`dendrite.yaml`. It already contains the defaults used in `docker-compose` for -networking so you will only have to change things like the `server_name` or to -toggle `naffka`. +Create a directory named `cfg` in the root of the project. Copy the +`dendrite-docker.yaml` file into that directory and rename it to `dendrite.yaml`. +It already contains the defaults used in `docker-compose` for networking so you will +only have to change things like the `server_name` or to toggle `naffka`. You can run the following `docker-compose` commands either from the top directory specifying the `docker-compose` file From 6bac7e5efddea05aa68a56e44423d2bc157ec364 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 24 Mar 2020 12:20:10 +0000 Subject: [PATCH 53/86] Implement backfill over federation (#938) * Implement history visibility checks for /backfill Required for p2p to show history correctly. * Add sytest * Logging * Fix two backfill bugs which prevented backfill from working correctly - When receiving backfill requests, do not send the event that was in the original request. - When storing backfill results, correctly update the backwards extremity for the room. * hack: make backfill work multiple times * add sqlite impl and remove logging * Linting --- go.sum | 6 + roomserver/auth/auth.go | 70 ++++++- roomserver/query/query.go | 89 ++++++++- syncapi/routing/messages.go | 176 +++++++++--------- .../postgres/backward_extremities_table.go | 58 +++--- .../postgres/output_room_events_table.go | 1 - syncapi/storage/postgres/syncserver.go | 21 +-- .../sqlite3/backward_extremities_table.go | 72 +++---- syncapi/storage/sqlite3/syncserver.go | 17 +- syncapi/sync/requestpool.go | 13 +- sytest-whitelist | 1 + 11 files changed, 322 insertions(+), 202 deletions(-) diff --git a/go.sum b/go.sum index 0f4542b28..48e0efd3e 100644 --- a/go.sum +++ b/go.sum @@ -276,6 +276,8 @@ github.com/matrix-org/gomatrixserverlib v0.0.0-20200318145320-bc896516d72a h1:7+ github.com/matrix-org/gomatrixserverlib v0.0.0-20200318145320-bc896516d72a/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= github.com/matrix-org/gomatrixserverlib v0.0.0-20200318150006-bc27294f9203 h1:7HkL6bF7/M2cYteNFVtvGW5qjD4wHIiR0HsdCm2Rqao= github.com/matrix-org/gomatrixserverlib v0.0.0-20200318150006-bc27294f9203/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200318180716-bc4ff56961e2 h1:y4DOMbhgPAnATHJ4lNxTWxIlJG0SlIPhvukx1sQkty4= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200318180716-bc4ff56961e2/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 h1:W7l5CP4V7wPyPb4tYE11dbmeAOwtFQBTW0rf4OonOS8= @@ -470,6 +472,8 @@ go.uber.org/atomic v1.3.0 h1:vs7fgriifsPbGdK3bNuMWapNn3qnZhCRXc19NRdq010= go.uber.org/atomic v1.3.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -491,6 +495,7 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -559,6 +564,7 @@ golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20190911230505-6bfd74cf029c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190912215617-3720d1ec3678 h1:rM1Udd0CgtYI3KUIhu9ROz0QCqjW+n/ODp/hH7c60Xc= golang.org/x/tools v0.0.0-20190912215617-3720d1ec3678/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/roomserver/auth/auth.go b/roomserver/auth/auth.go index 5ff1fadad..615a94b3c 100644 --- a/roomserver/auth/auth.go +++ b/roomserver/auth/auth.go @@ -12,18 +12,76 @@ package auth -import "github.com/matrix-org/gomatrixserverlib" +import ( + "encoding/json" -// IsServerAllowed returns true if there exists a event in authEvents -// which allows server to view this event. That is true when a client on the server -// can view the event. Otherwise returns false. + "github.com/matrix-org/gomatrixserverlib" +) + +// TODO: This logic should live in gomatrixserverlib + +// IsServerAllowed returns true if the server is allowed to see events in the room +// at this particular state. This function implements https://matrix.org/docs/spec/client_server/r0.6.0#id87 func IsServerAllowed( serverName gomatrixserverlib.ServerName, + serverCurrentlyInRoom bool, authEvents []gomatrixserverlib.Event, ) bool { + historyVisibility := historyVisibilityForRoom(authEvents) + + // 1. If the history_visibility was set to world_readable, allow. + if historyVisibility == "world_readable" { + return true + } + // 2. If the user's membership was join, allow. + joinedUserExists := IsAnyUserOnServerWithMembership(serverName, authEvents, gomatrixserverlib.Join) + if joinedUserExists { + return true + } + // 3. If history_visibility was set to shared, and the user joined the room at any point after the event was sent, allow. + if historyVisibility == "shared" && serverCurrentlyInRoom { + return true + } + // 4. If the user's membership was invite, and the history_visibility was set to invited, allow. + invitedUserExists := IsAnyUserOnServerWithMembership(serverName, authEvents, gomatrixserverlib.Invite) + if invitedUserExists && historyVisibility == "invited" { + return true + } + + // 5. Otherwise, deny. + return false +} + +func historyVisibilityForRoom(authEvents []gomatrixserverlib.Event) string { + // https://matrix.org/docs/spec/client_server/r0.6.0#id87 + // By default if no history_visibility is set, or if the value is not understood, the visibility is assumed to be shared. + visibility := "shared" + knownStates := []string{"invited", "joined", "shared", "world_readable"} + for _, ev := range authEvents { + if ev.Type() != gomatrixserverlib.MRoomHistoryVisibility { + continue + } + // TODO: This should be HistoryVisibilityContent to match things like 'MemberContent'. Do this when moving to GMSL + content := struct { + HistoryVisibility string `json:"history_visibility"` + }{} + if err := json.Unmarshal(ev.Content(), &content); err != nil { + break // value is not understood + } + for _, s := range knownStates { + if s == content.HistoryVisibility { + visibility = s + break + } + } + } + return visibility +} + +func IsAnyUserOnServerWithMembership(serverName gomatrixserverlib.ServerName, authEvents []gomatrixserverlib.Event, wantMembership string) bool { for _, ev := range authEvents { membership, err := ev.Membership() - if err != nil || membership != gomatrixserverlib.Join { + if err != nil || membership != wantMembership { continue } @@ -41,7 +99,5 @@ func IsServerAllowed( return true } } - - // TODO: Check if history visibility is shared and if the server is currently in the room return false } diff --git a/roomserver/query/query.go b/roomserver/query/query.go index 2638919ad..7fe9dc980 100644 --- a/roomserver/query/query.go +++ b/roomserver/query/query.go @@ -447,14 +447,26 @@ func (r *RoomserverQueryAPI) QueryServerAllowedToSeeEvent( request *api.QueryServerAllowedToSeeEventRequest, response *api.QueryServerAllowedToSeeEventResponse, ) (err error) { + events, err := r.DB.EventsFromIDs(ctx, []string{request.EventID}) + if err != nil { + return + } + if len(events) == 0 { + response.AllowedToSeeEvent = false // event doesn't exist so not allowed to see + return + } + isServerInRoom, err := r.isServerCurrentlyInRoom(ctx, request.ServerName, events[0].RoomID()) + if err != nil { + return + } response.AllowedToSeeEvent, err = r.checkServerAllowedToSeeEvent( - ctx, request.EventID, request.ServerName, + ctx, request.EventID, request.ServerName, isServerInRoom, ) return } func (r *RoomserverQueryAPI) checkServerAllowedToSeeEvent( - ctx context.Context, eventID string, serverName gomatrixserverlib.ServerName, + ctx context.Context, eventID string, serverName gomatrixserverlib.ServerName, isServerInRoom bool, ) (bool, error) { roomState := state.NewStateResolution(r.DB) stateEntries, err := roomState.LoadStateAtEvent(ctx, eventID) @@ -469,7 +481,7 @@ func (r *RoomserverQueryAPI) checkServerAllowedToSeeEvent( return false, err } - return auth.IsServerAllowed(serverName, stateAtEvent), nil + return auth.IsServerAllowed(serverName, isServerInRoom, stateAtEvent), nil } // QueryMissingEvents implements api.RoomserverQueryAPI @@ -564,17 +576,55 @@ func (r *RoomserverQueryAPI) QueryBackfill( return err } +func (r *RoomserverQueryAPI) isServerCurrentlyInRoom(ctx context.Context, serverName gomatrixserverlib.ServerName, roomID string) (bool, error) { + roomNID, err := r.DB.RoomNID(ctx, roomID) + if err != nil { + return false, err + } + + eventNIDs, err := r.DB.GetMembershipEventNIDsForRoom(ctx, roomNID, true) + if err != nil { + return false, err + } + + events, err := r.DB.Events(ctx, eventNIDs) + if err != nil { + return false, err + } + gmslEvents := make([]gomatrixserverlib.Event, len(events)) + for i := range events { + gmslEvents[i] = events[i].Event + } + return auth.IsAnyUserOnServerWithMembership(serverName, gmslEvents, gomatrixserverlib.Join), nil +} + +// TODO: Remove this when we have tests to assert correctness of this function +// nolint:gocyclo func (r *RoomserverQueryAPI) scanEventTree( ctx context.Context, front []string, visited map[string]bool, limit int, serverName gomatrixserverlib.ServerName, -) (resultNIDs []types.EventNID, err error) { +) ([]types.EventNID, error) { + var resultNIDs []types.EventNID + var err error var allowed bool var events []types.Event var next []string var pre string + // TODO: add tests for this function to ensure it meets the contract that callers expect (and doc what that is supposed to be) + // Currently, callers like QueryBackfill will call scanEventTree with a pre-populated `visited` map, assuming that by doing + // so means that the events in that map will NOT be returned from this function. That is not currently true, resulting in + // duplicate events being sent in response to /backfill requests. + initialIgnoreList := make(map[string]bool, len(visited)) + for k, v := range visited { + initialIgnoreList[k] = v + } + resultNIDs = make([]types.EventNID, 0, limit) + var checkedServerInRoom bool + var isServerInRoom bool + // Loop through the event IDs to retrieve the requested events and go // through the whole tree (up to the provided limit) using the events' // "prev_event" key. @@ -587,7 +637,18 @@ BFSLoop: // Retrieve the events to process from the database. events, err = r.DB.EventsFromIDs(ctx, front) if err != nil { - return + return resultNIDs, err + } + + if !checkedServerInRoom && len(events) > 0 { + // It's nasty that we have to extract the room ID from an event, but many federation requests + // only talk in event IDs, no room IDs at all (!!!) + ev := events[0] + isServerInRoom, err = r.isServerCurrentlyInRoom(ctx, serverName, ev.RoomID()) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("Failed to check if server is currently in room, assuming not.") + } + checkedServerInRoom = true } for _, ev := range events { @@ -595,17 +656,23 @@ BFSLoop: if len(resultNIDs) == limit { break BFSLoop } - // Update the list of events to retrieve. - resultNIDs = append(resultNIDs, ev.EventNID) + + if !initialIgnoreList[ev.EventID()] { + // Update the list of events to retrieve. + resultNIDs = append(resultNIDs, ev.EventNID) + } // Loop through the event's parents. for _, pre = range ev.PrevEventIDs() { // Only add an event to the list of next events to process if it // hasn't been seen before. if !visited[pre] { visited[pre] = true - allowed, err = r.checkServerAllowedToSeeEvent(ctx, pre, serverName) + allowed, err = r.checkServerAllowedToSeeEvent(ctx, pre, serverName, isServerInRoom) if err != nil { - return + util.GetLogger(ctx).WithField("server", serverName).WithField("event_id", pre).WithError(err).Error( + "Error checking if allowed to see event", + ) + return resultNIDs, err } // If the event hasn't been seen before and the HS @@ -613,6 +680,8 @@ BFSLoop: // the list of events to retrieve. if allowed { next = append(next, pre) + } else { + util.GetLogger(ctx).WithField("server", serverName).WithField("event_id", pre).Info("Not allowed to see event") } } } @@ -621,7 +690,7 @@ BFSLoop: front = next } - return + return resultNIDs, err } // QueryStateAndAuthChain implements api.RoomserverQueryAPI diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index cd21c3bd8..5f2e4f171 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -28,7 +28,7 @@ import ( "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" - log "github.com/sirupsen/logrus" + "github.com/sirupsen/logrus" ) type messagesReq struct { @@ -151,6 +151,14 @@ func OnIncomingMessagesRequest( util.GetLogger(req.Context()).WithError(err).Error("mreq.retrieveEvents failed") return jsonerror.InternalServerError() } + util.GetLogger(req.Context()).WithFields(logrus.Fields{ + "from": from.String(), + "to": to.String(), + "limit": limit, + "backwards": backwardOrdering, + "return_start": start.String(), + "return_end": end.String(), + }).Info("Responding") // Respond with the events. return util.JSONResponse{ @@ -302,8 +310,9 @@ func (r *messagesReq) handleNonEmptyEventsSlice(streamEvents []types.StreamEvent events []gomatrixserverlib.HeaderedEvent, err error, ) { // Check if we have enough events. - isSetLargeEnough := true - if len(streamEvents) < r.limit { + isSetLargeEnough := len(streamEvents) >= r.limit + if !isSetLargeEnough { + // it might be fine we don't have up to 'limit' events, let's find out if r.backwardOrdering { if r.wasToProvided { // The condition in the SQL query is a strict "greater than" so @@ -343,54 +352,6 @@ func (r *messagesReq) handleNonEmptyEventsSlice(streamEvents []types.StreamEvent return } -// containsBackwardExtremity checks if a slice of StreamEvent contains a -// backward extremity. It does so by selecting the earliest event in the slice -// and by checking the presence in the database of all of its parent events, and -// considers the event itself a backward extremity if at least one of the parent -// events doesn't exist in the database. -// Returns an error if there was an issue with talking to the database. -// -// This function is unused but currently set to nolint for now until we are -// absolutely sure that the changes in matrix-org/dendrite#847 are behaving -// properly. -// nolint:unused -func (r *messagesReq) containsBackwardExtremity(events []types.StreamEvent) (bool, error) { - // Select the earliest retrieved event. - var ev *types.StreamEvent - if r.backwardOrdering { - ev = &(events[len(events)-1]) - } else { - ev = &(events[0]) - } - // Get the earliest retrieved event's parents. - prevIDs := ev.PrevEventIDs() - prevs, err := r.db.Events(r.ctx, prevIDs) - if err != nil { - return false, nil - } - // Check if we have all of the events we requested. If not, it means we've - // reached a backward extremity. - var eventInDB bool - var id string - // Iterate over the IDs we used in the request. - for _, id = range prevIDs { - eventInDB = false - // Iterate over the events we got in response. - for _, ev := range prevs { - if ev.EventID() == id { - eventInDB = true - } - } - // One occurrence of one the event's parents not being present in the - // database is enough to say that the event is a backward extremity. - if !eventInDB { - return true, nil - } - } - - return false, nil -} - // backfill performs a backfill request over the federation on another // homeserver in the room. // See: https://matrix.org/docs/spec/server_server/latest#get-matrix-federation-v1-backfill-roomid @@ -401,6 +362,48 @@ func (r *messagesReq) containsBackwardExtremity(events []types.StreamEvent) (boo // Returns an error if there was an issue with retrieving the list of servers in // the room or sending the request. func (r *messagesReq) backfill(fromEventIDs []string, limit int) ([]gomatrixserverlib.HeaderedEvent, error) { + srvToBackfillFrom, err := r.serverToBackfillFrom(fromEventIDs) + if err != nil { + return nil, fmt.Errorf("Cannot find server to backfill from: %w", err) + } + + pdus := make([]gomatrixserverlib.HeaderedEvent, 0) + + // If the roomserver responded with at least one server that isn't us, + // send it a request for backfill. + util.GetLogger(r.ctx).WithField("server", srvToBackfillFrom).WithField("limit", limit).Info("Backfilling from server") + txn, err := r.federation.Backfill( + r.ctx, srvToBackfillFrom, r.roomID, limit, fromEventIDs, + ) + if err != nil { + return nil, err + } + + for _, p := range txn.PDUs { + pdus = append(pdus, p.Headered(gomatrixserverlib.RoomVersionV1)) + } + util.GetLogger(r.ctx).WithField("server", srvToBackfillFrom).WithField("new_events", len(pdus)).Info("Storing new events from backfill") + + // Store the events in the database, while marking them as unfit to show + // up in responses to sync requests. + for _, pdu := range pdus { + headered := pdu.Headered(gomatrixserverlib.RoomVersionV1) + if _, err = r.db.WriteEvent( + r.ctx, + &headered, + []gomatrixserverlib.HeaderedEvent{}, + []string{}, + []string{}, + nil, true, + ); err != nil { + return nil, err + } + } + + return pdus, nil +} + +func (r *messagesReq) serverToBackfillFrom(fromEventIDs []string) (gomatrixserverlib.ServerName, error) { // Query the list of servers in the room when one of the backward extremities // was sent. var serversResponse api.QueryServersInRoomAtEventResponse @@ -409,7 +412,33 @@ func (r *messagesReq) backfill(fromEventIDs []string, limit int) ([]gomatrixserv EventID: fromEventIDs[0], } if err := r.queryAPI.QueryServersInRoomAtEvent(r.ctx, &serversRequest, &serversResponse); err != nil { - return nil, err + util.GetLogger(r.ctx).WithError(err).Warn("Failed to query servers in room at event, falling back to event sender") + // FIXME: We shouldn't be doing this but in situations where we have already backfilled once + // the query API doesn't work as backfilled events do not make it to the room server. + // This means QueryServersInRoomAtEvent returns an error as it doesn't have the event ID in question. + // We need to inject backfilled events into the room server and store them appropriately. + events, err := r.db.Events(r.ctx, fromEventIDs) + if err != nil { + return "", err + } + if len(events) == 0 { + // should be impossible as these event IDs are backwards extremities + return "", fmt.Errorf("backfill: missing backwards extremities, event IDs: %s", fromEventIDs) + } + // The rationale here is that the last event was unlikely to be sent by us, so poke the server who sent it. + // We shouldn't be doing this really, but as a heuristic it should work pretty well for now. + for _, e := range events { + _, srv, srverr := gomatrixserverlib.SplitID('@', e.Sender()) + if srverr != nil { + util.GetLogger(r.ctx).WithError(srverr).Warn("Failed to extract domain from event sender") + continue + } + if srv != r.cfg.Matrix.ServerName { + return srv, nil + } + } + // no valid events which have a remote server, fail. + return "", err } // Use the first server from the response, except if that server is us. @@ -423,45 +452,11 @@ func (r *messagesReq) backfill(fromEventIDs []string, limit int) ([]gomatrixserv if len(serversResponse.Servers) > 1 { srvToBackfillFrom = serversResponse.Servers[1] } else { - srvToBackfillFrom = gomatrixserverlib.ServerName("") - log.Warn("Not enough servers to backfill from") + util.GetLogger(r.ctx).Info("Not enough servers to backfill from") + return "", nil } } - - pdus := make([]gomatrixserverlib.HeaderedEvent, 0) - - // If the roomserver responded with at least one server that isn't us, - // send it a request for backfill. - if len(srvToBackfillFrom) > 0 { - txn, err := r.federation.Backfill( - r.ctx, srvToBackfillFrom, r.roomID, limit, fromEventIDs, - ) - if err != nil { - return nil, err - } - - for _, p := range txn.PDUs { - pdus = append(pdus, p.Headered(gomatrixserverlib.RoomVersionV1)) - } - - // Store the events in the database, while marking them as unfit to show - // up in responses to sync requests. - for _, pdu := range pdus { - headered := pdu.Headered(gomatrixserverlib.RoomVersionV1) - if _, err = r.db.WriteEvent( - r.ctx, - &headered, - []gomatrixserverlib.HeaderedEvent{}, - []string{}, - []string{}, - nil, true, - ); err != nil { - return nil, err - } - } - } - - return pdus, nil + return srvToBackfillFrom, nil } // setToDefault returns the default value for the "to" query parameter of a @@ -475,7 +470,8 @@ func setToDefault( roomID string, ) (to *types.PaginationToken, err error) { if backwardOrdering { - to = types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, 1, 0) + // go 1 earlier than the first event so we correctly fetch the earliest event + to = types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, 0, 0) } else { var pos types.StreamPosition pos, err = db.MaxTopologicalPosition(ctx, roomID) diff --git a/syncapi/storage/postgres/backward_extremities_table.go b/syncapi/storage/postgres/backward_extremities_table.go index 8286ca434..b3ee28e01 100644 --- a/syncapi/storage/postgres/backward_extremities_table.go +++ b/syncapi/storage/postgres/backward_extremities_table.go @@ -21,39 +21,53 @@ import ( "github.com/matrix-org/dendrite/common" ) +// The purpose of this table is to keep track of backwards extremities for a room. +// Backwards extremities are the earliest (DAG-wise) known events which we have +// the entire event JSON. These event IDs are used in federation requests to fetch +// even earlier events. +// +// We persist the previous event IDs as well, one per row, so when we do fetch even +// earlier events we can simply delete rows which referenced it. Consider the graph: +// A +// | Event C has 1 prev_event ID: A. +// B C +// |___| Event D has 2 prev_event IDs: B and C. +// | +// D +// The earliest known event we have is D, so this table has 2 rows. +// A backfill request gives us C but not B. We delete rows where prev_event=C. This +// still means that D is a backwards extremity as we do not have event B. However, event +// C is *also* a backwards extremity at this point as we do not have event A. Later, +// when we fetch event B, we delete rows where prev_event=B. This then removes D as +// a backwards extremity because there are no more rows with event_id=B. const backwardExtremitiesSchema = ` -- Stores output room events received from the roomserver. CREATE TABLE IF NOT EXISTS syncapi_backward_extremities ( -- The 'room_id' key for the event. room_id TEXT NOT NULL, - -- The event ID for the event. + -- The event ID for the last known event. This is the backwards extremity. event_id TEXT NOT NULL, + -- The prev_events for the last known event. This is used to update extremities. + prev_event_id TEXT NOT NULL, - PRIMARY KEY(room_id, event_id) + PRIMARY KEY(room_id, event_id, prev_event_id) ); ` const insertBackwardExtremitySQL = "" + - "INSERT INTO syncapi_backward_extremities (room_id, event_id)" + - " VALUES ($1, $2)" + + "INSERT INTO syncapi_backward_extremities (room_id, event_id, prev_event_id)" + + " VALUES ($1, $2, $3)" + " ON CONFLICT DO NOTHING" const selectBackwardExtremitiesForRoomSQL = "" + "SELECT event_id FROM syncapi_backward_extremities WHERE room_id = $1" -const isBackwardExtremitySQL = "" + - "SELECT EXISTS (" + - " SELECT TRUE FROM syncapi_backward_extremities" + - " WHERE room_id = $1 AND event_id = $2" + - ")" - const deleteBackwardExtremitySQL = "" + - "DELETE FROM syncapi_backward_extremities WHERE room_id = $1 AND event_id = $2" + "DELETE FROM syncapi_backward_extremities WHERE room_id = $1 AND prev_event_id = $2" type backwardExtremitiesStatements struct { insertBackwardExtremityStmt *sql.Stmt selectBackwardExtremitiesForRoomStmt *sql.Stmt - isBackwardExtremityStmt *sql.Stmt deleteBackwardExtremityStmt *sql.Stmt } @@ -68,9 +82,6 @@ func (s *backwardExtremitiesStatements) prepare(db *sql.DB) (err error) { if s.selectBackwardExtremitiesForRoomStmt, err = db.Prepare(selectBackwardExtremitiesForRoomSQL); err != nil { return } - if s.isBackwardExtremityStmt, err = db.Prepare(isBackwardExtremitySQL); err != nil { - return - } if s.deleteBackwardExtremityStmt, err = db.Prepare(deleteBackwardExtremitySQL); err != nil { return } @@ -78,17 +89,15 @@ func (s *backwardExtremitiesStatements) prepare(db *sql.DB) (err error) { } func (s *backwardExtremitiesStatements) insertsBackwardExtremity( - ctx context.Context, roomID, eventID string, + ctx context.Context, txn *sql.Tx, roomID, eventID string, prevEventID string, ) (err error) { - _, err = s.insertBackwardExtremityStmt.ExecContext(ctx, roomID, eventID) + _, err = txn.Stmt(s.insertBackwardExtremityStmt).ExecContext(ctx, roomID, eventID, prevEventID) return } func (s *backwardExtremitiesStatements) selectBackwardExtremitiesForRoom( ctx context.Context, roomID string, ) (eventIDs []string, err error) { - eventIDs = make([]string, 0) - rows, err := s.selectBackwardExtremitiesForRoomStmt.QueryContext(ctx, roomID) if err != nil { return @@ -107,16 +116,9 @@ func (s *backwardExtremitiesStatements) selectBackwardExtremitiesForRoom( return eventIDs, rows.Err() } -func (s *backwardExtremitiesStatements) isBackwardExtremity( - ctx context.Context, roomID, eventID string, -) (isBE bool, err error) { - err = s.isBackwardExtremityStmt.QueryRowContext(ctx, roomID, eventID).Scan(&isBE) - return -} - func (s *backwardExtremitiesStatements) deleteBackwardExtremity( - ctx context.Context, roomID, eventID string, + ctx context.Context, txn *sql.Tx, roomID, knownEventID string, ) (err error) { - _, err = s.insertBackwardExtremityStmt.ExecContext(ctx, roomID, eventID) + _, err = txn.Stmt(s.deleteBackwardExtremityStmt).ExecContext(ctx, roomID, knownEventID) return } diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go index 5f9a1d0c6..0b53dfa9e 100644 --- a/syncapi/storage/postgres/output_room_events_table.go +++ b/syncapi/storage/postgres/output_room_events_table.go @@ -305,7 +305,6 @@ func (s *outputRoomEventsStatements) selectRecentEvents( } else { stmt = common.TxStmt(txn, s.selectRecentEventsStmt) } - rows, err := stmt.QueryContext(ctx, roomID, fromPos, toPos, limit) if err != nil { return nil, err diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go index ccf1c5656..f3f1aabc7 100644 --- a/syncapi/storage/postgres/syncserver.go +++ b/syncapi/storage/postgres/syncserver.go @@ -111,22 +111,17 @@ func (d *SyncServerDatasource) Events(ctx context.Context, eventIDs []string) ([ return d.StreamEventsToEvents(nil, streamEvents), nil } -func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, ev *gomatrixserverlib.HeaderedEvent) error { - // If the event is already known as a backward extremity, don't consider - // it as such anymore now that we have it. - isBackwardExtremity, err := d.backwardExtremities.isBackwardExtremity(ctx, ev.RoomID(), ev.EventID()) - if err != nil { +// handleBackwardExtremities adds this event as a backwards extremity if and only if we do not have all of +// the events listed in the event's 'prev_events'. This function also updates the backwards extremities table +// to account for the fact that the given event is no longer a backwards extremity, but may be marked as such. +func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, txn *sql.Tx, ev *gomatrixserverlib.HeaderedEvent) error { + if err := d.backwardExtremities.deleteBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID()); err != nil { return err } - if isBackwardExtremity { - if err = d.backwardExtremities.deleteBackwardExtremity(ctx, ev.RoomID(), ev.EventID()); err != nil { - return err - } - } // Check if we have all of the event's previous events. If an event is // missing, add it to the room's backward extremities. - prevEvents, err := d.events.selectEvents(ctx, nil, ev.PrevEventIDs()) + prevEvents, err := d.events.selectEvents(ctx, txn, ev.PrevEventIDs()) if err != nil { return err } @@ -141,7 +136,7 @@ func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, ev // If the event is missing, consider it a backward extremity. if !found { - if err = d.backwardExtremities.insertsBackwardExtremity(ctx, ev.RoomID(), ev.EventID()); err != nil { + if err = d.backwardExtremities.insertsBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID(), eID); err != nil { return err } } @@ -174,7 +169,7 @@ func (d *SyncServerDatasource) WriteEvent( return err } - if err = d.handleBackwardExtremities(ctx, ev); err != nil { + if err = d.handleBackwardExtremities(ctx, txn, ev); err != nil { return err } diff --git a/syncapi/storage/sqlite3/backward_extremities_table.go b/syncapi/storage/sqlite3/backward_extremities_table.go index fcf15da25..0663e2a12 100644 --- a/syncapi/storage/sqlite3/backward_extremities_table.go +++ b/syncapi/storage/sqlite3/backward_extremities_table.go @@ -21,38 +21,53 @@ import ( "github.com/matrix-org/dendrite/common" ) +// The purpose of this table is to keep track of backwards extremities for a room. +// Backwards extremities are the earliest (DAG-wise) known events which we have +// the entire event JSON. These event IDs are used in federation requests to fetch +// even earlier events. +// +// We persist the previous event IDs as well, one per row, so when we do fetch even +// earlier events we can simply delete rows which referenced it. Consider the graph: +// A +// | Event C has 1 prev_event ID: A. +// B C +// |___| Event D has 2 prev_event IDs: B and C. +// | +// D +// The earliest known event we have is D, so this table has 2 rows. +// A backfill request gives us C but not B. We delete rows where prev_event=C. This +// still means that D is a backwards extremity as we do not have event B. However, event +// C is *also* a backwards extremity at this point as we do not have event A. Later, +// when we fetch event B, we delete rows where prev_event=B. This then removes D as +// a backwards extremity because there are no more rows with event_id=B. const backwardExtremitiesSchema = ` -- Stores output room events received from the roomserver. CREATE TABLE IF NOT EXISTS syncapi_backward_extremities ( + -- The 'room_id' key for the event. room_id TEXT NOT NULL, + -- The event ID for the last known event. This is the backwards extremity. event_id TEXT NOT NULL, + -- The prev_events for the last known event. This is used to update extremities. + prev_event_id TEXT NOT NULL, - PRIMARY KEY(room_id, event_id) + PRIMARY KEY(room_id, event_id, prev_event_id) ); ` const insertBackwardExtremitySQL = "" + - "INSERT INTO syncapi_backward_extremities (room_id, event_id)" + - " VALUES ($1, $2)" + - " ON CONFLICT (room_id, event_id) DO NOTHING" + "INSERT INTO syncapi_backward_extremities (room_id, event_id, prev_event_id)" + + " VALUES ($1, $2, $3)" + + " ON CONFLICT (room_id, event_id, prev_event_id) DO NOTHING" const selectBackwardExtremitiesForRoomSQL = "" + "SELECT event_id FROM syncapi_backward_extremities WHERE room_id = $1" -const isBackwardExtremitySQL = "" + - "SELECT EXISTS (" + - " SELECT TRUE FROM syncapi_backward_extremities" + - " WHERE room_id = $1 AND event_id = $2" + - ")" - const deleteBackwardExtremitySQL = "" + - "DELETE FROM syncapi_backward_extremities" + - " WHERE room_id = $1 AND event_id = $2" + "DELETE FROM syncapi_backward_extremities WHERE room_id = $1 AND prev_event_id = $2" type backwardExtremitiesStatements struct { insertBackwardExtremityStmt *sql.Stmt selectBackwardExtremitiesForRoomStmt *sql.Stmt - isBackwardExtremityStmt *sql.Stmt deleteBackwardExtremityStmt *sql.Stmt } @@ -67,9 +82,6 @@ func (s *backwardExtremitiesStatements) prepare(db *sql.DB) (err error) { if s.selectBackwardExtremitiesForRoomStmt, err = db.Prepare(selectBackwardExtremitiesForRoomSQL); err != nil { return } - if s.isBackwardExtremityStmt, err = db.Prepare(isBackwardExtremitySQL); err != nil { - return - } if s.deleteBackwardExtremityStmt, err = db.Prepare(deleteBackwardExtremitySQL); err != nil { return } @@ -77,23 +89,20 @@ func (s *backwardExtremitiesStatements) prepare(db *sql.DB) (err error) { } func (s *backwardExtremitiesStatements) insertsBackwardExtremity( - ctx context.Context, txn *sql.Tx, roomID, eventID string, + ctx context.Context, txn *sql.Tx, roomID, eventID string, prevEventID string, ) (err error) { - stmt := common.TxStmt(txn, s.insertBackwardExtremityStmt) - _, err = stmt.ExecContext(ctx, roomID, eventID) + _, err = txn.Stmt(s.insertBackwardExtremityStmt).ExecContext(ctx, roomID, eventID, prevEventID) return } func (s *backwardExtremitiesStatements) selectBackwardExtremitiesForRoom( - ctx context.Context, txn *sql.Tx, roomID string, + ctx context.Context, roomID string, ) (eventIDs []string, err error) { - eventIDs = make([]string, 0) - - stmt := common.TxStmt(txn, s.selectBackwardExtremitiesForRoomStmt) - rows, err := stmt.QueryContext(ctx, roomID) + rows, err := s.selectBackwardExtremitiesForRoomStmt.QueryContext(ctx, roomID) if err != nil { return } + defer common.CloseAndLogIfError(ctx, rows, "selectBackwardExtremitiesForRoom: rows.close() failed") for rows.Next() { var eID string @@ -104,21 +113,12 @@ func (s *backwardExtremitiesStatements) selectBackwardExtremitiesForRoom( eventIDs = append(eventIDs, eID) } - return -} - -func (s *backwardExtremitiesStatements) isBackwardExtremity( - ctx context.Context, txn *sql.Tx, roomID, eventID string, -) (isBE bool, err error) { - stmt := common.TxStmt(txn, s.isBackwardExtremityStmt) - err = stmt.QueryRowContext(ctx, roomID, eventID).Scan(&isBE) - return + return eventIDs, rows.Err() } func (s *backwardExtremitiesStatements) deleteBackwardExtremity( - ctx context.Context, txn *sql.Tx, roomID, eventID string, + ctx context.Context, txn *sql.Tx, roomID, knownEventID string, ) (err error) { - stmt := common.TxStmt(txn, s.deleteBackwardExtremityStmt) - _, err = stmt.ExecContext(ctx, roomID, eventID) + _, err = txn.Stmt(s.deleteBackwardExtremityStmt).ExecContext(ctx, roomID, knownEventID) return } diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index a06fc91f5..8ff189007 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -137,18 +137,13 @@ func (d *SyncServerDatasource) Events(ctx context.Context, eventIDs []string) ([ return d.StreamEventsToEvents(nil, streamEvents), nil } +// handleBackwardExtremities adds this event as a backwards extremity if and only if we do not have all of +// the events listed in the event's 'prev_events'. This function also updates the backwards extremities table +// to account for the fact that the given event is no longer a backwards extremity, but may be marked as such. func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, txn *sql.Tx, ev *gomatrixserverlib.HeaderedEvent) error { - // If the event is already known as a backward extremity, don't consider - // it as such anymore now that we have it. - isBackwardExtremity, err := d.backwardExtremities.isBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID()) - if err != nil { + if err := d.backwardExtremities.deleteBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID()); err != nil { return err } - if isBackwardExtremity { - if err = d.backwardExtremities.deleteBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID()); err != nil { - return err - } - } // Check if we have all of the event's previous events. If an event is // missing, add it to the room's backward extremities. @@ -167,7 +162,7 @@ func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, tx // If the event is missing, consider it a backward extremity. if !found { - if err = d.backwardExtremities.insertsBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID()); err != nil { + if err = d.backwardExtremities.insertsBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID(), eID); err != nil { return err } } @@ -348,7 +343,7 @@ func (d *SyncServerDatasource) SyncPosition(ctx context.Context) (tok types.Pagi func (d *SyncServerDatasource) BackwardExtremitiesForRoom( ctx context.Context, roomID string, ) (backwardExtremities []string, err error) { - return d.backwardExtremities.selectBackwardExtremitiesForRoom(ctx, nil, roomID) + return d.backwardExtremities.selectBackwardExtremitiesForRoom(ctx, roomID) } // MaxTopologicalPosition returns the highest topological position for a given diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index b4ccbd272..69efd8aa8 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -47,7 +47,6 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *authtype var syncData *types.Response // Extract values from request - logger := util.GetLogger(req.Context()) userID := device.UserID syncReq, err := newSyncRequest(req, *device) if err != nil { @@ -56,20 +55,21 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *authtype JSON: jsonerror.Unknown(err.Error()), } } - logger.WithFields(log.Fields{ + logger := util.GetLogger(req.Context()).WithFields(log.Fields{ "userID": userID, "since": syncReq.since, "timeout": syncReq.timeout, - }).Info("Incoming /sync request") + }) currPos := rp.notifier.CurrentPosition() if shouldReturnImmediately(syncReq) { syncData, err = rp.currentSyncForUser(*syncReq, currPos) if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("rp.currentSyncForUser failed") + logger.WithError(err).Error("rp.currentSyncForUser failed") return jsonerror.InternalServerError() } + logger.WithField("next", syncData.NextBatch).Info("Responding immediately") return util.JSONResponse{ Code: http.StatusOK, JSON: syncData, @@ -107,7 +107,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *authtype hasTimedOut = true // Or for the request to be cancelled case <-req.Context().Done(): - util.GetLogger(req.Context()).WithError(err).Error("request cancelled") + logger.WithError(err).Error("request cancelled") return jsonerror.InternalServerError() } @@ -118,11 +118,12 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *authtype syncData, err = rp.currentSyncForUser(*syncReq, currPos) if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("rp.currentSyncForUser failed") + logger.WithError(err).Error("rp.currentSyncForUser failed") return jsonerror.InternalServerError() } if !syncData.IsEmpty() || hasTimedOut { + logger.WithField("next", syncData.NextBatch).WithField("timed_out", hasTimedOut).Info("Responding") return util.JSONResponse{ Code: http.StatusOK, JSON: syncData, diff --git a/sytest-whitelist b/sytest-whitelist index 03e11b83b..95b000e16 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -220,3 +220,4 @@ Regular users can add and delete aliases when m.room.aliases is restricted GET /r0/capabilities is not public GET /joined_rooms lists newly-created room /joined_rooms returns only joined rooms +Message history can be paginated over federation From 951b5d5e6895209c1940bbad04a793b147b2648a Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 24 Mar 2020 13:39:50 +0000 Subject: [PATCH 54/86] Missing whitelist --- sytest-whitelist | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sytest-whitelist b/sytest-whitelist index 95b000e16..b9eef9c83 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -221,3 +221,5 @@ GET /r0/capabilities is not public GET /joined_rooms lists newly-created room /joined_rooms returns only joined rooms Message history can be paginated over federation +GET /rooms/:room_id/messages returns a message +Remote user can backfill in a room with version 1 From 0b732d6f45dc96041a85c227812ea0b53b19af68 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 24 Mar 2020 15:46:17 +0000 Subject: [PATCH 55/86] Use HeaderedEvents in appservice component (#939) * App service HeaderedEvents * Fix database queries * Fix lint error --- appservice/consumers/roomserver.go | 22 +++++++++---------- appservice/storage/interface.go | 4 ++-- .../postgres/appservice_events_table.go | 14 ++++++------ appservice/storage/postgres/storage.go | 6 ++--- .../sqlite3/appservice_events_table.go | 14 ++++++------ appservice/storage/sqlite3/storage.go | 6 ++--- appservice/workers/transaction_scheduler.go | 7 +++++- 7 files changed, 38 insertions(+), 35 deletions(-) diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go index 9180d9ef0..6ae58e85c 100644 --- a/appservice/consumers/roomserver.go +++ b/appservice/consumers/roomserver.go @@ -101,11 +101,11 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { "type": ev.Type(), }).Info("appservice received an event from roomserver") - missingEvents, err := s.lookupMissingStateEvents(output.NewRoomEvent.AddsStateEventIDs, ev.Event) + missingEvents, err := s.lookupMissingStateEvents(output.NewRoomEvent.AddsStateEventIDs, ev) if err != nil { return err } - events := append(missingEvents, ev.Event) + events := append(missingEvents, ev) // Send event to any relevant application services return s.filterRoomserverEvents(context.TODO(), events) @@ -114,19 +114,19 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { // lookupMissingStateEvents looks up the state events that are added by a new event, // and returns any not already present. func (s *OutputRoomEventConsumer) lookupMissingStateEvents( - addsStateEventIDs []string, event gomatrixserverlib.Event, -) ([]gomatrixserverlib.Event, error) { + addsStateEventIDs []string, event gomatrixserverlib.HeaderedEvent, +) ([]gomatrixserverlib.HeaderedEvent, error) { // Fast path if there aren't any new state events. if len(addsStateEventIDs) == 0 { - return []gomatrixserverlib.Event{}, nil + return []gomatrixserverlib.HeaderedEvent{}, nil } // Fast path if the only state event added is the event itself. if len(addsStateEventIDs) == 1 && addsStateEventIDs[0] == event.EventID() { - return []gomatrixserverlib.Event{}, nil + return []gomatrixserverlib.HeaderedEvent{}, nil } - result := []gomatrixserverlib.Event{} + result := []gomatrixserverlib.HeaderedEvent{} missing := []string{} for _, id := range addsStateEventIDs { if id != event.EventID() { @@ -143,9 +143,7 @@ func (s *OutputRoomEventConsumer) lookupMissingStateEvents( return nil, err } - for _, headeredEvent := range eventResp.Events { - result = append(result, headeredEvent.Event) - } + result = append(result, eventResp.Events...) return result, nil } @@ -157,7 +155,7 @@ func (s *OutputRoomEventConsumer) lookupMissingStateEvents( // application service. func (s *OutputRoomEventConsumer) filterRoomserverEvents( ctx context.Context, - events []gomatrixserverlib.Event, + events []gomatrixserverlib.HeaderedEvent, ) error { for _, ws := range s.workerStates { for _, event := range events { @@ -180,7 +178,7 @@ func (s *OutputRoomEventConsumer) filterRoomserverEvents( // appserviceIsInterestedInEvent returns a boolean depending on whether a given // event falls within one of a given application service's namespaces. -func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Context, event gomatrixserverlib.Event, appservice config.ApplicationService) bool { +func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Context, event gomatrixserverlib.HeaderedEvent, appservice config.ApplicationService) bool { // No reason to queue events if they'll never be sent to the application // service if appservice.URL == "" { diff --git a/appservice/storage/interface.go b/appservice/storage/interface.go index 4b75ff68e..25d35af6c 100644 --- a/appservice/storage/interface.go +++ b/appservice/storage/interface.go @@ -21,8 +21,8 @@ import ( ) type Database interface { - StoreEvent(ctx context.Context, appServiceID string, event *gomatrixserverlib.Event) error - GetEventsWithAppServiceID(ctx context.Context, appServiceID string, limit int) (int, int, []gomatrixserverlib.Event, bool, error) + StoreEvent(ctx context.Context, appServiceID string, event *gomatrixserverlib.HeaderedEvent) error + GetEventsWithAppServiceID(ctx context.Context, appServiceID string, limit int) (int, int, []gomatrixserverlib.HeaderedEvent, bool, error) CountEventsWithAppServiceID(ctx context.Context, appServiceID string) (int, error) UpdateTxnIDForEvents(ctx context.Context, appserviceID string, maxID, txnID int) error RemoveEventsBeforeAndIncludingID(ctx context.Context, appserviceID string, eventTableID int) error diff --git a/appservice/storage/postgres/appservice_events_table.go b/appservice/storage/postgres/appservice_events_table.go index d72faeeae..d33a83b14 100644 --- a/appservice/storage/postgres/appservice_events_table.go +++ b/appservice/storage/postgres/appservice_events_table.go @@ -33,7 +33,7 @@ CREATE TABLE IF NOT EXISTS appservice_events ( -- The ID of the application service the event will be sent to as_id TEXT NOT NULL, -- JSON representation of the event - event_json TEXT NOT NULL, + headered_event_json TEXT NOT NULL, -- The ID of the transaction that this event is a part of txn_id BIGINT NOT NULL ); @@ -42,14 +42,14 @@ CREATE INDEX IF NOT EXISTS appservice_events_as_id ON appservice_events(as_id); ` const selectEventsByApplicationServiceIDSQL = "" + - "SELECT id, event_json, txn_id " + + "SELECT id, headered_event_json, txn_id " + "FROM appservice_events WHERE as_id = $1 ORDER BY txn_id DESC, id ASC" const countEventsByApplicationServiceIDSQL = "" + "SELECT COUNT(id) FROM appservice_events WHERE as_id = $1" const insertEventSQL = "" + - "INSERT INTO appservice_events(as_id, event_json, txn_id) " + + "INSERT INTO appservice_events(as_id, headered_event_json, txn_id) " + "VALUES ($1, $2, $3)" const updateTxnIDForEventsSQL = "" + @@ -107,7 +107,7 @@ func (s *eventsStatements) selectEventsByApplicationServiceID( limit int, ) ( txnID, maxID int, - events []gomatrixserverlib.Event, + events []gomatrixserverlib.HeaderedEvent, eventsRemaining bool, err error, ) { @@ -132,7 +132,7 @@ func (s *eventsStatements) selectEventsByApplicationServiceID( return } -func retrieveEvents(eventRows *sql.Rows, limit int) (events []gomatrixserverlib.Event, maxID, txnID int, eventsRemaining bool, err error) { +func retrieveEvents(eventRows *sql.Rows, limit int) (events []gomatrixserverlib.HeaderedEvent, maxID, txnID int, eventsRemaining bool, err error) { // Get current time for use in calculating event age nowMilli := time.Now().UnixNano() / int64(time.Millisecond) @@ -141,7 +141,7 @@ func retrieveEvents(eventRows *sql.Rows, limit int) (events []gomatrixserverlib. // new ones. Send back those events first. lastTxnID := invalidTxnID for eventsProcessed := 0; eventRows.Next(); { - var event gomatrixserverlib.Event + var event gomatrixserverlib.HeaderedEvent var eventJSON []byte var id int err = eventRows.Scan( @@ -209,7 +209,7 @@ func (s *eventsStatements) countEventsByApplicationServiceID( func (s *eventsStatements) insertEvent( ctx context.Context, appServiceID string, - event *gomatrixserverlib.Event, + event *gomatrixserverlib.HeaderedEvent, ) (err error) { // Convert event to JSON before inserting eventJSON, err := json.Marshal(event) diff --git a/appservice/storage/postgres/storage.go b/appservice/storage/postgres/storage.go index c47564689..ef92db871 100644 --- a/appservice/storage/postgres/storage.go +++ b/appservice/storage/postgres/storage.go @@ -52,12 +52,12 @@ func (d *Database) prepare() error { return d.txnID.prepare(d.db) } -// StoreEvent takes in a gomatrixserverlib.Event and stores it in the database +// StoreEvent takes in a gomatrixserverlib.HeaderedEvent and stores it in the database // for a transaction worker to pull and later send to an application service. func (d *Database) StoreEvent( ctx context.Context, appServiceID string, - event *gomatrixserverlib.Event, + event *gomatrixserverlib.HeaderedEvent, ) error { return d.events.insertEvent(ctx, appServiceID, event) } @@ -68,7 +68,7 @@ func (d *Database) GetEventsWithAppServiceID( ctx context.Context, appServiceID string, limit int, -) (int, int, []gomatrixserverlib.Event, bool, error) { +) (int, int, []gomatrixserverlib.HeaderedEvent, bool, error) { return d.events.selectEventsByApplicationServiceID(ctx, appServiceID, limit) } diff --git a/appservice/storage/sqlite3/appservice_events_table.go b/appservice/storage/sqlite3/appservice_events_table.go index 846f09f7f..479f2213c 100644 --- a/appservice/storage/sqlite3/appservice_events_table.go +++ b/appservice/storage/sqlite3/appservice_events_table.go @@ -33,7 +33,7 @@ CREATE TABLE IF NOT EXISTS appservice_events ( -- The ID of the application service the event will be sent to as_id TEXT NOT NULL, -- JSON representation of the event - event_json TEXT NOT NULL, + headered_event_json TEXT NOT NULL, -- The ID of the transaction that this event is a part of txn_id INTEGER NOT NULL ); @@ -42,14 +42,14 @@ CREATE INDEX IF NOT EXISTS appservice_events_as_id ON appservice_events(as_id); ` const selectEventsByApplicationServiceIDSQL = "" + - "SELECT id, event_json, txn_id " + + "SELECT id, headered_event_json, txn_id " + "FROM appservice_events WHERE as_id = $1 ORDER BY txn_id DESC, id ASC" const countEventsByApplicationServiceIDSQL = "" + "SELECT COUNT(id) FROM appservice_events WHERE as_id = $1" const insertEventSQL = "" + - "INSERT INTO appservice_events(as_id, event_json, txn_id) " + + "INSERT INTO appservice_events(as_id, headered_event_json, txn_id) " + "VALUES ($1, $2, $3)" const updateTxnIDForEventsSQL = "" + @@ -107,7 +107,7 @@ func (s *eventsStatements) selectEventsByApplicationServiceID( limit int, ) ( txnID, maxID int, - events []gomatrixserverlib.Event, + events []gomatrixserverlib.HeaderedEvent, eventsRemaining bool, err error, ) { @@ -132,7 +132,7 @@ func (s *eventsStatements) selectEventsByApplicationServiceID( return } -func retrieveEvents(eventRows *sql.Rows, limit int) (events []gomatrixserverlib.Event, maxID, txnID int, eventsRemaining bool, err error) { +func retrieveEvents(eventRows *sql.Rows, limit int) (events []gomatrixserverlib.HeaderedEvent, maxID, txnID int, eventsRemaining bool, err error) { // Get current time for use in calculating event age nowMilli := time.Now().UnixNano() / int64(time.Millisecond) @@ -141,7 +141,7 @@ func retrieveEvents(eventRows *sql.Rows, limit int) (events []gomatrixserverlib. // new ones. Send back those events first. lastTxnID := invalidTxnID for eventsProcessed := 0; eventRows.Next(); { - var event gomatrixserverlib.Event + var event gomatrixserverlib.HeaderedEvent var eventJSON []byte var id int err = eventRows.Scan( @@ -209,7 +209,7 @@ func (s *eventsStatements) countEventsByApplicationServiceID( func (s *eventsStatements) insertEvent( ctx context.Context, appServiceID string, - event *gomatrixserverlib.Event, + event *gomatrixserverlib.HeaderedEvent, ) (err error) { // Convert event to JSON before inserting eventJSON, err := json.Marshal(event) diff --git a/appservice/storage/sqlite3/storage.go b/appservice/storage/sqlite3/storage.go index 5040b61b2..d0538e263 100644 --- a/appservice/storage/sqlite3/storage.go +++ b/appservice/storage/sqlite3/storage.go @@ -53,12 +53,12 @@ func (d *Database) prepare() error { return d.txnID.prepare(d.db) } -// StoreEvent takes in a gomatrixserverlib.Event and stores it in the database +// StoreEvent takes in a gomatrixserverlib.HeaderedEvent and stores it in the database // for a transaction worker to pull and later send to an application service. func (d *Database) StoreEvent( ctx context.Context, appServiceID string, - event *gomatrixserverlib.Event, + event *gomatrixserverlib.HeaderedEvent, ) error { return d.events.insertEvent(ctx, appServiceID, event) } @@ -69,7 +69,7 @@ func (d *Database) GetEventsWithAppServiceID( ctx context.Context, appServiceID string, limit int, -) (int, int, []gomatrixserverlib.Event, bool, error) { +) (int, int, []gomatrixserverlib.HeaderedEvent, bool, error) { return d.events.selectEventsByApplicationServiceID(ctx, appServiceID, limit) } diff --git a/appservice/workers/transaction_scheduler.go b/appservice/workers/transaction_scheduler.go index faa1e4a96..10c7ef911 100644 --- a/appservice/workers/transaction_scheduler.go +++ b/appservice/workers/transaction_scheduler.go @@ -181,9 +181,14 @@ func createTransaction( } } + var ev []gomatrixserverlib.Event + for _, e := range events { + ev = append(ev, e.Event) + } + // Create a transaction and store the events inside transaction := gomatrixserverlib.ApplicationServiceTransaction{ - Events: events, + Events: ev, } transactionJSON, err = json.Marshal(transaction) From 314da91f1dff5e4c3921b06180110e7a15b38f22 Mon Sep 17 00:00:00 2001 From: Suvid Sahay <42059321+suvidsahay@users.noreply.github.com> Date: Thu, 26 Mar 2020 15:27:18 +0530 Subject: [PATCH 56/86] Fixed Didn't call Close on UserStreamListener log messages when syncing (#940) Signed-off-by: Suvid Sahay Removing the extra imports --- syncapi/sync/userstream.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/syncapi/sync/userstream.go b/syncapi/sync/userstream.go index 6eef86440..88867005e 100644 --- a/syncapi/sync/userstream.go +++ b/syncapi/sync/userstream.go @@ -21,7 +21,6 @@ import ( "time" "github.com/matrix-org/dendrite/syncapi/types" - "github.com/matrix-org/util" ) // UserStream represents a communication mechanism between the /sync request goroutine @@ -76,7 +75,6 @@ func (s *UserStream) GetListener(ctx context.Context) UserStreamListener { // Lets be a bit paranoid here and check that Close() is being called runtime.SetFinalizer(&listener, func(l *UserStreamListener) { if !l.hasClosed { - util.GetLogger(ctx).Warn("Didn't call Close on UserStreamListener") l.Close() } }) From 05e1ae8745725245ee3b85a588000b0d26bae96c Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 27 Mar 2020 16:28:22 +0000 Subject: [PATCH 57/86] Further room version wiring (#936) * Room version 2 by default, other wiring updates, update gomatrixserverlib * Fix nil pointer exception * Fix some more nil pointer exceptions hopefully * Update gomatrixserverlib * Send all room versions when joining, not just stable ones * Remove room version cquery * Get room version when getting events from the roomserver database * Reset default back to room version 2 * Don't generate event IDs unless needed * Revert "Remove room version cquery" This reverts commit a170d5873360dd059614460acc8b21ab2cda9767. * Query room version in federation API, client API as needed * Improvements to make_join send_join dance * Make room server producers use headered events * Lint tweaks * Update gomatrixserverlib * Versioned SendJoin * Query room version in syncapi backfill * Handle transaction marshalling/unmarshalling within Dendrite * Sorta fix federation (kinda) * whoops commit federation API too * Use NewEventFromTrustedJSON when getting events from the database * Update gomatrixserverlib * Strip headers on federationapi endpoints * Fix bug in clientapi profile room version query * Update gomatrixserverlib * Return more useful error if room version query doesn't find the room * Update gomatrixserverlib * Update gomatrixserverlib * Maybe fix federation * Fix formatting directive * Update sytest whitelist and blacklist * Temporarily disable room versions 3 and 4 until gmsl is fixed * Fix count of EDUs in logging * Update gomatrixserverlib * Update gomatrixserverlib * Update gomatrixserverlib * Rely on EventBuilder in gmsl to generate the event IDs for us * Some review comments fixed * Move function out of common and into gmsl * Comment in federationsender destinationqueue * Update gomatrixserverlib --- clientapi/clientapi.go | 2 +- clientapi/producers/roomserver.go | 41 +-- clientapi/routing/createroom.go | 13 +- clientapi/routing/joinroom.go | 46 +++- clientapi/routing/membership.go | 15 +- clientapi/routing/profile.go | 12 +- clientapi/routing/sendevent.go | 23 +- clientapi/threepid/invites.go | 13 +- cmd/create-room-events/main.go | 8 +- common/events.go | 39 ++- federationapi/federationapi.go | 2 +- federationapi/routing/backfill.go | 8 +- federationapi/routing/invite.go | 19 +- federationapi/routing/join.go | 50 +++- federationapi/routing/leave.go | 27 +- federationapi/routing/send.go | 68 +++-- federationapi/routing/state.go | 12 +- federationapi/routing/threepid.go | 41 ++- federationsender/consumers/roomserver.go | 2 +- federationsender/queue/destinationqueue.go | 14 +- federationsender/queue/queue.go | 2 +- go.mod | 5 +- go.sum | 269 +------------------- roomserver/alias/alias.go | 5 +- roomserver/api/query.go | 33 +++ roomserver/input/authevents.go | 6 +- roomserver/input/events.go | 16 +- roomserver/query/query.go | 31 +++ roomserver/query/query_test.go | 4 +- roomserver/storage/postgres/events_table.go | 13 + roomserver/storage/postgres/rooms_table.go | 7 + roomserver/storage/postgres/storage.go | 15 +- roomserver/storage/sqlite3/events_table.go | 13 + roomserver/storage/sqlite3/rooms_table.go | 7 + roomserver/storage/sqlite3/storage.go | 15 +- roomserver/version/version.go | 6 +- syncapi/routing/messages.go | 23 +- sytest-blacklist | 4 + sytest-whitelist | 10 +- 39 files changed, 532 insertions(+), 407 deletions(-) diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go index bb44e016a..e608b69f3 100644 --- a/clientapi/clientapi.go +++ b/clientapi/clientapi.go @@ -46,7 +46,7 @@ func SetupClientAPIComponent( transactionsCache *transactions.Cache, fedSenderAPI federationSenderAPI.FederationSenderQueryAPI, ) { - roomserverProducer := producers.NewRoomserverProducer(inputAPI) + roomserverProducer := producers.NewRoomserverProducer(inputAPI, queryAPI) typingProducer := producers.NewTypingServerProducer(typingInputAPI) userUpdateProducer := &producers.UserUpdateProducer{ diff --git a/clientapi/producers/roomserver.go b/clientapi/producers/roomserver.go index 0fe2d556b..06af54404 100644 --- a/clientapi/producers/roomserver.go +++ b/clientapi/producers/roomserver.go @@ -24,27 +24,27 @@ import ( // RoomserverProducer produces events for the roomserver to consume. type RoomserverProducer struct { InputAPI api.RoomserverInputAPI + QueryAPI api.RoomserverQueryAPI } // NewRoomserverProducer creates a new RoomserverProducer -func NewRoomserverProducer(inputAPI api.RoomserverInputAPI) *RoomserverProducer { +func NewRoomserverProducer(inputAPI api.RoomserverInputAPI, queryAPI api.RoomserverQueryAPI) *RoomserverProducer { return &RoomserverProducer{ InputAPI: inputAPI, + QueryAPI: queryAPI, } } // SendEvents writes the given events to the roomserver input log. The events are written with KindNew. func (c *RoomserverProducer) SendEvents( - ctx context.Context, events []gomatrixserverlib.Event, sendAsServer gomatrixserverlib.ServerName, + ctx context.Context, events []gomatrixserverlib.HeaderedEvent, sendAsServer gomatrixserverlib.ServerName, txnID *api.TransactionID, ) (string, error) { ires := make([]api.InputRoomEvent, len(events)) for i, event := range events { - roomVersion := gomatrixserverlib.RoomVersionV1 - ires[i] = api.InputRoomEvent{ Kind: api.KindNew, - Event: event.Headered(roomVersion), + Event: event, AuthEventIDs: event.AuthEventIDs(), SendAsServer: string(sendAsServer), TransactionID: txnID, @@ -56,23 +56,20 @@ func (c *RoomserverProducer) SendEvents( // SendEventWithState writes an event with KindNew to the roomserver input log // with the state at the event as KindOutlier before it. func (c *RoomserverProducer) SendEventWithState( - ctx context.Context, state gomatrixserverlib.RespState, event gomatrixserverlib.Event, + ctx context.Context, state gomatrixserverlib.RespState, event gomatrixserverlib.HeaderedEvent, ) error { outliers, err := state.Events() if err != nil { return err } - // TODO: Room version here - roomVersion := gomatrixserverlib.RoomVersionV1 - - ires := make([]api.InputRoomEvent, len(outliers)+1) - for i, outlier := range outliers { - ires[i] = api.InputRoomEvent{ + var ires []api.InputRoomEvent + for _, outlier := range outliers { + ires = append(ires, api.InputRoomEvent{ Kind: api.KindOutlier, - Event: outlier.Headered(roomVersion), + Event: outlier.Headered(event.RoomVersion), AuthEventIDs: outlier.AuthEventIDs(), - } + }) } stateEventIDs := make([]string, len(state.StateEvents)) @@ -80,13 +77,13 @@ func (c *RoomserverProducer) SendEventWithState( stateEventIDs[i] = state.StateEvents[i].EventID() } - ires[len(outliers)] = api.InputRoomEvent{ + ires = append(ires, api.InputRoomEvent{ Kind: api.KindNew, - Event: event.Headered(roomVersion), + Event: event, AuthEventIDs: event.AuthEventIDs(), HasState: true, StateEventIDs: stateEventIDs, - } + }) _, err = c.SendInputRoomEvents(ctx, ires) return err @@ -109,12 +106,16 @@ func (c *RoomserverProducer) SendInputRoomEvents( func (c *RoomserverProducer) SendInvite( ctx context.Context, inviteEvent gomatrixserverlib.Event, ) error { - // TODO: Room version here - roomVersion := gomatrixserverlib.RoomVersionV1 + verReq := api.QueryRoomVersionForRoomRequest{RoomID: inviteEvent.RoomID()} + verRes := api.QueryRoomVersionForRoomResponse{} + err := c.QueryAPI.QueryRoomVersionForRoom(ctx, &verReq, &verRes) + if err != nil { + return err + } request := api.InputRoomEventsRequest{ InputInviteEvents: []api.InputInviteEvent{{ - Event: inviteEvent.Headered(roomVersion), + Event: inviteEvent.Headered(verRes.RoomVersion), }}, } var response api.InputRoomEventsResponse diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index 964200760..ef11e8b3e 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -237,7 +237,7 @@ func createRoom( historyVisibility = historyVisibilityShared } - var builtEvents []gomatrixserverlib.Event + var builtEvents []gomatrixserverlib.HeaderedEvent // send events into the room in order of: // 1- m.room.create @@ -299,7 +299,7 @@ func createRoom( builder.PrevEvents = []gomatrixserverlib.EventReference{builtEvents[i-1].EventReference()} } var ev *gomatrixserverlib.Event - ev, err = buildEvent(&builder, &authEvents, cfg, evTime) + ev, err = buildEvent(&builder, &authEvents, cfg, evTime, roomVersion) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("buildEvent failed") return jsonerror.InternalServerError() @@ -311,7 +311,7 @@ func createRoom( } // Add the event to the list of auth events - builtEvents = append(builtEvents, *ev) + builtEvents = append(builtEvents, (*ev).Headered(roomVersion)) err = authEvents.AddEvent(ev) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("authEvents.AddEvent failed") @@ -368,6 +368,7 @@ func buildEvent( provider gomatrixserverlib.AuthEventProvider, cfg *config.Dendrite, evTime time.Time, + roomVersion gomatrixserverlib.RoomVersion, ) (*gomatrixserverlib.Event, error) { eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(builder) if err != nil { @@ -378,8 +379,10 @@ func buildEvent( return nil, err } builder.AuthEvents = refs - eventID := fmt.Sprintf("$%s:%s", util.RandomString(16), cfg.Matrix.ServerName) - event, err := builder.Build(eventID, evTime, cfg.Matrix.ServerName, cfg.Matrix.KeyID, cfg.Matrix.PrivateKey) + event, err := builder.Build( + evTime, cfg.Matrix.ServerName, cfg.Matrix.KeyID, + cfg.Matrix.PrivateKey, roomVersion, + ) if err != nil { return nil, fmt.Errorf("cannot build event %s : Builder failed to build. %w", builder.Type, err) } diff --git a/clientapi/routing/joinroom.go b/clientapi/routing/joinroom.go index 7643c41f3..745b4eecc 100644 --- a/clientapi/routing/joinroom.go +++ b/clientapi/routing/joinroom.go @@ -27,6 +27,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/roomserver/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" @@ -238,10 +239,17 @@ func (r joinRoomReq) joinRoomUsingServers( return jsonerror.InternalServerError() } - var queryRes roomserverAPI.QueryLatestEventsAndStateResponse + queryRes := roomserverAPI.QueryLatestEventsAndStateResponse{} event, err := common.BuildEvent(r.req.Context(), &eb, r.cfg, r.evTime, r.queryAPI, &queryRes) if err == nil { - if _, err = r.producer.SendEvents(r.req.Context(), []gomatrixserverlib.Event{*event}, r.cfg.Matrix.ServerName, nil); err != nil { + if _, err = r.producer.SendEvents( + r.req.Context(), + []gomatrixserverlib.HeaderedEvent{ + (*event).Headered(queryRes.RoomVersion), + }, + r.cfg.Matrix.ServerName, + nil, + ); err != nil { util.GetLogger(r.req.Context()).WithError(err).Error("r.producer.SendEvents failed") return jsonerror.InternalServerError() } @@ -299,7 +307,17 @@ func (r joinRoomReq) joinRoomUsingServers( // server was invalid this returns an error. // Otherwise this returns a JSONResponse. func (r joinRoomReq) joinRoomUsingServer(roomID string, server gomatrixserverlib.ServerName) (*util.JSONResponse, error) { - respMakeJoin, err := r.federation.MakeJoin(r.req.Context(), server, roomID, r.userID, []int{1}) + // Ask the room server for information about room versions. + var request api.QueryRoomVersionCapabilitiesRequest + var response api.QueryRoomVersionCapabilitiesResponse + if err := r.queryAPI.QueryRoomVersionCapabilities(r.req.Context(), &request, &response); err != nil { + return nil, err + } + var supportedVersions []gomatrixserverlib.RoomVersion + for version := range response.AvailableRoomVersions { + supportedVersions = append(supportedVersions, version) + } + respMakeJoin, err := r.federation.MakeJoin(r.req.Context(), server, roomID, r.userID, supportedVersions) if err != nil { // TODO: Check if the user was not allowed to join the room. return nil, err @@ -312,9 +330,21 @@ func (r joinRoomReq) joinRoomUsingServer(roomID string, server gomatrixserverlib return nil, err } - eventID := fmt.Sprintf("$%s:%s", util.RandomString(16), r.cfg.Matrix.ServerName) + if respMakeJoin.RoomVersion == "" { + respMakeJoin.RoomVersion = gomatrixserverlib.RoomVersionV1 + } + if _, err = respMakeJoin.RoomVersion.EventFormat(); err != nil { + return &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.UnsupportedRoomVersion( + fmt.Sprintf("Room version '%s' is not supported", respMakeJoin.RoomVersion), + ), + }, nil + } + event, err := respMakeJoin.JoinEvent.Build( - eventID, r.evTime, r.cfg.Matrix.ServerName, r.cfg.Matrix.KeyID, r.cfg.Matrix.PrivateKey, + r.evTime, r.cfg.Matrix.ServerName, r.cfg.Matrix.KeyID, + r.cfg.Matrix.PrivateKey, respMakeJoin.RoomVersion, ) if err != nil { util.GetLogger(r.req.Context()).WithError(err).Error("respMakeJoin.JoinEvent.Build failed") @@ -322,7 +352,7 @@ func (r joinRoomReq) joinRoomUsingServer(roomID string, server gomatrixserverlib return &res, nil } - respSendJoin, err := r.federation.SendJoin(r.req.Context(), server, event) + respSendJoin, err := r.federation.SendJoin(r.req.Context(), server, event, respMakeJoin.RoomVersion) if err != nil { return nil, err } @@ -332,7 +362,9 @@ func (r joinRoomReq) joinRoomUsingServer(roomID string, server gomatrixserverlib } if err = r.producer.SendEventWithState( - r.req.Context(), gomatrixserverlib.RespState(respSendJoin.RespState), event, + r.req.Context(), + gomatrixserverlib.RespState(respSendJoin.RespState), + event.Headered(respMakeJoin.RoomVersion), ); err != nil { util.GetLogger(r.req.Context()).WithError(err).Error("gomatrixserverlib.RespState failed") res := jsonerror.InternalServerError() diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index 61d9e8e39..9f386b718 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -29,6 +29,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/threepid" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/roomserver/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" @@ -45,6 +46,15 @@ func SendMembership( queryAPI roomserverAPI.RoomserverQueryAPI, asAPI appserviceAPI.AppServiceQueryAPI, producer *producers.RoomserverProducer, ) util.JSONResponse { + verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} + verRes := api.QueryRoomVersionForRoomResponse{} + if err := queryAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.UnsupportedRoomVersion(err.Error()), + } + } + var body threepid.MembershipRequest if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil { return *reqErr @@ -95,7 +105,10 @@ func SendMembership( } if _, err := producer.SendEvents( - req.Context(), []gomatrixserverlib.Event{*event}, cfg.Matrix.ServerName, nil, + req.Context(), + []gomatrixserverlib.HeaderedEvent{(*event).Headered(verRes.RoomVersion)}, + cfg.Matrix.ServerName, + nil, ); err != nil { util.GetLogger(req.Context()).WithError(err).Error("producer.SendEvents failed") return jsonerror.InternalServerError() diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index b1909e7ab..a51c55ea5 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -338,10 +338,16 @@ func buildMembershipEvents( memberships []authtypes.Membership, newProfile authtypes.Profile, userID string, cfg *config.Dendrite, evTime time.Time, queryAPI api.RoomserverQueryAPI, -) ([]gomatrixserverlib.Event, error) { - evs := []gomatrixserverlib.Event{} +) ([]gomatrixserverlib.HeaderedEvent, error) { + evs := []gomatrixserverlib.HeaderedEvent{} for _, membership := range memberships { + verReq := api.QueryRoomVersionForRoomRequest{RoomID: membership.RoomID} + verRes := api.QueryRoomVersionForRoomResponse{} + if err := queryAPI.QueryRoomVersionForRoom(ctx, &verReq, &verRes); err != nil { + return []gomatrixserverlib.HeaderedEvent{}, err + } + builder := gomatrixserverlib.EventBuilder{ Sender: userID, RoomID: membership.RoomID, @@ -365,7 +371,7 @@ func buildMembershipEvents( return nil, err } - evs = append(evs, *event) + evs = append(evs, (*event).Headered(verRes.RoomVersion)) } return evs, nil diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go index dbfea09c5..5b2cd8ad4 100644 --- a/clientapi/routing/sendevent.go +++ b/clientapi/routing/sendevent.go @@ -27,6 +27,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" + "github.com/sirupsen/logrus" ) // http://matrix.org/docs/spec/client_server/r0.2.0.html#put-matrix-client-r0-rooms-roomid-send-eventtype-txnid @@ -48,6 +49,15 @@ func SendEvent( producer *producers.RoomserverProducer, txnCache *transactions.Cache, ) util.JSONResponse { + verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} + verRes := api.QueryRoomVersionForRoomResponse{} + if err := queryAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.UnsupportedRoomVersion(err.Error()), + } + } + if txnID != nil { // Try to fetch response from transactionsCache if res, ok := txnCache.FetchTransaction(device.AccessToken, *txnID); ok { @@ -71,13 +81,22 @@ func SendEvent( // pass the new event to the roomserver and receive the correct event ID // event ID in case of duplicate transaction is discarded eventID, err := producer.SendEvents( - req.Context(), []gomatrixserverlib.Event{*e}, cfg.Matrix.ServerName, txnAndSessionID, + req.Context(), + []gomatrixserverlib.HeaderedEvent{ + e.Headered(verRes.RoomVersion), + }, + cfg.Matrix.ServerName, + txnAndSessionID, ) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("producer.SendEvents failed") return jsonerror.InternalServerError() } - util.GetLogger(req.Context()).WithField("event_id", eventID).Info("Sent event") + util.GetLogger(req.Context()).WithFields(logrus.Fields{ + "event_id": eventID, + "room_id": roomID, + "room_version": verRes.RoomVersion, + }).Info("Sent event to roomserver") res := util.JSONResponse{ Code: http.StatusOK, diff --git a/clientapi/threepid/invites.go b/clientapi/threepid/invites.go index aa54aa9fa..e34e91b56 100644 --- a/clientapi/threepid/invites.go +++ b/clientapi/threepid/invites.go @@ -353,12 +353,19 @@ func emit3PIDInviteEvent( return err } - var queryRes *api.QueryLatestEventsAndStateResponse - event, err := common.BuildEvent(ctx, builder, cfg, evTime, queryAPI, queryRes) + queryRes := api.QueryLatestEventsAndStateResponse{} + event, err := common.BuildEvent(ctx, builder, cfg, evTime, queryAPI, &queryRes) if err != nil { return err } - _, err = producer.SendEvents(ctx, []gomatrixserverlib.Event{*event}, cfg.Matrix.ServerName, nil) + _, err = producer.SendEvents( + ctx, + []gomatrixserverlib.HeaderedEvent{ + (*event).Headered(queryRes.RoomVersion), + }, + cfg.Matrix.ServerName, + nil, + ) return err } diff --git a/cmd/create-room-events/main.go b/cmd/create-room-events/main.go index ef442a0c4..ebce953ce 100644 --- a/cmd/create-room-events/main.go +++ b/cmd/create-room-events/main.go @@ -103,12 +103,14 @@ func main() { // Build an event and write the event to the output. func buildAndOutput() gomatrixserverlib.EventReference { eventID++ - id := fmt.Sprintf("$%d:%s", eventID, *serverName) now = time.Unix(0, 0) name := gomatrixserverlib.ServerName(*serverName) key := gomatrixserverlib.KeyID(*keyID) - event, err := b.Build(id, now, name, key, privateKey) + event, err := b.Build( + now, name, key, privateKey, + gomatrixserverlib.RoomVersionV1, + ) if err != nil { panic(err) } @@ -127,7 +129,7 @@ func writeEvent(event gomatrixserverlib.Event) { ire.Kind = api.KindNew ire.Event = event.Headered(gomatrixserverlib.RoomVersionV1) authEventIDs := []string{} - for _, ref := range b.AuthEvents { + for _, ref := range b.AuthEvents.([]gomatrixserverlib.EventReference) { authEventIDs = append(authEventIDs, ref.EventID) } ire.AuthEventIDs = authEventIDs diff --git a/common/events.go b/common/events.go index ff46e8e0b..b79998a73 100644 --- a/common/events.go +++ b/common/events.go @@ -17,14 +17,12 @@ package common import ( "context" "errors" - "fmt" "time" "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" ) // ErrRoomNoExists is returned when trying to lookup the state of a room that @@ -42,13 +40,19 @@ func BuildEvent( builder *gomatrixserverlib.EventBuilder, cfg *config.Dendrite, evTime time.Time, queryAPI api.RoomserverQueryAPI, queryRes *api.QueryLatestEventsAndStateResponse, ) (*gomatrixserverlib.Event, error) { + if queryRes == nil { + queryRes = &api.QueryLatestEventsAndStateResponse{} + } + err := AddPrevEventsToEvent(ctx, builder, queryAPI, queryRes) if err != nil { return nil, err } - eventID := fmt.Sprintf("$%s:%s", util.RandomString(16), cfg.Matrix.ServerName) - event, err := builder.Build(eventID, evTime, cfg.Matrix.ServerName, cfg.Matrix.KeyID, cfg.Matrix.PrivateKey) + event, err := builder.Build( + evTime, cfg.Matrix.ServerName, cfg.Matrix.KeyID, + cfg.Matrix.PrivateKey, queryRes.RoomVersion, + ) if err != nil { return nil, err } @@ -72,9 +76,6 @@ func AddPrevEventsToEvent( RoomID: builder.RoomID, StateToFetch: eventsNeeded.Tuples(), } - if queryRes == nil { - queryRes = &api.QueryLatestEventsAndStateResponse{} - } if err = queryAPI.QueryLatestEventsAndState(ctx, &queryReq, queryRes); err != nil { return err } @@ -83,8 +84,12 @@ func AddPrevEventsToEvent( return ErrRoomNoExists } + eventFormat, err := queryRes.RoomVersion.EventFormat() + if err != nil { + return err + } + builder.Depth = queryRes.Depth - builder.PrevEvents = queryRes.LatestEvents authEvents := gomatrixserverlib.NewAuthEvents(nil) @@ -99,7 +104,23 @@ func AddPrevEventsToEvent( if err != nil { return err } - builder.AuthEvents = refs + + switch eventFormat { + case gomatrixserverlib.EventFormatV1: + builder.AuthEvents = refs + builder.PrevEvents = queryRes.LatestEvents + case gomatrixserverlib.EventFormatV2: + v2AuthRefs := []string{} + v2PrevRefs := []string{} + for _, ref := range refs { + v2AuthRefs = append(v2AuthRefs, ref.EventID) + } + for _, ref := range queryRes.LatestEvents { + v2PrevRefs = append(v2PrevRefs, ref.EventID) + } + builder.AuthEvents = v2AuthRefs + builder.PrevEvents = v2PrevRefs + } return nil } diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go index ef57da881..90db95b3a 100644 --- a/federationapi/federationapi.go +++ b/federationapi/federationapi.go @@ -42,7 +42,7 @@ func SetupFederationAPIComponent( asAPI appserviceAPI.AppServiceQueryAPI, federationSenderAPI federationSenderAPI.FederationSenderQueryAPI, ) { - roomserverProducer := producers.NewRoomserverProducer(inputAPI) + roomserverProducer := producers.NewRoomserverProducer(inputAPI, queryAPI) routing.Setup( base.APIMux, base.Cfg, queryAPI, aliasAPI, asAPI, diff --git a/federationapi/routing/backfill.go b/federationapi/routing/backfill.go index a4bc3c67d..72ce0c669 100644 --- a/federationapi/routing/backfill.go +++ b/federationapi/routing/backfill.go @@ -15,6 +15,7 @@ package routing import ( + "encoding/json" "net/http" "strconv" "time" @@ -91,9 +92,14 @@ func Backfill( } } + var eventJSONs []json.RawMessage + for _, e := range evs { + eventJSONs = append(eventJSONs, e.JSON()) + } + txn := gomatrixserverlib.Transaction{ Origin: cfg.Matrix.ServerName, - PDUs: evs, + PDUs: eventJSONs, OriginServerTS: gomatrixserverlib.AsTimestamp(time.Now()), } diff --git a/federationapi/routing/invite.go b/federationapi/routing/invite.go index 946103462..09c3734be 100644 --- a/federationapi/routing/invite.go +++ b/federationapi/routing/invite.go @@ -15,12 +15,13 @@ package routing import ( - "encoding/json" + "context" "net/http" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -35,10 +36,19 @@ func Invite( producer *producers.RoomserverProducer, keys gomatrixserverlib.KeyRing, ) util.JSONResponse { + // Look up the room version for the room. + verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} + verRes := api.QueryRoomVersionForRoomResponse{} + if err := producer.QueryAPI.QueryRoomVersionForRoom(context.Background(), &verReq, &verRes); err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.UnsupportedRoomVersion(err.Error()), + } + } // Decode the event JSON from the request. - var event gomatrixserverlib.Event - if err := json.Unmarshal(request.Content(), &event); err != nil { + event, err := gomatrixserverlib.NewEventFromUntrustedJSON(request.Content(), verRes.RoomVersion) + if err != nil { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.NotJSON("The request body could not be decoded into valid JSON. " + err.Error()), @@ -70,9 +80,10 @@ func Invite( } // Check that the event is signed by the server sending the request. + redacted := event.Redact() verifyRequests := []gomatrixserverlib.VerifyJSONRequest{{ ServerName: event.Origin(), - Message: event.Redact().JSON(), + Message: redacted.JSON(), AtTS: event.OriginServerTS(), }} verifyResults, err := keys.VerifyJSONs(httpReq.Context(), verifyRequests) diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go index 7d48c86dc..a39ff6394 100644 --- a/federationapi/routing/join.go +++ b/federationapi/routing/join.go @@ -15,7 +15,6 @@ package routing import ( - "encoding/json" "net/http" "time" @@ -36,6 +35,15 @@ func MakeJoin( query api.RoomserverQueryAPI, roomID, userID string, ) util.JSONResponse { + verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} + verRes := api.QueryRoomVersionForRoomResponse{} + if err := query.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil { + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: jsonerror.InternalServerError(), + } + } + _, domain, err := gomatrixserverlib.SplitID('@', userID) if err != nil { return util.JSONResponse{ @@ -63,7 +71,9 @@ func MakeJoin( return jsonerror.InternalServerError() } - var queryRes api.QueryLatestEventsAndStateResponse + queryRes := api.QueryLatestEventsAndStateResponse{ + RoomVersion: verRes.RoomVersion, + } event, err := common.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), query, &queryRes) if err == common.ErrRoomNoExists { return util.JSONResponse{ @@ -80,6 +90,7 @@ func MakeJoin( for i := range queryRes.StateEvents { stateEvents[i] = &queryRes.StateEvents[i].Event } + provider := gomatrixserverlib.NewAuthEvents(stateEvents) if err = gomatrixserverlib.Allowed(*event, &provider); err != nil { return util.JSONResponse{ @@ -90,7 +101,10 @@ func MakeJoin( return util.JSONResponse{ Code: http.StatusOK, - JSON: map[string]interface{}{"event": builder}, + JSON: map[string]interface{}{ + "event": builder, + "room_version": verRes.RoomVersion, + }, } } @@ -104,8 +118,18 @@ func SendJoin( keys gomatrixserverlib.KeyRing, roomID, eventID string, ) util.JSONResponse { - var event gomatrixserverlib.Event - if err := json.Unmarshal(request.Content(), &event); err != nil { + verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} + verRes := api.QueryRoomVersionForRoomResponse{} + if err := query.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil { + util.GetLogger(httpReq.Context()).WithError(err).Error("query.QueryRoomVersionForRoom failed") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: jsonerror.InternalServerError(), + } + } + + event, err := gomatrixserverlib.NewEventFromUntrustedJSON(request.Content(), verRes.RoomVersion) + if err != nil { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.NotJSON("The request body could not be decoded into valid JSON. " + err.Error()), @@ -137,9 +161,10 @@ func SendJoin( } // Check that the event is signed by the server sending the request. + redacted := event.Redact() verifyRequests := []gomatrixserverlib.VerifyJSONRequest{{ ServerName: event.Origin(), - Message: event.Redact().JSON(), + Message: redacted.JSON(), AtTS: event.OriginServerTS(), }} verifyResults, err := keys.VerifyJSONs(httpReq.Context(), verifyRequests) @@ -150,7 +175,7 @@ func SendJoin( if verifyResults[0].Error != nil { return util.JSONResponse{ Code: http.StatusForbidden, - JSON: jsonerror.Forbidden("The join must be signed by the server it originated on"), + JSON: jsonerror.Forbidden("Signature check failed: " + verifyResults[0].Error.Error()), } } @@ -178,7 +203,12 @@ func SendJoin( // We are responsible for notifying other servers that the user has joined // the room, so set SendAsServer to cfg.Matrix.ServerName _, err = producer.SendEvents( - httpReq.Context(), []gomatrixserverlib.Event{event}, cfg.Matrix.ServerName, nil, + httpReq.Context(), + []gomatrixserverlib.HeaderedEvent{ + event.Headered(stateAndAuthChainResponse.RoomVersion), + }, + cfg.Matrix.ServerName, + nil, ) if err != nil { util.GetLogger(httpReq.Context()).WithError(err).Error("producer.SendEvents failed") @@ -188,8 +218,8 @@ func SendJoin( return util.JSONResponse{ Code: http.StatusOK, JSON: map[string]interface{}{ - "state": stateAndAuthChainResponse.StateEvents, - "auth_chain": stateAndAuthChainResponse.AuthChainEvents, + "state": gomatrixserverlib.UnwrapEventHeaders(stateAndAuthChainResponse.StateEvents), + "auth_chain": gomatrixserverlib.UnwrapEventHeaders(stateAndAuthChainResponse.AuthChainEvents), }, } } diff --git a/federationapi/routing/leave.go b/federationapi/routing/leave.go index 3eceb6f25..e0a142631 100644 --- a/federationapi/routing/leave.go +++ b/federationapi/routing/leave.go @@ -13,7 +13,6 @@ package routing import ( - "encoding/json" "net/http" "time" @@ -101,8 +100,18 @@ func SendLeave( keys gomatrixserverlib.KeyRing, roomID, eventID string, ) util.JSONResponse { - var event gomatrixserverlib.Event - if err := json.Unmarshal(request.Content(), &event); err != nil { + verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} + verRes := api.QueryRoomVersionForRoomResponse{} + if err := producer.QueryAPI.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.UnsupportedRoomVersion(err.Error()), + } + } + + // Decode the event JSON from the request. + event, err := gomatrixserverlib.NewEventFromUntrustedJSON(request.Content(), verRes.RoomVersion) + if err != nil { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.NotJSON("The request body could not be decoded into valid JSON. " + err.Error()), @@ -134,9 +143,10 @@ func SendLeave( } // Check that the event is signed by the server sending the request. + redacted := event.Redact() verifyRequests := []gomatrixserverlib.VerifyJSONRequest{{ ServerName: event.Origin(), - Message: event.Redact().JSON(), + Message: redacted.JSON(), AtTS: event.OriginServerTS(), }} verifyResults, err := keys.VerifyJSONs(httpReq.Context(), verifyRequests) @@ -166,7 +176,14 @@ func SendLeave( // Send the events to the room server. // We are responsible for notifying other servers that the user has left // the room, so set SendAsServer to cfg.Matrix.ServerName - _, err = producer.SendEvents(httpReq.Context(), []gomatrixserverlib.Event{event}, cfg.Matrix.ServerName, nil) + _, err = producer.SendEvents( + httpReq.Context(), + []gomatrixserverlib.HeaderedEvent{ + event.Headered(verRes.RoomVersion), + }, + cfg.Matrix.ServerName, + nil, + ) if err != nil { util.GetLogger(httpReq.Context()).WithError(err).Error("producer.SendEvents failed") return jsonerror.InternalServerError() diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index d3e060ac2..4c92c7e5e 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -39,7 +39,6 @@ func Send( keys gomatrixserverlib.KeyRing, federation *gomatrixserverlib.FederationClient, ) util.JSONResponse { - t := txnReq{ context: httpReq.Context(), query: query, @@ -47,17 +46,26 @@ func Send( keys: keys, federation: federation, } - if err := json.Unmarshal(request.Content(), &t); err != nil { + + var txnEvents struct { + PDUs []json.RawMessage `json:"pdus"` + EDUs []json.RawMessage `json:"edus"` + } + + if err := json.Unmarshal(request.Content(), &txnEvents); err != nil { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.NotJSON("The request body could not be decoded into valid JSON. " + err.Error()), } } + t.PDUs = txnEvents.PDUs t.Origin = request.Origin() t.TransactionID = txnID t.Destination = cfg.Matrix.ServerName + util.GetLogger(httpReq.Context()).Infof("Received transaction %q containing %d PDUs, %d EDUs", txnID, len(t.PDUs), len(t.EDUs)) + resp, err := t.processTransaction() if err != nil { util.GetLogger(httpReq.Context()).WithError(err).Error("t.processTransaction failed") @@ -80,15 +88,37 @@ type txnReq struct { } func (t *txnReq) processTransaction() (*gomatrixserverlib.RespSend, error) { - // Check the event signatures - if err := gomatrixserverlib.VerifyAllEventSignatures(t.context, t.PDUs, t.keys); err != nil { - return nil, err + var pdus []gomatrixserverlib.HeaderedEvent + for _, pdu := range t.PDUs { + var header struct { + RoomID string `json:"room_id"` + } + if err := json.Unmarshal(pdu, &header); err != nil { + util.GetLogger(t.context).WithError(err).Warn("Transaction: Failed to extract room ID from event") + return nil, err + } + verReq := api.QueryRoomVersionForRoomRequest{RoomID: header.RoomID} + verRes := api.QueryRoomVersionForRoomResponse{} + if err := t.query.QueryRoomVersionForRoom(t.context, &verReq, &verRes); err != nil { + util.GetLogger(t.context).WithError(err).Warn("Transaction: Failed to query room version for room", verReq.RoomID) + return nil, err + } + event, err := gomatrixserverlib.NewEventFromUntrustedJSON(pdu, verRes.RoomVersion) + if err != nil { + util.GetLogger(t.context).WithError(err).Warnf("Transaction: Failed to parse event JSON of event %q", event.EventID()) + return nil, err + } + if err := gomatrixserverlib.VerifyAllEventSignatures(t.context, []gomatrixserverlib.Event{event}, t.keys); err != nil { + util.GetLogger(t.context).WithError(err).Warnf("Transaction: Couldn't validate signature of event %q", event.EventID()) + return nil, err + } + pdus = append(pdus, event.Headered(verRes.RoomVersion)) } // Process the events. results := map[string]gomatrixserverlib.PDUResult{} - for _, e := range t.PDUs { - err := t.processEvent(e) + for _, e := range pdus { + err := t.processEvent(e.Unwrap()) if err != nil { // If the error is due to the event itself being bad then we skip // it and move onto the next event. We report an error so that the @@ -123,7 +153,7 @@ func (t *txnReq) processTransaction() (*gomatrixserverlib.RespSend, error) { } // TODO: Process the EDUs. - + util.GetLogger(t.context).Infof("Processed %d PDUs from transaction %q", len(results), t.TransactionID) return &gomatrixserverlib.RespSend{PDUs: results}, nil } @@ -159,13 +189,13 @@ func (t *txnReq) processEvent(e gomatrixserverlib.Event) error { } if !stateResp.PrevEventsExist { - return t.processEventWithMissingState(e) + return t.processEventWithMissingState(e, stateResp.RoomVersion) } // Check that the event is allowed by the state at the event. var events []gomatrixserverlib.Event for _, headeredEvent := range stateResp.StateEvents { - events = append(events, headeredEvent.Event) + events = append(events, headeredEvent.Unwrap()) } if err := checkAllowedByState(e, events); err != nil { return err @@ -175,7 +205,14 @@ func (t *txnReq) processEvent(e gomatrixserverlib.Event) error { // TODO: Check that the event is allowed by its auth_events. // pass the event to the roomserver - _, err := t.producer.SendEvents(t.context, []gomatrixserverlib.Event{e}, api.DoNotSendToOtherServers, nil) + _, err := t.producer.SendEvents( + t.context, + []gomatrixserverlib.HeaderedEvent{ + e.Headered(stateResp.RoomVersion), + }, + api.DoNotSendToOtherServers, + nil, + ) return err } @@ -190,7 +227,7 @@ func checkAllowedByState(e gomatrixserverlib.Event, stateEvents []gomatrixserver return gomatrixserverlib.Allowed(e, &authUsingState) } -func (t *txnReq) processEventWithMissingState(e gomatrixserverlib.Event) error { +func (t *txnReq) processEventWithMissingState(e gomatrixserverlib.Event, roomVersion gomatrixserverlib.RoomVersion) error { // We are missing the previous events for this events. // This means that there is a gap in our view of the history of the // room. There two ways that we can handle such a gap: @@ -207,7 +244,7 @@ func (t *txnReq) processEventWithMissingState(e gomatrixserverlib.Event) error { // need to fallback to /state. // TODO: Attempt to fill in the gap using /get_missing_events // TODO: Attempt to fetch the state using /state_ids and /events - state, err := t.federation.LookupState(t.context, t.Origin, e.RoomID(), e.EventID()) + state, err := t.federation.LookupState(t.context, t.Origin, e.RoomID(), e.EventID(), roomVersion) if err != nil { return err } @@ -225,7 +262,7 @@ retryAllowedState: if s.EventID() != missing.AuthEventID { continue } - err = t.processEventWithMissingState(s) + err = t.processEventWithMissingState(s, roomVersion) // If there was no error retrieving the event from federation then // we assume that it succeeded, so retry the original state check if err == nil { @@ -236,6 +273,7 @@ retryAllowedState: } return err } + // pass the event along with the state to the roomserver - return t.producer.SendEventWithState(t.context, state, e) + return t.producer.SendEventWithState(t.context, state, e.Headered(roomVersion)) } diff --git a/federationapi/routing/state.go b/federationapi/routing/state.go index 86a1e08d2..6a47882b7 100644 --- a/federationapi/routing/state.go +++ b/federationapi/routing/state.go @@ -129,17 +129,9 @@ func getState( return nil, &util.JSONResponse{Code: http.StatusNotFound, JSON: nil} } - var stateEvents, authEvents []gomatrixserverlib.Event - for _, headeredEvent := range response.StateEvents { - stateEvents = append(stateEvents, headeredEvent.Event) - } - for _, headeredEvent := range response.AuthChainEvents { - authEvents = append(authEvents, headeredEvent.Event) - } - return &gomatrixserverlib.RespState{ - StateEvents: stateEvents, - AuthEvents: authEvents, + StateEvents: gomatrixserverlib.UnwrapEventHeaders(response.StateEvents), + AuthEvents: gomatrixserverlib.UnwrapEventHeaders(response.AuthChainEvents), }, nil } diff --git a/federationapi/routing/threepid.go b/federationapi/routing/threepid.go index 18ebc07e1..da7174730 100644 --- a/federationapi/routing/threepid.go +++ b/federationapi/routing/threepid.go @@ -18,7 +18,6 @@ import ( "context" "encoding/json" "errors" - "fmt" "net/http" "time" @@ -28,6 +27,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/roomserver/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" @@ -68,8 +68,17 @@ func CreateInvitesFrom3PIDInvites( return *reqErr } - evs := []gomatrixserverlib.Event{} + evs := []gomatrixserverlib.HeaderedEvent{} for _, inv := range body.Invites { + verReq := api.QueryRoomVersionForRoomRequest{RoomID: inv.RoomID} + verRes := api.QueryRoomVersionForRoomResponse{} + if err := queryAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.UnsupportedRoomVersion(err.Error()), + } + } + event, err := createInviteFrom3PIDInvite( req.Context(), queryAPI, asAPI, cfg, inv, federation, accountDB, ) @@ -78,7 +87,7 @@ func CreateInvitesFrom3PIDInvites( return jsonerror.InternalServerError() } if event != nil { - evs = append(evs, *event) + evs = append(evs, (*event).Headered(verRes.RoomVersion)) } } @@ -137,6 +146,15 @@ func ExchangeThirdPartyInvite( } } + verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} + verRes := api.QueryRoomVersionForRoomResponse{} + if err = queryAPI.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.UnsupportedRoomVersion(err.Error()), + } + } + // Auth and build the event from what the remote server sent us event, err := buildMembershipEvent(httpReq.Context(), &builder, queryAPI, cfg) if err == errNotInRoom { @@ -159,7 +177,12 @@ func ExchangeThirdPartyInvite( // Send the event to the roomserver if _, err = producer.SendEvents( - httpReq.Context(), []gomatrixserverlib.Event{signedEvent.Event}, cfg.Matrix.ServerName, nil, + httpReq.Context(), + []gomatrixserverlib.HeaderedEvent{ + signedEvent.Event.Headered(verRes.RoomVersion), + }, + cfg.Matrix.ServerName, + nil, ); err != nil { util.GetLogger(httpReq.Context()).WithError(err).Error("producer.SendEvents failed") return jsonerror.InternalServerError() @@ -181,6 +204,12 @@ func createInviteFrom3PIDInvite( inv invite, federation *gomatrixserverlib.FederationClient, accountDB accounts.Database, ) (*gomatrixserverlib.Event, error) { + verReq := api.QueryRoomVersionForRoomRequest{RoomID: inv.RoomID} + verRes := api.QueryRoomVersionForRoomResponse{} + if err := queryAPI.QueryRoomVersionForRoom(ctx, &verReq, &verRes); err != nil { + return nil, err + } + _, server, err := gomatrixserverlib.SplitID('@', inv.MXID) if err != nil { return nil, err @@ -280,9 +309,9 @@ func buildMembershipEvent( } builder.AuthEvents = refs - eventID := fmt.Sprintf("$%s:%s", util.RandomString(16), cfg.Matrix.ServerName) event, err := builder.Build( - eventID, time.Now(), cfg.Matrix.ServerName, cfg.Matrix.KeyID, cfg.Matrix.PrivateKey, + time.Now(), cfg.Matrix.ServerName, cfg.Matrix.KeyID, + cfg.Matrix.PrivateKey, queryRes.RoomVersion, ) return &event, err diff --git a/federationsender/consumers/roomserver.go b/federationsender/consumers/roomserver.go index 2d4b1b2ef..8ab2affe2 100644 --- a/federationsender/consumers/roomserver.go +++ b/federationsender/consumers/roomserver.go @@ -155,7 +155,7 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent) err // Send the event. return s.queues.SendEvent( - &ore.Event.Event, gomatrixserverlib.ServerName(ore.SendAsServer), joinedHostsAtEvent, + &ore.Event, gomatrixserverlib.ServerName(ore.SendAsServer), joinedHostsAtEvent, ) } diff --git a/federationsender/queue/destinationqueue.go b/federationsender/queue/destinationqueue.go index f412cb19b..b4a6da1a3 100644 --- a/federationsender/queue/destinationqueue.go +++ b/federationsender/queue/destinationqueue.go @@ -16,11 +16,13 @@ package queue import ( "context" + "encoding/json" "fmt" "sync" "time" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" log "github.com/sirupsen/logrus" ) @@ -38,14 +40,14 @@ type destinationQueue struct { running bool sentCounter int lastTransactionIDs []gomatrixserverlib.TransactionID - pendingEvents []*gomatrixserverlib.Event + pendingEvents []*gomatrixserverlib.HeaderedEvent pendingEDUs []*gomatrixserverlib.EDU } // Send event adds the event to the pending queue for the destination. // If the queue is empty then it starts a background goroutine to // start sending events to that destination. -func (oq *destinationQueue) sendEvent(ev *gomatrixserverlib.Event) { +func (oq *destinationQueue) sendEvent(ev *gomatrixserverlib.HeaderedEvent) { oq.runningMutex.Lock() defer oq.runningMutex.Unlock() oq.pendingEvents = append(oq.pendingEvents, ev) @@ -80,6 +82,8 @@ func (oq *destinationQueue) backgroundSend() { // TODO: handle retries. // TODO: blacklist uncooperative servers. + util.GetLogger(context.TODO()).Infof("Sending transaction %q containing %d PDUs, %d EDUs", t.TransactionID, len(t.PDUs), len(t.EDUs)) + _, err := oq.client.SendTransaction(context.TODO(), *t) if err != nil { log.WithFields(log.Fields{ @@ -103,7 +107,7 @@ func (oq *destinationQueue) next() *gomatrixserverlib.Transaction { } t := gomatrixserverlib.Transaction{ - PDUs: []gomatrixserverlib.Event{}, + PDUs: []json.RawMessage{}, EDUs: []gomatrixserverlib.EDU{}, } now := gomatrixserverlib.AsTimestamp(time.Now()) @@ -119,7 +123,9 @@ func (oq *destinationQueue) next() *gomatrixserverlib.Transaction { oq.lastTransactionIDs = []gomatrixserverlib.TransactionID{t.TransactionID} for _, pdu := range oq.pendingEvents { - t.PDUs = append(t.PDUs, *pdu) + // Append the JSON of the event, since this is a json.RawMessage type in the + // gomatrixserverlib.Transaction struct + t.PDUs = append(t.PDUs, (*pdu).JSON()) } oq.pendingEvents = nil oq.sentCounter += len(t.PDUs) diff --git a/federationsender/queue/queue.go b/federationsender/queue/queue.go index 6a05c5f07..840fe4afe 100644 --- a/federationsender/queue/queue.go +++ b/federationsender/queue/queue.go @@ -43,7 +43,7 @@ func NewOutgoingQueues(origin gomatrixserverlib.ServerName, client *gomatrixserv // SendEvent sends an event to the destinations func (oqs *OutgoingQueues) SendEvent( - ev *gomatrixserverlib.Event, origin gomatrixserverlib.ServerName, + ev *gomatrixserverlib.HeaderedEvent, origin gomatrixserverlib.ServerName, destinations []gomatrixserverlib.ServerName, ) error { if origin != oqs.origin { diff --git a/go.mod b/go.mod index 3ceaea528..97c0e44ef 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,6 @@ module github.com/matrix-org/dendrite require ( - github.com/btcsuite/btcutil v1.0.1 - github.com/golangci/golangci-lint v1.19.1 // indirect github.com/gorilla/mux v1.7.3 github.com/hashicorp/golang-lru v0.5.4 github.com/lib/pq v1.2.0 @@ -11,7 +9,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200318180716-bc4ff56961e2 + github.com/matrix-org/gomatrixserverlib v0.0.0-20200327155501-33fb4c7049dc github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible @@ -27,6 +25,7 @@ require ( github.com/uber/jaeger-lib v2.2.0+incompatible go.uber.org/atomic v1.6.0 // indirect golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d + golang.org/x/net v0.0.0-20190909003024-a7b16738d86b // indirect gopkg.in/Shopify/sarama.v1 v1.20.1 gopkg.in/h2non/bimg.v1 v1.0.18 gopkg.in/yaml.v2 v2.2.5 diff --git a/go.sum b/go.sum index 48e0efd3e..aeedd6f62 100644 --- a/go.sum +++ b/go.sum @@ -3,18 +3,13 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/zstd v1.4.4 h1:+IawcoXhCBylN7ccwdwf8LOH2jKq7NavGpEPanrlTzE= github.com/DataDog/zstd v1.4.4/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/OpenPeeDeeP/depguard v1.0.1 h1:VlW4R6jmBIv3/u1JNlawEvJMM4J+dPORPaZasQee8Us= -github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmUx/1V+TNhjQvM= github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= -github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -30,82 +25,38 @@ github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVa github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q= github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/frankban/quicktest v1.0.0/go.mod h1:R98jIehRai+d1/3Hv2//jOVCTJhW1VBavT6B6CuGq2k= github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3BTYk= github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-critic/go-critic v0.3.5-0.20190904082202-d79a9f0c64db h1:GYXWx7Vr3+zv833u+8IoXbNnQY0AdXsxAgI0kX7xcwA= -github.com/go-critic/go-critic v0.3.5-0.20190904082202-d79a9f0c64db/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-lintpack/lintpack v0.5.2 h1:DI5mA3+eKdWeJ40nU4d6Wc26qmdG8RCi/btYq0TuRN0= -github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-toolsmith/astcast v1.0.0 h1:JojxlmI6STnFVG9yOImLeGREv8W2ocNUM+iOhR6jE7g= -github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4= -github.com/go-toolsmith/astcopy v1.0.0 h1:OMgl1b1MEpjFQ1m5ztEO06rz5CUd3oBv9RF7+DyvdG8= -github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ= -github.com/go-toolsmith/astequal v0.0.0-20180903214952-dcb477bfacd6/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY= -github.com/go-toolsmith/astequal v1.0.0 h1:4zxD8j3JRFNyLN46lodQuqz3xdKSrur7U/sr0SDS/gQ= -github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY= -github.com/go-toolsmith/astfmt v0.0.0-20180903215011-8f8ee99c3086/go.mod h1:mP93XdblcopXwlyN4X4uodxXQhldPGZbcEJIimQHrkg= -github.com/go-toolsmith/astfmt v1.0.0 h1:A0vDDXt+vsvLEdbMFJAUBI/uTbRw1ffOPnxsILnFL6k= -github.com/go-toolsmith/astfmt v1.0.0/go.mod h1:cnWmsOAuq4jJY6Ct5YWlVLmcmLMn1JUPuQIHCY7CJDw= -github.com/go-toolsmith/astinfo v0.0.0-20180906194353-9809ff7efb21/go.mod h1:dDStQCHtmZpYOmjRP/8gHHnCCch3Zz3oEgCdZVdtweU= -github.com/go-toolsmith/astp v0.0.0-20180903215135-0af7e3c24f30/go.mod h1:SV2ur98SGypH1UjcPpCatrV5hPazG6+IfNHbkDXBRrk= -github.com/go-toolsmith/astp v1.0.0 h1:alXE75TXgcmupDsMK1fRAy0YUzLzqPVvBKoyWV+KPXg= -github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdYrA8meBmLI= -github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8/go.mod h1:WoMrjiy4zvdS+Bg6z9jZH82QXwkcgCBX6nOfnmdaHks= -github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc= -github.com/go-toolsmith/strparse v1.0.0 h1:Vcw78DnpCAKlM20kSbAyO4mPfJn/lyYA4BJUDxe2Jb4= -github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8= -github.com/go-toolsmith/typep v1.0.0 h1:zKymWyA1TRYvqYrYDrfEMZULyrhcnGY3x7LDKU2XQaA= -github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU= -github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= -github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -114,37 +65,6 @@ github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 h1:23T5iq8rbUYlhpt5DB4XJkc6BU31uODLD1o1gKvZmD0= -github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4= -github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a h1:w8hkcTqaFpzKqonE9uMCefW1WDie15eSP/4MssdenaM= -github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk= -github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6 h1:YYWNAGTKWhKpcLLt7aSj/odlKrSrelQwlovBpDuf19w= -github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0= -github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613 h1:9kfjN3AdxcbsZBf8NjltjWihK2QfBBBZuv91cMFfDHw= -github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8= -github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3 h1:pe9JHs3cHHDQgOFXJJdYkK6fLz2PWyYtP4hthoCMvs8= -github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o= -github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee h1:J2XAy40+7yz70uaOiMbNnluTg7gyQhtGqLQncQh+4J8= -github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU= -github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98 h1:0OkFarm1Zy2CjCiDKfK9XHgmc2wbDlRMD2hD8anAJHU= -github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU= -github.com/golangci/golangci-lint v1.19.1 h1:g9xL8KW7UZDCkVlgHYJMA6F4Sj/sRVa0FoCeXI+Z3iM= -github.com/golangci/golangci-lint v1.19.1/go.mod h1:2CEc4Fxx3vxDv7g8DyXkHCBF73AOzAymcJAprs2vCps= -github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc h1:gLLhTLMk2/SutryVJ6D4VZCU3CUqr8YloG7FPIBWFpI= -github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU= -github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217 h1:En/tZdwhAn0JNwLuXzP3k2RVtMqMmOEK7Yu/g3tmtJE= -github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg= -github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca h1:kNY3/svz5T29MYHubXix4aDDuE3RWHkPvopM/EDv/MA= -github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o= -github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770 h1:EL/O5HGrF7Jaq0yNhBLucz9hTuRzj2LdwGBOaENgxIk= -github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA= -github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21 h1:leSNB7iYzLYSSx3J/s5sVf4Drkc68W2wm4Ixh/mr0us= -github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI= -github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0 h1:HVfrLniijszjS1aiNg8JbBMO2+E1WIQ+j/gL4SQqGPg= -github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4= -github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 h1:zwtduBRr5SSWhqsYNgcuWO2kFlpdOZbP0+yRjmvPGys= -github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= @@ -152,24 +72,13 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3 h1:JVnpOZS+qxli+rgVl98ILOXVNbW+kb5wcxeGx8ShUIw= -github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/go-cid v0.0.5 h1:o0Ix8e/ql7Zb5UVUJEUfjsWCIY8t48++9lR8qi6oiJU= github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog= github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= @@ -177,22 +86,16 @@ github.com/jbenet/goprocess v0.1.3 h1:YKyIEECS/XvcfHtBzxtjBBbWK+MbvA6dG8ASiqwvr1 github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= -github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -200,7 +103,6 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= @@ -210,32 +112,14 @@ github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoR github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= github.com/libp2p/go-libp2p-core v0.5.0 h1:FBQ1fpq2Fo/ClyjojVJ5AKXlKhvNc/B6U0O+7AN1ffE= github.com/libp2p/go-libp2p-core v0.5.0/go.mod h1:49XGI+kc38oGVwqSBhDEwytaAxgZasHhFfQKibzTls0= +github.com/libp2p/go-openssl v0.0.4 h1:d27YZvLoTyMhIN4njrkr8zMDOM4lfpHIp6A+TK9fovg= github.com/libp2p/go-openssl v0.0.4/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= -github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= -github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/matoous/godox v0.0.0-20190910121045-032ad8106c86 h1:q6SrfsK4FojRnJ1j8+8OJzyq3g9Y1oSVyL6nYGJXXBk= -github.com/matoous/godox v0.0.0-20190910121045-032ad8106c86/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s= github.com/matrix-org/dendrite v0.0.0-20200220135450-0352f250b857/go.mod h1:DZ35IoR+ViBNVPe9umdlOSnjvKl7wfyRmZg4QfWGvTo= github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 h1:nMX2t7hbGF0NYDYySx0pCqEKGKAeZIiSqlWSspetlhY= github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg= -github.com/matrix-org/go-http-js-libp2p v0.0.0-20200304160008-4ec1129a00c4 h1:oFjG7X1jS473zPDix1/FBZ2qd0anM1Ko4AlJCB6MUZs= -github.com/matrix-org/go-http-js-libp2p v0.0.0-20200304160008-4ec1129a00c4/go.mod h1:PJof7UbOKmVEEMsGqvbyIK0ldMMBjPH5EYia7MHR2RQ= -github.com/matrix-org/go-http-js-libp2p v0.0.0-20200306184112-c0b07c81904f h1:08U2fGZgEMzRYM2QmL2p9r+QmGryYUuZCJOSZWmxzN8= -github.com/matrix-org/go-http-js-libp2p v0.0.0-20200306184112-c0b07c81904f/go.mod h1:/giSXVd8D6DZGSfTmhQrLEoZZwsfkC14kSqP9MiLqIY= -github.com/matrix-org/go-http-js-libp2p v0.0.0-20200306190227-af44d7013315 h1:tExW6I0wSXy0CWz8QvvkQEuvtBO88CFKYf0MYkpKwek= -github.com/matrix-org/go-http-js-libp2p v0.0.0-20200306190227-af44d7013315/go.mod h1:/giSXVd8D6DZGSfTmhQrLEoZZwsfkC14kSqP9MiLqIY= -github.com/matrix-org/go-http-js-libp2p v0.0.0-20200306192008-b9e71eeaa437 h1:zcGpWvVV6swXw9LBMRsdDHPOugQYSwesH2RByUfBx2I= -github.com/matrix-org/go-http-js-libp2p v0.0.0-20200306192008-b9e71eeaa437/go.mod h1:/giSXVd8D6DZGSfTmhQrLEoZZwsfkC14kSqP9MiLqIY= -github.com/matrix-org/go-http-js-libp2p v0.0.0-20200310180544-7f3fad43b51c h1:jj/LIZKMO7GK6O0UarpRwse9L3ZyzozpyMtdPA7ddSk= -github.com/matrix-org/go-http-js-libp2p v0.0.0-20200310180544-7f3fad43b51c/go.mod h1:qK3LUW7RCLhFM7gC3pabj3EXT9A1DsCK33MHstUhhbk= github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f h1:5TOte9uk/epk8L+Pbp6qwaV8YsKYXKjyECPHUhJTWQc= github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f/go.mod h1:qK3LUW7RCLhFM7gC3pabj3EXT9A1DsCK33MHstUhhbk= -github.com/matrix-org/go-sqlite3-js v0.0.0-20200226144546-ea6ed5b90074 h1:UWz6vfhmQVshBuE67X1BCsdMhEDtd+uOz8CJ48Fc0F4= -github.com/matrix-org/go-sqlite3-js v0.0.0-20200226144546-ea6ed5b90074/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= -github.com/matrix-org/go-sqlite3-js v0.0.0-20200304163011-cfb4884075db h1:ERuFJq4DI8fakfBZlvXHltHZ0ix3K5YsLG0tQfQn6TI= -github.com/matrix-org/go-sqlite3-js v0.0.0-20200304163011-cfb4884075db/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658 h1:UlhTKClOgWnSB25Rv+BS/Vc1mRinjNUErfyGEVOBP04= github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20190130130140-385f072fe9af h1:piaIBNQGIHnni27xRB7VKkEwoWCgAmeuYf8pxAyG0bI= @@ -244,40 +128,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:kmRjpmFOenVpOaV/DRlo9p6z/IbOKlUC+hhKsAAh8Qg= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200306154041-df6bb9a3e424 h1:H61lT6ckUFIDl9qb636qNomfo0B52lFt73ecioiqF10= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200306154041-df6bb9a3e424/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200316100021-ac4a53d49690 h1:aQPPypOdoIsquJHhoalRYvKtDoiJfSyyJqOEn6R7yTY= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200316100021-ac4a53d49690/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200316144058-cc6847798a3f h1:JXSvzG4movqXT1KcdX+XZ9HM61m1FW4rIMFKUM9j/Dk= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200316144058-cc6847798a3f/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200316163031-36bb5f40ae9b h1:DdVa6Ledhi5dqwYMw8e9zG+kZmRlnWMgsYOxb3NaFOU= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200316163031-36bb5f40ae9b/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200316173242-f66834855555 h1:7M7QwpGbc5fJ6Uxn62ECsagekKkV36hjvnbaDQGJhls= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200316173242-f66834855555/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200316191245-60f1fa78147c h1:hbu6d/7LOKwPpISyE62cUXapx6askaDtKnpb5dtqr1I= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200316191245-60f1fa78147c/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200316200447-0820dfcdff10 h1:O6EHrD7m9Ah9SjMoHuEHHWh4moKllCYo0HRVtSSGzYc= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200316200447-0820dfcdff10/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200316202926-7ce8b3298af6 h1:ZeRF0DpsnaxwYL7b6U1J3XMyUBpB8GWiNCqz3bWdivs= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200316202926-7ce8b3298af6/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200316203446-64795a12215c h1:s7XUmnqbJTJmMHuZ9Ooad6vKlBsSpIe/C3aJc6G6cy8= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200316203446-64795a12215c/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200316222111-b66d3b63dd85 h1:I7ocq675X1B+RYDUQPvMcYcpfHkdTcreil48r3qs2KE= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200316222111-b66d3b63dd85/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200317103236-134415251e65 h1:llq2yETQhD75ImQhCwHbnivSKOgzOP6QdUqH9sse7XM= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200317103236-134415251e65/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200317114945-9a368ea4620d h1:0GYO2Jye1TNVzsn02IF5tqV80psDi0KIWC4+glH6+/Q= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200317114945-9a368ea4620d/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200317140257-ddc7feaaf2fd h1:n95A8YyiCZ8Nu2beqw4akCaPIRrZr/nesHYDZV8WkXI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200317140257-ddc7feaaf2fd/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200318141718-859b9f256ffd h1:381JBgdNYOQwWelV/LOPqDoUkdSBUCkVZY8Of+n5jAM= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200318141718-859b9f256ffd/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200318145320-bc896516d72a h1:7+e7ManmyiGNqo06sQIRIoUtoV92XWzSbc0o3e4aTGY= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200318145320-bc896516d72a/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200318150006-bc27294f9203 h1:7HkL6bF7/M2cYteNFVtvGW5qjD4wHIiR0HsdCm2Rqao= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200318150006-bc27294f9203/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200318180716-bc4ff56961e2 h1:y4DOMbhgPAnATHJ4lNxTWxIlJG0SlIPhvukx1sQkty4= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200318180716-bc4ff56961e2/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200327155501-33fb4c7049dc h1:qrRu4/AlulnldLiyGpYYm+ELIkrP51XCRlA3txWpN30= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200327155501-33fb4c7049dc/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 h1:W7l5CP4V7wPyPb4tYE11dbmeAOwtFQBTW0rf4OonOS8= @@ -285,14 +137,9 @@ github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5/go.mod h1:lePuOiXL github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= -github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-sqlite3 v2.0.2+incompatible h1:qzw9c2GNT8UFrgWNDhCTqRqYUSmu/Dav/9Z58LGpk7U= github.com/mattn/go-sqlite3 v2.0.2+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= @@ -305,16 +152,10 @@ github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8Rv github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk= -github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/mozilla/tls-observatory v0.0.0-20190404164649-a3c1b6cfecfd/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc= github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= @@ -332,22 +173,15 @@ github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXS github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= -github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d h1:AREM5mwr4u1ORQBMvzfzBgpsctsbQikCVpvC+tX285E= -github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU= github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 h1:BvoENQQU+fZ9uukda/RzCAL/191HHwJA5b13R6diVlY= github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pierrec/lz4 v2.4.1+incompatible h1:mFe7ttWaflA46Mhqh+jUfjp2qTbPYxLB2/OyBppH9dg= github.com/pierrec/lz4 v2.4.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -356,7 +190,6 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U= github.com/prometheus/client_golang v1.4.1 h1:FFSuS004yOQEtDdTq+TAOLP5xUq63KqAFYyOi8zA+Y8= @@ -366,65 +199,32 @@ github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1: github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ= github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/securego/gosec v0.0.0-20190912120752-140048b2a218 h1:O0yPHYL49quNL4Oj2wVq+zbGMu4dAM6iLoOQtm49TrQ= -github.com/securego/gosec v0.0.0-20190912120752-140048b2a218/go.mod h1:q6oYAujd2qyeU4cJqIri4LBIgdHXGvxWHZ1E29HNFRE= -github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc= -github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= -github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= -github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/sourcegraph/go-diff v0.5.1 h1:gO6i5zugwzo1RVTvgvfwCOSVegNuvnNi6bAD1QCmkHs= -github.com/sourcegraph/go-diff v0.5.1/go.mod h1:j2dHj3m8aZgQO8lMTcTnBcXkRRRqi34cd2MNlA9u1mE= +github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU= -github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= @@ -440,9 +240,6 @@ github.com/tidwall/pretty v1.0.1 h1:WE4RBSZ1x6McVVC8S/Md+Qse8YUv6HRObAx6ke00NY8= github.com/tidwall/pretty v1.0.1/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/sjson v1.0.3 h1:DeF+0LZqvIt4fKYw41aPB29ZGlvwVkHKktoXJ1YW9Y8= github.com/tidwall/sjson v1.0.3/go.mod h1:bURseu1nuBkFpIES5cz6zBtjmYeOQmEESshn7VpF15Y= -github.com/timakin/bodyclose v0.0.0-20190721030226-87058b9bfcec h1:AmoEvWAO3nDx1MEcMzPh+GzOOIA5Znpv6++c7bePPY0= -github.com/timakin/bodyclose v0.0.0-20190721030226-87058b9bfcec/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/uber-go/atomic v1.3.0 h1:ylWoWcs+jXihgo3Us1Sdsatf2R6+OlBGm8fexR3oFG4= github.com/uber-go/atomic v1.3.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= github.com/uber/jaeger-client-go v2.15.0+incompatible h1:NP3qsSqNxh8VYr956ur1N/1C1PjvOJnJykCzcD5QHbk= @@ -453,40 +250,20 @@ github.com/uber/jaeger-lib v1.5.0 h1:OHbgr8l656Ub3Fw5k9SWnBfIEwvoHQ+W2y+Aa9D1Uyo github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw= github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/ultraware/funlen v0.0.2 h1:Av96YVBwwNSe4MLR7iI/BIa3VyI7/djnto/pK3Uxbdo= -github.com/ultraware/funlen v0.0.2/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA= -github.com/ultraware/whitespace v0.0.3 h1:S5BCRRB5sttNy0bSOhbpw+0mb+cHiCmWfrvxpEzuUk0= -github.com/ultraware/whitespace v0.0.3/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA= -github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s= -github.com/valyala/quicktemplate v1.2.0/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4= -github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.0 h1:vs7fgriifsPbGdK3bNuMWapNn3qnZhCRXc19NRdq010= go.uber.org/atomic v1.3.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613 h1:MQ/ZZiDsUapFFiMS+vzwXkCTeEKaum+Do5rINYJDmxc= golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d h1:2+ZP7EfsZV7Vvmx3TIqSlSzATMkTAKqM14YGFPoSKjI= @@ -495,20 +272,17 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190301231341-16b79f2e4e95 h1:fY7Dsw114eJN4boqzVSbpVHO6rTdhq6/GnXeu+PKnzU= golang.org/x/net v0.0.0-20190301231341-16b79f2e4e95/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -527,43 +301,26 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 h1:I6FyU15t786LL7oL/hn43zqTuEGr4PN7F4XJ1p4E3Y8= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190911201528-7ad0cfa0b7b5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 h1:ywK/j/KkyTHcdyYSZNXGjMwgmDSfjglYZ3vStQ/gSCU= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190719005602-e377ae9d6386/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= -golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911230505-6bfd74cf029c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190912215617-3720d1ec3678 h1:rM1Udd0CgtYI3KUIhu9ROz0QCqjW+n/ODp/hH7c60Xc= -golang.org/x/tools v0.0.0-20190912215617-3720d1ec3678/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c h1:IGkKhmfzcztjm6gYkykvu/NiS8kaqbCWAEWWAyf8J5U= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= @@ -574,7 +331,6 @@ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoA google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= gopkg.in/Shopify/sarama.v1 v1.20.1 h1:Gi09A3fJXm0Jgt8kuKZ8YK+r60GfYn7MQuEmI3oq6hE= gopkg.in/Shopify/sarama.v1 v1.20.1/go.mod h1:AxnvoaevB2nBjNK17cG61A3LleFcWFwVBHBt+cot4Oc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -584,7 +340,6 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/h2non/bimg.v1 v1.0.18 h1:qn6/RpBHt+7WQqoBcK+aF2puc6nC78eZj5LexxoalT4= gopkg.in/h2non/bimg.v1 v1.0.18/go.mod h1:PgsZL7dLwUbsGm1NYps320GxGgvQNTnecMCZqxV11So= @@ -592,11 +347,9 @@ gopkg.in/h2non/gock.v1 v1.0.14 h1:fTeu9fcUvSnLNacYvYI54h+1/XEteDyHvrVCZEEEYNM= gopkg.in/h2non/gock.v1 v1.0.14/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE= gopkg.in/macaroon.v2 v2.1.0 h1:HZcsjBCzq9t0eBPMKqTN/uSN6JOm78ZJ2INbqcBQOUI= gopkg.in/macaroon.v2 v2.1.0/go.mod h1:OUb+TQP/OP0WOerC2Jp/3CwhIKyIa9kQjuc7H24e6/o= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -604,13 +357,3 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed h1:WX1yoOaKQfddO/mLzdV4wptyWgoH/6hwLs7QHTixo0I= -mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc= -mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b h1:DxJ5nJdkhDlLok9K6qO+5290kphDJbHOQO1DFFFTeBo= -mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4= -mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f h1:Cq7MalBHYACRd6EesksG1Q8EoIAKOsiZviGKbOLIej4= -mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f/go.mod h1:4G1h5nDURzA3bwVMZIVpwbkw+04kSxk3rAtzlimaUJw= -sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4 h1:JPJh2pk3+X4lXAkZIk2RuE/7/FoK9maXw+TNPJhVS/c= -sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= diff --git a/roomserver/alias/alias.go b/roomserver/alias/alias.go index dfb7e76a7..032b8da1c 100644 --- a/roomserver/alias/alias.go +++ b/roomserver/alias/alias.go @@ -17,7 +17,6 @@ package alias import ( "context" "encoding/json" - "fmt" "net/http" "time" @@ -250,10 +249,10 @@ func (r *RoomserverAliasAPI) sendUpdatedAliasesEvent( } // Build the event - eventID := fmt.Sprintf("$%s:%s", util.RandomString(16), r.Cfg.Matrix.ServerName) now := time.Now() event, err := builder.Build( - eventID, now, r.Cfg.Matrix.ServerName, r.Cfg.Matrix.KeyID, r.Cfg.Matrix.PrivateKey, + now, r.Cfg.Matrix.ServerName, r.Cfg.Matrix.KeyID, + r.Cfg.Matrix.PrivateKey, roomVersion, ) if err != nil { return err diff --git a/roomserver/api/query.go b/roomserver/api/query.go index 9003a95a3..3cb1b8a7b 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -261,6 +261,16 @@ type QueryRoomVersionCapabilitiesResponse struct { AvailableRoomVersions map[gomatrixserverlib.RoomVersion]string `json:"available"` } +// QueryRoomVersionForRoom asks for the room version for a given room. +type QueryRoomVersionForRoomRequest struct { + RoomID string `json:"room_id"` +} + +// QueryRoomVersionCapabilitiesResponse is a response to QueryServersInRoomAtEventResponse +type QueryRoomVersionForRoomResponse struct { + RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"` +} + // RoomserverQueryAPI is used to query information from the room server. type RoomserverQueryAPI interface { // Query the latest events and state for a room from the room server. @@ -347,6 +357,13 @@ type RoomserverQueryAPI interface { request *QueryRoomVersionCapabilitiesRequest, response *QueryRoomVersionCapabilitiesResponse, ) error + + // Asks for the room version for a given room. + QueryRoomVersionForRoom( + ctx context.Context, + request *QueryRoomVersionForRoomRequest, + response *QueryRoomVersionForRoomResponse, + ) error } // RoomserverQueryLatestEventsAndStatePath is the HTTP path for the QueryLatestEventsAndState API. @@ -385,6 +402,9 @@ const RoomserverQueryServersInRoomAtEventPath = "/api/roomserver/queryServersInR // RoomserverQueryRoomVersionCapabilitiesPath is the HTTP path for the QueryRoomVersionCapabilities API const RoomserverQueryRoomVersionCapabilitiesPath = "/api/roomserver/queryRoomVersionCapabilities" +// RoomserverQueryRoomVersionCapabilitiesPath is the HTTP path for the QueryRoomVersionCapabilities API +const RoomserverQueryRoomVersionForRoomPath = "/api/roomserver/queryRoomVersionForRoom" + // NewRoomserverQueryAPIHTTP creates a RoomserverQueryAPI implemented by talking to a HTTP POST API. // If httpClient is nil then it uses the http.DefaultClient func NewRoomserverQueryAPIHTTP(roomserverURL string, httpClient *http.Client) RoomserverQueryAPI { @@ -554,3 +574,16 @@ func (h *httpRoomserverQueryAPI) QueryRoomVersionCapabilities( apiURL := h.roomserverURL + RoomserverQueryRoomVersionCapabilitiesPath return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } + +// QueryRoomVersionForRoom implements RoomServerQueryAPI +func (h *httpRoomserverQueryAPI) QueryRoomVersionForRoom( + ctx context.Context, + request *QueryRoomVersionForRoomRequest, + response *QueryRoomVersionForRoomResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "QueryRoomVersionForRoom") + defer span.Finish() + + apiURL := h.roomserverURL + RoomserverQueryRoomVersionForRoomPath + return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} diff --git a/roomserver/input/authevents.go b/roomserver/input/authevents.go index 74be2ed33..456a01c79 100644 --- a/roomserver/input/authevents.go +++ b/roomserver/input/authevents.go @@ -27,7 +27,7 @@ import ( func checkAuthEvents( ctx context.Context, db RoomEventDatabase, - event gomatrixserverlib.Event, + event gomatrixserverlib.HeaderedEvent, authEventIDs []string, ) ([]types.EventNID, error) { // Grab the numeric IDs for the supplied auth state events from the database. @@ -38,7 +38,7 @@ func checkAuthEvents( // TODO: check for duplicate state keys here. // Work out which of the state events we actually need. - stateNeeded := gomatrixserverlib.StateNeededForAuth([]gomatrixserverlib.Event{event}) + stateNeeded := gomatrixserverlib.StateNeededForAuth([]gomatrixserverlib.Event{event.Unwrap()}) // Load the actual auth events from the database. authEvents, err := loadAuthEvents(ctx, db, stateNeeded, authStateEntries) @@ -47,7 +47,7 @@ func checkAuthEvents( } // Check if the event is allowed. - if err = gomatrixserverlib.Allowed(event, &authEvents); err != nil { + if err = gomatrixserverlib.Allowed(event.Event, &authEvents); err != nil { return nil, err } diff --git a/roomserver/input/events.go b/roomserver/input/events.go index ab2bbe1cd..c75a3acd9 100644 --- a/roomserver/input/events.go +++ b/roomserver/input/events.go @@ -96,10 +96,11 @@ func processRoomEvent( input api.InputRoomEvent, ) (eventID string, err error) { // Parse and validate the event JSON - event := input.Event + headered := input.Event + event := headered.Unwrap() // Check that the event passes authentication checks and work out the numeric IDs for the auth events. - authEventNIDs, err := checkAuthEvents(ctx, db, event.Event, input.AuthEventIDs) + authEventNIDs, err := checkAuthEvents(ctx, db, headered, input.AuthEventIDs) if err != nil { return } @@ -107,7 +108,7 @@ func processRoomEvent( if input.TransactionID != nil { tdID := input.TransactionID eventID, err = db.GetTransactionEventID( - ctx, tdID.TransactionID, tdID.SessionID, input.Event.Sender(), + ctx, tdID.TransactionID, tdID.SessionID, event.Sender(), ) // On error OR event with the transaction already processed/processesing if err != nil || eventID != "" { @@ -116,7 +117,7 @@ func processRoomEvent( } // Store the event - roomNID, stateAtEvent, err := db.StoreEvent(ctx, event.Event, input.TransactionID, authEventNIDs) + roomNID, stateAtEvent, err := db.StoreEvent(ctx, event, input.TransactionID, authEventNIDs) if err != nil { return } @@ -131,7 +132,7 @@ func processRoomEvent( if stateAtEvent.BeforeStateSnapshotNID == 0 { // We haven't calculated a state for this event yet. // Lets calculate one. - err = calculateAndSetState(ctx, db, input, roomNID, &stateAtEvent, event.Event) + err = calculateAndSetState(ctx, db, input, roomNID, &stateAtEvent, event) if err != nil { return } @@ -144,7 +145,7 @@ func processRoomEvent( // Update the extremities of the event graph for the room return event.EventID(), updateLatestEvents( - ctx, db, ow, roomNID, stateAtEvent, event.Event, input.SendAsServer, input.TransactionID, + ctx, db, ow, roomNID, stateAtEvent, event, input.SendAsServer, input.TransactionID, ) } @@ -235,7 +236,8 @@ func processInviteEvent( return nil } - outputUpdates, err := updateToInviteMembership(updater, &input.Event.Event, nil) + event := input.Event.Unwrap() + outputUpdates, err := updateToInviteMembership(updater, &event, nil) if err != nil { return err } diff --git a/roomserver/query/query.go b/roomserver/query/query.go index 7fe9dc980..b7cdf1507 100644 --- a/roomserver/query/query.go +++ b/roomserver/query/query.go @@ -123,6 +123,7 @@ func (r *RoomserverQueryAPI) QueryLatestEventsAndState( return nil } response.RoomExists = true + response.RoomVersion = roomVersion var currentStateSnapshotNID types.StateSnapshotNID response.LatestEvents, currentStateSnapshotNID, response.Depth, err = @@ -174,6 +175,7 @@ func (r *RoomserverQueryAPI) QueryStateAfterEvents( return nil } response.RoomExists = true + response.RoomVersion = roomVersion prevStates, err := r.DB.StateAtEventIDs(ctx, request.PrevEventIDs) if err != nil { @@ -713,6 +715,7 @@ func (r *RoomserverQueryAPI) QueryStateAndAuthChain( if err != nil { return err } + response.RoomVersion = roomVersion stateEvents, err := r.loadStateAtEventIDs(ctx, request.PrevEventIDs) if err != nil { @@ -871,6 +874,20 @@ func (r *RoomserverQueryAPI) QueryRoomVersionCapabilities( return nil } +// QueryRoomVersionCapabilities implements api.RoomserverQueryAPI +func (r *RoomserverQueryAPI) QueryRoomVersionForRoom( + ctx context.Context, + request *api.QueryRoomVersionForRoomRequest, + response *api.QueryRoomVersionForRoomResponse, +) error { + roomVersion, err := r.DB.GetRoomVersionForRoom(ctx, request.RoomID) + if err != nil { + return err + } + response.RoomVersion = roomVersion + return nil +} + // SetupHTTP adds the RoomserverQueryAPI handlers to the http.ServeMux. // nolint: gocyclo func (r *RoomserverQueryAPI) SetupHTTP(servMux *http.ServeMux) { @@ -1042,4 +1059,18 @@ func (r *RoomserverQueryAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) + servMux.Handle( + api.RoomserverQueryRoomVersionForRoomPath, + common.MakeInternalAPI("QueryRoomVersionForRoom", func(req *http.Request) util.JSONResponse { + var request api.QueryRoomVersionForRoomRequest + var response api.QueryRoomVersionForRoomResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryRoomVersionForRoom(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) } diff --git a/roomserver/query/query_test.go b/roomserver/query/query_test.go index 76c2e158f..7e040c6fb 100644 --- a/roomserver/query/query_test.go +++ b/roomserver/query/query_test.go @@ -54,7 +54,9 @@ func (db *getEventDB) addFakeEvent(eventID string, authIDs []string) error { return err } - event, err := gomatrixserverlib.NewEventFromTrustedJSON(eventJSON, false) + event, err := gomatrixserverlib.NewEventFromTrustedJSON( + eventJSON, false, gomatrixserverlib.RoomVersionV1, + ) if err != nil { return err } diff --git a/roomserver/storage/postgres/events_table.go b/roomserver/storage/postgres/events_table.go index 0caa8199a..ecc35f37a 100644 --- a/roomserver/storage/postgres/events_table.go +++ b/roomserver/storage/postgres/events_table.go @@ -116,6 +116,9 @@ const bulkSelectEventNIDSQL = "" + const selectMaxEventDepthSQL = "" + "SELECT COALESCE(MAX(depth) + 1, 0) FROM roomserver_events WHERE event_nid = ANY($1)" +const selectRoomNIDForEventNIDSQL = "" + + "SELECT room_nid FROM roomserver_events WHERE event_nid = $1" + type eventStatements struct { insertEventStmt *sql.Stmt selectEventStmt *sql.Stmt @@ -130,6 +133,7 @@ type eventStatements struct { bulkSelectEventIDStmt *sql.Stmt bulkSelectEventNIDStmt *sql.Stmt selectMaxEventDepthStmt *sql.Stmt + selectRoomNIDForEventNIDStmt *sql.Stmt } func (s *eventStatements) prepare(db *sql.DB) (err error) { @@ -152,6 +156,7 @@ func (s *eventStatements) prepare(db *sql.DB) (err error) { {&s.bulkSelectEventIDStmt, bulkSelectEventIDSQL}, {&s.bulkSelectEventNIDStmt, bulkSelectEventNIDSQL}, {&s.selectMaxEventDepthStmt, selectMaxEventDepthSQL}, + {&s.selectRoomNIDForEventNIDStmt, selectRoomNIDForEventNIDSQL}, }.prepare(db) } @@ -417,6 +422,14 @@ func (s *eventStatements) selectMaxEventDepth(ctx context.Context, eventNIDs []t return result, nil } +func (s *eventStatements) selectRoomNIDForEventNID( + ctx context.Context, txn *sql.Tx, eventNID types.EventNID, +) (roomNID types.RoomNID, err error) { + selectStmt := common.TxStmt(txn, s.selectRoomNIDForEventNIDStmt) + err = selectStmt.QueryRowContext(ctx, int64(eventNID)).Scan(&roomNID) + return +} + func eventNIDsAsArray(eventNIDs []types.EventNID) pq.Int64Array { nids := make([]int64, len(eventNIDs)) for i := range eventNIDs { diff --git a/roomserver/storage/postgres/rooms_table.go b/roomserver/storage/postgres/rooms_table.go index 0fd9d5b53..ef5b510c9 100644 --- a/roomserver/storage/postgres/rooms_table.go +++ b/roomserver/storage/postgres/rooms_table.go @@ -18,6 +18,7 @@ package postgres import ( "context" "database/sql" + "errors" "github.com/lib/pq" "github.com/matrix-org/dendrite/common" @@ -176,6 +177,9 @@ func (s *roomStatements) selectRoomVersionForRoomID( var roomVersion gomatrixserverlib.RoomVersion stmt := common.TxStmt(txn, s.selectRoomVersionForRoomIDStmt) err := stmt.QueryRowContext(ctx, roomID).Scan(&roomVersion) + if err == sql.ErrNoRows { + return roomVersion, errors.New("room not found") + } return roomVersion, err } @@ -185,5 +189,8 @@ func (s *roomStatements) selectRoomVersionForRoomNID( var roomVersion gomatrixserverlib.RoomVersion stmt := common.TxStmt(txn, s.selectRoomVersionForRoomNIDStmt) err := stmt.QueryRowContext(ctx, roomNID).Scan(&roomVersion) + if err == sql.ErrNoRows { + return roomVersion, errors.New("room not found") + } return roomVersion, err } diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go index 084b83ce6..83a17b1a1 100644 --- a/roomserver/storage/postgres/storage.go +++ b/roomserver/storage/postgres/storage.go @@ -254,10 +254,21 @@ func (d *Database) Events( } results := make([]types.Event, len(eventJSONs)) for i, eventJSON := range eventJSONs { + var roomNID types.RoomNID + var roomVersion gomatrixserverlib.RoomVersion result := &results[i] result.EventNID = eventJSON.EventNID - // TODO: Use NewEventFromTrustedJSON for efficiency - result.Event, err = gomatrixserverlib.NewEventFromUntrustedJSON(eventJSON.EventJSON) + roomNID, err = d.statements.selectRoomNIDForEventNID(ctx, nil, eventJSON.EventNID) + if err != nil { + return nil, err + } + roomVersion, err = d.statements.selectRoomVersionForRoomNID(ctx, nil, roomNID) + if err != nil { + return nil, err + } + result.Event, err = gomatrixserverlib.NewEventFromTrustedJSON( + eventJSON.EventJSON, false, roomVersion, + ) if err != nil { return nil, err } diff --git a/roomserver/storage/sqlite3/events_table.go b/roomserver/storage/sqlite3/events_table.go index 1e4ed448f..d881fa91f 100644 --- a/roomserver/storage/sqlite3/events_table.go +++ b/roomserver/storage/sqlite3/events_table.go @@ -96,6 +96,9 @@ const bulkSelectEventNIDSQL = "" + const selectMaxEventDepthSQL = "" + "SELECT COALESCE(MAX(depth) + 1, 0) FROM roomserver_events WHERE event_nid IN ($1)" +const selectRoomNIDForEventNIDSQL = "" + + "SELECT room_nid FROM roomserver_events WHERE event_nid = $1" + type eventStatements struct { db *sql.DB insertEventStmt *sql.Stmt @@ -111,6 +114,7 @@ type eventStatements struct { bulkSelectEventReferenceStmt *sql.Stmt bulkSelectEventIDStmt *sql.Stmt bulkSelectEventNIDStmt *sql.Stmt + selectRoomNIDForEventNIDStmt *sql.Stmt } func (s *eventStatements) prepare(db *sql.DB) (err error) { @@ -134,6 +138,7 @@ func (s *eventStatements) prepare(db *sql.DB) (err error) { {&s.bulkSelectEventReferenceStmt, bulkSelectEventReferenceSQL}, {&s.bulkSelectEventIDStmt, bulkSelectEventIDSQL}, {&s.bulkSelectEventNIDStmt, bulkSelectEventNIDSQL}, + {&s.selectRoomNIDForEventNIDStmt, selectRoomNIDForEventNIDSQL}, }.prepare(db) } @@ -472,6 +477,14 @@ func (s *eventStatements) selectMaxEventDepth(ctx context.Context, txn *sql.Tx, return result, nil } +func (s *eventStatements) selectRoomNIDForEventNID( + ctx context.Context, txn *sql.Tx, eventNID types.EventNID, +) (roomNID types.RoomNID, err error) { + selectStmt := common.TxStmt(txn, s.selectRoomNIDForEventNIDStmt) + err = selectStmt.QueryRowContext(ctx, int64(eventNID)).Scan(&roomNID) + return +} + func eventNIDsAsArray(eventNIDs []types.EventNID) string { b, _ := json.Marshal(eventNIDs) return string(b) diff --git a/roomserver/storage/sqlite3/rooms_table.go b/roomserver/storage/sqlite3/rooms_table.go index 512b98137..427eeeb70 100644 --- a/roomserver/storage/sqlite3/rooms_table.go +++ b/roomserver/storage/sqlite3/rooms_table.go @@ -19,6 +19,7 @@ import ( "context" "database/sql" "encoding/json" + "errors" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/types" @@ -168,6 +169,9 @@ func (s *roomStatements) selectRoomVersionForRoomID( var roomVersion gomatrixserverlib.RoomVersion stmt := common.TxStmt(txn, s.selectRoomVersionForRoomIDStmt) err := stmt.QueryRowContext(ctx, roomID).Scan(&roomVersion) + if err == sql.ErrNoRows { + return roomVersion, errors.New("room not found") + } return roomVersion, err } @@ -177,5 +181,8 @@ func (s *roomStatements) selectRoomVersionForRoomNID( var roomVersion gomatrixserverlib.RoomVersion stmt := common.TxStmt(txn, s.selectRoomVersionForRoomNIDStmt) err := stmt.QueryRowContext(ctx, roomNID).Scan(&roomVersion) + if err == sql.ErrNoRows { + return roomVersion, errors.New("room not found") + } return roomVersion, err } diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index 28d608ca5..6d6743393 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -307,10 +307,21 @@ func (d *Database) Events( } results = make([]types.Event, len(eventJSONs)) for i, eventJSON := range eventJSONs { + var roomNID types.RoomNID + var roomVersion gomatrixserverlib.RoomVersion result := &results[i] result.EventNID = eventJSON.EventNID - // TODO: Use NewEventFromTrustedJSON for efficiency - result.Event, err = gomatrixserverlib.NewEventFromUntrustedJSON(eventJSON.EventJSON) + roomNID, err = d.statements.selectRoomNIDForEventNID(ctx, txn, eventJSON.EventNID) + if err != nil { + return err + } + roomVersion, err = d.statements.selectRoomVersionForRoomNID(ctx, txn, roomNID) + if err != nil { + return err + } + result.Event, err = gomatrixserverlib.NewEventFromTrustedJSON( + eventJSON.EventJSON, false, roomVersion, + ) if err != nil { return nil } diff --git a/roomserver/version/version.go b/roomserver/version/version.go index b18df49f5..ed16ecca0 100644 --- a/roomserver/version/version.go +++ b/roomserver/version/version.go @@ -39,8 +39,8 @@ var roomVersions = map[gomatrixserverlib.RoomVersion]RoomVersionDescription{ Stable: true, }, gomatrixserverlib.RoomVersionV2: RoomVersionDescription{ - Supported: false, - Stable: false, + Supported: true, + Stable: true, }, gomatrixserverlib.RoomVersionV3: RoomVersionDescription{ Supported: false, @@ -59,7 +59,7 @@ var roomVersions = map[gomatrixserverlib.RoomVersion]RoomVersionDescription{ // DefaultRoomVersion contains the room version that will, by // default, be used to create new rooms on this server. func DefaultRoomVersion() gomatrixserverlib.RoomVersion { - return gomatrixserverlib.RoomVersionV1 + return gomatrixserverlib.RoomVersionV2 } // RoomVersions returns a map of all known room versions to this diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index 5f2e4f171..c9d62477d 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -362,12 +362,18 @@ func (r *messagesReq) handleNonEmptyEventsSlice(streamEvents []types.StreamEvent // Returns an error if there was an issue with retrieving the list of servers in // the room or sending the request. func (r *messagesReq) backfill(fromEventIDs []string, limit int) ([]gomatrixserverlib.HeaderedEvent, error) { + verReq := api.QueryRoomVersionForRoomRequest{RoomID: r.roomID} + verRes := api.QueryRoomVersionForRoomResponse{} + if err := r.queryAPI.QueryRoomVersionForRoom(r.ctx, &verReq, &verRes); err != nil { + return nil, err + } + srvToBackfillFrom, err := r.serverToBackfillFrom(fromEventIDs) if err != nil { return nil, fmt.Errorf("Cannot find server to backfill from: %w", err) } - pdus := make([]gomatrixserverlib.HeaderedEvent, 0) + headered := make([]gomatrixserverlib.HeaderedEvent, 0) // If the roomserver responded with at least one server that isn't us, // send it a request for backfill. @@ -380,17 +386,20 @@ func (r *messagesReq) backfill(fromEventIDs []string, limit int) ([]gomatrixserv } for _, p := range txn.PDUs { - pdus = append(pdus, p.Headered(gomatrixserverlib.RoomVersionV1)) + event, e := gomatrixserverlib.NewEventFromUntrustedJSON(p, verRes.RoomVersion) + if e != nil { + continue + } + headered = append(headered, event.Headered(verRes.RoomVersion)) } - util.GetLogger(r.ctx).WithField("server", srvToBackfillFrom).WithField("new_events", len(pdus)).Info("Storing new events from backfill") + util.GetLogger(r.ctx).WithField("server", srvToBackfillFrom).WithField("new_events", len(headered)).Info("Storing new events from backfill") // Store the events in the database, while marking them as unfit to show // up in responses to sync requests. - for _, pdu := range pdus { - headered := pdu.Headered(gomatrixserverlib.RoomVersionV1) + for i := range headered { if _, err = r.db.WriteEvent( r.ctx, - &headered, + &headered[i], []gomatrixserverlib.HeaderedEvent{}, []string{}, []string{}, @@ -400,7 +409,7 @@ func (r *messagesReq) backfill(fromEventIDs []string, limit int) ([]gomatrixserv } } - return pdus, nil + return headered, nil } func (r *messagesReq) serverToBackfillFrom(fromEventIDs []string) (gomatrixserverlib.ServerName, error) { diff --git a/sytest-blacklist b/sytest-blacklist index 2df2b3a86..2e6f2057c 100644 --- a/sytest-blacklist +++ b/sytest-blacklist @@ -28,3 +28,7 @@ Outbound federation can backfill events # Blacklisted due to alias work on Synapse Alias creators can delete canonical alias with no ops + +# Blacklisted because we need to implement v2 invite endpoints for room versions +# to be supported (currently fails with M_UNSUPPORTED_ROOM_VERSION) +Inbound federation rejects invites which are not signed by the sender \ No newline at end of file diff --git a/sytest-whitelist b/sytest-whitelist index b9eef9c83..a2e7b2a6d 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -183,7 +183,6 @@ After deactivating account, can't log in with an email Remote room alias queries can handle Unicode Newly joined room is included in an incremental sync after invite Inbound /v1/make_join rejects remote attempts to join local users to rooms -Inbound federation rejects invites which are not signed by the sender Local room members see posted message events Fetching eventstream a second time doesn't yield the message again Local non-members don't see posted message events @@ -223,3 +222,12 @@ GET /joined_rooms lists newly-created room Message history can be paginated over federation GET /rooms/:room_id/messages returns a message Remote user can backfill in a room with version 1 +POST /createRoom creates a room with the given version +POST /createRoom rejects attempts to create rooms with numeric versions +POST /createRoom rejects attempts to create rooms with unknown versions +User can create and send/receive messages in a room with version 2 +local user can join room with version 2 +remote user can join room with version 2 +User can invite local user to room with version 2 +Remote user can backfill in a room with version 2 +Inbound federation accepts attempts to join v2 rooms from servers with support \ No newline at end of file From 664f31ec988180341f007b4479ea83a19e68d8e7 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 30 Mar 2020 09:51:45 +0100 Subject: [PATCH 58/86] Ensure state res results are unique --- roomserver/state/state.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roomserver/state/state.go b/roomserver/state/state.go index b8e3e18a1..94873dbeb 100644 --- a/roomserver/state/state.go +++ b/roomserver/state/state.go @@ -672,7 +672,7 @@ func (v StateResolution) calculateStateAfterManyEvents( return } algorithm = "full_state_with_conflicts" - state = resolved + state = resolved[:util.SortAndUnique(stateEntrySorter(resolved))] } else { algorithm = "full_state_no_conflicts" // 6) There weren't any conflicts From f72b759426c1895b76c16ace526cc788713c2fea Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 30 Mar 2020 12:47:09 +0100 Subject: [PATCH 59/86] Update to latest go-sqite-js --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 97c0e44ef..8743437ec 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/libp2p/go-libp2p-core v0.5.0 github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f - github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658 + github.com/matrix-org/go-sqlite3-js v0.0.0-20200325174927-327088cdef10 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 github.com/matrix-org/gomatrixserverlib v0.0.0-20200327155501-33fb4c7049dc github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 diff --git a/go.sum b/go.sum index aeedd6f62..1294f3b7b 100644 --- a/go.sum +++ b/go.sum @@ -122,6 +122,8 @@ github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f h1:5T github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f/go.mod h1:qK3LUW7RCLhFM7gC3pabj3EXT9A1DsCK33MHstUhhbk= github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658 h1:UlhTKClOgWnSB25Rv+BS/Vc1mRinjNUErfyGEVOBP04= github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= +github.com/matrix-org/go-sqlite3-js v0.0.0-20200325174927-327088cdef10 h1:SnhC7/o87ueVwEWI3mUYtrs+s8VnYq3KZtpWsFQOLFE= +github.com/matrix-org/go-sqlite3-js v0.0.0-20200325174927-327088cdef10/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20190130130140-385f072fe9af h1:piaIBNQGIHnni27xRB7VKkEwoWCgAmeuYf8pxAyG0bI= github.com/matrix-org/gomatrix v0.0.0-20190130130140-385f072fe9af/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bhrnp3Ky1qgx/fzCtCALOoGYylh2tpS9K4= From 11a8059bba9dbdc855f10e3b380c4a2f245635a2 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Mon, 30 Mar 2020 15:02:20 +0100 Subject: [PATCH 60/86] Rename the typing server to EDU server (#948) * Blunt move and sed rename * Rename common/ refs to typing * Rename internal stuff in eduserver * Rename docs and scripts * Rename constants/filenames, goimports everything to re-order imports --- WIRING.md | 6 ++-- clientapi/clientapi.go | 8 +++--- .../{typingserver.go => eduserver.go} | 18 ++++++------ clientapi/routing/routing.go | 4 +-- clientapi/routing/sendtyping.go | 6 ++-- cmd/dendrite-client-api-server/main.go | 8 +++--- .../main.go | 10 +++---- cmd/dendrite-monolith-server/main.go | 8 +++--- cmd/dendritejs/main.go | 8 +++--- common/basecomponent/base.go | 8 +++--- common/config/config.go | 20 ++++++------- common/config/config_test.go | 2 +- common/test/config.go | 4 +-- dendrite-config.yaml | 4 +-- docker/README.md | 2 +- docker/dendrite-docker.yml | 4 +-- docker/docker-compose.yml | 8 +++--- docker/services/edu-server.sh | 5 ++++ docker/services/typing-server.sh | 5 ---- {typingserver => eduserver}/api/input.go | 28 +++++++++---------- {typingserver => eduserver}/api/output.go | 0 {typingserver => eduserver}/cache/cache.go | 26 ++++++++--------- .../cache/cache_test.go | 12 ++++---- .../typingserver.go => eduserver/eduserver.go | 22 +++++++-------- {typingserver => eduserver}/input/input.go | 22 +++++++-------- .../{typingserver.go => eduserver.go} | 12 ++++---- .../{typingserver.go => eduserver.go} | 12 ++++---- syncapi/storage/interface.go | 2 +- syncapi/storage/postgres/syncserver.go | 16 +++++------ syncapi/storage/sqlite3/syncserver.go | 16 +++++------ 30 files changed, 153 insertions(+), 153 deletions(-) rename clientapi/producers/{typingserver.go => eduserver.go} (71%) rename cmd/{dendrite-typing-server => dendrite-edu-server}/main.go (72%) create mode 100644 docker/services/edu-server.sh delete mode 100644 docker/services/typing-server.sh rename {typingserver => eduserver}/api/input.go (71%) rename {typingserver => eduserver}/api/output.go (100%) rename {typingserver => eduserver}/cache/cache.go (87%) rename {typingserver => eduserver}/cache/cache_test.go (88%) rename typingserver/typingserver.go => eduserver/eduserver.go (66%) rename {typingserver => eduserver}/input/input.go (81%) rename federationsender/consumers/{typingserver.go => eduserver.go} (87%) rename syncapi/consumers/{typingserver.go => eduserver.go} (90%) diff --git a/WIRING.md b/WIRING.md index bddb1614c..8ec5b0432 100644 --- a/WIRING.md +++ b/WIRING.md @@ -72,7 +72,7 @@ Diagram: | | | | | | +---+ | | | | +----------| S | | | - | | | Typing +---+ | | + | | | EDU +---+ | | | |>=========================================>| Server |>=====================>| | +------------+ | | +----------+ +---+ | | @@ -156,7 +156,7 @@ choke-point to implement ratelimiting and backoff correctly. * It may be impossible to implement without folding it into the Room Server forever coupling the components together. -## Typing Server +## EDU Server * Reads new updates to typing from the logs written by the FS and CTS. * Updates the current list of people typing in a room. @@ -179,7 +179,7 @@ choke-point to implement ratelimiting and backoff correctly. * Reads new events and the current state of the rooms from logs written by the Room Server. * Reads new receipts positions from the logs written by the Receipts Server. * Reads changes to presence from the logs written by the Presence Server. - * Reads changes to typing from the logs written by the Typing Server. + * Reads changes to typing from the logs written by the EDU Server. * Writes when a client starts and stops syncing to the logs. ## Client Search diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go index e608b69f3..1339f7c8c 100644 --- a/clientapi/clientapi.go +++ b/clientapi/clientapi.go @@ -23,9 +23,9 @@ import ( "github.com/matrix-org/dendrite/clientapi/routing" "github.com/matrix-org/dendrite/common/basecomponent" "github.com/matrix-org/dendrite/common/transactions" + eduServerAPI "github.com/matrix-org/dendrite/eduserver/api" federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" - typingServerAPI "github.com/matrix-org/dendrite/typingserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" ) @@ -41,13 +41,13 @@ func SetupClientAPIComponent( aliasAPI roomserverAPI.RoomserverAliasAPI, inputAPI roomserverAPI.RoomserverInputAPI, queryAPI roomserverAPI.RoomserverQueryAPI, - typingInputAPI typingServerAPI.TypingServerInputAPI, + eduInputAPI eduServerAPI.EDUServerInputAPI, asAPI appserviceAPI.AppServiceQueryAPI, transactionsCache *transactions.Cache, fedSenderAPI federationSenderAPI.FederationSenderQueryAPI, ) { roomserverProducer := producers.NewRoomserverProducer(inputAPI, queryAPI) - typingProducer := producers.NewTypingServerProducer(typingInputAPI) + eduProducer := producers.NewEDUServerProducer(eduInputAPI) userUpdateProducer := &producers.UserUpdateProducer{ Producer: base.KafkaProducer, @@ -69,6 +69,6 @@ func SetupClientAPIComponent( routing.Setup( base.APIMux, base.Cfg, roomserverProducer, queryAPI, aliasAPI, asAPI, accountsDB, deviceDB, federation, *keyRing, userUpdateProducer, - syncProducer, typingProducer, transactionsCache, fedSenderAPI, + syncProducer, eduProducer, transactionsCache, fedSenderAPI, ) } diff --git a/clientapi/producers/typingserver.go b/clientapi/producers/eduserver.go similarity index 71% rename from clientapi/producers/typingserver.go rename to clientapi/producers/eduserver.go index f4d0bcba7..14414ec64 100644 --- a/clientapi/producers/typingserver.go +++ b/clientapi/producers/eduserver.go @@ -16,24 +16,24 @@ import ( "context" "time" - "github.com/matrix-org/dendrite/typingserver/api" + "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/gomatrixserverlib" ) -// TypingServerProducer produces events for the typing server to consume -type TypingServerProducer struct { - InputAPI api.TypingServerInputAPI +// EDUServerProducer produces events for the typing server to consume +type EDUServerProducer struct { + InputAPI api.EDUServerInputAPI } -// NewTypingServerProducer creates a new TypingServerProducer -func NewTypingServerProducer(inputAPI api.TypingServerInputAPI) *TypingServerProducer { - return &TypingServerProducer{ +// NewEDUServerProducer creates a new EDUServerProducer +func NewEDUServerProducer(inputAPI api.EDUServerInputAPI) *EDUServerProducer { + return &EDUServerProducer{ InputAPI: inputAPI, } } -// Send typing event to typing server -func (p *TypingServerProducer) Send( +// SendTyping sends a typing event to EDU server +func (p *EDUServerProducer) SendTyping( ctx context.Context, userID, roomID string, typing bool, timeout int64, ) error { diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 22ff12b02..91a1588cb 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -58,7 +58,7 @@ func Setup( keyRing gomatrixserverlib.KeyRing, userUpdateProducer *producers.UserUpdateProducer, syncProducer *producers.SyncAPIProducer, - typingProducer *producers.TypingServerProducer, + eduProducer *producers.EDUServerProducer, transactionsCache *transactions.Cache, federationSender federationSenderAPI.FederationSenderQueryAPI, ) { @@ -235,7 +235,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SendTyping(req, device, vars["roomID"], vars["userID"], accountDB, typingProducer) + return SendTyping(req, device, vars["roomID"], vars["userID"], accountDB, eduProducer) }), ).Methods(http.MethodPut, http.MethodOptions) diff --git a/clientapi/routing/sendtyping.go b/clientapi/routing/sendtyping.go index 29953c32d..ffaa0e662 100644 --- a/clientapi/routing/sendtyping.go +++ b/clientapi/routing/sendtyping.go @@ -35,7 +35,7 @@ type typingContentJSON struct { func SendTyping( req *http.Request, device *authtypes.Device, roomID string, userID string, accountDB accounts.Database, - typingProducer *producers.TypingServerProducer, + eduProducer *producers.EDUServerProducer, ) util.JSONResponse { if device.UserID != userID { return util.JSONResponse{ @@ -69,10 +69,10 @@ func SendTyping( return *resErr } - if err = typingProducer.Send( + if err = eduProducer.SendTyping( req.Context(), userID, roomID, r.Typing, r.Timeout, ); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("typingProducer.Send failed") + util.GetLogger(req.Context()).WithError(err).Error("eduProducer.Send failed") return jsonerror.InternalServerError() } diff --git a/cmd/dendrite-client-api-server/main.go b/cmd/dendrite-client-api-server/main.go index 2bde0f4cf..a7e241b13 100644 --- a/cmd/dendrite-client-api-server/main.go +++ b/cmd/dendrite-client-api-server/main.go @@ -19,8 +19,8 @@ import ( "github.com/matrix-org/dendrite/common/basecomponent" "github.com/matrix-org/dendrite/common/keydb" "github.com/matrix-org/dendrite/common/transactions" - "github.com/matrix-org/dendrite/typingserver" - "github.com/matrix-org/dendrite/typingserver/cache" + "github.com/matrix-org/dendrite/eduserver" + "github.com/matrix-org/dendrite/eduserver/cache" ) func main() { @@ -38,11 +38,11 @@ func main() { asQuery := base.CreateHTTPAppServiceAPIs() alias, input, query := base.CreateHTTPRoomserverAPIs() fedSenderAPI := base.CreateHTTPFederationSenderAPIs() - typingInputAPI := typingserver.SetupTypingServerComponent(base, cache.NewTypingCache()) + eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New()) clientapi.SetupClientAPIComponent( base, deviceDB, accountDB, federation, &keyRing, - alias, input, query, typingInputAPI, asQuery, transactions.New(), fedSenderAPI, + alias, input, query, eduInputAPI, asQuery, transactions.New(), fedSenderAPI, ) base.SetupAndServeHTTP(string(base.Cfg.Bind.ClientAPI), string(base.Cfg.Listen.ClientAPI)) diff --git a/cmd/dendrite-typing-server/main.go b/cmd/dendrite-edu-server/main.go similarity index 72% rename from cmd/dendrite-typing-server/main.go rename to cmd/dendrite-edu-server/main.go index 461eb7144..a4511f1ba 100644 --- a/cmd/dendrite-typing-server/main.go +++ b/cmd/dendrite-edu-server/main.go @@ -16,22 +16,22 @@ import ( _ "net/http/pprof" "github.com/matrix-org/dendrite/common/basecomponent" - "github.com/matrix-org/dendrite/typingserver" - "github.com/matrix-org/dendrite/typingserver/cache" + "github.com/matrix-org/dendrite/eduserver" + "github.com/matrix-org/dendrite/eduserver/cache" "github.com/sirupsen/logrus" ) func main() { cfg := basecomponent.ParseFlags() - base := basecomponent.NewBaseDendrite(cfg, "TypingServerAPI") + base := basecomponent.NewBaseDendrite(cfg, "EDUServerAPI") defer func() { if err := base.Close(); err != nil { logrus.WithError(err).Warn("BaseDendrite close failed") } }() - typingserver.SetupTypingServerComponent(base, cache.NewTypingCache()) + eduserver.SetupEDUServerComponent(base, cache.New()) - base.SetupAndServeHTTP(string(base.Cfg.Bind.TypingServer), string(base.Cfg.Listen.TypingServer)) + base.SetupAndServeHTTP(string(base.Cfg.Bind.EDUServer), string(base.Cfg.Listen.EDUServer)) } diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 27c3054b8..0aceef029 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -24,14 +24,14 @@ import ( "github.com/matrix-org/dendrite/common/basecomponent" "github.com/matrix-org/dendrite/common/keydb" "github.com/matrix-org/dendrite/common/transactions" + "github.com/matrix-org/dendrite/eduserver" + "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/federationsender" "github.com/matrix-org/dendrite/mediaapi" "github.com/matrix-org/dendrite/publicroomsapi" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/syncapi" - "github.com/matrix-org/dendrite/typingserver" - "github.com/matrix-org/dendrite/typingserver/cache" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" @@ -56,7 +56,7 @@ func main() { keyRing := keydb.CreateKeyRing(federation.Client, keyDB) alias, input, query := roomserver.SetupRoomServerComponent(base) - typingInputAPI := typingserver.SetupTypingServerComponent(base, cache.NewTypingCache()) + eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New()) asQuery := appservice.SetupAppServiceAPIComponent( base, accountDB, deviceDB, federation, alias, query, transactions.New(), ) @@ -65,7 +65,7 @@ func main() { clientapi.SetupClientAPIComponent( base, deviceDB, accountDB, federation, &keyRing, alias, input, query, - typingInputAPI, asQuery, transactions.New(), fedSenderAPI, + eduInputAPI, asQuery, transactions.New(), fedSenderAPI, ) federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fedSenderAPI) mediaapi.SetupMediaAPIComponent(base, deviceDB) diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 7c8526715..0f72dc1ec 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -27,14 +27,14 @@ import ( "github.com/matrix-org/dendrite/common/basecomponent" "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/common/transactions" + "github.com/matrix-org/dendrite/eduserver" + "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/federationsender" "github.com/matrix-org/dendrite/mediaapi" "github.com/matrix-org/dendrite/publicroomsapi" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/syncapi" - "github.com/matrix-org/dendrite/typingserver" - "github.com/matrix-org/dendrite/typingserver/cache" "github.com/matrix-org/go-http-js-libp2p/go_http_js_libp2p" "github.com/matrix-org/gomatrixserverlib" @@ -122,7 +122,7 @@ func main() { p2pPublicRoomProvider := NewLibP2PPublicRoomsProvider(node) alias, input, query := roomserver.SetupRoomServerComponent(base) - typingInputAPI := typingserver.SetupTypingServerComponent(base, cache.NewTypingCache()) + eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New()) asQuery := appservice.SetupAppServiceAPIComponent( base, accountDB, deviceDB, federation, alias, query, transactions.New(), ) @@ -131,7 +131,7 @@ func main() { clientapi.SetupClientAPIComponent( base, deviceDB, accountDB, federation, &keyRing, alias, input, query, - typingInputAPI, asQuery, transactions.New(), fedSenderAPI, + eduInputAPI, asQuery, transactions.New(), fedSenderAPI, ) federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fedSenderAPI) mediaapi.SetupMediaAPIComponent(base, deviceDB) diff --git a/common/basecomponent/base.go b/common/basecomponent/base.go index d1d953f7b..8d559f4dc 100644 --- a/common/basecomponent/base.go +++ b/common/basecomponent/base.go @@ -35,9 +35,9 @@ import ( appserviceAPI "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/common/config" + eduServerAPI "github.com/matrix-org/dendrite/eduserver/api" federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" - typingServerAPI "github.com/matrix-org/dendrite/typingserver/api" "github.com/sirupsen/logrus" ) @@ -111,10 +111,10 @@ func (b *BaseDendrite) CreateHTTPRoomserverAPIs() ( return alias, input, query } -// CreateHTTPTypingServerAPIs returns typingInputAPI for hitting the typing +// CreateHTTPEDUServerAPIs returns eduInputAPI for hitting the EDU // server over HTTP -func (b *BaseDendrite) CreateHTTPTypingServerAPIs() typingServerAPI.TypingServerInputAPI { - return typingServerAPI.NewTypingServerInputAPIHTTP(b.Cfg.TypingServerURL(), nil) +func (b *BaseDendrite) CreateHTTPEDUServerAPIs() eduServerAPI.EDUServerInputAPI { + return eduServerAPI.NewEDUServerInputAPIHTTP(b.Cfg.EDUServerURL(), nil) } // CreateHTTPFederationSenderAPIs returns FederationSenderQueryAPI for hitting diff --git a/common/config/config.go b/common/config/config.go index bd83cbf8b..e2f5e6635 100644 --- a/common/config/config.go +++ b/common/config/config.go @@ -134,7 +134,7 @@ type Dendrite struct { OutputRoomEvent Topic `yaml:"output_room_event"` // Topic for sending account data from client API to sync API OutputClientData Topic `yaml:"output_client_data"` - // Topic for typingserver/api.OutputTypingEvent events. + // Topic for eduserver/api.OutputTypingEvent events. OutputTypingEvent Topic `yaml:"output_typing_event"` // Topic for user updates (profile, presence) UserUpdates Topic `yaml:"user_updates"` @@ -206,7 +206,7 @@ type Dendrite struct { RoomServer Address `yaml:"room_server"` FederationSender Address `yaml:"federation_sender"` PublicRoomsAPI Address `yaml:"public_rooms_api"` - TypingServer Address `yaml:"typing_server"` + EDUServer Address `yaml:"edu_server"` } `yaml:"bind"` // The addresses for talking to other microservices. @@ -219,7 +219,7 @@ type Dendrite struct { RoomServer Address `yaml:"room_server"` FederationSender Address `yaml:"federation_sender"` PublicRoomsAPI Address `yaml:"public_rooms_api"` - TypingServer Address `yaml:"typing_server"` + EDUServer Address `yaml:"edu_server"` } `yaml:"listen"` // The config for tracing the dendrite servers. @@ -571,7 +571,7 @@ func (config *Dendrite) checkListen(configErrs *configErrors) { checkNotEmpty(configErrs, "listen.federation_api", string(config.Listen.FederationAPI)) checkNotEmpty(configErrs, "listen.sync_api", string(config.Listen.SyncAPI)) checkNotEmpty(configErrs, "listen.room_server", string(config.Listen.RoomServer)) - checkNotEmpty(configErrs, "listen.typing_server", string(config.Listen.TypingServer)) + checkNotEmpty(configErrs, "listen.edu_server", string(config.Listen.EDUServer)) } // checkLogging verifies the parameters logging.* are valid. @@ -669,7 +669,7 @@ func fingerprintPEM(data []byte) *gomatrixserverlib.TLSFingerprint { // AppServiceURL returns a HTTP URL for where the appservice component is listening. func (config *Dendrite) AppServiceURL() string { - // Hard code the roomserver to talk HTTP for now. + // Hard code the appservice server to talk HTTP for now. // If we support HTTPS we need to think of a practical way to do certificate validation. // People setting up servers shouldn't need to get a certificate valid for the public // internet for an internal API. @@ -685,18 +685,18 @@ func (config *Dendrite) RoomServerURL() string { return "http://" + string(config.Listen.RoomServer) } -// TypingServerURL returns an HTTP URL for where the typing server is listening. -func (config *Dendrite) TypingServerURL() string { - // Hard code the typing server to talk HTTP for now. +// EDUServerURL returns an HTTP URL for where the EDU server is listening. +func (config *Dendrite) EDUServerURL() string { + // Hard code the EDU server to talk HTTP for now. // If we support HTTPS we need to think of a practical way to do certificate validation. // People setting up servers shouldn't need to get a certificate valid for the public // internet for an internal API. - return "http://" + string(config.Listen.TypingServer) + return "http://" + string(config.Listen.EDUServer) } // FederationSenderURL returns an HTTP URL for where the federation sender is listening. func (config *Dendrite) FederationSenderURL() string { - // Hard code the typing server to talk HTTP for now. + // Hard code the federation sender server to talk HTTP for now. // If we support HTTPS we need to think of a practical way to do certificate validation. // People setting up servers shouldn't need to get a certificate valid for the public // internet for an internal API. diff --git a/common/config/config_test.go b/common/config/config_test.go index 110c8b84c..b72f5fad0 100644 --- a/common/config/config_test.go +++ b/common/config/config_test.go @@ -62,7 +62,7 @@ listen: sync_api: "localhost:7773" media_api: "localhost:7774" appservice_api: "localhost:7777" - typing_server: "localhost:7778" + edu_server: "localhost:7778" logging: - type: "file" level: "info" diff --git a/common/test/config.go b/common/test/config.go index 0fed252ae..f88e45125 100644 --- a/common/test/config.go +++ b/common/test/config.go @@ -106,7 +106,7 @@ func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*con cfg.Listen.RoomServer = assignAddress() cfg.Listen.SyncAPI = assignAddress() cfg.Listen.PublicRoomsAPI = assignAddress() - cfg.Listen.TypingServer = assignAddress() + cfg.Listen.EDUServer = assignAddress() // Bind to the same address as the listen address // All microservices are run on the same host in testing @@ -117,7 +117,7 @@ func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*con cfg.Bind.RoomServer = cfg.Listen.RoomServer cfg.Bind.SyncAPI = cfg.Listen.SyncAPI cfg.Bind.PublicRoomsAPI = cfg.Listen.PublicRoomsAPI - cfg.Bind.TypingServer = cfg.Listen.TypingServer + cfg.Bind.EDUServer = cfg.Listen.EDUServer return &cfg, port, nil } diff --git a/dendrite-config.yaml b/dendrite-config.yaml index a8d39aa1e..7436af7a3 100644 --- a/dendrite-config.yaml +++ b/dendrite-config.yaml @@ -85,7 +85,7 @@ kafka: topics: output_room_event: roomserverOutput output_client_data: clientapiOutput - output_typing_event: typingServerOutput + output_typing_event: eduServerOutput user_updates: userUpdates # The postgres connection configs for connecting to the databases e.g a postgres:// URI @@ -114,7 +114,7 @@ listen: public_rooms_api: "localhost:7775" federation_sender: "localhost:7776" appservice_api: "localhost:7777" - typing_server: "localhost:7778" + edu_server: "localhost:7778" # The configuration for tracing the dendrite components. tracing: diff --git a/docker/README.md b/docker/README.md index ee4f0f96f..83d0b6a87 100644 --- a/docker/README.md +++ b/docker/README.md @@ -58,7 +58,7 @@ docker-compose up kafka zookeeper postgres and the following dendrite components ``` -docker-compose up client_api media_api sync_api room_server public_rooms_api typing_server +docker-compose up client_api media_api sync_api room_server public_rooms_api edu_server docker-compose up client_api_proxy ``` diff --git a/docker/dendrite-docker.yml b/docker/dendrite-docker.yml index abb8c3307..a72ff3ddc 100644 --- a/docker/dendrite-docker.yml +++ b/docker/dendrite-docker.yml @@ -85,7 +85,7 @@ kafka: topics: output_room_event: roomserverOutput output_client_data: clientapiOutput - output_typing_event: typingServerOutput + output_typing_event: eduServerOutput user_updates: userUpdates @@ -114,7 +114,7 @@ listen: media_api: "media_api:7774" public_rooms_api: "public_rooms_api:7775" federation_sender: "federation_sender:7776" - typing_server: "typing_server:7777" + edu_server: "typing_server:7777" # The configuration for tracing the dendrite components. tracing: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index d738ed3f0..957c3bf3f 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -103,10 +103,10 @@ services: networks: - internal - typing_server: - container_name: dendrite_typing_server - hostname: typing_server - entrypoint: ["bash", "./docker/services/typing-server.sh"] + edu_server: + container_name: dendrite_edu_server + hostname: edu_server + entrypoint: ["bash", "./docker/services/edu-server.sh"] build: ./ volumes: - ..:/build diff --git a/docker/services/edu-server.sh b/docker/services/edu-server.sh new file mode 100644 index 000000000..d40b9fa7e --- /dev/null +++ b/docker/services/edu-server.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +bash ./docker/build.sh + +./bin/dendrite-edu-server --config=dendrite.yaml diff --git a/docker/services/typing-server.sh b/docker/services/typing-server.sh deleted file mode 100644 index 16ee0fa62..000000000 --- a/docker/services/typing-server.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -bash ./docker/build.sh - -./bin/dendrite-typing-server --config=dendrite.yaml diff --git a/typingserver/api/input.go b/eduserver/api/input.go similarity index 71% rename from typingserver/api/input.go rename to eduserver/api/input.go index 25e2ea228..c95acaf16 100644 --- a/typingserver/api/input.go +++ b/eduserver/api/input.go @@ -36,7 +36,7 @@ type InputTypingEvent struct { OriginServerTS gomatrixserverlib.Timestamp `json:"origin_server_ts"` } -// InputTypingEventRequest is a request to TypingServerInputAPI +// InputTypingEventRequest is a request to EDUServerInputAPI type InputTypingEventRequest struct { InputTypingEvent InputTypingEvent `json:"input_typing_event"` } @@ -44,8 +44,8 @@ type InputTypingEventRequest struct { // InputTypingEventResponse is a response to InputTypingEvents type InputTypingEventResponse struct{} -// TypingServerInputAPI is used to write events to the typing server. -type TypingServerInputAPI interface { +// EDUServerInputAPI is used to write events to the typing server. +type EDUServerInputAPI interface { InputTypingEvent( ctx context.Context, request *InputTypingEventRequest, @@ -53,24 +53,24 @@ type TypingServerInputAPI interface { ) error } -// TypingServerInputTypingEventPath is the HTTP path for the InputTypingEvent API. -const TypingServerInputTypingEventPath = "/api/typingserver/input" +// EDUServerInputTypingEventPath is the HTTP path for the InputTypingEvent API. +const EDUServerInputTypingEventPath = "/api/eduserver/input" -// NewTypingServerInputAPIHTTP creates a TypingServerInputAPI implemented by talking to a HTTP POST API. -func NewTypingServerInputAPIHTTP(typingServerURL string, httpClient *http.Client) TypingServerInputAPI { +// NewEDUServerInputAPIHTTP creates a EDUServerInputAPI implemented by talking to a HTTP POST API. +func NewEDUServerInputAPIHTTP(eduServerURL string, httpClient *http.Client) EDUServerInputAPI { if httpClient == nil { httpClient = http.DefaultClient } - return &httpTypingServerInputAPI{typingServerURL, httpClient} + return &httpEDUServerInputAPI{eduServerURL, httpClient} } -type httpTypingServerInputAPI struct { - typingServerURL string - httpClient *http.Client +type httpEDUServerInputAPI struct { + eduServerURL string + httpClient *http.Client } -// InputRoomEvents implements TypingServerInputAPI -func (h *httpTypingServerInputAPI) InputTypingEvent( +// InputRoomEvents implements EDUServerInputAPI +func (h *httpEDUServerInputAPI) InputTypingEvent( ctx context.Context, request *InputTypingEventRequest, response *InputTypingEventResponse, @@ -78,6 +78,6 @@ func (h *httpTypingServerInputAPI) InputTypingEvent( span, ctx := opentracing.StartSpanFromContext(ctx, "InputTypingEvent") defer span.Finish() - apiURL := h.typingServerURL + TypingServerInputTypingEventPath + apiURL := h.eduServerURL + EDUServerInputTypingEventPath return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } diff --git a/typingserver/api/output.go b/eduserver/api/output.go similarity index 100% rename from typingserver/api/output.go rename to eduserver/api/output.go diff --git a/typingserver/cache/cache.go b/eduserver/cache/cache.go similarity index 87% rename from typingserver/cache/cache.go rename to eduserver/cache/cache.go index 3f05c938e..46f7a2b13 100644 --- a/typingserver/cache/cache.go +++ b/eduserver/cache/cache.go @@ -32,8 +32,8 @@ type roomData struct { userSet userSet } -// TypingCache maintains a list of users typing in each room. -type TypingCache struct { +// EDUCache maintains a list of users typing in each room. +type EDUCache struct { sync.RWMutex latestSyncPosition int64 data map[string]*roomData @@ -42,26 +42,26 @@ type TypingCache struct { // Create a roomData with its sync position set to the latest sync position. // Must only be called after locking the cache. -func (t *TypingCache) newRoomData() *roomData { +func (t *EDUCache) newRoomData() *roomData { return &roomData{ syncPosition: t.latestSyncPosition, userSet: make(userSet), } } -// NewTypingCache returns a new TypingCache initialised for use. -func NewTypingCache() *TypingCache { - return &TypingCache{data: make(map[string]*roomData)} +// New returns a new EDUCache initialised for use. +func New() *EDUCache { + return &EDUCache{data: make(map[string]*roomData)} } // SetTimeoutCallback sets a callback function that is called right after // a user is removed from the typing user list due to timeout. -func (t *TypingCache) SetTimeoutCallback(fn TimeoutCallbackFn) { +func (t *EDUCache) SetTimeoutCallback(fn TimeoutCallbackFn) { t.timeoutCallback = fn } // GetTypingUsers returns the list of users typing in a room. -func (t *TypingCache) GetTypingUsers(roomID string) []string { +func (t *EDUCache) GetTypingUsers(roomID string) []string { users, _ := t.GetTypingUsersIfUpdatedAfter(roomID, 0) // 0 should work above because the first position used will be 1. return users @@ -70,7 +70,7 @@ func (t *TypingCache) GetTypingUsers(roomID string) []string { // GetTypingUsersIfUpdatedAfter returns all users typing in this room with // updated == true if the typing sync position of the room is after the given // position. Otherwise, returns an empty slice with updated == false. -func (t *TypingCache) GetTypingUsersIfUpdatedAfter( +func (t *EDUCache) GetTypingUsersIfUpdatedAfter( roomID string, position int64, ) (users []string, updated bool) { t.RLock() @@ -93,7 +93,7 @@ func (t *TypingCache) GetTypingUsersIfUpdatedAfter( // expire is the time when the user typing should time out. // if expire is nil, defaultTypingTimeout is assumed. // Returns the latest sync position for typing after update. -func (t *TypingCache) AddTypingUser( +func (t *EDUCache) AddTypingUser( userID, roomID string, expire *time.Time, ) int64 { expireTime := getExpireTime(expire) @@ -111,7 +111,7 @@ func (t *TypingCache) AddTypingUser( // addUser with mutex lock & replace the previous timer. // Returns the latest typing sync position after update. -func (t *TypingCache) addUser( +func (t *EDUCache) addUser( userID, roomID string, expiryTimer *time.Timer, ) int64 { t.Lock() @@ -143,7 +143,7 @@ func (t *TypingCache) addUser( // RemoveUser with mutex lock & stop the timer. // Returns the latest sync position for typing after update. -func (t *TypingCache) RemoveUser(userID, roomID string) int64 { +func (t *EDUCache) RemoveUser(userID, roomID string) int64 { t.Lock() defer t.Unlock() @@ -166,7 +166,7 @@ func (t *TypingCache) RemoveUser(userID, roomID string) int64 { return t.latestSyncPosition } -func (t *TypingCache) GetLatestSyncPosition() int64 { +func (t *EDUCache) GetLatestSyncPosition() int64 { t.Lock() defer t.Unlock() return t.latestSyncPosition diff --git a/typingserver/cache/cache_test.go b/eduserver/cache/cache_test.go similarity index 88% rename from typingserver/cache/cache_test.go rename to eduserver/cache/cache_test.go index 2a6ffa50e..8a1b6f797 100644 --- a/typingserver/cache/cache_test.go +++ b/eduserver/cache/cache_test.go @@ -19,10 +19,10 @@ import ( "github.com/matrix-org/dendrite/common/test" ) -func TestTypingCache(t *testing.T) { - tCache := NewTypingCache() +func TestEDUCache(t *testing.T) { + tCache := New() if tCache == nil { - t.Fatal("NewTypingCache failed") + t.Fatal("New failed") } t.Run("AddTypingUser", func(t *testing.T) { @@ -38,7 +38,7 @@ func TestTypingCache(t *testing.T) { }) } -func testAddTypingUser(t *testing.T, tCache *TypingCache) { // nolint: unparam +func testAddTypingUser(t *testing.T, tCache *EDUCache) { // nolint: unparam present := time.Now() tests := []struct { userID string @@ -58,7 +58,7 @@ func testAddTypingUser(t *testing.T, tCache *TypingCache) { // nolint: unparam } } -func testGetTypingUsers(t *testing.T, tCache *TypingCache) { +func testGetTypingUsers(t *testing.T, tCache *EDUCache) { tests := []struct { roomID string wantUsers []string @@ -75,7 +75,7 @@ func testGetTypingUsers(t *testing.T, tCache *TypingCache) { } } -func testRemoveUser(t *testing.T, tCache *TypingCache) { +func testRemoveUser(t *testing.T, tCache *EDUCache) { tests := []struct { roomID string userIDs []string diff --git a/typingserver/typingserver.go b/eduserver/eduserver.go similarity index 66% rename from typingserver/typingserver.go rename to eduserver/eduserver.go index b43f72f75..8ddd2c527 100644 --- a/typingserver/typingserver.go +++ b/eduserver/eduserver.go @@ -10,27 +10,27 @@ // See the License for the specific language governing permissions and // limitations under the License. -package typingserver +package eduserver import ( "net/http" "github.com/matrix-org/dendrite/common/basecomponent" - "github.com/matrix-org/dendrite/typingserver/api" - "github.com/matrix-org/dendrite/typingserver/cache" - "github.com/matrix-org/dendrite/typingserver/input" + "github.com/matrix-org/dendrite/eduserver/api" + "github.com/matrix-org/dendrite/eduserver/cache" + "github.com/matrix-org/dendrite/eduserver/input" ) -// SetupTypingServerComponent sets up and registers HTTP handlers for the -// TypingServer component. Returns instances of the various roomserver APIs, +// SetupEDUServerComponent sets up and registers HTTP handlers for the +// EDUServer component. Returns instances of the various roomserver APIs, // allowing other components running in the same process to hit the query the // APIs directly instead of having to use HTTP. -func SetupTypingServerComponent( +func SetupEDUServerComponent( base *basecomponent.BaseDendrite, - typingCache *cache.TypingCache, -) api.TypingServerInputAPI { - inputAPI := &input.TypingServerInputAPI{ - Cache: typingCache, + eduCache *cache.EDUCache, +) api.EDUServerInputAPI { + inputAPI := &input.EDUServerInputAPI{ + Cache: eduCache, Producer: base.KafkaProducer, OutputTypingEventTopic: string(base.Cfg.Kafka.Topics.OutputTypingEvent), } diff --git a/typingserver/input/input.go b/eduserver/input/input.go similarity index 81% rename from typingserver/input/input.go rename to eduserver/input/input.go index 0e2fbe51f..e0cc6922a 100644 --- a/typingserver/input/input.go +++ b/eduserver/input/input.go @@ -19,25 +19,25 @@ import ( "time" "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/typingserver/api" - "github.com/matrix-org/dendrite/typingserver/cache" + "github.com/matrix-org/dendrite/eduserver/api" + "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" "gopkg.in/Shopify/sarama.v1" ) -// TypingServerInputAPI implements api.TypingServerInputAPI -type TypingServerInputAPI struct { +// EDUServerInputAPI implements api.EDUServerInputAPI +type EDUServerInputAPI struct { // Cache to store the current typing members in each room. - Cache *cache.TypingCache + Cache *cache.EDUCache // The kafka topic to output new typing events to. OutputTypingEventTopic string // kafka producer Producer sarama.SyncProducer } -// InputTypingEvent implements api.TypingServerInputAPI -func (t *TypingServerInputAPI) InputTypingEvent( +// InputTypingEvent implements api.EDUServerInputAPI +func (t *EDUServerInputAPI) InputTypingEvent( ctx context.Context, request *api.InputTypingEventRequest, response *api.InputTypingEventResponse, @@ -56,7 +56,7 @@ func (t *TypingServerInputAPI) InputTypingEvent( return t.sendEvent(ite) } -func (t *TypingServerInputAPI) sendEvent(ite *api.InputTypingEvent) error { +func (t *EDUServerInputAPI) sendEvent(ite *api.InputTypingEvent) error { ev := &api.TypingEvent{ Type: gomatrixserverlib.MTyping, RoomID: ite.RoomID, @@ -89,9 +89,9 @@ func (t *TypingServerInputAPI) sendEvent(ite *api.InputTypingEvent) error { return err } -// SetupHTTP adds the TypingServerInputAPI handlers to the http.ServeMux. -func (t *TypingServerInputAPI) SetupHTTP(servMux *http.ServeMux) { - servMux.Handle(api.TypingServerInputTypingEventPath, +// SetupHTTP adds the EDUServerInputAPI handlers to the http.ServeMux. +func (t *EDUServerInputAPI) SetupHTTP(servMux *http.ServeMux) { + servMux.Handle(api.EDUServerInputTypingEventPath, common.MakeInternalAPI("inputTypingEvents", func(req *http.Request) util.JSONResponse { var request api.InputTypingEventRequest var response api.InputTypingEventResponse diff --git a/federationsender/consumers/typingserver.go b/federationsender/consumers/eduserver.go similarity index 87% rename from federationsender/consumers/typingserver.go rename to federationsender/consumers/eduserver.go index 590fcb257..ba45db7fa 100644 --- a/federationsender/consumers/typingserver.go +++ b/federationsender/consumers/eduserver.go @@ -18,15 +18,15 @@ import ( "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/federationsender/queue" "github.com/matrix-org/dendrite/federationsender/storage" - "github.com/matrix-org/dendrite/typingserver/api" "github.com/matrix-org/gomatrixserverlib" log "github.com/sirupsen/logrus" "gopkg.in/Shopify/sarama.v1" ) -// OutputTypingEventConsumer consumes events that originate in typing server. +// OutputTypingEventConsumer consumes events that originate in EDU server. type OutputTypingEventConsumer struct { consumer *common.ContinualConsumer db storage.Database @@ -34,7 +34,7 @@ type OutputTypingEventConsumer struct { ServerName gomatrixserverlib.ServerName } -// NewOutputTypingEventConsumer creates a new OutputTypingEventConsumer. Call Start() to begin consuming from typing servers. +// NewOutputTypingEventConsumer creates a new OutputTypingEventConsumer. Call Start() to begin consuming from EDU servers. func NewOutputTypingEventConsumer( cfg *config.Dendrite, kafkaConsumer sarama.Consumer, @@ -57,19 +57,19 @@ func NewOutputTypingEventConsumer( return c } -// Start consuming from typing servers +// Start consuming from EDU servers func (t *OutputTypingEventConsumer) Start() error { return t.consumer.Start() } -// onMessage is called for OutputTypingEvent received from the typing servers. +// onMessage is called for OutputTypingEvent received from the EDU servers. // Parses the msg, creates a matrix federation EDU and sends it to joined hosts. func (t *OutputTypingEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { // Extract the typing event from msg. var ote api.OutputTypingEvent if err := json.Unmarshal(msg.Value, &ote); err != nil { // Skip this msg but continue processing messages. - log.WithError(err).Errorf("typingserver output log: message parse failed") + log.WithError(err).Errorf("eduserver output log: message parse failed") return nil } diff --git a/syncapi/consumers/typingserver.go b/syncapi/consumers/eduserver.go similarity index 90% rename from syncapi/consumers/typingserver.go rename to syncapi/consumers/eduserver.go index 369254411..5491c1e9f 100644 --- a/syncapi/consumers/typingserver.go +++ b/syncapi/consumers/eduserver.go @@ -19,15 +19,15 @@ import ( "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/sync" "github.com/matrix-org/dendrite/syncapi/types" - "github.com/matrix-org/dendrite/typingserver/api" log "github.com/sirupsen/logrus" sarama "gopkg.in/Shopify/sarama.v1" ) -// OutputTypingEventConsumer consumes events that originated in the typing server. +// OutputTypingEventConsumer consumes events that originated in the EDU server. type OutputTypingEventConsumer struct { typingConsumer *common.ContinualConsumer db storage.Database @@ -35,7 +35,7 @@ type OutputTypingEventConsumer struct { } // NewOutputTypingEventConsumer creates a new OutputTypingEventConsumer. -// Call Start() to begin consuming from the typing server. +// Call Start() to begin consuming from the EDU server. func NewOutputTypingEventConsumer( cfg *config.Dendrite, kafkaConsumer sarama.Consumer, @@ -60,7 +60,7 @@ func NewOutputTypingEventConsumer( return s } -// Start consuming from typing api +// Start consuming from EDU api func (s *OutputTypingEventConsumer) Start() error { s.db.SetTypingTimeoutCallback(func(userID, roomID string, latestSyncPosition int64) { s.notifier.OnNewEvent( @@ -78,7 +78,7 @@ func (s *OutputTypingEventConsumer) onMessage(msg *sarama.ConsumerMessage) error var output api.OutputTypingEvent if err := json.Unmarshal(msg.Value, &output); err != nil { // If the message was invalid, log it and move on to the next message in the stream - log.WithError(err).Errorf("typing server output log: message parse failure") + log.WithError(err).Errorf("EDU server output log: message parse failure") return nil } @@ -86,7 +86,7 @@ func (s *OutputTypingEventConsumer) onMessage(msg *sarama.ConsumerMessage) error "room_id": output.Event.RoomID, "user_id": output.Event.UserID, "typing": output.Event.Typing, - }).Debug("received data from typing server") + }).Debug("received data from EDU server") var typingPos types.StreamPosition typingEvent := output.Event diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index b6dc19696..a3efd8d58 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -20,9 +20,9 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/types" - "github.com/matrix-org/dendrite/typingserver/cache" "github.com/matrix-org/gomatrixserverlib" ) diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go index f3f1aabc7..ead1bf335 100644 --- a/syncapi/storage/postgres/syncserver.go +++ b/syncapi/storage/postgres/syncserver.go @@ -30,8 +30,8 @@ import ( // Import the postgres database driver. _ "github.com/lib/pq" "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/syncapi/types" - "github.com/matrix-org/dendrite/typingserver/cache" "github.com/matrix-org/gomatrixserverlib" ) @@ -53,7 +53,7 @@ type SyncServerDatasource struct { events outputRoomEventsStatements roomstate currentRoomStateStatements invites inviteEventsStatements - typingCache *cache.TypingCache + eduCache *cache.EDUCache topology outputRoomEventsTopologyStatements backwardExtremities backwardExtremitiesStatements } @@ -86,7 +86,7 @@ func NewSyncServerDatasource(dbDataSourceName string) (*SyncServerDatasource, er if err := d.backwardExtremities.prepare(d.db); err != nil { return nil, err } - d.typingCache = cache.NewTypingCache() + d.eduCache = cache.New() return &d, nil } @@ -395,7 +395,7 @@ func (d *SyncServerDatasource) syncPositionTx( maxEventID = maxInviteID } sp.PDUPosition = types.StreamPosition(maxEventID) - sp.EDUTypingPosition = types.StreamPosition(d.typingCache.GetLatestSyncPosition()) + sp.EDUTypingPosition = types.StreamPosition(d.eduCache.GetLatestSyncPosition()) return } @@ -468,7 +468,7 @@ func (d *SyncServerDatasource) addTypingDeltaToResponse( var ok bool var err error for _, roomID := range joinedRoomIDs { - if typingUsers, updated := d.typingCache.GetTypingUsersIfUpdatedAfter( + if typingUsers, updated := d.eduCache.GetTypingUsersIfUpdatedAfter( roomID, int64(since.EDUTypingPosition), ); updated { ev := gomatrixserverlib.ClientEvent{ @@ -719,7 +719,7 @@ func (d *SyncServerDatasource) RetireInviteEvent( } func (d *SyncServerDatasource) SetTypingTimeoutCallback(fn cache.TimeoutCallbackFn) { - d.typingCache.SetTimeoutCallback(fn) + d.eduCache.SetTimeoutCallback(fn) } // AddTypingUser adds a typing user to the typing cache. @@ -727,7 +727,7 @@ func (d *SyncServerDatasource) SetTypingTimeoutCallback(fn cache.TimeoutCallback func (d *SyncServerDatasource) AddTypingUser( userID, roomID string, expireTime *time.Time, ) types.StreamPosition { - return types.StreamPosition(d.typingCache.AddTypingUser(userID, roomID, expireTime)) + return types.StreamPosition(d.eduCache.AddTypingUser(userID, roomID, expireTime)) } // RemoveTypingUser removes a typing user from the typing cache. @@ -735,7 +735,7 @@ func (d *SyncServerDatasource) AddTypingUser( func (d *SyncServerDatasource) RemoveTypingUser( userID, roomID string, ) types.StreamPosition { - return types.StreamPosition(d.typingCache.RemoveUser(userID, roomID)) + return types.StreamPosition(d.eduCache.RemoveUser(userID, roomID)) } func (d *SyncServerDatasource) addInvitesToResponse( diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index 8ff189007..30f77e54d 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -33,8 +33,8 @@ import ( _ "github.com/mattn/go-sqlite3" "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/syncapi/types" - "github.com/matrix-org/dendrite/typingserver/cache" "github.com/matrix-org/gomatrixserverlib" ) @@ -57,7 +57,7 @@ type SyncServerDatasource struct { events outputRoomEventsStatements roomstate currentRoomStateStatements invites inviteEventsStatements - typingCache *cache.TypingCache + eduCache *cache.EDUCache topology outputRoomEventsTopologyStatements backwardExtremities backwardExtremitiesStatements } @@ -84,7 +84,7 @@ func NewSyncServerDatasource(dataSourceName string) (*SyncServerDatasource, erro if err = d.prepare(); err != nil { return nil, err } - d.typingCache = cache.NewTypingCache() + d.eduCache = cache.New() return &d, nil } @@ -429,7 +429,7 @@ func (d *SyncServerDatasource) syncPositionTx( maxEventID = maxInviteID } sp.PDUPosition = types.StreamPosition(maxEventID) - sp.EDUTypingPosition = types.StreamPosition(d.typingCache.GetLatestSyncPosition()) + sp.EDUTypingPosition = types.StreamPosition(d.eduCache.GetLatestSyncPosition()) return } @@ -502,7 +502,7 @@ func (d *SyncServerDatasource) addTypingDeltaToResponse( var ok bool var err error for _, roomID := range joinedRoomIDs { - if typingUsers, updated := d.typingCache.GetTypingUsersIfUpdatedAfter( + if typingUsers, updated := d.eduCache.GetTypingUsersIfUpdatedAfter( roomID, int64(since.EDUTypingPosition), ); updated { ev := gomatrixserverlib.ClientEvent{ @@ -766,7 +766,7 @@ func (d *SyncServerDatasource) RetireInviteEvent( } func (d *SyncServerDatasource) SetTypingTimeoutCallback(fn cache.TimeoutCallbackFn) { - d.typingCache.SetTimeoutCallback(fn) + d.eduCache.SetTimeoutCallback(fn) } // AddTypingUser adds a typing user to the typing cache. @@ -774,7 +774,7 @@ func (d *SyncServerDatasource) SetTypingTimeoutCallback(fn cache.TimeoutCallback func (d *SyncServerDatasource) AddTypingUser( userID, roomID string, expireTime *time.Time, ) types.StreamPosition { - return types.StreamPosition(d.typingCache.AddTypingUser(userID, roomID, expireTime)) + return types.StreamPosition(d.eduCache.AddTypingUser(userID, roomID, expireTime)) } // RemoveTypingUser removes a typing user from the typing cache. @@ -782,7 +782,7 @@ func (d *SyncServerDatasource) AddTypingUser( func (d *SyncServerDatasource) RemoveTypingUser( userID, roomID string, ) types.StreamPosition { - return types.StreamPosition(d.typingCache.RemoveUser(userID, roomID)) + return types.StreamPosition(d.eduCache.RemoveUser(userID, roomID)) } func (d *SyncServerDatasource) addInvitesToResponse( From 8fbe9f40782bd56561c75ac74916900c11db3654 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Mon, 30 Mar 2020 16:40:28 +0100 Subject: [PATCH 61/86] Implement typing over federation (#949) Also fix a pet peeve of mine: not putting units on things!!! Manually tested on p2p and works well, with some fudge factor delay. --- clientapi/producers/eduserver.go | 6 +-- cmd/dendrite-federation-api-server/main.go | 7 ++- cmd/dendrite-monolith-server/main.go | 4 +- cmd/dendritejs/main.go | 4 +- eduserver/api/input.go | 4 +- eduserver/input/input.go | 4 +- federationapi/federationapi.go | 3 +- federationapi/routing/routing.go | 3 +- federationapi/routing/send.go | 54 ++++++++++++++++------ federationsender/consumers/eduserver.go | 11 +++++ 10 files changed, 75 insertions(+), 25 deletions(-) diff --git a/clientapi/producers/eduserver.go b/clientapi/producers/eduserver.go index 14414ec64..30c40fb7f 100644 --- a/clientapi/producers/eduserver.go +++ b/clientapi/producers/eduserver.go @@ -20,7 +20,7 @@ import ( "github.com/matrix-org/gomatrixserverlib" ) -// EDUServerProducer produces events for the typing server to consume +// EDUServerProducer produces events for the EDU server to consume type EDUServerProducer struct { InputAPI api.EDUServerInputAPI } @@ -35,13 +35,13 @@ func NewEDUServerProducer(inputAPI api.EDUServerInputAPI) *EDUServerProducer { // SendTyping sends a typing event to EDU server func (p *EDUServerProducer) SendTyping( ctx context.Context, userID, roomID string, - typing bool, timeout int64, + typing bool, timeoutMS int64, ) error { requestData := api.InputTypingEvent{ UserID: userID, RoomID: roomID, Typing: typing, - Timeout: timeout, + TimeoutMS: timeoutMS, OriginServerTS: gomatrixserverlib.AsTimestamp(time.Now()), } diff --git a/cmd/dendrite-federation-api-server/main.go b/cmd/dendrite-federation-api-server/main.go index 367f5dc0c..d18926a68 100644 --- a/cmd/dendrite-federation-api-server/main.go +++ b/cmd/dendrite-federation-api-server/main.go @@ -15,8 +15,11 @@ package main import ( + "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/common/basecomponent" "github.com/matrix-org/dendrite/common/keydb" + "github.com/matrix-org/dendrite/eduserver" + "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/federationapi" ) @@ -34,10 +37,12 @@ func main() { alias, input, query := base.CreateHTTPRoomserverAPIs() asQuery := base.CreateHTTPAppServiceAPIs() + eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New()) + eduProducer := producers.NewEDUServerProducer(eduInputAPI) federationapi.SetupFederationAPIComponent( base, accountDB, deviceDB, federation, &keyRing, - alias, input, query, asQuery, federationSender, + alias, input, query, asQuery, federationSender, eduProducer, ) base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationAPI), string(base.Cfg.Listen.FederationAPI)) diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 0aceef029..9f6531ed3 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -20,6 +20,7 @@ import ( "github.com/matrix-org/dendrite/appservice" "github.com/matrix-org/dendrite/clientapi" + "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/basecomponent" "github.com/matrix-org/dendrite/common/keydb" @@ -67,7 +68,8 @@ func main() { federation, &keyRing, alias, input, query, eduInputAPI, asQuery, transactions.New(), fedSenderAPI, ) - federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fedSenderAPI) + eduProducer := producers.NewEDUServerProducer(eduInputAPI) + federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fedSenderAPI, eduProducer) mediaapi.SetupMediaAPIComponent(base, deviceDB) publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, query, federation, nil) syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query, federation, cfg) diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 0f72dc1ec..05802725d 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -23,6 +23,7 @@ import ( "github.com/matrix-org/dendrite/appservice" "github.com/matrix-org/dendrite/clientapi" + "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/basecomponent" "github.com/matrix-org/dendrite/common/config" @@ -133,7 +134,8 @@ func main() { federation, &keyRing, alias, input, query, eduInputAPI, asQuery, transactions.New(), fedSenderAPI, ) - federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fedSenderAPI) + eduProducer := producers.NewEDUServerProducer(eduInputAPI) + federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fedSenderAPI, eduProducer) mediaapi.SetupMediaAPIComponent(base, deviceDB) publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, query, federation, p2pPublicRoomProvider) syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query, federation, cfg) diff --git a/eduserver/api/input.go b/eduserver/api/input.go index c95acaf16..ad3f1ed58 100644 --- a/eduserver/api/input.go +++ b/eduserver/api/input.go @@ -30,8 +30,8 @@ type InputTypingEvent struct { RoomID string `json:"room_id"` // Typing is true if the user is typing, false if they have stopped. Typing bool `json:"typing"` - // Timeout is the interval for which the user should be marked as typing. - Timeout int64 `json:"timeout"` + // Timeout is the interval in milliseconds for which the user should be marked as typing. + TimeoutMS int64 `json:"timeout"` // OriginServerTS when the server received the update. OriginServerTS gomatrixserverlib.Timestamp `json:"origin_server_ts"` } diff --git a/eduserver/input/input.go b/eduserver/input/input.go index e0cc6922a..845909452 100644 --- a/eduserver/input/input.go +++ b/eduserver/input/input.go @@ -46,7 +46,7 @@ func (t *EDUServerInputAPI) InputTypingEvent( if ite.Typing { // user is typing, update our current state of users typing. expireTime := ite.OriginServerTS.Time().Add( - time.Duration(ite.Timeout) * time.Millisecond, + time.Duration(ite.TimeoutMS) * time.Millisecond, ) t.Cache.AddTypingUser(ite.UserID, ite.RoomID, &expireTime) } else { @@ -69,7 +69,7 @@ func (t *EDUServerInputAPI) sendEvent(ite *api.InputTypingEvent) error { if ev.Typing { expireTime := ite.OriginServerTS.Time().Add( - time.Duration(ite.Timeout) * time.Millisecond, + time.Duration(ite.TimeoutMS) * time.Millisecond, ) ote.ExpireTime = &expireTime } diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go index 90db95b3a..ed96322b8 100644 --- a/federationapi/federationapi.go +++ b/federationapi/federationapi.go @@ -41,12 +41,13 @@ func SetupFederationAPIComponent( queryAPI roomserverAPI.RoomserverQueryAPI, asAPI appserviceAPI.AppServiceQueryAPI, federationSenderAPI federationSenderAPI.FederationSenderQueryAPI, + eduProducer *producers.EDUServerProducer, ) { roomserverProducer := producers.NewRoomserverProducer(inputAPI, queryAPI) routing.Setup( base.APIMux, base.Cfg, queryAPI, aliasAPI, asAPI, - roomserverProducer, federationSenderAPI, *keyRing, + roomserverProducer, eduProducer, federationSenderAPI, *keyRing, federation, accountsDB, deviceDB, ) } diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index b5c8e53de..9ac535767 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -48,6 +48,7 @@ func Setup( aliasAPI roomserverAPI.RoomserverAliasAPI, asAPI appserviceAPI.AppServiceQueryAPI, producer *producers.RoomserverProducer, + eduProducer *producers.EDUServerProducer, federationSenderAPI federationSenderAPI.FederationSenderQueryAPI, keys gomatrixserverlib.KeyRing, federation *gomatrixserverlib.FederationClient, @@ -79,7 +80,7 @@ func Setup( } return Send( httpReq, request, gomatrixserverlib.TransactionID(vars["txnID"]), - cfg, query, producer, keys, federation, + cfg, query, producer, eduProducer, keys, federation, ) }, )).Methods(http.MethodPut, http.MethodOptions) diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index 4c92c7e5e..1013a44cf 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -36,20 +36,22 @@ func Send( cfg *config.Dendrite, query api.RoomserverQueryAPI, producer *producers.RoomserverProducer, + eduProducer *producers.EDUServerProducer, keys gomatrixserverlib.KeyRing, federation *gomatrixserverlib.FederationClient, ) util.JSONResponse { t := txnReq{ - context: httpReq.Context(), - query: query, - producer: producer, - keys: keys, - federation: federation, + context: httpReq.Context(), + query: query, + producer: producer, + eduProducer: eduProducer, + keys: keys, + federation: federation, } var txnEvents struct { - PDUs []json.RawMessage `json:"pdus"` - EDUs []json.RawMessage `json:"edus"` + PDUs []json.RawMessage `json:"pdus"` + EDUs []gomatrixserverlib.EDU `json:"edus"` } if err := json.Unmarshal(request.Content(), &txnEvents); err != nil { @@ -59,7 +61,9 @@ func Send( } } + // TODO: Really we should have a function to convert FederationRequest to txnReq t.PDUs = txnEvents.PDUs + t.EDUs = txnEvents.EDUs t.Origin = request.Origin() t.TransactionID = txnID t.Destination = cfg.Matrix.ServerName @@ -80,11 +84,12 @@ func Send( type txnReq struct { gomatrixserverlib.Transaction - context context.Context - query api.RoomserverQueryAPI - producer *producers.RoomserverProducer - keys gomatrixserverlib.KeyRing - federation *gomatrixserverlib.FederationClient + context context.Context + query api.RoomserverQueryAPI + producer *producers.RoomserverProducer + eduProducer *producers.EDUServerProducer + keys gomatrixserverlib.KeyRing + federation *gomatrixserverlib.FederationClient } func (t *txnReq) processTransaction() (*gomatrixserverlib.RespSend, error) { @@ -152,7 +157,7 @@ func (t *txnReq) processTransaction() (*gomatrixserverlib.RespSend, error) { } } - // TODO: Process the EDUs. + t.processEDUs(t.EDUs) util.GetLogger(t.context).Infof("Processed %d PDUs from transaction %q", len(results), t.TransactionID) return &gomatrixserverlib.RespSend{PDUs: results}, nil } @@ -163,6 +168,29 @@ type unknownRoomError struct { func (e unknownRoomError) Error() string { return fmt.Sprintf("unknown room %q", e.roomID) } +func (t *txnReq) processEDUs(edus []gomatrixserverlib.EDU) { + for _, e := range edus { + switch e.Type { + case gomatrixserverlib.MTyping: + // https://matrix.org/docs/spec/server_server/latest#typing-notifications + var typingPayload struct { + RoomID string `json:"room_id"` + UserID string `json:"user_id"` + Typing bool `json:"typing"` + } + if err := json.Unmarshal(e.Content, &typingPayload); err != nil { + util.GetLogger(t.context).WithError(err).Error("Failed to unmarshal typing event") + continue + } + if err := t.eduProducer.SendTyping(t.context, typingPayload.UserID, typingPayload.RoomID, typingPayload.Typing, 30*1000); err != nil { + util.GetLogger(t.context).WithError(err).Error("Failed to send typing event to edu server") + } + default: + util.GetLogger(t.context).WithField("type", e.Type).Warn("unhandled edu") + } + } +} + func (t *txnReq) processEvent(e gomatrixserverlib.Event) error { prevEventIDs := e.PrevEventIDs() diff --git a/federationsender/consumers/eduserver.go b/federationsender/consumers/eduserver.go index ba45db7fa..4d2445f3c 100644 --- a/federationsender/consumers/eduserver.go +++ b/federationsender/consumers/eduserver.go @@ -73,6 +73,17 @@ func (t *OutputTypingEventConsumer) onMessage(msg *sarama.ConsumerMessage) error return nil } + // only send typing events which originated from us + _, typingServerName, err := gomatrixserverlib.SplitID('@', ote.Event.UserID) + if err != nil { + log.WithError(err).WithField("user_id", ote.Event.UserID).Error("Failed to extract domain from typing sender") + return nil + } + if typingServerName != t.ServerName { + log.WithField("other_server", typingServerName).Info("Suppressing typing notif: originated elsewhere") + return nil + } + joined, err := t.db.GetJoinedHosts(context.TODO(), ote.Event.RoomID) if err != nil { return err From 2c8950221ef47cf4298dbd1f6e94450e4c35a81e Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Tue, 31 Mar 2020 22:53:09 +0800 Subject: [PATCH 62/86] Fix: show-expected-fail-tests.sh ignores # in whitelist and strips quotes in test names in output (#943) Signed-off-by: Alex Chen --- show-expected-fail-tests.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/show-expected-fail-tests.sh b/show-expected-fail-tests.sh index 9cd51b007..0a4c7be87 100755 --- a/show-expected-fail-tests.sh +++ b/show-expected-fail-tests.sh @@ -60,7 +60,7 @@ while read -r test_name; do # Ignore empty lines [ "${test_name}" = "" ] && continue - grep "${test_name}" "${whitelist_file}" > /dev/null 2>&1 + grep "^${test_name}" "${whitelist_file}" > /dev/null 2>&1 if [ "$?" != "0" ]; then # Check if this test name is blacklisted if printf '%s\n' "${blacklisted_tests[@]}" | grep -q -P "^${test_name}$"; then @@ -80,8 +80,8 @@ done <<< "${passed_but_expected_fail}" # TODO: Check that the same test doesn't appear twice in the whitelist|blacklist # Trim test output strings -tests_to_add=$(echo -e $tests_to_add | xargs) -already_in_whitelist=$(echo -e $already_in_whitelist | xargs) +tests_to_add=$(echo -e $tests_to_add | xargs -d '\n') +already_in_whitelist=$(echo -e $already_in_whitelist | xargs -d '\n') # Format output with markdown for buildkite annotation rendering purposes if [ -n "${tests_to_add}" ] && [ -n "${already_in_whitelist}" ]; then From 955244c09298d0e6c870377dad3af2ffa1f5e578 Mon Sep 17 00:00:00 2001 From: Ben B Date: Fri, 3 Apr 2020 12:40:50 +0200 Subject: [PATCH 63/86] use custom http client instead of the http DefaultClient (#823) This commit replaces the default client from the http lib with a custom one. The previously used default client doesn't come with a timeout. This could cause unwanted locks. That solution chosen here creates a http client in the base component dendrite with a constant timeout of 30 seconds. If it should be necessary to overwrite this, we could include the timeout in the dendrite configuration. Here it would be a good idea to extend the type "Address" by a timeout and create an http client for each service. Closes #820 Signed-off-by: Benedikt Bongartz Co-authored-by: Kegsay --- appservice/api/query.go | 9 +++--- cmd/roomserver-integration-tests/main.go | 18 +++++++++-- common/basecomponent/base.go | 39 ++++++++++++++++++++---- eduserver/api/input.go | 7 +++-- federationsender/api/query.go | 9 +++--- roomserver/api/alias.go | 9 +++--- roomserver/api/input.go | 9 +++--- roomserver/api/query.go | 9 +++--- 8 files changed, 77 insertions(+), 32 deletions(-) diff --git a/appservice/api/query.go b/appservice/api/query.go index 7e61d6233..afd5c5d76 100644 --- a/appservice/api/query.go +++ b/appservice/api/query.go @@ -20,6 +20,7 @@ package api import ( "context" "database/sql" + "errors" "net/http" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" @@ -97,15 +98,15 @@ type httpAppServiceQueryAPI struct { // NewAppServiceQueryAPIHTTP creates a AppServiceQueryAPI implemented by talking // to a HTTP POST API. -// If httpClient is nil then it uses http.DefaultClient +// If httpClient is nil an error is returned func NewAppServiceQueryAPIHTTP( appserviceURL string, httpClient *http.Client, -) AppServiceQueryAPI { +) (AppServiceQueryAPI, error) { if httpClient == nil { - httpClient = http.DefaultClient + return nil, errors.New("NewRoomserverAliasAPIHTTP: httpClient is ") } - return &httpAppServiceQueryAPI{appserviceURL, httpClient} + return &httpAppServiceQueryAPI{appserviceURL, httpClient}, nil } // RoomAliasExists implements AppServiceQueryAPI diff --git a/cmd/roomserver-integration-tests/main.go b/cmd/roomserver-integration-tests/main.go index d4a8a1d10..df5607bcb 100644 --- a/cmd/roomserver-integration-tests/main.go +++ b/cmd/roomserver-integration-tests/main.go @@ -44,6 +44,8 @@ var ( // This needs to be high enough to account for the time it takes to create // the postgres database tables which can take a while on travis. timeoutString = defaulting(os.Getenv("TIMEOUT"), "60s") + // Timeout for http client + timeoutHTTPClient = defaulting(os.Getenv("TIMEOUT_HTTP"), "30s") // The name of maintenance database to connect to in order to create the test database. postgresDatabase = defaulting(os.Getenv("POSTGRES_DATABASE"), "postgres") // The name of the test database to create. @@ -68,7 +70,10 @@ func defaulting(value, defaultValue string) string { return value } -var timeout time.Duration +var ( + timeout time.Duration + timeoutHTTP time.Duration +) func init() { var err error @@ -76,6 +81,10 @@ func init() { if err != nil { panic(err) } + timeoutHTTP, err = time.ParseDuration(timeoutHTTPClient) + if err != nil { + panic(err) + } } func createDatabase(database string) error { @@ -199,7 +208,10 @@ func writeToRoomServer(input []string, roomserverURL string) error { return err } } - x := api.NewRoomserverInputAPIHTTP(roomserverURL, nil) + x, err := api.NewRoomserverInputAPIHTTP(roomserverURL, &http.Client{Timeout: timeoutHTTP}) + if err != nil { + return err + } return x.InputRoomEvents(context.Background(), &request, &response) } @@ -258,7 +270,7 @@ func testRoomserver(input []string, wantOutput []string, checkQueries func(api.R cmd.Args = []string{"dendrite-room-server", "--config", filepath.Join(dir, test.ConfigFile)} gotOutput, err := runAndReadFromTopic(cmd, cfg.RoomServerURL()+"/metrics", doInput, outputTopic, len(wantOutput), func() { - queryAPI := api.NewRoomserverQueryAPIHTTP("http://"+string(cfg.Listen.RoomServer), nil) + queryAPI, _ := api.NewRoomserverQueryAPIHTTP("http://"+string(cfg.Listen.RoomServer), &http.Client{Timeout: timeoutHTTP}) checkQueries(queryAPI) }) if err != nil { diff --git a/common/basecomponent/base.go b/common/basecomponent/base.go index 8d559f4dc..432819a23 100644 --- a/common/basecomponent/base.go +++ b/common/basecomponent/base.go @@ -19,6 +19,7 @@ import ( "io" "net/http" "net/url" + "time" "golang.org/x/crypto/ed25519" @@ -52,6 +53,7 @@ type BaseDendrite struct { // APIMux should be used to register new public matrix api endpoints APIMux *mux.Router + httpClient *http.Client Cfg *config.Dendrite KafkaConsumer sarama.Consumer KafkaProducer sarama.SyncProducer @@ -77,11 +79,14 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string) *BaseDendrite { kafkaConsumer, kafkaProducer = setupKafka(cfg) } + const defaultHTTPTimeout = 30 * time.Second + return &BaseDendrite{ componentName: componentName, tracerCloser: closer, Cfg: cfg, APIMux: mux.NewRouter().UseEncodedPath(), + httpClient: &http.Client{Timeout: defaultHTTPTimeout}, KafkaConsumer: kafkaConsumer, KafkaProducer: kafkaProducer, } @@ -95,7 +100,11 @@ func (b *BaseDendrite) Close() error { // CreateHTTPAppServiceAPIs returns the QueryAPI for hitting the appservice // component over HTTP. func (b *BaseDendrite) CreateHTTPAppServiceAPIs() appserviceAPI.AppServiceQueryAPI { - return appserviceAPI.NewAppServiceQueryAPIHTTP(b.Cfg.AppServiceURL(), nil) + a, err := appserviceAPI.NewAppServiceQueryAPIHTTP(b.Cfg.AppServiceURL(), b.httpClient) + if err != nil { + logrus.WithError(err).Panic("CreateHTTPAppServiceAPIs failed") + } + return a } // CreateHTTPRoomserverAPIs returns the AliasAPI, InputAPI and QueryAPI for hitting @@ -105,22 +114,40 @@ func (b *BaseDendrite) CreateHTTPRoomserverAPIs() ( roomserverAPI.RoomserverInputAPI, roomserverAPI.RoomserverQueryAPI, ) { - alias := roomserverAPI.NewRoomserverAliasAPIHTTP(b.Cfg.RoomServerURL(), nil) - input := roomserverAPI.NewRoomserverInputAPIHTTP(b.Cfg.RoomServerURL(), nil) - query := roomserverAPI.NewRoomserverQueryAPIHTTP(b.Cfg.RoomServerURL(), nil) + + alias, err := roomserverAPI.NewRoomserverAliasAPIHTTP(b.Cfg.RoomServerURL(), b.httpClient) + if err != nil { + logrus.WithError(err).Panic("NewRoomserverAliasAPIHTTP failed") + } + input, err := roomserverAPI.NewRoomserverInputAPIHTTP(b.Cfg.RoomServerURL(), b.httpClient) + if err != nil { + logrus.WithError(err).Panic("NewRoomserverInputAPIHTTP failed", b.httpClient) + } + query, err := roomserverAPI.NewRoomserverQueryAPIHTTP(b.Cfg.RoomServerURL(), nil) + if err != nil { + logrus.WithError(err).Panic("NewRoomserverQueryAPIHTTP failed", b.httpClient) + } return alias, input, query } // CreateHTTPEDUServerAPIs returns eduInputAPI for hitting the EDU // server over HTTP func (b *BaseDendrite) CreateHTTPEDUServerAPIs() eduServerAPI.EDUServerInputAPI { - return eduServerAPI.NewEDUServerInputAPIHTTP(b.Cfg.EDUServerURL(), nil) + e, err := eduServerAPI.NewEDUServerInputAPIHTTP(b.Cfg.EDUServerURL(), nil) + if err != nil { + logrus.WithError(err).Panic("NewEDUServerInputAPIHTTP failed", b.httpClient) + } + return e } // CreateHTTPFederationSenderAPIs returns FederationSenderQueryAPI for hitting // the federation sender over HTTP func (b *BaseDendrite) CreateHTTPFederationSenderAPIs() federationSenderAPI.FederationSenderQueryAPI { - return federationSenderAPI.NewFederationSenderQueryAPIHTTP(b.Cfg.FederationSenderURL(), nil) + f, err := federationSenderAPI.NewFederationSenderQueryAPIHTTP(b.Cfg.FederationSenderURL(), nil) + if err != nil { + logrus.WithError(err).Panic("NewFederationSenderQueryAPIHTTP failed", b.httpClient) + } + return f } // CreateDeviceDB creates a new instance of the device database. Should only be diff --git a/eduserver/api/input.go b/eduserver/api/input.go index ad3f1ed58..be2d4c56a 100644 --- a/eduserver/api/input.go +++ b/eduserver/api/input.go @@ -15,6 +15,7 @@ package api import ( "context" + "errors" "net/http" commonHTTP "github.com/matrix-org/dendrite/common/http" @@ -57,11 +58,11 @@ type EDUServerInputAPI interface { const EDUServerInputTypingEventPath = "/api/eduserver/input" // NewEDUServerInputAPIHTTP creates a EDUServerInputAPI implemented by talking to a HTTP POST API. -func NewEDUServerInputAPIHTTP(eduServerURL string, httpClient *http.Client) EDUServerInputAPI { +func NewEDUServerInputAPIHTTP(eduServerURL string, httpClient *http.Client) (EDUServerInputAPI, error) { if httpClient == nil { - httpClient = http.DefaultClient + return nil, errors.New("NewTypingServerInputAPIHTTP: httpClient is ") } - return &httpEDUServerInputAPI{eduServerURL, httpClient} + return &httpEDUServerInputAPI{eduServerURL, httpClient}, nil } type httpEDUServerInputAPI struct { diff --git a/federationsender/api/query.go b/federationsender/api/query.go index ebc6e833f..7c0ca7ff2 100644 --- a/federationsender/api/query.go +++ b/federationsender/api/query.go @@ -2,6 +2,7 @@ package api import ( "context" + "errors" "net/http" commonHTTP "github.com/matrix-org/dendrite/common/http" @@ -58,12 +59,12 @@ const FederationSenderQueryJoinedHostsInRoomPath = "/api/federationsender/queryJ const FederationSenderQueryJoinedHostServerNamesInRoomPath = "/api/federationsender/queryJoinedHostServerNamesInRoom" // NewFederationSenderQueryAPIHTTP creates a FederationSenderQueryAPI implemented by talking to a HTTP POST API. -// If httpClient is nil then it uses the http.DefaultClient -func NewFederationSenderQueryAPIHTTP(federationSenderURL string, httpClient *http.Client) FederationSenderQueryAPI { +// If httpClient is nil an error is returned +func NewFederationSenderQueryAPIHTTP(federationSenderURL string, httpClient *http.Client) (FederationSenderQueryAPI, error) { if httpClient == nil { - httpClient = http.DefaultClient + return nil, errors.New("NewFederationSenderQueryAPIHTTP: httpClient is ") } - return &httpFederationSenderQueryAPI{federationSenderURL, httpClient} + return &httpFederationSenderQueryAPI{federationSenderURL, httpClient}, nil } type httpFederationSenderQueryAPI struct { diff --git a/roomserver/api/alias.go b/roomserver/api/alias.go index cb78f726a..ad375a830 100644 --- a/roomserver/api/alias.go +++ b/roomserver/api/alias.go @@ -16,6 +16,7 @@ package api import ( "context" + "errors" "net/http" commonHTTP "github.com/matrix-org/dendrite/common/http" @@ -139,12 +140,12 @@ const RoomserverGetCreatorIDForAliasPath = "/api/roomserver/GetCreatorIDForAlias const RoomserverRemoveRoomAliasPath = "/api/roomserver/removeRoomAlias" // NewRoomserverAliasAPIHTTP creates a RoomserverAliasAPI implemented by talking to a HTTP POST API. -// If httpClient is nil then it uses the http.DefaultClient -func NewRoomserverAliasAPIHTTP(roomserverURL string, httpClient *http.Client) RoomserverAliasAPI { +// If httpClient is nil an error is returned +func NewRoomserverAliasAPIHTTP(roomserverURL string, httpClient *http.Client) (RoomserverAliasAPI, error) { if httpClient == nil { - httpClient = http.DefaultClient + return nil, errors.New("NewRoomserverAliasAPIHTTP: httpClient is ") } - return &httpRoomserverAliasAPI{roomserverURL, httpClient} + return &httpRoomserverAliasAPI{roomserverURL, httpClient}, nil } type httpRoomserverAliasAPI struct { diff --git a/roomserver/api/input.go b/roomserver/api/input.go index f07cc0221..42e8385d0 100644 --- a/roomserver/api/input.go +++ b/roomserver/api/input.go @@ -17,6 +17,7 @@ package api import ( "context" + "errors" "net/http" commonHTTP "github.com/matrix-org/dendrite/common/http" @@ -112,12 +113,12 @@ type RoomserverInputAPI interface { const RoomserverInputRoomEventsPath = "/api/roomserver/inputRoomEvents" // NewRoomserverInputAPIHTTP creates a RoomserverInputAPI implemented by talking to a HTTP POST API. -// If httpClient is nil then it uses the http.DefaultClient -func NewRoomserverInputAPIHTTP(roomserverURL string, httpClient *http.Client) RoomserverInputAPI { +// If httpClient is nil an error is returned +func NewRoomserverInputAPIHTTP(roomserverURL string, httpClient *http.Client) (RoomserverInputAPI, error) { if httpClient == nil { - httpClient = http.DefaultClient + return nil, errors.New("NewRoomserverInputAPIHTTP: httpClient is ") } - return &httpRoomserverInputAPI{roomserverURL, httpClient} + return &httpRoomserverInputAPI{roomserverURL, httpClient}, nil } type httpRoomserverInputAPI struct { diff --git a/roomserver/api/query.go b/roomserver/api/query.go index 3cb1b8a7b..9120da4bb 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -18,6 +18,7 @@ package api import ( "context" + "errors" "net/http" commonHTTP "github.com/matrix-org/dendrite/common/http" @@ -406,12 +407,12 @@ const RoomserverQueryRoomVersionCapabilitiesPath = "/api/roomserver/queryRoomVer const RoomserverQueryRoomVersionForRoomPath = "/api/roomserver/queryRoomVersionForRoom" // NewRoomserverQueryAPIHTTP creates a RoomserverQueryAPI implemented by talking to a HTTP POST API. -// If httpClient is nil then it uses the http.DefaultClient -func NewRoomserverQueryAPIHTTP(roomserverURL string, httpClient *http.Client) RoomserverQueryAPI { +// If httpClient is nil an error is returned +func NewRoomserverQueryAPIHTTP(roomserverURL string, httpClient *http.Client) (RoomserverQueryAPI, error) { if httpClient == nil { - httpClient = http.DefaultClient + return nil, errors.New("NewRoomserverQueryAPIHTTP: httpClient is ") } - return &httpRoomserverQueryAPI{roomserverURL, httpClient} + return &httpRoomserverQueryAPI{roomserverURL, httpClient}, nil } type httpRoomserverQueryAPI struct { From 067b87506357c996fd6ddb11271db9469ad4ce80 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 3 Apr 2020 14:29:06 +0100 Subject: [PATCH 64/86] Invites v2 endpoint (#952) * Start converting v1 invite endpoint to v2 * Update gomatrixserverlib * Early federationsender code for sending invites * Sending invites sorta happens now * Populate invite request with stripped state * Remodel a bit, don't reflect received invites * Handle invite_room_state * Handle room versions a bit better * Update gomatrixserverlib * Tweak order in destinationQueue.next * Revert check in processMessage * Tweak federation sender destination queue code a bit * Add comments --- clientapi/producers/roomserver.go | 14 +-- clientapi/routing/joinroom.go | 3 + federationapi/routing/invite.go | 37 ++----- federationapi/routing/routing.go | 2 +- federationsender/consumers/roomserver.go | 120 +++++++++++++++++---- federationsender/queue/destinationqueue.go | 98 ++++++++++++----- federationsender/queue/queue.go | 43 ++++++++ go.mod | 2 +- go.sum | 4 +- roomserver/api/input.go | 4 +- roomserver/api/output.go | 2 + roomserver/input/events.go | 18 +++- roomserver/input/membership.go | 7 +- roomserver/storage/interface.go | 2 +- roomserver/storage/postgres/storage.go | 10 +- roomserver/storage/sqlite3/storage.go | 10 +- roomserver/types/types.go | 2 + sytest-whitelist | 3 +- 18 files changed, 286 insertions(+), 95 deletions(-) diff --git a/clientapi/producers/roomserver.go b/clientapi/producers/roomserver.go index 06af54404..391ea07bf 100644 --- a/clientapi/producers/roomserver.go +++ b/clientapi/producers/roomserver.go @@ -104,18 +104,14 @@ func (c *RoomserverProducer) SendInputRoomEvents( // This should only be needed for invite events that occur outside of a known room. // If we are in the room then the event should be sent using the SendEvents method. func (c *RoomserverProducer) SendInvite( - ctx context.Context, inviteEvent gomatrixserverlib.Event, + ctx context.Context, inviteEvent gomatrixserverlib.HeaderedEvent, + inviteRoomState []gomatrixserverlib.InviteV2StrippedState, ) error { - verReq := api.QueryRoomVersionForRoomRequest{RoomID: inviteEvent.RoomID()} - verRes := api.QueryRoomVersionForRoomResponse{} - err := c.QueryAPI.QueryRoomVersionForRoom(ctx, &verReq, &verRes) - if err != nil { - return err - } - request := api.InputRoomEventsRequest{ InputInviteEvents: []api.InputInviteEvent{{ - Event: inviteEvent.Headered(verRes.RoomVersion), + Event: inviteEvent, + InviteRoomState: inviteRoomState, + RoomVersion: inviteEvent.RoomVersion, }}, } var response api.InputRoomEventsResponse diff --git a/clientapi/routing/joinroom.go b/clientapi/routing/joinroom.go index 745b4eecc..3ca7d0522 100644 --- a/clientapi/routing/joinroom.go +++ b/clientapi/routing/joinroom.go @@ -260,6 +260,9 @@ func (r joinRoomReq) joinRoomUsingServers( }{roomID}, } } + // TODO: This needs to be re-thought, as in the case of an invite, the room + // will exist in the database in roomserver_rooms but won't have any state + // events, therefore this below check fails. if err != common.ErrRoomNoExists { util.GetLogger(r.req.Context()).WithError(err).Error("common.BuildEvent failed") return jsonerror.InternalServerError() diff --git a/federationapi/routing/invite.go b/federationapi/routing/invite.go index 09c3734be..6c3e12e23 100644 --- a/federationapi/routing/invite.go +++ b/federationapi/routing/invite.go @@ -15,18 +15,17 @@ package routing import ( - "context" + "encoding/json" "net/http" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/common/config" - "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) -// Invite implements /_matrix/federation/v1/invite/{roomID}/{eventID} +// Invite implements /_matrix/federation/v2/invite/{roomID}/{eventID} func Invite( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, @@ -36,24 +35,14 @@ func Invite( producer *producers.RoomserverProducer, keys gomatrixserverlib.KeyRing, ) util.JSONResponse { - // Look up the room version for the room. - verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} - verRes := api.QueryRoomVersionForRoomResponse{} - if err := producer.QueryAPI.QueryRoomVersionForRoom(context.Background(), &verReq, &verRes); err != nil { + inviteReq := gomatrixserverlib.InviteV2Request{} + if err := json.Unmarshal(request.Content(), &inviteReq); err != nil { return util.JSONResponse{ Code: http.StatusBadRequest, - JSON: jsonerror.UnsupportedRoomVersion(err.Error()), - } - } - - // Decode the event JSON from the request. - event, err := gomatrixserverlib.NewEventFromUntrustedJSON(request.Content(), verRes.RoomVersion) - if err != nil { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.NotJSON("The request body could not be decoded into valid JSON. " + err.Error()), + JSON: jsonerror.NotJSON("The request body could not be decoded into an invite request. " + err.Error()), } } + event := inviteReq.Event() // Check that the room ID is correct. if event.RoomID() != roomID { @@ -71,14 +60,6 @@ func Invite( } } - // Check that the event is from the server sending the request. - if event.Origin() != request.Origin() { - return util.JSONResponse{ - Code: http.StatusForbidden, - JSON: jsonerror.Forbidden("The invite must be sent by the server it originated on"), - } - } - // Check that the event is signed by the server sending the request. redacted := event.Redact() verifyRequests := []gomatrixserverlib.VerifyJSONRequest{{ @@ -104,7 +85,11 @@ func Invite( ) // Add the invite event to the roomserver. - if err = producer.SendInvite(httpReq.Context(), signedEvent); err != nil { + if err = producer.SendInvite( + httpReq.Context(), + signedEvent.Headered(inviteReq.RoomVersion()), + inviteReq.InviteRoomState(), + ); err != nil { util.GetLogger(httpReq.Context()).WithError(err).Error("producer.SendInvite failed") return jsonerror.InternalServerError() } diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index 9ac535767..a2b9dc210 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -85,7 +85,7 @@ func Setup( }, )).Methods(http.MethodPut, http.MethodOptions) - v1fedmux.Handle("/invite/{roomID}/{eventID}", common.MakeFedAPI( + v2fedmux.Handle("/invite/{roomID}/{eventID}", common.MakeFedAPI( "federation_invite", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) diff --git a/federationsender/consumers/roomserver.go b/federationsender/consumers/roomserver.go index 8ab2affe2..f59405af0 100644 --- a/federationsender/consumers/roomserver.go +++ b/federationsender/consumers/roomserver.go @@ -32,6 +32,7 @@ import ( // OutputRoomEventConsumer consumes events that originated in the room server. type OutputRoomEventConsumer struct { + cfg *config.Dendrite roomServerConsumer *common.ContinualConsumer db storage.Database queues *queue.OutgoingQueues @@ -52,6 +53,7 @@ func NewOutputRoomEventConsumer( PartitionStore: store, } s := &OutputRoomEventConsumer{ + cfg: cfg, roomServerConsumer: &consumer, db: store, queues: queues, @@ -79,29 +81,48 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { log.WithError(err).Errorf("roomserver output log: message parse failure") return nil } - if output.Type != api.OutputTypeNewRoomEvent { + + switch output.Type { + case api.OutputTypeNewRoomEvent: + ev := &output.NewRoomEvent.Event + log.WithFields(log.Fields{ + "event_id": ev.EventID(), + "room_id": ev.RoomID(), + "send_as_server": output.NewRoomEvent.SendAsServer, + }).Info("received room event from roomserver") + + if err := s.processMessage(*output.NewRoomEvent); err != nil { + // panic rather than continue with an inconsistent database + log.WithFields(log.Fields{ + "event": string(ev.JSON()), + "add": output.NewRoomEvent.AddsStateEventIDs, + "del": output.NewRoomEvent.RemovesStateEventIDs, + log.ErrorKey: err, + }).Panicf("roomserver output log: write room event failure") + return nil + } + case api.OutputTypeNewInviteEvent: + ev := &output.NewInviteEvent.Event + log.WithFields(log.Fields{ + "event_id": ev.EventID(), + "room_id": ev.RoomID(), + "state_key": ev.StateKey(), + }).Info("received invite event from roomserver") + + if err := s.processInvite(*output.NewInviteEvent); err != nil { + // panic rather than continue with an inconsistent database + log.WithFields(log.Fields{ + "event": string(ev.JSON()), + log.ErrorKey: err, + }).Panicf("roomserver output log: write invite event failure") + return nil + } + default: log.WithField("type", output.Type).Debug( "roomserver output log: ignoring unknown output type", ) return nil } - ev := &output.NewRoomEvent.Event - log.WithFields(log.Fields{ - "event_id": ev.EventID(), - "room_id": ev.RoomID(), - "send_as_server": output.NewRoomEvent.SendAsServer, - }).Info("received event from roomserver") - - if err := s.processMessage(*output.NewRoomEvent); err != nil { - // panic rather than continue with an inconsistent database - log.WithFields(log.Fields{ - "event": string(ev.JSON()), - log.ErrorKey: err, - "add": output.NewRoomEvent.AddsStateEventIDs, - "del": output.NewRoomEvent.RemovesStateEventIDs, - }).Panicf("roomserver output log: write event failure") - return nil - } return nil } @@ -159,6 +180,69 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent) err ) } +// processInvite handles an invite event for sending over federation. +func (s *OutputRoomEventConsumer) processInvite(oie api.OutputNewInviteEvent) error { + // Don't try to reflect and resend invites that didn't originate from us. + if s.cfg.Matrix.ServerName != oie.Event.Origin() { + return nil + } + + // When sending a v2 invite, the inviting server should try and include + // a "stripped down" version of the room state. This is pretty much just + // enough information for the remote side to show something useful to the + // user, like the room name, aliases etc. + strippedState := []gomatrixserverlib.InviteV2StrippedState{} + stateWanted := []string{ + gomatrixserverlib.MRoomName, gomatrixserverlib.MRoomCanonicalAlias, + gomatrixserverlib.MRoomAliases, gomatrixserverlib.MRoomJoinRules, + } + + // For each of the state keys that we want to try and send, ask the + // roomserver if we have a state event for that room that matches the + // state key. + for _, wanted := range stateWanted { + queryReq := api.QueryLatestEventsAndStateRequest{ + RoomID: oie.Event.RoomID(), + StateToFetch: []gomatrixserverlib.StateKeyTuple{ + gomatrixserverlib.StateKeyTuple{ + EventType: wanted, + StateKey: "", + }, + }, + } + // If this fails then we just move onto the next event - we don't + // actually know at this point whether the room even has that type + // of state. + queryRes := api.QueryLatestEventsAndStateResponse{} + if err := s.query.QueryLatestEventsAndState(context.TODO(), &queryReq, &queryRes); err != nil { + log.WithFields(log.Fields{ + "room_id": queryReq.RoomID, + "event_type": wanted, + }).WithError(err).Info("couldn't find state to strip") + continue + } + // Append the stripped down copy of the state to our list. + for _, headeredEvent := range queryRes.StateEvents { + event := headeredEvent.Unwrap() + strippedState = append(strippedState, gomatrixserverlib.NewInviteV2StrippedState(&event)) + + log.WithFields(log.Fields{ + "room_id": queryReq.RoomID, + "event_type": event.Type(), + }).Info("adding stripped state") + } + } + + // Build the invite request with the info we've got. + inviteReq, err := gomatrixserverlib.NewInviteV2Request(&oie.Event, strippedState) + if err != nil { + return fmt.Errorf("gomatrixserverlib.NewInviteV2Request: %w", err) + } + + // Send the event. + return s.queues.SendInvite(&inviteReq) +} + // joinedHostsAtEvent works out a list of matrix servers that were joined to // the room at the event. // It is important to use the state at the event for sending messages because: diff --git a/federationsender/queue/destinationqueue.go b/federationsender/queue/destinationqueue.go index b4a6da1a3..7d4dc850b 100644 --- a/federationsender/queue/destinationqueue.go +++ b/federationsender/queue/destinationqueue.go @@ -24,6 +24,7 @@ import ( "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" log "github.com/sirupsen/logrus" + "go.uber.org/atomic" ) // destinationQueue is a queue of events for a single destination. @@ -34,14 +35,15 @@ type destinationQueue struct { client *gomatrixserverlib.FederationClient origin gomatrixserverlib.ServerName destination gomatrixserverlib.ServerName - // The running mutex protects running, sentCounter, lastTransactionIDs and + running atomic.Bool + // The running mutex protects sentCounter, lastTransactionIDs and // pendingEvents, pendingEDUs. runningMutex sync.Mutex - running bool sentCounter int lastTransactionIDs []gomatrixserverlib.TransactionID pendingEvents []*gomatrixserverlib.HeaderedEvent pendingEDUs []*gomatrixserverlib.EDU + pendingInvites []*gomatrixserverlib.InviteV2Request } // Send event adds the event to the pending queue for the destination. @@ -51,29 +53,43 @@ func (oq *destinationQueue) sendEvent(ev *gomatrixserverlib.HeaderedEvent) { oq.runningMutex.Lock() defer oq.runningMutex.Unlock() oq.pendingEvents = append(oq.pendingEvents, ev) - if !oq.running { - oq.running = true + if !oq.running.Load() { go oq.backgroundSend() } } // sendEDU adds the EDU event to the pending queue for the destination. // If the queue is empty then it starts a background goroutine to -// start sending event to that destination. +// start sending events to that destination. func (oq *destinationQueue) sendEDU(e *gomatrixserverlib.EDU) { oq.runningMutex.Lock() defer oq.runningMutex.Unlock() oq.pendingEDUs = append(oq.pendingEDUs, e) - if !oq.running { - oq.running = true + if !oq.running.Load() { go oq.backgroundSend() } } +// sendInvite adds the invite event to the pending queue for the +// destination. If the queue is empty then it starts a background +// goroutine to start sending events to that destination. +func (oq *destinationQueue) sendInvite(ev *gomatrixserverlib.InviteV2Request) { + oq.runningMutex.Lock() + defer oq.runningMutex.Unlock() + oq.pendingInvites = append(oq.pendingInvites, ev) + if !oq.running.Load() { + go oq.backgroundSend() + } +} + +// backgroundSend is the worker goroutine for sending events. func (oq *destinationQueue) backgroundSend() { + oq.running.Store(true) + defer oq.running.Store(false) + for { - t := oq.next() - if t == nil { + transaction, invites := oq.nextTransaction(), oq.nextInvites() + if !transaction && !invites { // If the queue is empty then stop processing for this destination. // TODO: Remove this destination from the queue map. return @@ -81,29 +97,18 @@ func (oq *destinationQueue) backgroundSend() { // TODO: handle retries. // TODO: blacklist uncooperative servers. - - util.GetLogger(context.TODO()).Infof("Sending transaction %q containing %d PDUs, %d EDUs", t.TransactionID, len(t.PDUs), len(t.EDUs)) - - _, err := oq.client.SendTransaction(context.TODO(), *t) - if err != nil { - log.WithFields(log.Fields{ - "destination": oq.destination, - log.ErrorKey: err, - }).Info("problem sending transaction") - } } } -// next creates a new transaction from the pending event queue -// and flushes the queue. -// Returns nil if the queue was empty. -func (oq *destinationQueue) next() *gomatrixserverlib.Transaction { +// nextTransaction creates a new transaction from the pending event +// queue and sends it. Returns true if a transaction was sent or +// false otherwise. +func (oq *destinationQueue) nextTransaction() bool { oq.runningMutex.Lock() defer oq.runningMutex.Unlock() if len(oq.pendingEvents) == 0 && len(oq.pendingEDUs) == 0 { - oq.running = false - return nil + return false } t := gomatrixserverlib.Transaction{ @@ -136,5 +141,46 @@ func (oq *destinationQueue) next() *gomatrixserverlib.Transaction { oq.pendingEDUs = nil oq.sentCounter += len(t.EDUs) - return &t + util.GetLogger(context.TODO()).Infof("Sending transaction %q containing %d PDUs, %d EDUs", t.TransactionID, len(t.PDUs), len(t.EDUs)) + + _, err := oq.client.SendTransaction(context.TODO(), t) + if err != nil { + log.WithFields(log.Fields{ + "destination": oq.destination, + log.ErrorKey: err, + }).Info("problem sending transaction") + } + + return true +} + +// nextInvite takes pending invite events from the queue and sends +// them. Returns true if a transaction was sent or false otherwise. +func (oq *destinationQueue) nextInvites() bool { + oq.runningMutex.Lock() + defer oq.runningMutex.Unlock() + + if len(oq.pendingInvites) == 0 { + return false + } + + for _, inviteReq := range oq.pendingInvites { + ev := inviteReq.Event() + + if _, err := oq.client.SendInviteV2( + context.TODO(), + oq.destination, + *inviteReq, + ); err != nil { + log.WithFields(log.Fields{ + "event_id": ev.EventID(), + "state_key": ev.StateKey(), + "destination": oq.destination, + }).WithError(err).Error("failed to send invite") + } + } + + oq.pendingInvites = nil + + return true } diff --git a/federationsender/queue/queue.go b/federationsender/queue/queue.go index 840fe4afe..88d47f120 100644 --- a/federationsender/queue/queue.go +++ b/federationsender/queue/queue.go @@ -80,6 +80,49 @@ func (oqs *OutgoingQueues) SendEvent( return nil } +// SendEvent sends an event to the destinations +func (oqs *OutgoingQueues) SendInvite( + inviteReq *gomatrixserverlib.InviteV2Request, +) error { + ev := inviteReq.Event() + stateKey := ev.StateKey() + if stateKey == nil { + log.WithFields(log.Fields{ + "event_id": ev.EventID(), + }).Info("invite had no state key, dropping") + return nil + } + + _, destination, err := gomatrixserverlib.SplitID('@', *stateKey) + if err != nil { + log.WithFields(log.Fields{ + "event_id": ev.EventID(), + "state_key": stateKey, + }).Info("failed to split destination from state key") + return nil + } + + log.WithFields(log.Fields{ + "event_id": ev.EventID(), + }).Info("Sending invite") + + oqs.queuesMutex.Lock() + defer oqs.queuesMutex.Unlock() + oq := oqs.queues[destination] + if oq == nil { + oq = &destinationQueue{ + origin: oqs.origin, + destination: destination, + client: oqs.client, + } + oqs.queues[destination] = oq + } + + oq.sendInvite(inviteReq) + + return nil +} + // SendEDU sends an EDU event to the destinations func (oqs *OutgoingQueues) SendEDU( e *gomatrixserverlib.EDU, origin gomatrixserverlib.ServerName, diff --git a/go.mod b/go.mod index 8743437ec..16350bb93 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f github.com/matrix-org/go-sqlite3-js v0.0.0-20200325174927-327088cdef10 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200327155501-33fb4c7049dc + github.com/matrix-org/gomatrixserverlib v0.0.0-20200402141635-4a6e1ade46f8 github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible diff --git a/go.sum b/go.sum index 1294f3b7b..f6b268c4a 100644 --- a/go.sum +++ b/go.sum @@ -130,8 +130,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:kmRjpmFOenVpOaV/DRlo9p6z/IbOKlUC+hhKsAAh8Qg= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200327155501-33fb4c7049dc h1:qrRu4/AlulnldLiyGpYYm+ELIkrP51XCRlA3txWpN30= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200327155501-33fb4c7049dc/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200402141635-4a6e1ade46f8 h1:VZ7xGklSuzU9geMekuxKO4FvUBUaPjP+8IkcwzQtqOI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200402141635-4a6e1ade46f8/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 h1:W7l5CP4V7wPyPb4tYE11dbmeAOwtFQBTW0rf4OonOS8= diff --git a/roomserver/api/input.go b/roomserver/api/input.go index 42e8385d0..87e3983e3 100644 --- a/roomserver/api/input.go +++ b/roomserver/api/input.go @@ -86,7 +86,9 @@ type TransactionID struct { // the usual context a matrix room event would have. We usually do not have // access to the events needed to check the event auth rules for the invite. type InputInviteEvent struct { - Event gomatrixserverlib.HeaderedEvent `json:"event"` + RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"` + Event gomatrixserverlib.HeaderedEvent `json:"event"` + InviteRoomState []gomatrixserverlib.InviteV2StrippedState `json:"invite_room_state"` } // InputRoomEventsRequest is a request to InputRoomEvents diff --git a/roomserver/api/output.go b/roomserver/api/output.go index 4e7adff79..92a468a96 100644 --- a/roomserver/api/output.go +++ b/roomserver/api/output.go @@ -116,6 +116,8 @@ type OutputNewRoomEvent struct { // Invite events can be received outside of an existing room so have to be // tracked separately from the room events themselves. type OutputNewInviteEvent struct { + // The room version of the invited room. + RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"` // The "m.room.member" invite event. Event gomatrixserverlib.HeaderedEvent `json:"event"` } diff --git a/roomserver/input/events.go b/roomserver/input/events.go index c75a3acd9..2bb0d0a05 100644 --- a/roomserver/input/events.go +++ b/roomserver/input/events.go @@ -26,6 +26,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/state/database" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" + log "github.com/sirupsen/logrus" ) // A RoomEventDatabase has the storage APIs needed to store a room event. @@ -64,6 +65,7 @@ type RoomEventDatabase interface { // Build a membership updater for the target user in a room. MembershipUpdater( ctx context.Context, roomID, targerUserID string, + roomVersion gomatrixserverlib.RoomVersion, ) (types.MembershipUpdater, error) // Look up event ID by transaction's info. // This is used to determine if the room event is processed/processing already. @@ -193,7 +195,14 @@ func processInviteEvent( roomID := input.Event.RoomID() targetUserID := *input.Event.StateKey() - updater, err := db.MembershipUpdater(ctx, roomID, targetUserID) + log.WithFields(log.Fields{ + "event_id": input.Event.EventID(), + "room_id": roomID, + "room_version": input.RoomVersion, + "target_user_id": targetUserID, + }).Info("processing invite event") + + updater, err := db.MembershipUpdater(ctx, roomID, targetUserID, input.RoomVersion) if err != nil { return err } @@ -237,7 +246,12 @@ func processInviteEvent( } event := input.Event.Unwrap() - outputUpdates, err := updateToInviteMembership(updater, &event, nil) + + if err = event.SetUnsignedField("invite_room_state", input.InviteRoomState); err != nil { + return err + } + + outputUpdates, err := updateToInviteMembership(updater, &event, nil, input.Event.RoomVersion) if err != nil { return err } diff --git a/roomserver/input/membership.go b/roomserver/input/membership.go index f2ac3b510..ee39ff5eb 100644 --- a/roomserver/input/membership.go +++ b/roomserver/input/membership.go @@ -112,7 +112,7 @@ func updateMembership( switch newMembership { case gomatrixserverlib.Invite: - return updateToInviteMembership(mu, add, updates) + return updateToInviteMembership(mu, add, updates, updater.RoomVersion()) case gomatrixserverlib.Join: return updateToJoinMembership(mu, add, updates) case gomatrixserverlib.Leave, gomatrixserverlib.Ban: @@ -126,6 +126,7 @@ func updateMembership( func updateToInviteMembership( mu types.MembershipUpdater, add *gomatrixserverlib.Event, updates []api.OutputEvent, + roomVersion gomatrixserverlib.RoomVersion, ) ([]api.OutputEvent, error) { // We may have already sent the invite to the user, either because we are // reprocessing this event, or because the we received this invite from a @@ -136,14 +137,14 @@ func updateToInviteMembership( return nil, err } if needsSending { - roomVersion := gomatrixserverlib.RoomVersionV1 // We notify the consumers using a special event even though we will // notify them about the change in current state as part of the normal // room event stream. This ensures that the consumers only have to // consider a single stream of events when determining whether a user // is invited, rather than having to combine multiple streams themselves. onie := api.OutputNewInviteEvent{ - Event: (*add).Headered(roomVersion), + Event: (*add).Headered(roomVersion), + RoomVersion: roomVersion, } updates = append(updates, api.OutputEvent{ Type: api.OutputTypeNewInviteEvent, diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go index 20db7ef7f..50369d806 100644 --- a/roomserver/storage/interface.go +++ b/roomserver/storage/interface.go @@ -41,7 +41,7 @@ type Database interface { GetAliasesForRoomID(ctx context.Context, roomID string) ([]string, error) GetCreatorIDForAlias(ctx context.Context, alias string) (string, error) RemoveRoomAlias(ctx context.Context, alias string) error - MembershipUpdater(ctx context.Context, roomID, targetUserID string) (types.MembershipUpdater, error) + MembershipUpdater(ctx context.Context, roomID, targetUserID string, roomVersion gomatrixserverlib.RoomVersion) (types.MembershipUpdater, error) GetMembership(ctx context.Context, roomNID types.RoomNID, requestSenderUserID string) (membershipEventNID types.EventNID, stillInRoom bool, err error) GetMembershipEventNIDsForRoom(ctx context.Context, roomNID types.RoomNID, joinOnly bool) ([]types.EventNID, error) EventsFromIDs(ctx context.Context, eventIDs []string) ([]types.Event, error) diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go index 83a17b1a1..c91c59ebc 100644 --- a/roomserver/storage/postgres/storage.go +++ b/roomserver/storage/postgres/storage.go @@ -393,6 +393,12 @@ type roomRecentEventsUpdater struct { currentStateSnapshotNID types.StateSnapshotNID } +// RoomVersion implements types.RoomRecentEventsUpdater +func (u *roomRecentEventsUpdater) RoomVersion() (version gomatrixserverlib.RoomVersion) { + version, _ = u.d.GetRoomVersionForRoomNID(u.ctx, u.roomNID) + return +} + // LatestEvents implements types.RoomRecentEventsUpdater func (u *roomRecentEventsUpdater) LatestEvents() []types.StateAtEventAndReference { return u.latestEvents @@ -534,6 +540,7 @@ func (d *Database) StateEntriesForTuples( // MembershipUpdater implements input.RoomEventDatabase func (d *Database) MembershipUpdater( ctx context.Context, roomID, targetUserID string, + roomVersion gomatrixserverlib.RoomVersion, ) (types.MembershipUpdater, error) { txn, err := d.db.Begin() if err != nil { @@ -546,8 +553,7 @@ func (d *Database) MembershipUpdater( } }() - // TODO: Room version here - roomNID, err := d.assignRoomNID(ctx, txn, roomID, "1") + roomNID, err := d.assignRoomNID(ctx, txn, roomID, roomVersion) if err != nil { return nil, err } diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index 6d6743393..f6c692fd1 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -486,6 +486,12 @@ type roomRecentEventsUpdater struct { currentStateSnapshotNID types.StateSnapshotNID } +// RoomVersion implements types.RoomRecentEventsUpdater +func (u *roomRecentEventsUpdater) RoomVersion() (version gomatrixserverlib.RoomVersion) { + version, _ = u.d.GetRoomVersionForRoomNID(u.ctx, u.roomNID) + return +} + // LatestEvents implements types.RoomRecentEventsUpdater func (u *roomRecentEventsUpdater) LatestEvents() []types.StateAtEventAndReference { return u.latestEvents @@ -657,6 +663,7 @@ func (d *Database) StateEntriesForTuples( // MembershipUpdater implements input.RoomEventDatabase func (d *Database) MembershipUpdater( ctx context.Context, roomID, targetUserID string, + roomVersion gomatrixserverlib.RoomVersion, ) (updater types.MembershipUpdater, err error) { var txn *sql.Tx txn, err = d.db.Begin() @@ -682,8 +689,7 @@ func (d *Database) MembershipUpdater( } }() - // TODO: Room version here - roomNID, err := d.assignRoomNID(ctx, txn, roomID, "1") + roomNID, err := d.assignRoomNID(ctx, txn, roomID, roomVersion) if err != nil { return nil, err } diff --git a/roomserver/types/types.go b/roomserver/types/types.go index d5fe32762..dfc112cfd 100644 --- a/roomserver/types/types.go +++ b/roomserver/types/types.go @@ -140,6 +140,8 @@ type StateEntryList struct { // (On postgresql this wraps a database transaction that holds a "FOR UPDATE" // lock on the row in the rooms table holding the latest events for the room.) type RoomRecentEventsUpdater interface { + // The room version of the room. + RoomVersion() gomatrixserverlib.RoomVersion // The latest event IDs and state in the room. LatestEvents() []StateAtEventAndReference // The event ID of the latest event written to the output log in the room. diff --git a/sytest-whitelist b/sytest-whitelist index a2e7b2a6d..38062bb59 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -230,4 +230,5 @@ local user can join room with version 2 remote user can join room with version 2 User can invite local user to room with version 2 Remote user can backfill in a room with version 2 -Inbound federation accepts attempts to join v2 rooms from servers with support \ No newline at end of file +Inbound federation accepts attempts to join v2 rooms from servers with support +Outbound federation can send invites via v2 API \ No newline at end of file From dacee648f7b6a44636271709cc62b93e25b0f451 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 9 Apr 2020 15:46:06 +0100 Subject: [PATCH 65/86] Federation for v3/v4 rooms (#954) * Update gomatrixserverlib * Default to room version 4 * Update gomatrixserverlib * Limit prev_events and auth_events * Fix auth_events, prev_events * Fix linter issues * Update gomatrixserverlib * Fix getState * Update sytest-whitelist * Squashed commit of the following: commit 067b87506357c996fd6ddb11271db9469ad4ce80 Author: Neil Alexander Date: Fri Apr 3 14:29:06 2020 +0100 Invites v2 endpoint (#952) * Start converting v1 invite endpoint to v2 * Update gomatrixserverlib * Early federationsender code for sending invites * Sending invites sorta happens now * Populate invite request with stripped state * Remodel a bit, don't reflect received invites * Handle invite_room_state * Handle room versions a bit better * Update gomatrixserverlib * Tweak order in destinationQueue.next * Revert check in processMessage * Tweak federation sender destination queue code a bit * Add comments commit 955244c09298d0e6c870377dad3af2ffa1f5e578 Author: Ben B Date: Fri Apr 3 12:40:50 2020 +0200 use custom http client instead of the http DefaultClient (#823) This commit replaces the default client from the http lib with a custom one. The previously used default client doesn't come with a timeout. This could cause unwanted locks. That solution chosen here creates a http client in the base component dendrite with a constant timeout of 30 seconds. If it should be necessary to overwrite this, we could include the timeout in the dendrite configuration. Here it would be a good idea to extend the type "Address" by a timeout and create an http client for each service. Closes #820 Signed-off-by: Benedikt Bongartz Co-authored-by: Kegsay * Update sytest-whitelist, sytest-blacklist * Update go.mod/go.sum * Add some error wrapping for debug * Add a NOTSPEC to common/events.go * Perform state resolution at send_join * Set default room version to v2 again * Tweak GetCapabilities * Add comments to ResolveConflictsAdhoc * Update sytest-blacklist * go mod tidy * Update sytest-whitelist, sytest-blacklist * Update versions * Updates from review comments * Update sytest-blacklist, sytest-whitelist * Check room versions compatible at make_join, add some comments, update gomatrixserverlib, other tweaks * Set default room version back to v2 * Update gomatrixserverlib, sytest-whitelist --- clientapi/routing/capabilities.go | 2 +- clientapi/routing/joinroom.go | 23 +++++---- common/events.go | 42 +++++++++++----- federationapi/routing/join.go | 38 ++++++++++++++- federationapi/routing/routing.go | 38 ++++++++++----- federationapi/routing/state.go | 3 +- go.mod | 6 +-- go.sum | 15 ++++-- roomserver/api/query.go | 3 ++ roomserver/query/query.go | 10 +++- roomserver/state/state.go | 80 ++++++++++++++++++++++++++++++- roomserver/version/version.go | 8 ++-- sytest-blacklist | 5 +- sytest-whitelist | 17 ++++++- 14 files changed, 235 insertions(+), 55 deletions(-) diff --git a/clientapi/routing/capabilities.go b/clientapi/routing/capabilities.go index 0c583055e..1792c6308 100644 --- a/clientapi/routing/capabilities.go +++ b/clientapi/routing/capabilities.go @@ -29,7 +29,7 @@ func GetCapabilities( req *http.Request, queryAPI roomserverAPI.RoomserverQueryAPI, ) util.JSONResponse { roomVersionsQueryReq := roomserverAPI.QueryRoomVersionCapabilitiesRequest{} - var roomVersionsQueryRes roomserverAPI.QueryRoomVersionCapabilitiesResponse + roomVersionsQueryRes := roomserverAPI.QueryRoomVersionCapabilitiesResponse{} if err := queryAPI.QueryRoomVersionCapabilities( req.Context(), &roomVersionsQueryReq, diff --git a/clientapi/routing/joinroom.go b/clientapi/routing/joinroom.go index 3ca7d0522..0f1a9ba4d 100644 --- a/clientapi/routing/joinroom.go +++ b/clientapi/routing/joinroom.go @@ -242,6 +242,9 @@ func (r joinRoomReq) joinRoomUsingServers( queryRes := roomserverAPI.QueryLatestEventsAndStateResponse{} event, err := common.BuildEvent(r.req.Context(), &eb, r.cfg, r.evTime, r.queryAPI, &queryRes) if err == nil { + // If we have successfully built an event at this point then we can + // assert that the room is a local room, as BuildEvent was able to + // add prev_events etc successfully. if _, err = r.producer.SendEvents( r.req.Context(), []gomatrixserverlib.HeaderedEvent{ @@ -260,6 +263,10 @@ func (r joinRoomReq) joinRoomUsingServers( }{roomID}, } } + + // Otherwise, if we've reached here, then we haven't been able to populate + // prev_events etc for the room, therefore the room is probably federated. + // TODO: This needs to be re-thought, as in the case of an invite, the room // will exist in the database in roomserver_rooms but won't have any state // events, therefore this below check fails. @@ -323,14 +330,14 @@ func (r joinRoomReq) joinRoomUsingServer(roomID string, server gomatrixserverlib respMakeJoin, err := r.federation.MakeJoin(r.req.Context(), server, roomID, r.userID, supportedVersions) if err != nil { // TODO: Check if the user was not allowed to join the room. - return nil, err + return nil, fmt.Errorf("r.federation.MakeJoin: %w", err) } // Set all the fields to be what they should be, this should be a no-op // but it's possible that the remote server returned us something "odd" err = r.writeToBuilder(&respMakeJoin.JoinEvent, roomID) if err != nil { - return nil, err + return nil, fmt.Errorf("r.writeToBuilder: %w", err) } if respMakeJoin.RoomVersion == "" { @@ -350,18 +357,16 @@ func (r joinRoomReq) joinRoomUsingServer(roomID string, server gomatrixserverlib r.cfg.Matrix.PrivateKey, respMakeJoin.RoomVersion, ) if err != nil { - util.GetLogger(r.req.Context()).WithError(err).Error("respMakeJoin.JoinEvent.Build failed") - res := jsonerror.InternalServerError() - return &res, nil + return nil, fmt.Errorf("respMakeJoin.JoinEvent.Build: %w", err) } respSendJoin, err := r.federation.SendJoin(r.req.Context(), server, event, respMakeJoin.RoomVersion) if err != nil { - return nil, err + return nil, fmt.Errorf("r.federation.SendJoin: %w", err) } if err = respSendJoin.Check(r.req.Context(), r.keyRing, event); err != nil { - return nil, err + return nil, fmt.Errorf("respSendJoin: %w", err) } if err = r.producer.SendEventWithState( @@ -369,9 +374,7 @@ func (r joinRoomReq) joinRoomUsingServer(roomID string, server gomatrixserverlib gomatrixserverlib.RespState(respSendJoin.RespState), event.Headered(respMakeJoin.RoomVersion), ); err != nil { - util.GetLogger(r.req.Context()).WithError(err).Error("gomatrixserverlib.RespState failed") - res := jsonerror.InternalServerError() - return &res, nil + return nil, fmt.Errorf("r.producer.SendEventWithState: %w", err) } return &util.JSONResponse{ diff --git a/common/events.go b/common/events.go index b79998a73..adbdf3389 100644 --- a/common/events.go +++ b/common/events.go @@ -17,6 +17,7 @@ package common import ( "context" "errors" + "fmt" "time" "github.com/matrix-org/dendrite/common/config" @@ -46,6 +47,7 @@ func BuildEvent( err := AddPrevEventsToEvent(ctx, builder, queryAPI, queryRes) if err != nil { + // This can pass through a ErrRoomNoExists to the caller return nil, err } @@ -68,7 +70,7 @@ func AddPrevEventsToEvent( ) error { eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(builder) if err != nil { - return err + return fmt.Errorf("gomatrixserverlib.StateNeededForEventBuilder: %w", err) } // Ask the roomserver for information about this room @@ -77,7 +79,7 @@ func AddPrevEventsToEvent( StateToFetch: eventsNeeded.Tuples(), } if err = queryAPI.QueryLatestEventsAndState(ctx, &queryReq, queryRes); err != nil { - return err + return fmt.Errorf("queryAPI.QueryLatestEventsAndState: %w", err) } if !queryRes.RoomExists { @@ -86,7 +88,7 @@ func AddPrevEventsToEvent( eventFormat, err := queryRes.RoomVersion.EventFormat() if err != nil { - return err + return fmt.Errorf("queryRes.RoomVersion.EventFormat: %w", err) } builder.Depth = queryRes.Depth @@ -96,26 +98,26 @@ func AddPrevEventsToEvent( for i := range queryRes.StateEvents { err = authEvents.AddEvent(&queryRes.StateEvents[i].Event) if err != nil { - return err + return fmt.Errorf("authEvents.AddEvent: %w", err) } } refs, err := eventsNeeded.AuthEventReferences(&authEvents) if err != nil { - return err + return fmt.Errorf("eventsNeeded.AuthEventReferences: %w", err) } + truncAuth, truncPrev := truncateAuthAndPrevEvents(refs, queryRes.LatestEvents) switch eventFormat { case gomatrixserverlib.EventFormatV1: - builder.AuthEvents = refs - builder.PrevEvents = queryRes.LatestEvents + builder.AuthEvents = truncAuth + builder.PrevEvents = truncPrev case gomatrixserverlib.EventFormatV2: - v2AuthRefs := []string{} - v2PrevRefs := []string{} - for _, ref := range refs { + v2AuthRefs, v2PrevRefs := []string{}, []string{} + for _, ref := range truncAuth { v2AuthRefs = append(v2AuthRefs, ref.EventID) } - for _, ref := range queryRes.LatestEvents { + for _, ref := range truncPrev { v2PrevRefs = append(v2PrevRefs, ref.EventID) } builder.AuthEvents = v2AuthRefs @@ -124,3 +126,21 @@ func AddPrevEventsToEvent( return nil } + +// truncateAuthAndPrevEvents limits the number of events we add into +// an event as prev_events or auth_events. +// NOTSPEC: The limits here feel a bit arbitrary but they are currently +// here because of https://github.com/matrix-org/matrix-doc/issues/2307 +// and because Synapse will just drop events that don't comply. +func truncateAuthAndPrevEvents(auth, prev []gomatrixserverlib.EventReference) ( + truncAuth, truncPrev []gomatrixserverlib.EventReference, +) { + truncAuth, truncPrev = auth, prev + if len(truncAuth) > 10 { + truncAuth = truncAuth[:10] + } + if len(truncPrev) > 20 { + truncPrev = truncPrev[:20] + } + return +} diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go index a39ff6394..0a7b23000 100644 --- a/federationapi/routing/join.go +++ b/federationapi/routing/join.go @@ -15,6 +15,7 @@ package routing import ( + "fmt" "net/http" "time" @@ -34,6 +35,7 @@ func MakeJoin( cfg *config.Dendrite, query api.RoomserverQueryAPI, roomID, userID string, + remoteVersions []gomatrixserverlib.RoomVersion, ) util.JSONResponse { verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} verRes := api.QueryRoomVersionForRoomResponse{} @@ -44,6 +46,27 @@ func MakeJoin( } } + // Check that the room that the remote side is trying to join is actually + // one of the room versions that they listed in their supported ?ver= in + // the make_join URL. + // https://matrix.org/docs/spec/server_server/r0.1.3#get-matrix-federation-v1-make-join-roomid-userid + remoteSupportsVersion := false + for _, v := range remoteVersions { + if v == verRes.RoomVersion { + remoteSupportsVersion = true + break + } + } + // If it isn't, stop trying to join the room. + if !remoteSupportsVersion { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.UnsupportedRoomVersion( + fmt.Sprintf("Joining server does not support room version %s", verRes.RoomVersion), + ), + } + } + _, domain, err := gomatrixserverlib.SplitID('@', userID) if err != nil { return util.JSONResponse{ @@ -140,7 +163,12 @@ func SendJoin( if event.RoomID() != roomID { return util.JSONResponse{ Code: http.StatusBadRequest, - JSON: jsonerror.BadJSON("The room ID in the request path must match the room ID in the join event JSON"), + JSON: jsonerror.BadJSON( + fmt.Sprintf( + "The room ID in the request path (%q) must match the room ID in the join event JSON (%q)", + roomID, event.RoomID(), + ), + ), } } @@ -148,7 +176,12 @@ func SendJoin( if event.EventID() != eventID { return util.JSONResponse{ Code: http.StatusBadRequest, - JSON: jsonerror.BadJSON("The event ID in the request path must match the event ID in the join event JSON"), + JSON: jsonerror.BadJSON( + fmt.Sprintf( + "The event ID in the request path (%q) must match the event ID in the join event JSON (%q)", + eventID, event.EventID(), + ), + ), } } @@ -186,6 +219,7 @@ func SendJoin( PrevEventIDs: event.PrevEventIDs(), AuthEventIDs: event.AuthEventIDs(), RoomID: roomID, + ResolveState: true, }, &stateAndAuthChainResponse) if err != nil { util.GetLogger(httpReq.Context()).WithError(err).Error("query.QueryStateAndAuthChain failed") diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index a2b9dc210..83bac5550 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -198,7 +198,7 @@ func Setup( }, )).Methods(http.MethodGet) - v1fedmux.Handle("/make_join/{roomID}/{userID}", common.MakeFedAPI( + v1fedmux.Handle("/make_join/{roomID}/{eventID}", common.MakeFedAPI( "federation_make_join", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) @@ -206,14 +206,28 @@ func Setup( return util.ErrorResponse(err) } roomID := vars["roomID"] - userID := vars["userID"] + eventID := vars["eventID"] + queryVars := httpReq.URL.Query() + remoteVersions := []gomatrixserverlib.RoomVersion{} + if vers, ok := queryVars["ver"]; ok { + // The remote side supplied a ?=ver so use that to build up the list + // of supported room versions + for _, v := range vers { + remoteVersions = append(remoteVersions, gomatrixserverlib.RoomVersion(v)) + } + } else { + // The remote side didn't supply a ?ver= so just assume that they only + // support room version 1, as per the spec + // https://matrix.org/docs/spec/server_server/r0.1.3#get-matrix-federation-v1-make-join-roomid-userid + remoteVersions = append(remoteVersions, gomatrixserverlib.RoomVersionV1) + } return MakeJoin( - httpReq, request, cfg, query, roomID, userID, + httpReq, request, cfg, query, roomID, eventID, remoteVersions, ) }, )).Methods(http.MethodGet) - v2fedmux.Handle("/send_join/{roomID}/{userID}", common.MakeFedAPI( + v2fedmux.Handle("/send_join/{roomID}/{eventID}", common.MakeFedAPI( "federation_send_join", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) @@ -221,14 +235,14 @@ func Setup( return util.ErrorResponse(err) } roomID := vars["roomID"] - userID := vars["userID"] + eventID := vars["eventID"] return SendJoin( - httpReq, request, cfg, query, producer, keys, roomID, userID, + httpReq, request, cfg, query, producer, keys, roomID, eventID, ) }, )).Methods(http.MethodPut) - v1fedmux.Handle("/make_leave/{roomID}/{userID}", common.MakeFedAPI( + v1fedmux.Handle("/make_leave/{roomID}/{eventID}", common.MakeFedAPI( "federation_make_leave", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) @@ -236,14 +250,14 @@ func Setup( return util.ErrorResponse(err) } roomID := vars["roomID"] - userID := vars["userID"] + eventID := vars["eventID"] return MakeLeave( - httpReq, request, cfg, query, roomID, userID, + httpReq, request, cfg, query, roomID, eventID, ) }, )).Methods(http.MethodGet) - v2fedmux.Handle("/send_leave/{roomID}/{userID}", common.MakeFedAPI( + v2fedmux.Handle("/send_leave/{roomID}/{eventID}", common.MakeFedAPI( "federation_send_leave", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) @@ -251,9 +265,9 @@ func Setup( return util.ErrorResponse(err) } roomID := vars["roomID"] - userID := vars["userID"] + eventID := vars["eventID"] return SendLeave( - httpReq, request, cfg, producer, keys, roomID, userID, + httpReq, request, cfg, producer, keys, roomID, eventID, ) }, )).Methods(http.MethodPut) diff --git a/federationapi/routing/state.go b/federationapi/routing/state.go index 6a47882b7..548598dd7 100644 --- a/federationapi/routing/state.go +++ b/federationapi/routing/state.go @@ -107,7 +107,6 @@ func getState( return nil, &util.JSONResponse{Code: http.StatusNotFound, JSON: nil} } - prevEventIDs := getIDsFromEventRef(event.PrevEvents()) authEventIDs := getIDsFromEventRef(event.AuthEvents()) var response api.QueryStateAndAuthChainResponse @@ -115,7 +114,7 @@ func getState( ctx, &api.QueryStateAndAuthChainRequest{ RoomID: roomID, - PrevEventIDs: prevEventIDs, + PrevEventIDs: []string{eventID}, AuthEventIDs: authEventIDs, }, &response, diff --git a/go.mod b/go.mod index 16350bb93..5854e552b 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f github.com/matrix-org/go-sqlite3-js v0.0.0-20200325174927-327088cdef10 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200402141635-4a6e1ade46f8 + github.com/matrix-org/gomatrixserverlib v0.0.0-20200409140603-8b9a51fe9b89 github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible @@ -23,9 +23,9 @@ require ( github.com/tidwall/pretty v1.0.1 // indirect github.com/uber/jaeger-client-go v2.22.1+incompatible github.com/uber/jaeger-lib v2.2.0+incompatible - go.uber.org/atomic v1.6.0 // indirect + go.uber.org/atomic v1.6.0 golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d - golang.org/x/net v0.0.0-20190909003024-a7b16738d86b // indirect + golang.org/x/tools v0.0.0-20200402223321-bcf690261a44 // indirect gopkg.in/Shopify/sarama.v1 v1.20.1 gopkg.in/h2non/bimg.v1 v1.0.18 gopkg.in/yaml.v2 v2.2.5 diff --git a/go.sum b/go.sum index f6b268c4a..b1f5ac2f4 100644 --- a/go.sum +++ b/go.sum @@ -130,8 +130,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:kmRjpmFOenVpOaV/DRlo9p6z/IbOKlUC+hhKsAAh8Qg= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200402141635-4a6e1ade46f8 h1:VZ7xGklSuzU9geMekuxKO4FvUBUaPjP+8IkcwzQtqOI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200402141635-4a6e1ade46f8/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200409140603-8b9a51fe9b89 h1:YAlUJK/Ty2ZrP/DL41CiR0Cp3pteshnyIS420KVs220= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200409140603-8b9a51fe9b89/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 h1:W7l5CP4V7wPyPb4tYE11dbmeAOwtFQBTW0rf4OonOS8= @@ -253,6 +253,7 @@ github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/Aaua github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw= github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.0 h1:vs7fgriifsPbGdK3bNuMWapNn3qnZhCRXc19NRdq010= go.uber.org/atomic v1.3.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -276,6 +277,8 @@ golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvx golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -288,8 +291,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190909003024-a7b16738d86b h1:XfVGCX+0T4WOStkaOsJRllbsiImhB2jgVBGc9L0lPGc= -golang.org/x/net v0.0.0-20190909003024-a7b16738d86b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -324,7 +327,11 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c h1:IGkKhmfzcztjm6gYkykvu/NiS8kaqbCWAEWWAyf8J5U= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200402223321-bcf690261a44 h1:bMm0eoDiGkM5VfIyKjxDvoflW5GLp7+VCo+60n8F+TE= +golang.org/x/tools v0.0.0-20200402223321-bcf690261a44/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= diff --git a/roomserver/api/query.go b/roomserver/api/query.go index 9120da4bb..5f024d266 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -203,6 +203,9 @@ type QueryStateAndAuthChainRequest struct { PrevEventIDs []string `json:"prev_event_ids"` // The list of auth events for the event. Used to calculate the auth chain AuthEventIDs []string `json:"auth_event_ids"` + // Should state resolution be ran on the result events? + // TODO: check call sites and remove if we always want to do state res + ResolveState bool `json:"resolve_state"` } // QueryStateAndAuthChainResponse is a response to QueryStateAndAuthChain diff --git a/roomserver/query/query.go b/roomserver/query/query.go index b7cdf1507..7e05fe36f 100644 --- a/roomserver/query/query.go +++ b/roomserver/query/query.go @@ -132,7 +132,7 @@ func (r *RoomserverQueryAPI) QueryLatestEventsAndState( return err } - // Look up the currrent state for the requested tuples. + // Look up the current state for the requested tuples. stateEntries, err := roomState.LoadStateAtSnapshotForStringTuples( ctx, currentStateSnapshotNID, request.StateToFetch, ) @@ -736,6 +736,14 @@ func (r *RoomserverQueryAPI) QueryStateAndAuthChain( return err } + if request.ResolveState { + if stateEvents, err = state.ResolveConflictsAdhoc( + roomVersion, stateEvents, authEvents, + ); err != nil { + return err + } + } + for _, event := range stateEvents { response.StateEvents = append(response.StateEvents, event.Headered(roomVersion)) } diff --git a/roomserver/state/state.go b/roomserver/state/state.go index 94873dbeb..3f68e0747 100644 --- a/roomserver/state/state.go +++ b/roomserver/state/state.go @@ -18,7 +18,6 @@ package state import ( "context" - "errors" "fmt" "sort" "time" @@ -681,6 +680,83 @@ func (v StateResolution) calculateStateAfterManyEvents( return } +// ResolveConflictsAdhoc is a helper function to assist the query API in +// performing state resolution when requested. This is a different code +// path to the rest of state.go because this assumes you already have +// gomatrixserverlib.Event objects and not just a bunch of NIDs like +// elsewhere in the state resolution. +// TODO: Some of this can possibly be deduplicated +func ResolveConflictsAdhoc( + version gomatrixserverlib.RoomVersion, + events []gomatrixserverlib.Event, + authEvents []gomatrixserverlib.Event, +) ([]gomatrixserverlib.Event, error) { + type stateKeyTuple struct { + Type string + StateKey string + } + + // Prepare our data structures. + eventMap := make(map[stateKeyTuple][]gomatrixserverlib.Event) + var conflicted, notConflicted, resolved []gomatrixserverlib.Event + + // Run through all of the events that we were given and sort them + // into a map, sorted by (event_type, state_key) tuple. This means + // that we can easily spot events that are "conflicted", e.g. + // there are duplicate values for the same tuple key. + for _, event := range events { + if event.StateKey() == nil { + // Ignore events that are not state events. + continue + } + // Append the events if there is already a conflicted list for + // this tuple key, create it if not. + tuple := stateKeyTuple{event.Type(), *event.StateKey()} + if _, ok := eventMap[tuple]; ok { + eventMap[tuple] = append(eventMap[tuple], event) + } else { + eventMap[tuple] = []gomatrixserverlib.Event{event} + } + } + + // Split out the events in the map into conflicted and unconflicted + // buckets. The conflicted events will be ran through state res, + // whereas unconfliced events will always going to appear in the + // final resolved state. + for _, list := range eventMap { + if len(list) > 1 { + conflicted = append(conflicted, list...) + } else { + notConflicted = append(notConflicted, list...) + } + } + + // Work out which state resolution algorithm we want to run for + // the room version. + stateResAlgo, err := version.StateResAlgorithm() + if err != nil { + return nil, err + } + switch stateResAlgo { + case gomatrixserverlib.StateResV1: + // Currently state res v1 doesn't handle unconflicted events + // for us, like state res v2 does, so we will need to add the + // unconflicted events into the state ourselves. + // TODO: Fix state res v1 so this is handled for the caller. + resolved = gomatrixserverlib.ResolveStateConflicts(conflicted, authEvents) + resolved = append(resolved, notConflicted...) + case gomatrixserverlib.StateResV2: + // TODO: auth difference here? + resolved = gomatrixserverlib.ResolveStateConflictsV2(conflicted, notConflicted, authEvents, authEvents) + default: + return nil, fmt.Errorf("unsupported state resolution algorithm %v", stateResAlgo) + } + + // Return the final resolved state events, including both the + // resolved set of conflicted events, and the unconflicted events. + return resolved, nil +} + func (v StateResolution) resolveConflicts( ctx context.Context, version gomatrixserverlib.RoomVersion, notConflicted, conflicted []types.StateEntry, @@ -695,7 +771,7 @@ func (v StateResolution) resolveConflicts( case gomatrixserverlib.StateResV2: return v.resolveConflictsV2(ctx, notConflicted, conflicted) } - return nil, errors.New("unsupported state resolution algorithm") + return nil, fmt.Errorf("unsupported state resolution algorithm %v", stateResAlgo) } // resolveConflicts resolves a list of conflicted state entries. It takes two lists. diff --git a/roomserver/version/version.go b/roomserver/version/version.go index ed16ecca0..e60b5ef7a 100644 --- a/roomserver/version/version.go +++ b/roomserver/version/version.go @@ -43,12 +43,12 @@ var roomVersions = map[gomatrixserverlib.RoomVersion]RoomVersionDescription{ Stable: true, }, gomatrixserverlib.RoomVersionV3: RoomVersionDescription{ - Supported: false, - Stable: false, + Supported: true, + Stable: true, }, gomatrixserverlib.RoomVersionV4: RoomVersionDescription{ - Supported: false, - Stable: false, + Supported: true, + Stable: true, }, gomatrixserverlib.RoomVersionV5: RoomVersionDescription{ Supported: false, diff --git a/sytest-blacklist b/sytest-blacklist index 2e6f2057c..da0667a1d 100644 --- a/sytest-blacklist +++ b/sytest-blacklist @@ -31,4 +31,7 @@ Alias creators can delete canonical alias with no ops # Blacklisted because we need to implement v2 invite endpoints for room versions # to be supported (currently fails with M_UNSUPPORTED_ROOM_VERSION) -Inbound federation rejects invites which are not signed by the sender \ No newline at end of file +Inbound federation rejects invites which are not signed by the sender + +# Blacklisted because we don't support ignores yet +Ignore invite in incremental sync diff --git a/sytest-whitelist b/sytest-whitelist index 38062bb59..97ded9bb8 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -215,7 +215,6 @@ Guest users can sync from default guest_access rooms if joined Real non-joined users cannot room initalSync for non-world_readable rooms Push rules come down in an initial /sync Regular users can add and delete aliases in the default room configuration -Regular users can add and delete aliases when m.room.aliases is restricted GET /r0/capabilities is not public GET /joined_rooms lists newly-created room /joined_rooms returns only joined rooms @@ -225,10 +224,24 @@ Remote user can backfill in a room with version 1 POST /createRoom creates a room with the given version POST /createRoom rejects attempts to create rooms with numeric versions POST /createRoom rejects attempts to create rooms with unknown versions +Regular users can add and delete aliases when m.room.aliases is restricted User can create and send/receive messages in a room with version 2 local user can join room with version 2 remote user can join room with version 2 User can invite local user to room with version 2 Remote user can backfill in a room with version 2 Inbound federation accepts attempts to join v2 rooms from servers with support -Outbound federation can send invites via v2 API \ No newline at end of file +Outbound federation can send invites via v2 API +User can create and send/receive messages in a room with version 3 +local user can join room with version 3 +Remote user can backfill in a room with version 3 +User can create and send/receive messages in a room with version 4 +local user can join room with version 4 +remote user can join room with version 3 +remote user can join room with version 4 +Remote user can backfill in a room with version 4 +# We don't support ignores yet, so ignore this for now - ha ha. +# Ignore invite in incremental sync +Outbound federation can send invites via v2 API +User can invite local user to room with version 3 +User can invite local user to room with version 4 From 317658aceae207624e46a19318e2c39781d4e7ae Mon Sep 17 00:00:00 2001 From: Prateek Sachan <42961174+prateek2211@users.noreply.github.com> Date: Sat, 11 Apr 2020 22:17:05 +0530 Subject: [PATCH 66/86] Added checks for JSON body in accounts_data endpoint (#863) Signed-off-by: Prateek Sachan --- clientapi/routing/account_data.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/clientapi/routing/account_data.go b/clientapi/routing/account_data.go index 24db41f5f..a5d53c326 100644 --- a/clientapi/routing/account_data.go +++ b/clientapi/routing/account_data.go @@ -15,6 +15,7 @@ package routing import ( + "encoding/json" "io/ioutil" "net/http" @@ -80,12 +81,26 @@ func SaveAccountData( defer req.Body.Close() // nolint: errcheck + if req.Body == http.NoBody { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.NotJSON("Content not JSON"), + } + } + body, err := ioutil.ReadAll(req.Body) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("ioutil.ReadAll failed") return jsonerror.InternalServerError() } + if !json.Valid(body) { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.BadJSON("Bad JSON content"), + } + } + if err := accountDB.SaveAccountData( req.Context(), localpart, roomID, dataType, string(body), ); err != nil { From d35985170884e619508572465117be41bd5d45ae Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Sun, 12 Apr 2020 20:48:24 +0800 Subject: [PATCH 67/86] Update guides for running sytest (#958) * Update guides for running sytest Bring the details for running sytest against postgres up to date; prefer docker over manual setup Signed-off-by: Alex Chen * Better flags Signed-off-by: Alex Chen --- docs/sytest.md | 69 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 47 insertions(+), 22 deletions(-) diff --git a/docs/sytest.md b/docs/sytest.md index 9385ebff3..26a46c0bb 100644 --- a/docs/sytest.md +++ b/docs/sytest.md @@ -10,9 +10,9 @@ passes. ## Finding out which tests to add -We recommend you run the tests locally by manually setting up SyTest or using a -SyTest docker image. After running the tests, a script will print the tests you -need to add to `sytest-whitelist`. +We recommend you run the tests locally by using the SyTest docker image or +manually setting up SyTest. After running the tests, a script will print the +tests you need to add to `sytest-whitelist`. You should proceed after you see no build problems for dendrite after running: @@ -20,9 +20,32 @@ You should proceed after you see no build problems for dendrite after running: ./build.sh ``` +### Using the SyTest Docker image + +Use the following commands to pull the latest SyTest image and run the tests: + +```sh +docker pull matrixdotorg/sytest-dendrite +docker run --rm -v /path/to/dendrite/:/src/ -v /path/to/log/output/:/logs/ matrixdotorg/sytest-dendrite +``` + +`/path/to/dendrite/` should be replaced with the actual path to your dendrite +source code. The test results TAP file and homeserver logging output will go to +`/path/to/log/output`. The output of the command should tell you if you need to +add any tests to `sytest-whitelist`. + +When debugging, the following Docker `run` options may also be useful: +* `-v /path/to/sytest/:/sytest/`: Use your local SyTest repository at + `/path/to/sytest` instead of pulling from GitHub. This is useful when you want + to speed things up or make modifications to SyTest. +* `--entrypoint bash`: Prevent the container from automatically starting the + tests. When used, you need to manually run `/bootstrap.sh dendrite` inside + the container to start them. + ### Manually Setting up SyTest -Make sure you have Perl v5+ installed, and get SyTest with: +If you don't want to use the Docker image, you can also run SyTest by hand. Make +sure you have Perl 5 or above, and get SyTest with: (Note that this guide assumes your SyTest checkout is next to your `dendrite` checkout.) @@ -37,12 +60,23 @@ Set up the database: ```sh sudo -u postgres psql -c "CREATE USER dendrite PASSWORD 'itsasecret'" -sudo -u postgres psql -c "CREATE DATABASE sytest_template OWNER dendrite" +for i in dendrite0 dendrite1 sytest_template; do sudo -u postgres psql -c "CREATE DATABASE $i OWNER dendrite;"; done mkdir -p "server-0" cat > "server-0/database.yaml" << EOF args: user: dendrite - database: dendrite + password: itsasecret + database: dendrite0 + host: 127.0.0.1 + sslmode: disable +type: pg +EOF +mkdir -p "server-1" +cat > "server-1/database.yaml" << EOF +args: + user: dendrite + password: itsasecret + database: dendrite1 host: 127.0.0.1 sslmode: disable type: pg @@ -52,29 +86,20 @@ EOF Run the tests: ```sh -./run-tests.pl -I Dendrite::Monolith -d ../dendrite/bin -W ../dendrite/sytest-whitelist -O tap --all | tee results.tap +POSTGRES=1 ./run-tests.pl -I Dendrite::Monolith -d ../dendrite/bin -W ../dendrite/sytest-whitelist -O tap --all | tee results.tap ``` -where `tee` lets you see the results while they're being piped to the file. +where `tee` lets you see the results while they're being piped to the file, and +`POSTGRES=1` enables testing with PostgeSQL. If the `POSTGRES` environment +variable is not set or is set to 0, SyTest will fall back to SQLite 3. For more +flags and options, see https://github.com/matrix-org/sytest#running. Once the tests are complete, run the helper script to see if you need to add -any newly passing test names to `sytest-whitelist` in the project's root directory: +any newly passing test names to `sytest-whitelist` in the project's root +directory: ```sh ../dendrite/show-expected-fail-tests.sh results.tap ../dendrite/sytest-whitelist ../dendrite/sytest-blacklist ``` If the script prints nothing/exits with 0, then you're good to go. - -### Using a SyTest Docker image - -Ensure you have the latest image for SyTest, then run the tests: - -```sh -docker pull matrixdotorg/sytest-dendrite -docker run --rm -v /path/to/dendrite/:/src/ matrixdotorg/sytest-dendrite -``` - -where `/path/to/dendrite/` should be replaced with the actual path to your -dendrite source code. The output should tell you if you need to add any tests to -`sytest-whitelist`. From 1321f8da80010fe9c6aa08b7580ad5484f10d2f9 Mon Sep 17 00:00:00 2001 From: Prateek Sachan <42961174+prateek2211@users.noreply.github.com> Date: Tue, 14 Apr 2020 18:37:03 +0530 Subject: [PATCH 68/86] Check if user has the power level to edit the room visibility (#900) * Check if user has the power level to edit the room visibility * fix review changes * Add nil check for queryEventsRes.StateEvents Co-authored-by: Alex Chen Co-authored-by: Neil Alexander --- publicroomsapi/directory/directory.go | 47 +++++++++++++++++++++++++-- publicroomsapi/publicroomsapi.go | 2 +- publicroomsapi/routing/routing.go | 6 ++-- 3 files changed, 50 insertions(+), 5 deletions(-) diff --git a/publicroomsapi/directory/directory.go b/publicroomsapi/directory/directory.go index e56fc6cce..1e305f3ca 100644 --- a/publicroomsapi/directory/directory.go +++ b/publicroomsapi/directory/directory.go @@ -17,6 +17,9 @@ package directory import ( "net/http" + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/publicroomsapi/storage" @@ -54,11 +57,51 @@ func GetVisibility( } // SetVisibility implements PUT /directory/list/room/{roomID} -// TODO: Check if user has the power level to edit the room visibility +// TODO: Allow admin users to edit the room visibility func SetVisibility( - req *http.Request, publicRoomsDatabase storage.Database, + req *http.Request, publicRoomsDatabase storage.Database, queryAPI api.RoomserverQueryAPI, dev *authtypes.Device, roomID string, ) util.JSONResponse { + queryMembershipReq := api.QueryMembershipForUserRequest{ + RoomID: roomID, + UserID: dev.UserID, + } + var queryMembershipRes api.QueryMembershipForUserResponse + err := queryAPI.QueryMembershipForUser(req.Context(), &queryMembershipReq, &queryMembershipRes) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("could not query membership for user") + return jsonerror.InternalServerError() + } + // Check if user id is in room + if !queryMembershipRes.IsInRoom { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("user does not belong to room"), + } + } + queryEventsReq := api.QueryLatestEventsAndStateRequest{ + RoomID: roomID, + StateToFetch: []gomatrixserverlib.StateKeyTuple{{ + EventType: gomatrixserverlib.MRoomPowerLevels, + StateKey: "", + }}, + } + var queryEventsRes api.QueryLatestEventsAndStateResponse + err = queryAPI.QueryLatestEventsAndState(req.Context(), &queryEventsReq, &queryEventsRes) + if err != nil || len(queryEventsRes.StateEvents) == 0 { + util.GetLogger(req.Context()).WithError(err).Error("could not query events from room") + return jsonerror.InternalServerError() + } + power, _ := gomatrixserverlib.NewPowerLevelContentFromEvent(queryEventsRes.StateEvents[0].Event) + + // Check if the user's power is greater than power required to change m.room.aliases event + if power.UserLevel(dev.UserID) < power.EventLevel(gomatrixserverlib.MRoomAliases, true) { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("userID doesn't have power level to change visibility"), + } + } + var v roomVisibility if reqErr := httputil.UnmarshalJSONRequest(req, &v); reqErr != nil { return *reqErr diff --git a/publicroomsapi/publicroomsapi.go b/publicroomsapi/publicroomsapi.go index 399c0cc57..4d8a26018 100644 --- a/publicroomsapi/publicroomsapi.go +++ b/publicroomsapi/publicroomsapi.go @@ -47,5 +47,5 @@ func SetupPublicRoomsAPIComponent( logrus.WithError(err).Panic("failed to start public rooms server consumer") } - routing.Setup(base.APIMux, deviceDB, publicRoomsDB, fedClient, extRoomsProvider) + routing.Setup(base.APIMux, deviceDB, publicRoomsDB, rsQueryAPI, fedClient, extRoomsProvider) } diff --git a/publicroomsapi/routing/routing.go b/publicroomsapi/routing/routing.go index 321b61b89..da5ea90d6 100644 --- a/publicroomsapi/routing/routing.go +++ b/publicroomsapi/routing/routing.go @@ -17,6 +17,8 @@ package routing import ( "net/http" + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/gorilla/mux" "github.com/matrix-org/dendrite/clientapi/auth" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" @@ -37,7 +39,7 @@ const pathPrefixR0 = "/_matrix/client/r0" // applied: // nolint: gocyclo func Setup( - apiMux *mux.Router, deviceDB devices.Database, publicRoomsDB storage.Database, + apiMux *mux.Router, deviceDB devices.Database, publicRoomsDB storage.Database, queryAPI api.RoomserverQueryAPI, fedClient *gomatrixserverlib.FederationClient, extRoomsProvider types.ExternalPublicRoomsProvider, ) { r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter() @@ -64,7 +66,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return directory.SetVisibility(req, publicRoomsDB, vars["roomID"]) + return directory.SetVisibility(req, publicRoomsDB, queryAPI, device, vars["roomID"]) }), ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/publicRooms", From 2c43e222bd57cbb1e278ff776f7a2aa2221ac1e8 Mon Sep 17 00:00:00 2001 From: mohit kumar singh Date: Tue, 14 Apr 2020 20:01:27 +0530 Subject: [PATCH 69/86] support non-exclusive namespaces for AS (#828) Signed-off-by: MohitKS5 Co-authored-by: Neil Alexander Co-authored-by: Kegsay --- clientapi/routing/register.go | 4 ++-- common/config/appservice.go | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index 2de7b2733..b67e68e19 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -362,7 +362,7 @@ func UsernameMatchesMultipleExclusiveNamespaces( // Check namespaces and see if more than one match matchCount := 0 for _, appservice := range cfg.Derived.ApplicationServices { - if appservice.IsInterestedInUserID(userID) { + if appservice.OwnsNamespaceCoveringUserId(userID) { if matchCount++; matchCount > 1 { return true } @@ -1000,7 +1000,7 @@ func RegisterAvailable( // Check if this username is reserved by an application service userID := userutil.MakeUserID(username, cfg.Matrix.ServerName) for _, appservice := range cfg.Derived.ApplicationServices { - if appservice.IsInterestedInUserID(userID) { + if appservice.OwnsNamespaceCoveringUserId(userID) { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.UserInUse("Desired user ID is reserved by an application service."), diff --git a/common/config/appservice.go b/common/config/appservice.go index 7a43d48fb..bf5f089b7 100644 --- a/common/config/appservice.go +++ b/common/config/appservice.go @@ -98,6 +98,22 @@ func (a *ApplicationService) IsInterestedInUserID( return false } +// OwnsNamespaceCoveringUserId returns a bool on whether an application service's +// namespace is exclusive and includes the given user ID +func (a *ApplicationService) OwnsNamespaceCoveringUserId( + userID string, +) bool { + if namespaceSlice, ok := a.NamespaceMap["users"]; ok { + for _, namespace := range namespaceSlice { + if namespace.Exclusive && namespace.RegexpObject.MatchString(userID) { + return true + } + } + } + + return false +} + // IsInterestedInRoomAlias returns a bool on whether an application service's // namespace includes the given room alias func (a *ApplicationService) IsInterestedInRoomAlias( From 609f034bfb241934dc0653a7af0df08417688640 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 14 Apr 2020 15:54:35 +0100 Subject: [PATCH 70/86] S7evinK: basicauth metrics (#961) * Add setting to enable/disable metrics (#461) Add basic auth to /metric handlers Signed-off-by: Till Faelligen * Add warning message if metrics are exposed without protection * Remove redundant type conversion Signed-off-by: Till Faelligen * SetBasicAuth per test case * Update warning message and change loglevel to warn * Update common/config/config.go * Update dendrite-config.yaml Co-authored-by: Till Faelligen Co-authored-by: Neil Alexander --- cmd/dendrite-monolith-server/main.go | 6 +- common/basecomponent/base.go | 2 +- common/config/config.go | 13 ++++ common/httpapi.go | 35 +++++++++- common/httpapi_test.go | 95 ++++++++++++++++++++++++++++ dendrite-config.yaml | 9 +++ 6 files changed, 155 insertions(+), 5 deletions(-) create mode 100644 common/httpapi_test.go diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 9f6531ed3..c71d956b2 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -33,8 +33,8 @@ import ( "github.com/matrix-org/dendrite/publicroomsapi" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/syncapi" - "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/sirupsen/logrus" ) @@ -78,7 +78,9 @@ func main() { // Set up the API endpoints we handle. /metrics is for prometheus, and is // not wrapped by CORS, while everything else is - http.Handle("/metrics", promhttp.Handler()) + if cfg.Metrics.Enabled { + http.Handle("/metrics", common.WrapHandlerInBasicAuth(promhttp.Handler(), cfg.Metrics.BasicAuth)) + } http.Handle("/", httpHandler) // Expose the matrix APIs directly rather than putting them under a /api path. diff --git a/common/basecomponent/base.go b/common/basecomponent/base.go index 432819a23..dc27f5409 100644 --- a/common/basecomponent/base.go +++ b/common/basecomponent/base.go @@ -208,7 +208,7 @@ func (b *BaseDendrite) SetupAndServeHTTP(bindaddr string, listenaddr string) { addr = listenaddr } - common.SetupHTTPAPI(http.DefaultServeMux, common.WrapHandlerInCORS(b.APIMux)) + common.SetupHTTPAPI(http.DefaultServeMux, common.WrapHandlerInCORS(b.APIMux), b.Cfg) logrus.Infof("Starting %s server on %s", b.componentName, addr) err := http.ListenAndServe(addr, nil) diff --git a/common/config/config.go b/common/config/config.go index e2f5e6635..a1a844252 100644 --- a/common/config/config.go +++ b/common/config/config.go @@ -119,6 +119,19 @@ type Dendrite struct { ThumbnailSizes []ThumbnailSize `yaml:"thumbnail_sizes"` } `yaml:"media"` + // The configuration to use for Prometheus metrics + Metrics struct { + // Whether or not the metrics are enabled + Enabled bool `yaml:"enabled"` + // Use BasicAuth for Authorization + BasicAuth struct { + // Authorization via Static Username & Password + // Hardcoded Username and Password + Username string `yaml:"username"` + Password string `yaml:"password"` + } `yaml:"basic_auth"` + } `yaml:"metrics"` + // The configuration for talking to kafka. Kafka struct { // A list of kafka addresses to connect to. diff --git a/common/httpapi.go b/common/httpapi.go index 22c774475..843336f5b 100644 --- a/common/httpapi.go +++ b/common/httpapi.go @@ -6,6 +6,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" opentracing "github.com/opentracing/opentracing-go" @@ -13,8 +14,15 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/sirupsen/logrus" ) +// BasicAuth is used for authorization on /metrics handlers +type BasicAuth struct { + Username string `yaml:"username"` + Password string `yaml:"password"` +} + // MakeAuthAPI turns a util.JSONRequestHandler function into an http.Handler which authenticates the request. func MakeAuthAPI( metricsName string, data auth.Data, @@ -123,11 +131,34 @@ func MakeFedAPI( // SetupHTTPAPI registers an HTTP API mux under /api and sets up a metrics // listener. -func SetupHTTPAPI(servMux *http.ServeMux, apiMux http.Handler) { - servMux.Handle("/metrics", promhttp.Handler()) +func SetupHTTPAPI(servMux *http.ServeMux, apiMux http.Handler, cfg *config.Dendrite) { + if cfg.Metrics.Enabled { + servMux.Handle("/metrics", WrapHandlerInBasicAuth(promhttp.Handler(), cfg.Metrics.BasicAuth)) + } servMux.Handle("/api/", http.StripPrefix("/api", apiMux)) } +// WrapHandlerInBasicAuth adds basic auth to a handler. Only used for /metrics +func WrapHandlerInBasicAuth(h http.Handler, b BasicAuth) http.HandlerFunc { + if b.Username == "" || b.Password == "" { + logrus.Warn("Metrics are exposed without protection. Make sure you set up protection at proxy level.") + } + return func(w http.ResponseWriter, r *http.Request) { + // Serve without authorization if either Username or Password is unset + if b.Username == "" || b.Password == "" { + h.ServeHTTP(w, r) + return + } + user, pass, ok := r.BasicAuth() + + if !ok || user != b.Username || pass != b.Password { + http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden) + return + } + h.ServeHTTP(w, r) + } +} + // WrapHandlerInCORS adds CORS headers to all responses, including all error // responses. // Handles OPTIONS requests directly. diff --git a/common/httpapi_test.go b/common/httpapi_test.go new file mode 100644 index 000000000..7de7ce33c --- /dev/null +++ b/common/httpapi_test.go @@ -0,0 +1,95 @@ +package common + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func TestWrapHandlerInBasicAuth(t *testing.T) { + type args struct { + h http.Handler + b BasicAuth + } + + dummyHandler := http.HandlerFunc(func(h http.ResponseWriter, r *http.Request) { + h.WriteHeader(http.StatusOK) + }) + + tests := []struct { + name string + args args + want int + reqAuth bool + }{ + { + name: "no user or password setup", + args: args{h: dummyHandler}, + want: http.StatusOK, + reqAuth: false, + }, + { + name: "only user set", + args: args{ + h: dummyHandler, + b: BasicAuth{Username: "test"}, // no basic auth + }, + want: http.StatusOK, + reqAuth: false, + }, + { + name: "only pass set", + args: args{ + h: dummyHandler, + b: BasicAuth{Password: "test"}, // no basic auth + }, + want: http.StatusOK, + reqAuth: false, + }, + { + name: "credentials correct", + args: args{ + h: dummyHandler, + b: BasicAuth{Username: "test", Password: "test"}, // basic auth enabled + }, + want: http.StatusOK, + reqAuth: true, + }, + { + name: "credentials wrong", + args: args{ + h: dummyHandler, + b: BasicAuth{Username: "test1", Password: "test"}, // basic auth enabled + }, + want: http.StatusForbidden, + reqAuth: true, + }, + { + name: "no basic auth in request", + args: args{ + h: dummyHandler, + b: BasicAuth{Username: "test", Password: "test"}, // basic auth enabled + }, + want: http.StatusForbidden, + reqAuth: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + baHandler := WrapHandlerInBasicAuth(tt.args.h, tt.args.b) + + req := httptest.NewRequest("GET", "http://localhost/metrics", nil) + if tt.reqAuth { + req.SetBasicAuth("test", "test") + } + + w := httptest.NewRecorder() + baHandler(w, req) + resp := w.Result() + + if resp.StatusCode != tt.want { + t.Errorf("Expected status code %d, got %d", resp.StatusCode, tt.want) + } + }) + } +} diff --git a/dendrite-config.yaml b/dendrite-config.yaml index 7436af7a3..86a208d7f 100644 --- a/dendrite-config.yaml +++ b/dendrite-config.yaml @@ -53,6 +53,15 @@ media: height: 600 method: scale +# Metrics config for Prometheus +metrics: + # Whether or not metrics are enabled + enabled: false + # Use basic auth to protect the metrics. Uncomment to the complete block to enable. + #basic_auth: + # username: prometheusUser + # password: y0ursecr3tPa$$w0rd + # The config for the TURN server turn: # Whether or not guests can request TURN credentials From 48303d06cb91d19582f776af32ee22e5d820f031 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 14 Apr 2020 16:05:15 +0100 Subject: [PATCH 71/86] Are we synapse yet? (#960) * Add a script which tracks synapse parity based on sytests Processes results.tap to give you a percentage check. * Pretty * argparse and -v output --- are-we-synapse-yet.list | 825 ++++++++++++++++++++++++++++++++++++++++ are-we-synapse-yet.py | 252 ++++++++++++ 2 files changed, 1077 insertions(+) create mode 100644 are-we-synapse-yet.list create mode 100755 are-we-synapse-yet.py diff --git a/are-we-synapse-yet.list b/are-we-synapse-yet.list new file mode 100644 index 000000000..5d41c6ad5 --- /dev/null +++ b/are-we-synapse-yet.list @@ -0,0 +1,825 @@ +reg GET /register yields a set of flows +reg POST /register can create a user +reg POST /register downcases capitals in usernames +reg POST /register returns the same device_id as that in the request +reg POST /register rejects registration of usernames with '!' +reg POST /register rejects registration of usernames with '"' +reg POST /register rejects registration of usernames with ':' +reg POST /register rejects registration of usernames with '?' +reg POST /register rejects registration of usernames with '\' +reg POST /register rejects registration of usernames with '@' +reg POST /register rejects registration of usernames with '[' +reg POST /register rejects registration of usernames with ']' +reg POST /register rejects registration of usernames with '{' +reg POST /register rejects registration of usernames with '|' +reg POST /register rejects registration of usernames with '}' +reg POST /register rejects registration of usernames with '£' +reg POST /register rejects registration of usernames with 'é' +reg POST /register rejects registration of usernames with '\n' +reg POST /register rejects registration of usernames with ''' +reg POST /r0/admin/register with shared secret +reg POST /r0/admin/register admin with shared secret +reg POST /r0/admin/register with shared secret downcases capitals +reg POST /r0/admin/register with shared secret disallows symbols +reg POST rejects invalid utf-8 in JSON +log GET /login yields a set of flows +log POST /login can log in as a user +log POST /login returns the same device_id as that in the request +log POST /login can log in as a user with just the local part of the id +log POST /login as non-existing user is rejected +log POST /login wrong password is rejected +v1s GET /events initially +v1s GET /initialSync initially +csa Version responds 200 OK with valid structure +pro PUT /profile/:user_id/displayname sets my name +pro GET /profile/:user_id/displayname publicly accessible +pro PUT /profile/:user_id/avatar_url sets my avatar +pro GET /profile/:user_id/avatar_url publicly accessible +dev GET /device/{deviceId} +dev GET /device/{deviceId} gives a 404 for unknown devices +dev GET /devices +dev PUT /device/{deviceId} updates device fields +dev PUT /device/{deviceId} gives a 404 for unknown devices +dev DELETE /device/{deviceId} +dev DELETE /device/{deviceId} requires UI auth user to match device owner +dev DELETE /device/{deviceId} with no body gives a 401 +dev The deleted device must be consistent through an interactive auth session +pre GET /presence/:user_id/status fetches initial status +pre PUT /presence/:user_id/status updates my presence +crm POST /createRoom makes a public room +crm POST /createRoom makes a private room +crm POST /createRoom makes a private room with invites +crm POST /createRoom makes a room with a name +crm POST /createRoom makes a room with a topic +syn Can /sync newly created room +crm POST /createRoom creates a room with the given version +crm POST /createRoom rejects attempts to create rooms with numeric versions +crm POST /createRoom rejects attempts to create rooms with unknown versions +crm POST /createRoom ignores attempts to set the room version via creation_content +mem GET /rooms/:room_id/state/m.room.member/:user_id fetches my membership +mem GET /rooms/:room_id/state/m.room.member/:user_id?format=event fetches my membership event +rst GET /rooms/:room_id/state/m.room.power_levels fetches powerlevels +mem GET /rooms/:room_id/joined_members fetches my membership +v1s GET /rooms/:room_id/initialSync fetches initial sync state +pub GET /publicRooms lists newly-created room +ali GET /directory/room/:room_alias yields room ID +mem GET /joined_rooms lists newly-created room +rst POST /rooms/:room_id/state/m.room.name sets name +rst GET /rooms/:room_id/state/m.room.name gets name +rst POST /rooms/:room_id/state/m.room.topic sets topic +rst GET /rooms/:room_id/state/m.room.topic gets topic +rst GET /rooms/:room_id/state fetches entire room state +crm POST /createRoom with creation content +ali PUT /directory/room/:room_alias creates alias +nsp GET /rooms/:room_id/aliases lists aliases +jon POST /rooms/:room_id/join can join a room +jon POST /join/:room_alias can join a room +jon POST /join/:room_id can join a room +jon POST /join/:room_id can join a room with custom content +jon POST /join/:room_alias can join a room with custom content +lev POST /rooms/:room_id/leave can leave a room +inv POST /rooms/:room_id/invite can send an invite +ban POST /rooms/:room_id/ban can ban a user +snd POST /rooms/:room_id/send/:event_type sends a message +snd PUT /rooms/:room_id/send/:event_type/:txn_id sends a message +snd PUT /rooms/:room_id/send/:event_type/:txn_id deduplicates the same txn id +get GET /rooms/:room_id/messages returns a message +get GET /rooms/:room_id/messages lazy loads members correctly +typ PUT /rooms/:room_id/typing/:user_id sets typing notification +rst GET /rooms/:room_id/state/m.room.power_levels can fetch levels +rst PUT /rooms/:room_id/state/m.room.power_levels can set levels +rst PUT power_levels should not explode if the old power levels were empty +rst Both GET and PUT work +rct POST /rooms/:room_id/receipt can create receipts +red POST /rooms/:room_id/read_markers can create read marker +med POST /media/v1/upload can create an upload +med GET /media/v1/download can fetch the value again +cap GET /capabilities is present and well formed for registered user +cap GET /r0/capabilities is not public +reg Register with a recaptcha +reg registration is idempotent, without username specified +reg registration is idempotent, with username specified +reg registration remembers parameters +reg registration accepts non-ascii passwords +reg registration with inhibit_login inhibits login +reg User signups are forbidden from starting with '_' +reg Can register using an email address +log Can login with 3pid and password using m.login.password +log login types include SSO +log /login/cas/redirect redirects if the old m.login.cas login type is listed +log Can login with new user via CAS +lox Can logout current device +lox Can logout all devices +lox Request to logout with invalid an access token is rejected +lox Request to logout without an access token is rejected +log After changing password, can't log in with old password +log After changing password, can log in with new password +log After changing password, existing session still works +log After changing password, a different session no longer works by default +log After changing password, different sessions can optionally be kept +psh Pushers created with a different access token are deleted on password change +psh Pushers created with a the same access token are not deleted on password change +acc Can deactivate account +acc Can't deactivate account with wrong password +acc After deactivating account, can't log in with password +acc After deactivating account, can't log in with an email +v1s initialSync sees my presence status +pre Presence change reports an event to myself +pre Friends presence changes reports events +crm Room creation reports m.room.create to myself +crm Room creation reports m.room.member to myself +rst Setting room topic reports m.room.topic to myself +v1s Global initialSync +v1s Global initialSync with limit=0 gives no messages +v1s Room initialSync +v1s Room initialSync with limit=0 gives no messages +rst Setting state twice is idempotent +jon Joining room twice is idempotent +syn New room members see their own join event +v1s New room members see existing users' presence in room initialSync +syn Existing members see new members' join events +syn Existing members see new members' presence +v1s All room members see all room members' presence in global initialSync +f,jon Remote users can join room by alias +syn New room members see their own join event +v1s New room members see existing members' presence in room initialSync +syn Existing members see new members' join events +syn Existing members see new member's presence +v1s New room members see first user's profile information in global initialSync +v1s New room members see first user's profile information in per-room initialSync +f,jon Remote users may not join unfederated rooms +syn Local room members see posted message events +v1s Fetching eventstream a second time doesn't yield the message again +syn Local non-members don't see posted message events +get Local room members can get room messages +f,syn Remote room members also see posted message events +f,get Remote room members can get room messages +get Message history can be paginated +f,get Message history can be paginated over federation +eph Ephemeral messages received from clients are correctly expired +ali Room aliases can contain Unicode +f,ali Remote room alias queries can handle Unicode +ali Canonical alias can be set +ali Canonical alias can include alt_aliases +ali Regular users can add and delete aliases in the default room configuration +ali Regular users can add and delete aliases when m.room.aliases is restricted +ali Deleting a non-existent alias should return a 404 +ali Users can't delete other's aliases +ali Users with sufficient power-level can delete other's aliases +ali Can delete canonical alias +ali Alias creators can delete alias with no ops +ali Alias creators can delete canonical alias with no ops +ali Only room members can list aliases of a room +inv Can invite users to invite-only rooms +inv Uninvited users cannot join the room +inv Invited user can reject invite +f,inv Invited user can reject invite over federation +f,inv Invited user can reject invite over federation several times +inv Invited user can reject invite for empty room +f,inv Invited user can reject invite over federation for empty room +inv Invited user can reject local invite after originator leaves +inv Invited user can see room metadata +f,inv Remote invited user can see room metadata +inv Users cannot invite themselves to a room +inv Users cannot invite a user that is already in the room +ban Banned user is kicked and may not rejoin until unbanned +f,ban Remote banned user is kicked and may not rejoin until unbanned +ban 'ban' event respects room powerlevel +plv setting 'm.room.name' respects room powerlevel +plv setting 'm.room.power_levels' respects room powerlevel (2 subtests) +plv Unprivileged users can set m.room.topic if it only needs level 0 +plv Users cannot set ban powerlevel higher than their own (2 subtests) +plv Users cannot set kick powerlevel higher than their own (2 subtests) +plv Users cannot set redact powerlevel higher than their own (2 subtests) +v1s Check that event streams started after a client joined a room work (SYT-1) +v1s Event stream catches up fully after many messages +xxx POST /rooms/:room_id/redact/:event_id as power user redacts message +xxx POST /rooms/:room_id/redact/:event_id as original message sender redacts message +xxx POST /rooms/:room_id/redact/:event_id as random user does not redact message +xxx POST /redact disallows redaction of event in different room +xxx Redaction of a redaction redacts the redaction reason +v1s A departed room is still included in /initialSync (SPEC-216) +v1s Can get rooms/{roomId}/initialSync for a departed room (SPEC-216) +rst Can get rooms/{roomId}/state for a departed room (SPEC-216) +mem Can get rooms/{roomId}/members for a departed room (SPEC-216) +get Can get rooms/{roomId}/messages for a departed room (SPEC-216) +rst Can get 'm.room.name' state for a departed room (SPEC-216) +syn Getting messages going forward is limited for a departed room (SPEC-216) +3pd Can invite existing 3pid +3pd Can invite existing 3pid with no ops into a private room +3pd Can invite existing 3pid in createRoom +3pd Can invite unbound 3pid +f,3pd Can invite unbound 3pid over federation +3pd Can invite unbound 3pid with no ops into a private room +f,3pd Can invite unbound 3pid over federation with no ops into a private room +f,3pd Can invite unbound 3pid over federation with users from both servers +3pd Can accept unbound 3pid invite after inviter leaves +3pd Can accept third party invite with /join +3pd 3pid invite join with wrong but valid signature are rejected +3pd 3pid invite join valid signature but revoked keys are rejected +3pd 3pid invite join valid signature but unreachable ID server are rejected +gst Guest user cannot call /events globally +gst Guest users can join guest_access rooms +gst Guest users can send messages to guest_access rooms if joined +gst Guest user calling /events doesn't tightloop +gst Guest users are kicked from guest_access rooms on revocation of guest_access +gst Guest user can set display names +gst Guest users are kicked from guest_access rooms on revocation of guest_access over federation +gst Guest user can upgrade to fully featured user +gst Guest user cannot upgrade other users +pub GET /publicRooms lists rooms +pub GET /publicRooms includes avatar URLs +gst Guest users can accept invites to private rooms over federation +gst Guest users denied access over federation if guest access prohibited +mem Room members can override their displayname on a room-specific basis +mem Room members can join a room with an overridden displayname +mem Users cannot kick users from a room they are not in +mem Users cannot kick users who have already left a room +typ Typing notification sent to local room members +f,typ Typing notifications also sent to remote room members +typ Typing can be explicitly stopped +rct Read receipts are visible to /initialSync +rct Read receipts are sent as events +rct Receipts must be m.read +pro displayname updates affect room member events +pro avatar_url updates affect room member events +gst m.room.history_visibility == "world_readable" allows/forbids appropriately for Guest users +gst m.room.history_visibility == "shared" allows/forbids appropriately for Guest users +gst m.room.history_visibility == "invited" allows/forbids appropriately for Guest users +gst m.room.history_visibility == "joined" allows/forbids appropriately for Guest users +gst m.room.history_visibility == "default" allows/forbids appropriately for Guest users +gst Guest non-joined user cannot call /events on shared room +gst Guest non-joined user cannot call /events on invited room +gst Guest non-joined user cannot call /events on joined room +gst Guest non-joined user cannot call /events on default room +gst Guest non-joined user can call /events on world_readable room +gst Guest non-joined users can get state for world_readable rooms +gst Guest non-joined users can get individual state for world_readable rooms +gst Guest non-joined users cannot room initalSync for non-world_readable rooms +gst Guest non-joined users can room initialSync for world_readable rooms +gst Guest non-joined users can get individual state for world_readable rooms after leaving +gst Guest non-joined users cannot send messages to guest_access rooms if not joined +gst Guest users can sync from world_readable guest_access rooms if joined +gst Guest users can sync from shared guest_access rooms if joined +gst Guest users can sync from invited guest_access rooms if joined +gst Guest users can sync from joined guest_access rooms if joined +gst Guest users can sync from default guest_access rooms if joined +ath m.room.history_visibility == "world_readable" allows/forbids appropriately for Real users +ath m.room.history_visibility == "shared" allows/forbids appropriately for Real users +ath m.room.history_visibility == "invited" allows/forbids appropriately for Real users +ath m.room.history_visibility == "joined" allows/forbids appropriately for Real users +ath m.room.history_visibility == "default" allows/forbids appropriately for Real users +ath Real non-joined user cannot call /events on shared room +ath Real non-joined user cannot call /events on invited room +ath Real non-joined user cannot call /events on joined room +ath Real non-joined user cannot call /events on default room +ath Real non-joined user can call /events on world_readable room +ath Real non-joined users can get state for world_readable rooms +ath Real non-joined users can get individual state for world_readable rooms +ath Real non-joined users cannot room initalSync for non-world_readable rooms +ath Real non-joined users can room initialSync for world_readable rooms +ath Real non-joined users can get individual state for world_readable rooms after leaving +ath Real non-joined users cannot send messages to guest_access rooms if not joined +ath Real users can sync from world_readable guest_access rooms if joined +ath Real users can sync from shared guest_access rooms if joined +ath Real users can sync from invited guest_access rooms if joined +ath Real users can sync from joined guest_access rooms if joined +ath Real users can sync from default guest_access rooms if joined +ath Only see history_visibility changes on boundaries +f,ath Backfill works correctly with history visibility set to joined +fgt Forgotten room messages cannot be paginated +fgt Forgetting room does not show up in v2 /sync +fgt Can forget room you've been kicked from +fgt Can't forget room you're still in +mem Can re-join room if re-invited +ath Only original members of the room can see messages from erased users +mem /joined_rooms returns only joined rooms +mem /joined_members return joined members +ctx /context/ on joined room works +ctx /context/ on non world readable room does not work +ctx /context/ returns correct number of events +ctx /context/ with lazy_load_members filter works +get /event/ on joined room works +get /event/ on non world readable room does not work +get /event/ does not allow access to events before the user joined +mem Can get rooms/{roomId}/members +mem Can get rooms/{roomId}/members at a given point +mem Can filter rooms/{roomId}/members +upg /upgrade creates a new room +upg /upgrade should preserve room visibility for public rooms +upg /upgrade should preserve room visibility for private rooms +upg /upgrade copies >100 power levels to the new room +upg /upgrade copies the power levels to the new room +upg /upgrade preserves the power level of the upgrading user in old and new rooms +upg /upgrade copies important state to the new room +upg /upgrade copies ban events to the new room +upg local user has push rules copied to upgraded room +f,upg remote user has push rules copied to upgraded room +upg /upgrade moves aliases to the new room +upg /upgrade moves remote aliases to the new room +upg /upgrade preserves direct room state +upg /upgrade preserves room federation ability +upg /upgrade restricts power levels in the old room +upg /upgrade restricts power levels in the old room when the old PLs are unusual +upg /upgrade to an unknown version is rejected +upg /upgrade is rejected if the user can't send state events +upg /upgrade of a bogus room fails gracefully +upg Cannot send tombstone event that points to the same room +f,upg Local and remote users' homeservers remove a room from their public directory on upgrade +rst Name/topic keys are correct +f,pub Can get remote public room list +pub Can paginate public room list +pub Can search public room list +syn Can create filter +syn Can download filter +syn Can sync +syn Can sync a joined room +syn Full state sync includes joined rooms +syn Newly joined room is included in an incremental sync +syn Newly joined room has correct timeline in incremental sync +syn Newly joined room includes presence in incremental sync +syn Get presence for newly joined members in incremental sync +syn Can sync a room with a single message +syn Can sync a room with a message with a transaction id +syn A message sent after an initial sync appears in the timeline of an incremental sync. +syn A filtered timeline reaches its limit +syn Syncing a new room with a large timeline limit isn't limited +syn A full_state incremental update returns only recent timeline +syn A prev_batch token can be used in the v1 messages API +syn A next_batch token can be used in the v1 messages API +syn User sees their own presence in a sync +syn User is offline if they set_presence=offline in their sync +syn User sees updates to presence from other users in the incremental sync. +syn State is included in the timeline in the initial sync +f,syn State from remote users is included in the state in the initial sync +syn Changes to state are included in an incremental sync +syn Changes to state are included in an gapped incremental sync +f,syn State from remote users is included in the timeline in an incremental sync +syn A full_state incremental update returns all state +syn When user joins a room the state is included in the next sync +syn A change to displayname should not result in a full state sync +syn A change to displayname should appear in incremental /sync +syn When user joins a room the state is included in a gapped sync +syn When user joins and leaves a room in the same batch, the full state is still included in the next sync +syn Current state appears in timeline in private history +syn Current state appears in timeline in private history with many messages before +syn Current state appears in timeline in private history with many messages after +syn Rooms a user is invited to appear in an initial sync +syn Rooms a user is invited to appear in an incremental sync +syn Newly joined room is included in an incremental sync after invite +syn Sync can be polled for updates +syn Sync is woken up for leaves +syn Left rooms appear in the leave section of sync +syn Newly left rooms appear in the leave section of incremental sync +syn We should see our own leave event, even if history_visibility is restricted (SYN-662) +syn We should see our own leave event when rejecting an invite, even if history_visibility is restricted (riot-web/3462) +syn Newly left rooms appear in the leave section of gapped sync +syn Previously left rooms don't appear in the leave section of sync +syn Left rooms appear in the leave section of full state sync +syn Archived rooms only contain history from before the user left +syn Banned rooms appear in the leave section of sync +syn Newly banned rooms appear in the leave section of incremental sync +syn Newly banned rooms appear in the leave section of incremental sync +syn Typing events appear in initial sync +syn Typing events appear in incremental sync +syn Typing events appear in gapped sync +syn Read receipts appear in initial v2 /sync +syn New read receipts appear in incremental v2 /sync +syn Can pass a JSON filter as a query parameter +syn Can request federation format via the filter +syn Read markers appear in incremental v2 /sync +syn Read markers appear in initial v2 /sync +syn Read markers can be updated +syn Lazy loading parameters in the filter are strictly boolean +syn The only membership state included in an initial sync is for all the senders in the timeline +syn The only membership state included in an incremental sync is for senders in the timeline +syn The only membership state included in a gapped incremental sync is for senders in the timeline +syn Gapped incremental syncs include all state changes +syn Old leaves are present in gapped incremental syncs +syn Leaves are present in non-gapped incremental syncs +syn Old members are included in gappy incr LL sync if they start speaking +syn Members from the gap are included in gappy incr LL sync +syn We don't send redundant membership state across incremental syncs by default +syn We do send redundant membership state across incremental syncs if asked +syn Unnamed room comes with a name summary +syn Named room comes with just joined member count summary +syn Room summary only has 5 heroes +syn Room summary counts change when membership changes +rmv User can create and send/receive messages in a room with version 1 (2 subtests) +rmv local user can join room with version 1 +rmv User can invite local user to room with version 1 +rmv remote user can join room with version 1 +rmv User can invite remote user to room with version 1 +rmv Remote user can backfill in a room with version 1 +rmv Can reject invites over federation for rooms with version 1 +rmv Can receive redactions from regular users over federation in room version 1 +rmv User can create and send/receive messages in a room with version 2 (2 subtests) +rmv local user can join room with version 2 +rmv User can invite local user to room with version 2 +rmv remote user can join room with version 2 +rmv User can invite remote user to room with version 2 +rmv Remote user can backfill in a room with version 2 +rmv Can reject invites over federation for rooms with version 2 +rmv Can receive redactions from regular users over federation in room version 2 +rmv User can create and send/receive messages in a room with version 3 +rmv local user can join room with version 3 +rmv User can invite local user to room with version 3 +rmv remote user can join room with version 3 +rmv User can invite remote user to room with version 3 +rmv Remote user can backfill in a room with version 3 +rmv Can reject invites over federation for rooms with version 3 +rmv Can receive redactions from regular users over federation in room version 3 +rmv User can create and send/receive messages in a room with version 4 +rmv local user can join room with version 4 +rmv User can invite local user to room with version 4 +rmv remote user can join room with version 4 +rmv User can invite remote user to room with version 4 +rmv Remote user can backfill in a room with version 4 +rmv Can reject invites over federation for rooms with version 4 +rmv Can receive redactions from regular users over federation in room version 4 +rmv User can create and send/receive messages in a room with version 5 +rmv local user can join room with version 5 +rmv User can invite local user to room with version 5 +rmv remote user can join room with version 5 +rmv User can invite remote user to room with version 5 +rmv Remote user can backfill in a room with version 5 +rmv Can reject invites over federation for rooms with version 5 +rmv Can receive redactions from regular users over federation in room version 5 +pre Presence changes are reported to local room members +f,pre Presence changes are also reported to remote room members +pre Presence changes to UNAVAILABLE are reported to local room members +f,pre Presence changes to UNAVAILABLE are reported to remote room members +v1s Newly created users see their own presence in /initialSync (SYT-34) +dvk Can upload device keys +dvk Should reject keys claiming to belong to a different user +dvk Can query device keys using POST +dvk Can query specific device keys using POST +dvk query for user with no keys returns empty key dict +dvk Can claim one time key using POST +f,dvk Can query remote device keys using POST +f,dvk Can claim remote one time key using POST +dvk Local device key changes appear in v2 /sync +dvk Local new device changes appear in v2 /sync +dvk Local delete device changes appear in v2 /sync +dvk Local update device changes appear in v2 /sync +dvk Can query remote device keys using POST after notification +f,dev Device deletion propagates over federation +f,dev If remote user leaves room, changes device and rejoins we see update in sync +f,dev If remote user leaves room we no longer receive device updates +dvk Local device key changes appear in /keys/changes +dvk New users appear in /keys/changes +f,dvk If remote user leaves room, changes device and rejoins we see update in /keys/changes +dvk Get left notifs in sync and /keys/changes when other user leaves +dvk Get left notifs for other users in sync and /keys/changes when user leaves +f,dvk If user leaves room, remote user changes device and rejoins we see update in /sync and /keys/changes +dvk Can create backup version +dvk Can update backup version +dvk Responds correctly when backup is empty +dvk Can backup keys +dvk Can update keys with better versions +dvk Will not update keys with worse versions +dvk Will not back up to an old backup version +dvk Can delete backup +dvk Deleted & recreated backups are empty +dvk Can create more than 10 backup versions +dvk Can upload self-signing keys +dvk Fails to upload self-signing keys with no auth +dvk Fails to upload self-signing key without master key +dvk Changing master key notifies local users +dvk Changing user-signing key notifies local users +f,dvk can fetch self-signing keys over federation +f,dvk uploading self-signing key notifies over federation +f,dvk uploading signed devices gets propagated over federation +tag Can add tag +tag Can remove tag +tag Can list tags for a room +v1s Tags appear in the v1 /events stream +v1s Tags appear in the v1 /initalSync +v1s Tags appear in the v1 room initial sync +tag Tags appear in an initial v2 /sync +tag Newly updated tags appear in an incremental v2 /sync +tag Deleted tags appear in an incremental v2 /sync +tag local user has tags copied to the new room +f,tag remote user has tags copied to the new room +sch Can search for an event by body +sch Can get context around search results +sch Can back-paginate search results +sch Search works across an upgraded room and its predecessor +sch Search results with rank ordering do not include redacted events +sch Search results with recent ordering do not include redacted events +acc Can add account data +acc Can add account data to room +acc Can get account data without syncing +acc Can get room account data without syncing +v1s Latest account data comes down in /initialSync +v1s Latest account data comes down in room initialSync +v1s Account data appears in v1 /events stream +v1s Room account data appears in v1 /events stream +acc Latest account data appears in v2 /sync +acc New account data appears in incremental v2 /sync +oid Can generate a openid access_token that can be exchanged for information about a user +oid Invalid openid access tokens are rejected +oid Requests to userinfo without access tokens are rejected +std Can send a message directly to a device using PUT /sendToDevice +std Can recv a device message using /sync +std Can recv device messages until they are acknowledged +std Device messages with the same txn_id are deduplicated +std Device messages wake up /sync +std Can recv device messages over federation +std Device messages over federation wake up /sync +std Can send messages with a wildcard device id +std Can send messages with a wildcard device id to two devices +std Wildcard device messages wake up /sync +std Wildcard device messages over federation wake up /sync +adm /whois +nsp /purge_history +nsp /purge_history by ts +nsp Can backfill purged history +nsp Shutdown room +ign Ignore user in existing room +ign Ignore invite in full sync +ign Ignore invite in incremental sync +fky Checking local federation server +fky Federation key API allows unsigned requests for keys +fky Federation key API can act as a notary server via a GET request +fky Federation key API can act as a notary server via a POST request +fky Key notary server should return an expired key if it can't find any others +fky Key notary server must not overwrite a valid key with a spurious result from the origin server +fqu Non-numeric ports in server names are rejected +fqu Outbound federation can query profile data +fqu Inbound federation can query profile data +fqu Outbound federation can query room alias directory +fqu Inbound federation can query room alias directory +fsj Outbound federation can query v1 /send_join +fsj Outbound federation can query v2 /send_join +fmj Outbound federation passes make_join failures through to the client +fsj Inbound federation can receive v1 /send_join +fsj Inbound federation can receive v2 /send_join +fmj Inbound /v1/make_join rejects remote attempts to join local users to rooms +fsj Inbound /v1/send_join rejects incorrectly-signed joins +fsj Inbound /v1/send_join rejects joins from other servers +fau Inbound federation rejects remote attempts to kick local users to rooms +frv Inbound federation rejects attempts to join v1 rooms from servers without v1 support +frv Inbound federation rejects attempts to join v2 rooms from servers lacking version support +frv Inbound federation rejects attempts to join v2 rooms from servers only supporting v1 +frv Inbound federation accepts attempts to join v2 rooms from servers with support +frv Outbound federation correctly handles unsupported room versions +frv A pair of servers can establish a join in a v2 room +fsj Outbound federation rejects send_join responses with no m.room.create event +frv Outbound federation rejects m.room.create events with an unknown room version +fsj Event with an invalid signature in the send_join response should not cause room join to fail +fed Outbound federation can send events +fed Inbound federation can receive events +fed Inbound federation can receive redacted events +fed Ephemeral messages received from servers are correctly expired +fed Events whose auth_events are in the wrong room do not mess up the room state +fed Inbound federation can return events +fed Inbound federation redacts events from erased users +fme Outbound federation can request missing events +fme Inbound federation can return missing events for world_readable visibility +fme Inbound federation can return missing events for shared visibility +fme Inbound federation can return missing events for invite visibility +fme Inbound federation can return missing events for joined visibility +fme outliers whose auth_events are in a different room are correctly rejected +fbk Outbound federation can backfill events +fbk Inbound federation can backfill events +fbk Backfill checks the events requested belong to the room +fbk Backfilled events whose prev_events are in a different room do not allow cross-room back-pagination +fiv Outbound federation can send invites via v1 API +fiv Outbound federation can send invites via v2 API +fiv Inbound federation can receive invites via v1 API +fiv Inbound federation can receive invites via v2 API +fiv Inbound federation can receive invite and reject when remote replies with a 403 +fiv Inbound federation can receive invite and reject when remote replies with a 500 +fiv Inbound federation can receive invite and reject when remote is unreachable +fiv Inbound federation rejects invites which are not signed by the sender +fiv Inbound federation can receive invite rejections +fiv Inbound federation rejects incorrectly-signed invite rejections +fsl Inbound /v1/send_leave rejects leaves from other servers +fst Inbound federation can get state for a room +fst Inbound federation of state requires event_id as a mandatory paramater +fst Inbound federation can get state_ids for a room +fst Inbound federation of state_ids requires event_id as a mandatory paramater +fst Federation rejects inbound events where the prev_events cannot be found +fst Room state at a rejected message event is the same as its predecessor +fst Room state at a rejected state event is the same as its predecessor +fst Outbound federation requests missing prev_events and then asks for /state_ids and resolves the state +fst Federation handles empty auth_events in state_ids sanely +fst Getting state checks the events requested belong to the room +fst Getting state IDs checks the events requested belong to the room +fst Should not be able to take over the room by pretending there is no PL event +fpb Inbound federation can get public room list +fed Outbound federation sends receipts +fed Inbound federation rejects receipts from wrong remote +fed Inbound federation ignores redactions from invalid servers room > v3 +fed An event which redacts an event in a different room should be ignored +fed An event which redacts itself should be ignored +fed A pair of events which redact each other should be ignored +fdk Local device key changes get to remote servers +fdk Server correctly handles incoming m.device_list_update +fdk Server correctly resyncs when client query keys and there is no remote cache +fdk Server correctly resyncs when server leaves and rejoins a room +fdk Local device key changes get to remote servers with correct prev_id +fdk Device list doesn't change if remote server is down +fdk If a device list update goes missing, the server resyncs on the next one +fst Name/topic keys are correct +fau Remote servers cannot set power levels in rooms without existing powerlevels +fau Remote servers should reject attempts by non-creators to set the power levels +fau Inbound federation rejects typing notifications from wrong remote +fed Forward extremities remain so even after the next events are populated as outliers +fau Banned servers cannot send events +fau Banned servers cannot /make_join +fau Banned servers cannot /send_join +fau Banned servers cannot /make_leave +fau Banned servers cannot /send_leave +fau Banned servers cannot /invite +fau Banned servers cannot get room state +fau Banned servers cannot get room state ids +fau Banned servers cannot backfill +fau Banned servers cannot /event_auth +fau Banned servers cannot get missing events +fau Server correctly handles transactions that break edu limits +fau Inbound federation correctly soft fails events +fau Inbound federation accepts a second soft-failed event +fau Inbound federation correctly handles soft failed events as extremities +med Can upload with Unicode file name +med Can download with Unicode file name locally +f,med Can download with Unicode file name over federation +med Alternative server names do not cause a routing loop +med Can download specifying a different Unicode file name +med Can upload without a file name +med Can download without a file name locally +f,med Can download without a file name over federation +med Can upload with ASCII file name +med Can download file 'ascii' +med Can download file 'name with spaces' +med Can download file 'name;with;semicolons' +med Can download specifying a different ASCII file name +med Can send image in room message +med Can fetch images in room +med POSTed media can be thumbnailed +f,med Remote media can be thumbnailed +med Test URL preview +med Can read configuration endpoint +nsp Can quarantine media in rooms +udr User appears in user directory +udr User in private room doesn't appear in user directory +udr User joining then leaving public room appears and dissappears from directory +udr Users appear/disappear from directory when join_rules are changed +udr Users appear/disappear from directory when history_visibility are changed +udr Users stay in directory when join_rules are changed but history_visibility is world_readable +f,udr User in remote room doesn't appear in user directory after server left room +udr User directory correctly update on display name change +udr User in shared private room does appear in user directory +udr User in shared private room does appear in user directory until leave +udr User in dir while user still shares private rooms +nsp Create group +nsp Add group rooms +nsp Remove group rooms +nsp Get local group profile +nsp Get local group users +nsp Add/remove local group rooms +nsp Get local group summary +nsp Get remote group profile +nsp Get remote group users +nsp Add/remove remote group rooms +nsp Get remote group summary +nsp Add local group users +nsp Remove self from local group +nsp Remove other from local group +nsp Add remote group users +nsp Remove self from remote group +nsp Listing invited users of a remote group when not a member returns a 403 +nsp Add group category +nsp Remove group category +nsp Get group categories +nsp Add group role +nsp Remove group role +nsp Get group roles +nsp Add room to group summary +nsp Adding room to group summary keeps room_id when fetching rooms in group +nsp Adding multiple rooms to group summary have correct order +nsp Remove room from group summary +nsp Add room to group summary with category +nsp Remove room from group summary with category +nsp Add user to group summary +nsp Adding multiple users to group summary have correct order +nsp Remove user from group summary +nsp Add user to group summary with role +nsp Remove user from group summary with role +nsp Local group invites come down sync +nsp Group creator sees group in sync +nsp Group creator sees group in initial sync +nsp Get/set local group publicity +nsp Bulk get group publicity +nsp Joinability comes down summary +nsp Set group joinable and join it +nsp Group is not joinable by default +nsp Group is joinable over federation +nsp Room is transitioned on local and remote groups upon room upgrade +3pd Can bind 3PID via home server +3pd Can bind and unbind 3PID via homeserver +3pd Can unbind 3PID via homeserver when bound out of band +3pd 3PIDs are unbound after account deactivation +3pd Can bind and unbind 3PID via /unbind by specifying the identity server +3pd Can bind and unbind 3PID via /unbind without specifying the identity server +app AS can create a user +app AS can create a user with an underscore +app AS can create a user with inhibit_login +app AS cannot create users outside its own namespace +app Regular users cannot register within the AS namespace +app AS can make room aliases +app Regular users cannot create room aliases within the AS namespace +app AS-ghosted users can use rooms via AS +app AS-ghosted users can use rooms themselves +app Ghost user must register before joining room +app AS can set avatar for ghosted users +app AS can set displayname for ghosted users +app AS can't set displayname for random users +app Inviting an AS-hosted user asks the AS server +app Accesing an AS-hosted room alias asks the AS server +app Events in rooms with AS-hosted room aliases are sent to AS server +app AS user (not ghost) can join room without registering +app AS user (not ghost) can join room without registering, with user_id query param +app HS provides query metadata +app HS can provide query metadata on a single protocol +app HS will proxy request for 3PU mapping +app HS will proxy request for 3PL mapping +app AS can publish rooms in their own list +app AS and main public room lists are separate +app AS can deactivate a user +psh Test that a message is pushed +psh Invites are pushed +psh Rooms with names are correctly named in pushed +psh Rooms with canonical alias are correctly named in pushed +psh Rooms with many users are correctly pushed +psh Don't get pushed for rooms you've muted +psh Rejected events are not pushed +psh Can add global push rule for room +psh Can add global push rule for sender +psh Can add global push rule for content +psh Can add global push rule for override +psh Can add global push rule for underride +psh Can add global push rule for content +psh New rules appear before old rules by default +psh Can add global push rule before an existing rule +psh Can add global push rule after an existing rule +psh Can delete a push rule +psh Can disable a push rule +psh Adding the same push rule twice is idempotent +psh Messages that notify from another user increment unread notification count +psh Messages that highlight from another user increment unread highlight count +psh Can change the actions of default rules +psh Changing the actions of an unknown default rule fails with 404 +psh Can change the actions of a user specified rule +psh Changing the actions of an unknown rule fails with 404 +psh Can fetch a user's pushers +psh Push rules come down in an initial /sync +psh Adding a push rule wakes up an incremental /sync +psh Disabling a push rule wakes up an incremental /sync +psh Enabling a push rule wakes up an incremental /sync +psh Setting actions for a push rule wakes up an incremental /sync +psh Can enable/disable default rules +psh Enabling an unknown default rule fails with 404 +psh Test that rejected pushers are removed. +psh Notifications can be viewed with GET /notifications +psh Trying to add push rule with no scope fails with 400 +psh Trying to add push rule with invalid scope fails with 400 +psh Trying to add push rule with missing template fails with 400 +psh Trying to add push rule with missing rule_id fails with 400 +psh Trying to add push rule with empty rule_id fails with 400 +psh Trying to add push rule with invalid template fails with 400 +psh Trying to add push rule with rule_id with slashes fails with 400 +psh Trying to add push rule with override rule without conditions fails with 400 +psh Trying to add push rule with underride rule without conditions fails with 400 +psh Trying to add push rule with condition without kind fails with 400 +psh Trying to add push rule with content rule without pattern fails with 400 +psh Trying to add push rule with no actions fails with 400 +psh Trying to add push rule with invalid action fails with 400 +psh Trying to add push rule with invalid attr fails with 400 +psh Trying to add push rule with invalid value for enabled fails with 400 +psh Trying to get push rules with no trailing slash fails with 400 +psh Trying to get push rules with scope without trailing slash fails with 400 +psh Trying to get push rules with template without tailing slash fails with 400 +psh Trying to get push rules with unknown scope fails with 400 +psh Trying to get push rules with unknown template fails with 400 +psh Trying to get push rules with unknown attribute fails with 400 +psh Trying to get push rules with unknown rule_id fails with 404 +v1s GET /initialSync with non-numeric 'limit' +v1s GET /events with non-numeric 'limit' +v1s GET /events with negative 'limit' +v1s GET /events with non-numeric 'timeout' +ath Event size limits +syn Check creating invalid filters returns 4xx +f,pre New federated private chats get full presence information (SYN-115) +pre Left room members do not cause problems for presence +crm Rooms can be created with an initial invite list (SYN-205) +typ Typing notifications don't leak +ban Non-present room members cannot ban others +psh Getting push rules doesn't corrupt the cache SYN-390 +inv Test that we can be reinvited to a room we created +syn Multiple calls to /sync should not cause 500 errors +gst Guest user can call /events on another world_readable room (SYN-606) +gst Real user can call /events on another world_readable room (SYN-606) +gst Events come down the correct room +pub Asking for a remote rooms list, but supplying the local server's name, returns the local rooms list \ No newline at end of file diff --git a/are-we-synapse-yet.py b/are-we-synapse-yet.py new file mode 100755 index 000000000..ffed8d384 --- /dev/null +++ b/are-we-synapse-yet.py @@ -0,0 +1,252 @@ +#!/usr/bin/env python3 + +from __future__ import division +import argparse +import re +import sys + +# Usage: $ ./are-we-synapse-yet.py [-v] results.tap +# This script scans a results.tap file from Dendrite's CI process and spits out +# a rating of how close we are to Synapse parity, based purely on SyTests. +# The main complexity is grouping tests sensibly into features like 'Registration' +# and 'Federation'. Then it just checks the ones which are passing and calculates +# percentages for each group. Produces results like: +# +# Client-Server APIs: 29% (196/666 tests) +# ------------------- +# Registration : 62% (20/32 tests) +# Login : 7% (1/15 tests) +# V1 CS APIs : 10% (3/30 tests) +# ... +# +# or in verbose mode: +# +# Client-Server APIs: 29% (196/666 tests) +# ------------------- +# Registration : 62% (20/32 tests) +# ✓ GET /register yields a set of flows +# ✓ POST /register can create a user +# ✓ POST /register downcases capitals in usernames +# ... +# +# You can also tack `-v` on to see exactly which tests each category falls under. + +test_mappings = { + "nsp": "Non-Spec API", + "f": "Federation", # flag to mark test involves federation + + "federation_apis": { + "fky": "Key API", + "fsj": "send_join API", + "fmj": "make_join API", + "fsl": "send_leave API", + "fiv": "Invite API", + "fqu": "Query API", + "frv": "room versions", + "fau": "Auth", + "fbk": "Backfill API", + "fme": "get_missing_events API", + "fst": "State APIs", + "fpb": "Public Room API", + "fdk": "Device Key APIs", + "fed": "Federation API", + }, + + "client_apis": { + "reg": "Registration", + "log": "Login", + "lox": "Logout", + "v1s": "V1 CS APIs", + "csa": "Misc CS APIs", + "pro": "Profile", + "dev": "Devices", + "dvk": "Device Keys", + "pre": "Presence", + "crm": "Create Room", + "syn": "Sync API", + "rmv": "Room Versions", + "rst": "Room State APIs", + "pub": "Public Room APIs", + "mem": "Room Membership", + "ali": "Room Aliases", + "jon": "Joining Rooms", + "lev": "Leaving Rooms", + "inv": "Inviting users to Rooms", + "ban": "Banning users", + "snd": "Sending events", + "get": "Getting events for Rooms", + "rct": "Receipts", + "red": "Read markers", + "med": "Media APIs", + "cap": "Capabilities API", + "typ": "Typing API", + "psh": "Push APIs", + "acc": "Account APIs", + "eph": "Ephemeral Events", + "plv": "Power Levels", + "xxx": "Redaction", + "3pd": "Third-Party ID APIs", + "gst": "Guest APIs", + "ath": "Room Auth", + "fgt": "Forget APIs", + "ctx": "Context APIs", + "upg": "Room Upgrade APIs", + "tag": "Tagging APIs", + "sch": "Search APIs", + "oid": "OpenID API", + "std": "Send-to-Device APIs", + "adm": "Server Admin API", + "ign": "Ignore Users", + "udr": "User Directory APIs", + "app": "Application Services API", + }, +} + +# optional 'not ' with test number then anything but '#' +re_testname = re.compile(r"^(not )?ok [0-9]+ ([^#]+)") + +# Parses lines like the following: +# +# SUCCESS: ok 3 POST /register downcases capitals in usernames +# FAIL: not ok 54 (expected fail) POST /createRoom creates a room with the given version +# SKIP: ok 821 Multiple calls to /sync should not cause 500 errors # skip lack of can_post_room_receipts +# EXPECT FAIL: not ok 822 (expected fail) Guest user can call /events on another world_readable room (SYN-606) # TODO expected fail +# +# Only SUCCESS lines are treated as success, the rest are not implemented. +# +# Returns a dict like: +# { name: "...", ok: True } +def parse_test_line(line): + if not line.startswith("ok ") and not line.startswith("not ok "): + return + re_match = re_testname.match(line) + test_name = re_match.groups()[1].replace("(expected fail) ", "").strip() + test_pass = False + if line.startswith("ok ") and not "# skip " in line: + test_pass = True + return { + "name": test_name, + "ok": test_pass, + } + +# Prints the stats for a complete section. +# header_name => "Client-Server APIs" +# gid_to_tests => { gid: { : True|False }} +# gid_to_name => { gid: "Group Name" } +# verbose => True|False +# Produces: +# Client-Server APIs: 29% (196/666 tests) +# ------------------- +# Registration : 62% (20/32 tests) +# Login : 7% (1/15 tests) +# V1 CS APIs : 10% (3/30 tests) +# ... +# or in verbose mode: +# Client-Server APIs: 29% (196/666 tests) +# ------------------- +# Registration : 62% (20/32 tests) +# ✓ GET /register yields a set of flows +# ✓ POST /register can create a user +# ✓ POST /register downcases capitals in usernames +# ... +def print_stats(header_name, gid_to_tests, gid_to_name, verbose): + subsections = [] # Registration: 100% (13/13 tests) + subsection_test_names = {} # 'subsection name': ["✓ Test 1", "✓ Test 2", "× Test 3"] + total_passing = 0 + total_tests = 0 + for gid, tests in gid_to_tests.items(): + group_total = len(tests) + group_passing = 0 + test_names_and_marks = [] + for name, passing in tests.items(): + if passing: + group_passing += 1 + test_names_and_marks.append(f"{'✓' if passing else '×'} {name}") + + total_tests += group_total + total_passing += group_passing + pct = "{0:.0f}%".format(group_passing/group_total * 100) + line = "%s: %s (%d/%d tests)" % (gid_to_name[gid].ljust(25, ' '), pct.rjust(4, ' '), group_passing, group_total) + subsections.append(line) + subsection_test_names[line] = test_names_and_marks + + pct = "{0:.0f}%".format(total_passing/total_tests * 100) + print("%s: %s (%d/%d tests)" % (header_name, pct, total_passing, total_tests)) + print("-" * (len(header_name)+1)) + for line in subsections: + print(" %s" % (line,)) + if verbose: + for test_name_and_pass_mark in subsection_test_names[line]: + print(" %s" % (test_name_and_pass_mark,)) + print("") + print("") + +def main(results_tap_path, verbose): + # Load up test mappings + test_name_to_group_id = {} + fed_tests = set() + client_tests = set() + with open("./are-we-synapse-yet.list", "r") as f: + for line in f.readlines(): + test_name = " ".join(line.split(" ")[1:]).strip() + groups = line.split(" ")[0].split(",") + for gid in groups: + if gid == "f" or gid in test_mappings["federation_apis"]: + fed_tests.add(test_name) + else: + client_tests.add(test_name) + if gid == "f": + continue # we expect another group ID + test_name_to_group_id[test_name] = gid + + # parse results.tap + summary = { + "client": { + # gid: { + # test_name: OK + # } + }, + "federation": { + # gid: { + # test_name: OK + # } + }, + "nonspec": { + "nsp": {} + }, + } + with open(results_tap_path, "r") as f: + for line in f.readlines(): + test_result = parse_test_line(line) + if not test_result: + continue + name = test_result["name"] + group_id = test_name_to_group_id.get(name) + if not group_id: + raise Exception("The test '%s' doesn't have a group" % (name,)) + if group_id == "nsp": + summary["nonspec"]["nsp"][name] = test_result["ok"] + elif group_id in test_mappings["federation_apis"]: + group = summary["federation"].get(group_id, {}) + group[name] = test_result["ok"] + summary["federation"][group_id] = group + elif group_id in test_mappings["client_apis"]: + group = summary["client"].get(group_id, {}) + group[name] = test_result["ok"] + summary["client"][group_id] = group + + print("Are We Synapse Yet?") + print("===================") + print("") + print_stats("Non-Spec APIs", summary["nonspec"], test_mappings, verbose) + print_stats("Client-Server APIs", summary["client"], test_mappings["client_apis"], verbose) + print_stats("Federation APIs", summary["federation"], test_mappings["federation_apis"], verbose) + + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument("tap_file", help="path to results.tap") + parser.add_argument("-v", action="store_true", help="show individual test names in output") + args = parser.parse_args() + main(args.tap_file, args.v) \ No newline at end of file From 73d2f59e303fa998f997c483bb6843bf77e069e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hilmar=20G=C3=BAstafsson?= Date: Tue, 14 Apr 2020 17:15:59 +0200 Subject: [PATCH 72/86] WIP: Add libp2p-go (#956) * Add libp2p-go * Some tweaks, tidying up (cherry picked from commit 1a5bb121f8121c4f68a27abbf25a9a35a1b7c63e) * Move p2p dockerfile (cherry picked from commit 8d3bf44ea1bf37f950034e73bcdc315afdabe79a) * Remove containsBackwardsExtremity * Fix some linter errors, update some libp2p packages/calls, other tidying up * Add -port for dendrite-p2p-demo * Use instance name as key ID * Remove P2P demo docker stuff, no longer needed now that we have SQLite * Remove Dockerfile-p2p too * Remove p2p logic from dendrite-monolith-server * Inject publicRoomsDB in publicroomsapi Inject publicRoomsDB instead of switching on base.libP2P. See: https://github.com/matrix-org/dendrite/pull/956/files?file-filters%5B%5D=.go#r406276914 * Fix lint warning * Extract mDNSListener from base.go * Extract CreateFederationClient into demo * Create P2PDendrite from BaseDendrite Extract logic specific to P2PDendrite from base.go * Set base.go to upstream/master * Move pubsub to demo cmd * Move PostgreswithDHT to cmd * Remove unstable features * Add copyrights * Move libp2pvalidator into p2pdendrite * Rename dendrite-p2p-demo -> dendrite-demo-libp2p * Update copyrights * go mod tidy Co-authored-by: Neil Alexander --- cmd/dendrite-demo-libp2p/main.go | 203 ++++++++++ cmd/dendrite-demo-libp2p/mdnslistener.go | 63 ++++ cmd/dendrite-demo-libp2p/p2pdendrite.go | 126 +++++++ .../storage/postgreswithdht/storage.go | 164 ++++++++ .../storage/postgreswithpubsub/storage.go | 179 +++++++++ cmd/dendrite-demo-libp2p/storage/storage.go | 61 +++ cmd/dendrite-monolith-server/main.go | 7 +- cmd/dendrite-public-rooms-api-server/main.go | 9 +- cmd/dendritejs/main.go | 7 +- go.mod | 17 +- go.sum | 351 ++++++++++++++++++ publicroomsapi/publicroomsapi.go | 8 +- publicroomsapi/storage/storage.go | 7 +- 13 files changed, 1186 insertions(+), 16 deletions(-) create mode 100644 cmd/dendrite-demo-libp2p/main.go create mode 100644 cmd/dendrite-demo-libp2p/mdnslistener.go create mode 100644 cmd/dendrite-demo-libp2p/p2pdendrite.go create mode 100644 cmd/dendrite-demo-libp2p/storage/postgreswithdht/storage.go create mode 100644 cmd/dendrite-demo-libp2p/storage/postgreswithpubsub/storage.go create mode 100644 cmd/dendrite-demo-libp2p/storage/storage.go diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go new file mode 100644 index 000000000..df3b48adf --- /dev/null +++ b/cmd/dendrite-demo-libp2p/main.go @@ -0,0 +1,203 @@ +// Copyright 2017 Vector Creations Ltd +// Copyright 2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "crypto/ed25519" + "flag" + "fmt" + "io/ioutil" + "net/http" + "os" + "time" + + gostream "github.com/libp2p/go-libp2p-gostream" + p2phttp "github.com/libp2p/go-libp2p-http" + p2pdisc "github.com/libp2p/go-libp2p/p2p/discovery" + "github.com/matrix-org/dendrite/appservice" + "github.com/matrix-org/dendrite/clientapi" + "github.com/matrix-org/dendrite/clientapi/producers" + "github.com/matrix-org/dendrite/cmd/dendrite-demo-libp2p/storage" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/common/keydb" + "github.com/matrix-org/dendrite/common/transactions" + "github.com/matrix-org/dendrite/eduserver" + "github.com/matrix-org/dendrite/federationapi" + "github.com/matrix-org/dendrite/federationsender" + "github.com/matrix-org/dendrite/mediaapi" + "github.com/matrix-org/dendrite/publicroomsapi" + "github.com/matrix-org/dendrite/roomserver" + "github.com/matrix-org/dendrite/syncapi" + "github.com/matrix-org/gomatrixserverlib" + + "github.com/matrix-org/dendrite/eduserver/cache" + + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/sirupsen/logrus" +) + +func createKeyDB( + base *P2PDendrite, +) keydb.Database { + db, err := keydb.NewDatabase( + string(base.Base.Cfg.Database.ServerKey), + base.Base.Cfg.Matrix.ServerName, + base.Base.Cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey), + base.Base.Cfg.Matrix.KeyID, + ) + if err != nil { + logrus.WithError(err).Panicf("failed to connect to keys db") + } + mdns := mDNSListener{ + host: base.LibP2P, + keydb: db, + } + serv, err := p2pdisc.NewMdnsService( + base.LibP2PContext, + base.LibP2P, + time.Second*10, + "_matrix-dendrite-p2p._tcp", + ) + if err != nil { + panic(err) + } + serv.RegisterNotifee(&mdns) + return db +} + +func createFederationClient( + base *P2PDendrite, +) *gomatrixserverlib.FederationClient { + fmt.Println("Running in libp2p federation mode") + fmt.Println("Warning: Federation with non-libp2p homeservers will not work in this mode yet!") + tr := &http.Transport{} + tr.RegisterProtocol( + "matrix", + p2phttp.NewTransport(base.LibP2P, p2phttp.ProtocolOption("/matrix")), + ) + return gomatrixserverlib.NewFederationClientWithTransport( + base.Base.Cfg.Matrix.ServerName, base.Base.Cfg.Matrix.KeyID, base.Base.Cfg.Matrix.PrivateKey, tr, + ) +} + +func main() { + instanceName := flag.String("name", "dendrite-p2p", "the name of this P2P demo instance") + instancePort := flag.Int("port", 8080, "the port that the client API will listen on") + flag.Parse() + + filename := fmt.Sprintf("%s-private.key", *instanceName) + _, err := os.Stat(filename) + var privKey ed25519.PrivateKey + if os.IsNotExist(err) { + _, privKey, _ = ed25519.GenerateKey(nil) + if err = ioutil.WriteFile(filename, privKey, 0600); err != nil { + fmt.Printf("Couldn't write private key to file '%s': %s\n", filename, err) + } + } else { + privKey, err = ioutil.ReadFile(filename) + if err != nil { + fmt.Printf("Couldn't read private key from file '%s': %s\n", filename, err) + _, privKey, _ = ed25519.GenerateKey(nil) + } + } + + cfg := config.Dendrite{} + cfg.Matrix.ServerName = "p2p" + cfg.Matrix.PrivateKey = privKey + cfg.Matrix.KeyID = gomatrixserverlib.KeyID(fmt.Sprintf("ed25519:%s", *instanceName)) + cfg.Kafka.UseNaffka = true + cfg.Kafka.Topics.OutputRoomEvent = "roomserverOutput" + cfg.Kafka.Topics.OutputClientData = "clientapiOutput" + cfg.Kafka.Topics.OutputTypingEvent = "typingServerOutput" + cfg.Kafka.Topics.UserUpdates = "userUpdates" + cfg.Database.Account = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName)) + cfg.Database.Device = config.DataSource(fmt.Sprintf("file:%s-device.db", *instanceName)) + cfg.Database.MediaAPI = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName)) + cfg.Database.SyncAPI = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName)) + cfg.Database.RoomServer = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName)) + cfg.Database.ServerKey = config.DataSource(fmt.Sprintf("file:%s-serverkey.db", *instanceName)) + cfg.Database.FederationSender = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", *instanceName)) + cfg.Database.AppService = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName)) + cfg.Database.PublicRoomsAPI = config.DataSource(fmt.Sprintf("file:%s-publicroomsa.db", *instanceName)) + cfg.Database.Naffka = config.DataSource(fmt.Sprintf("file:%s-naffka.db", *instanceName)) + if err = cfg.Derive(); err != nil { + panic(err) + } + + base := NewP2PDendrite(&cfg, "Monolith") + defer base.Base.Close() // nolint: errcheck + + accountDB := base.Base.CreateAccountsDB() + deviceDB := base.Base.CreateDeviceDB() + keyDB := createKeyDB(base) + federation := createFederationClient(base) + keyRing := keydb.CreateKeyRing(federation.Client, keyDB) + + alias, input, query := roomserver.SetupRoomServerComponent(&base.Base) + eduInputAPI := eduserver.SetupEDUServerComponent(&base.Base, cache.New()) + asQuery := appservice.SetupAppServiceAPIComponent( + &base.Base, accountDB, deviceDB, federation, alias, query, transactions.New(), + ) + fedSenderAPI := federationsender.SetupFederationSenderComponent(&base.Base, federation, query) + + clientapi.SetupClientAPIComponent( + &base.Base, deviceDB, accountDB, + federation, &keyRing, alias, input, query, + eduInputAPI, asQuery, transactions.New(), fedSenderAPI, + ) + eduProducer := producers.NewEDUServerProducer(eduInputAPI) + federationapi.SetupFederationAPIComponent(&base.Base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fedSenderAPI, eduProducer) + mediaapi.SetupMediaAPIComponent(&base.Base, deviceDB) + publicRoomsDB, err := storage.NewPublicRoomsServerDatabaseWithPubSub(string(base.Base.Cfg.Database.PublicRoomsAPI), base.LibP2PPubsub) + if err != nil { + logrus.WithError(err).Panicf("failed to connect to public rooms db") + } + publicroomsapi.SetupPublicRoomsAPIComponent(&base.Base, deviceDB, publicRoomsDB, query, federation, nil) // Check this later + syncapi.SetupSyncAPIComponent(&base.Base, deviceDB, accountDB, query, federation, &cfg) + + httpHandler := common.WrapHandlerInCORS(base.Base.APIMux) + + // Set up the API endpoints we handle. /metrics is for prometheus, and is + // not wrapped by CORS, while everything else is + http.Handle("/metrics", promhttp.Handler()) + http.Handle("/", httpHandler) + + // Expose the matrix APIs directly rather than putting them under a /api path. + go func() { + httpBindAddr := fmt.Sprintf(":%d", *instancePort) + logrus.Info("Listening on ", httpBindAddr) + logrus.Fatal(http.ListenAndServe(httpBindAddr, nil)) + }() + // Expose the matrix APIs also via libp2p + if base.LibP2P != nil { + go func() { + logrus.Info("Listening on libp2p host ID ", base.LibP2P.ID()) + listener, err := gostream.Listen(base.LibP2P, "/matrix") + if err != nil { + panic(err) + } + defer func() { + logrus.Fatal(listener.Close()) + }() + logrus.Fatal(http.Serve(listener, nil)) + }() + } + + // We want to block forever to let the HTTP and HTTPS handler serve the APIs + select {} +} diff --git a/cmd/dendrite-demo-libp2p/mdnslistener.go b/cmd/dendrite-demo-libp2p/mdnslistener.go new file mode 100644 index 000000000..3fefbec2c --- /dev/null +++ b/cmd/dendrite-demo-libp2p/mdnslistener.go @@ -0,0 +1,63 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "fmt" + "math" + + "github.com/libp2p/go-libp2p-core/host" + "github.com/libp2p/go-libp2p-core/peer" + "github.com/matrix-org/dendrite/common/keydb" + "github.com/matrix-org/gomatrixserverlib" +) + +type mDNSListener struct { + keydb keydb.Database + host host.Host +} + +func (n *mDNSListener) HandlePeerFound(p peer.AddrInfo) { + if err := n.host.Connect(context.Background(), p); err != nil { + fmt.Println("Error adding peer", p.ID.String(), "via mDNS:", err) + } + if pubkey, err := p.ID.ExtractPublicKey(); err == nil { + raw, _ := pubkey.Raw() + if err := n.keydb.StoreKeys( + context.Background(), + map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult{ + { + ServerName: gomatrixserverlib.ServerName(p.ID.String()), + KeyID: "ed25519:p2pdemo", + }: { + VerifyKey: gomatrixserverlib.VerifyKey{ + Key: gomatrixserverlib.Base64String(raw), + }, + ValidUntilTS: math.MaxUint64 >> 1, + ExpiredTS: gomatrixserverlib.PublicKeyNotExpired, + }, + }, + ); err != nil { + fmt.Println("Failed to store keys:", err) + } + } + fmt.Println("Discovered", len(n.host.Peerstore().Peers())-1, "other libp2p peer(s):") + for _, peer := range n.host.Peerstore().Peers() { + if peer != n.host.ID() { + fmt.Println("-", peer) + } + } +} diff --git a/cmd/dendrite-demo-libp2p/p2pdendrite.go b/cmd/dendrite-demo-libp2p/p2pdendrite.go new file mode 100644 index 000000000..a9db3b39c --- /dev/null +++ b/cmd/dendrite-demo-libp2p/p2pdendrite.go @@ -0,0 +1,126 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "fmt" + + "errors" + + pstore "github.com/libp2p/go-libp2p-core/peerstore" + record "github.com/libp2p/go-libp2p-record" + "github.com/matrix-org/dendrite/common/basecomponent" + + "github.com/libp2p/go-libp2p" + circuit "github.com/libp2p/go-libp2p-circuit" + crypto "github.com/libp2p/go-libp2p-core/crypto" + routing "github.com/libp2p/go-libp2p-core/routing" + + host "github.com/libp2p/go-libp2p-core/host" + dht "github.com/libp2p/go-libp2p-kad-dht" + pubsub "github.com/libp2p/go-libp2p-pubsub" + "github.com/matrix-org/gomatrixserverlib" + + "github.com/matrix-org/dendrite/common/config" +) + +// P2PDendrite is a Peer-to-Peer variant of BaseDendrite. +type P2PDendrite struct { + Base basecomponent.BaseDendrite + + // Store our libp2p object so that we can make outgoing connections from it + // later + LibP2P host.Host + LibP2PContext context.Context + LibP2PCancel context.CancelFunc + LibP2PDHT *dht.IpfsDHT + LibP2PPubsub *pubsub.PubSub +} + +// NewP2PDendrite creates a new instance to be used by a component. +// The componentName is used for logging purposes, and should be a friendly name +// of the component running, e.g. SyncAPI. +func NewP2PDendrite(cfg *config.Dendrite, componentName string) *P2PDendrite { + baseDendrite := basecomponent.NewBaseDendrite(cfg, componentName) + + ctx, cancel := context.WithCancel(context.Background()) + + privKey, err := crypto.UnmarshalEd25519PrivateKey(cfg.Matrix.PrivateKey[:]) + if err != nil { + panic(err) + } + + //defaultIP6ListenAddr, _ := multiaddr.NewMultiaddr("/ip6/::/tcp/0") + var libp2pdht *dht.IpfsDHT + libp2p, err := libp2p.New(ctx, + libp2p.Identity(privKey), + libp2p.DefaultListenAddrs, + //libp2p.ListenAddrs(defaultIP6ListenAddr), + libp2p.DefaultTransports, + libp2p.Routing(func(h host.Host) (r routing.PeerRouting, err error) { + libp2pdht, err = dht.New(ctx, h) + if err != nil { + return nil, err + } + libp2pdht.Validator = libP2PValidator{} + r = libp2pdht + return + }), + libp2p.EnableAutoRelay(), + libp2p.EnableRelay(circuit.OptHop), + ) + if err != nil { + panic(err) + } + + libp2ppubsub, err := pubsub.NewFloodSub(context.Background(), libp2p, []pubsub.Option{ + pubsub.WithMessageSigning(true), + }...) + if err != nil { + panic(err) + } + + fmt.Println("Our public key:", privKey.GetPublic()) + fmt.Println("Our node ID:", libp2p.ID()) + fmt.Println("Our addresses:", libp2p.Addrs()) + + cfg.Matrix.ServerName = gomatrixserverlib.ServerName(libp2p.ID().String()) + + return &P2PDendrite{ + Base: *baseDendrite, + LibP2P: libp2p, + LibP2PContext: ctx, + LibP2PCancel: cancel, + LibP2PDHT: libp2pdht, + LibP2PPubsub: libp2ppubsub, + } +} + +type libP2PValidator struct { + KeyBook pstore.KeyBook +} + +func (v libP2PValidator) Validate(key string, value []byte) error { + ns, _, err := record.SplitKey(key) + if err != nil || ns != "matrix" { + return errors.New("not Matrix path") + } + return nil +} + +func (v libP2PValidator) Select(k string, vals [][]byte) (int, error) { + return 0, nil +} diff --git a/cmd/dendrite-demo-libp2p/storage/postgreswithdht/storage.go b/cmd/dendrite-demo-libp2p/storage/postgreswithdht/storage.go new file mode 100644 index 000000000..819469ee8 --- /dev/null +++ b/cmd/dendrite-demo-libp2p/storage/postgreswithdht/storage.go @@ -0,0 +1,164 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package postgreswithdht + +import ( + "context" + "encoding/json" + "fmt" + "sync" + "sync/atomic" + "time" + + "github.com/matrix-org/dendrite/publicroomsapi/storage/postgres" + "github.com/matrix-org/gomatrixserverlib" + + dht "github.com/libp2p/go-libp2p-kad-dht" +) + +const DHTInterval = time.Second * 10 + +// PublicRoomsServerDatabase represents a public rooms server database. +type PublicRoomsServerDatabase struct { + dht *dht.IpfsDHT + postgres.PublicRoomsServerDatabase + ourRoomsContext context.Context // our current value in the DHT + ourRoomsCancel context.CancelFunc // cancel when we want to expire our value + foundRooms map[string]gomatrixserverlib.PublicRoom // additional rooms we have learned about from the DHT + foundRoomsMutex sync.RWMutex // protects foundRooms + maintenanceTimer *time.Timer // + roomsAdvertised atomic.Value // stores int + roomsDiscovered atomic.Value // stores int +} + +// NewPublicRoomsServerDatabase creates a new public rooms server database. +func NewPublicRoomsServerDatabase(dataSourceName string, dht *dht.IpfsDHT) (*PublicRoomsServerDatabase, error) { + pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName) + if err != nil { + return nil, err + } + provider := PublicRoomsServerDatabase{ + dht: dht, + PublicRoomsServerDatabase: *pg, + } + go provider.ResetDHTMaintenance() + provider.roomsAdvertised.Store(0) + provider.roomsDiscovered.Store(0) + return &provider, nil +} + +func (d *PublicRoomsServerDatabase) GetRoomVisibility(ctx context.Context, roomID string) (bool, error) { + return d.PublicRoomsServerDatabase.GetRoomVisibility(ctx, roomID) +} + +func (d *PublicRoomsServerDatabase) SetRoomVisibility(ctx context.Context, visible bool, roomID string) error { + d.ResetDHTMaintenance() + return d.PublicRoomsServerDatabase.SetRoomVisibility(ctx, visible, roomID) +} + +func (d *PublicRoomsServerDatabase) CountPublicRooms(ctx context.Context) (int64, error) { + count, err := d.PublicRoomsServerDatabase.CountPublicRooms(ctx) + if err != nil { + return 0, err + } + d.foundRoomsMutex.RLock() + defer d.foundRoomsMutex.RUnlock() + return count + int64(len(d.foundRooms)), nil +} + +func (d *PublicRoomsServerDatabase) GetPublicRooms(ctx context.Context, offset int64, limit int16, filter string) ([]gomatrixserverlib.PublicRoom, error) { + realfilter := filter + if realfilter == "__local__" { + realfilter = "" + } + rooms, err := d.PublicRoomsServerDatabase.GetPublicRooms(ctx, offset, limit, realfilter) + if err != nil { + return []gomatrixserverlib.PublicRoom{}, err + } + if filter != "__local__" { + d.foundRoomsMutex.RLock() + defer d.foundRoomsMutex.RUnlock() + for _, room := range d.foundRooms { + rooms = append(rooms, room) + } + } + return rooms, nil +} + +func (d *PublicRoomsServerDatabase) UpdateRoomFromEvents(ctx context.Context, eventsToAdd []gomatrixserverlib.Event, eventsToRemove []gomatrixserverlib.Event) error { + return d.PublicRoomsServerDatabase.UpdateRoomFromEvents(ctx, eventsToAdd, eventsToRemove) +} + +func (d *PublicRoomsServerDatabase) UpdateRoomFromEvent(ctx context.Context, event gomatrixserverlib.Event) error { + return d.PublicRoomsServerDatabase.UpdateRoomFromEvent(ctx, event) +} + +func (d *PublicRoomsServerDatabase) ResetDHTMaintenance() { + if d.maintenanceTimer != nil && !d.maintenanceTimer.Stop() { + <-d.maintenanceTimer.C + } + d.Interval() +} + +func (d *PublicRoomsServerDatabase) Interval() { + if err := d.AdvertiseRoomsIntoDHT(); err != nil { + // fmt.Println("Failed to advertise room in DHT:", err) + } + if err := d.FindRoomsInDHT(); err != nil { + // fmt.Println("Failed to find rooms in DHT:", err) + } + fmt.Println("Found", d.roomsDiscovered.Load(), "room(s), advertised", d.roomsAdvertised.Load(), "room(s)") + d.maintenanceTimer = time.AfterFunc(DHTInterval, d.Interval) +} + +func (d *PublicRoomsServerDatabase) AdvertiseRoomsIntoDHT() error { + dbCtx, dbCancel := context.WithTimeout(context.Background(), 3*time.Second) + _ = dbCancel + ourRooms, err := d.GetPublicRooms(dbCtx, 0, 1024, "__local__") + if err != nil { + return err + } + if j, err := json.Marshal(ourRooms); err == nil { + d.roomsAdvertised.Store(len(ourRooms)) + d.ourRoomsContext, d.ourRoomsCancel = context.WithCancel(context.Background()) + if err := d.dht.PutValue(d.ourRoomsContext, "/matrix/publicRooms", j); err != nil { + return err + } + } + return nil +} + +func (d *PublicRoomsServerDatabase) FindRoomsInDHT() error { + d.foundRoomsMutex.Lock() + searchCtx, searchCancel := context.WithTimeout(context.Background(), 10*time.Second) + defer searchCancel() + defer d.foundRoomsMutex.Unlock() + results, err := d.dht.GetValues(searchCtx, "/matrix/publicRooms", 1024) + if err != nil { + return err + } + d.foundRooms = make(map[string]gomatrixserverlib.PublicRoom) + for _, result := range results { + var received []gomatrixserverlib.PublicRoom + if err := json.Unmarshal(result.Val, &received); err != nil { + return err + } + for _, room := range received { + d.foundRooms[room.RoomID] = room + } + } + d.roomsDiscovered.Store(len(d.foundRooms)) + return nil +} diff --git a/cmd/dendrite-demo-libp2p/storage/postgreswithpubsub/storage.go b/cmd/dendrite-demo-libp2p/storage/postgreswithpubsub/storage.go new file mode 100644 index 000000000..661192243 --- /dev/null +++ b/cmd/dendrite-demo-libp2p/storage/postgreswithpubsub/storage.go @@ -0,0 +1,179 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package postgreswithpubsub + +import ( + "context" + "encoding/json" + "fmt" + "sync" + "sync/atomic" + "time" + + "github.com/matrix-org/dendrite/publicroomsapi/storage/postgres" + "github.com/matrix-org/gomatrixserverlib" + + pubsub "github.com/libp2p/go-libp2p-pubsub" +) + +const MaintenanceInterval = time.Second * 10 + +type discoveredRoom struct { + time time.Time + room gomatrixserverlib.PublicRoom +} + +// PublicRoomsServerDatabase represents a public rooms server database. +type PublicRoomsServerDatabase struct { + postgres.PublicRoomsServerDatabase // + pubsub *pubsub.PubSub // + subscription *pubsub.Subscription // + foundRooms map[string]discoveredRoom // additional rooms we have learned about from the DHT + foundRoomsMutex sync.RWMutex // protects foundRooms + maintenanceTimer *time.Timer // + roomsAdvertised atomic.Value // stores int +} + +// NewPublicRoomsServerDatabase creates a new public rooms server database. +func NewPublicRoomsServerDatabase(dataSourceName string, pubsub *pubsub.PubSub) (*PublicRoomsServerDatabase, error) { + pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName) + if err != nil { + return nil, err + } + provider := PublicRoomsServerDatabase{ + pubsub: pubsub, + PublicRoomsServerDatabase: *pg, + foundRooms: make(map[string]discoveredRoom), + } + if topic, err := pubsub.Join("/matrix/publicRooms"); err != nil { + return nil, err + } else if sub, err := topic.Subscribe(); err == nil { + provider.subscription = sub + go provider.MaintenanceTimer() + go provider.FindRooms() + provider.roomsAdvertised.Store(0) + return &provider, nil + } else { + return nil, err + } +} + +func (d *PublicRoomsServerDatabase) GetRoomVisibility(ctx context.Context, roomID string) (bool, error) { + return d.PublicRoomsServerDatabase.GetRoomVisibility(ctx, roomID) +} + +func (d *PublicRoomsServerDatabase) SetRoomVisibility(ctx context.Context, visible bool, roomID string) error { + d.MaintenanceTimer() + return d.PublicRoomsServerDatabase.SetRoomVisibility(ctx, visible, roomID) +} + +func (d *PublicRoomsServerDatabase) CountPublicRooms(ctx context.Context) (int64, error) { + d.foundRoomsMutex.RLock() + defer d.foundRoomsMutex.RUnlock() + return int64(len(d.foundRooms)), nil +} + +func (d *PublicRoomsServerDatabase) GetPublicRooms(ctx context.Context, offset int64, limit int16, filter string) ([]gomatrixserverlib.PublicRoom, error) { + var rooms []gomatrixserverlib.PublicRoom + if filter == "__local__" { + if r, err := d.PublicRoomsServerDatabase.GetPublicRooms(ctx, offset, limit, ""); err == nil { + rooms = append(rooms, r...) + } else { + return []gomatrixserverlib.PublicRoom{}, err + } + } else { + d.foundRoomsMutex.RLock() + defer d.foundRoomsMutex.RUnlock() + for _, room := range d.foundRooms { + rooms = append(rooms, room.room) + } + } + return rooms, nil +} + +func (d *PublicRoomsServerDatabase) UpdateRoomFromEvents(ctx context.Context, eventsToAdd []gomatrixserverlib.Event, eventsToRemove []gomatrixserverlib.Event) error { + return d.PublicRoomsServerDatabase.UpdateRoomFromEvents(ctx, eventsToAdd, eventsToRemove) +} + +func (d *PublicRoomsServerDatabase) UpdateRoomFromEvent(ctx context.Context, event gomatrixserverlib.Event) error { + return d.PublicRoomsServerDatabase.UpdateRoomFromEvent(ctx, event) +} + +func (d *PublicRoomsServerDatabase) MaintenanceTimer() { + if d.maintenanceTimer != nil && !d.maintenanceTimer.Stop() { + <-d.maintenanceTimer.C + } + d.Interval() +} + +func (d *PublicRoomsServerDatabase) Interval() { + d.foundRoomsMutex.Lock() + for k, v := range d.foundRooms { + if time.Since(v.time) > time.Minute { + delete(d.foundRooms, k) + } + } + d.foundRoomsMutex.Unlock() + if err := d.AdvertiseRooms(); err != nil { + fmt.Println("Failed to advertise room in DHT:", err) + } + d.foundRoomsMutex.RLock() + defer d.foundRoomsMutex.RUnlock() + fmt.Println("Found", len(d.foundRooms), "room(s), advertised", d.roomsAdvertised.Load(), "room(s)") + d.maintenanceTimer = time.AfterFunc(MaintenanceInterval, d.Interval) +} + +func (d *PublicRoomsServerDatabase) AdvertiseRooms() error { + dbCtx, dbCancel := context.WithTimeout(context.Background(), 3*time.Second) + _ = dbCancel + ourRooms, err := d.GetPublicRooms(dbCtx, 0, 1024, "__local__") + if err != nil { + return err + } + advertised := 0 + for _, room := range ourRooms { + if j, err := json.Marshal(room); err == nil { + if topic, err := d.pubsub.Join("/matrix/publicRooms"); err != nil { + fmt.Println("Failed to subscribe to topic:", err) + } else if err := topic.Publish(context.TODO(), j); err != nil { + fmt.Println("Failed to publish public room:", err) + } else { + advertised++ + } + } + } + + d.roomsAdvertised.Store(advertised) + return nil +} + +func (d *PublicRoomsServerDatabase) FindRooms() { + for { + msg, err := d.subscription.Next(context.Background()) + if err != nil { + continue + } + received := discoveredRoom{ + time: time.Now(), + } + if err := json.Unmarshal(msg.Data, &received.room); err != nil { + fmt.Println("Unmarshal error:", err) + continue + } + d.foundRoomsMutex.Lock() + d.foundRooms[received.room.RoomID] = received + d.foundRoomsMutex.Unlock() + } +} diff --git a/cmd/dendrite-demo-libp2p/storage/storage.go b/cmd/dendrite-demo-libp2p/storage/storage.go new file mode 100644 index 000000000..668edbaa3 --- /dev/null +++ b/cmd/dendrite-demo-libp2p/storage/storage.go @@ -0,0 +1,61 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "net/url" + + dht "github.com/libp2p/go-libp2p-kad-dht" + pubsub "github.com/libp2p/go-libp2p-pubsub" + "github.com/matrix-org/dendrite/cmd/dendrite-demo-libp2p/storage/postgreswithdht" + "github.com/matrix-org/dendrite/cmd/dendrite-demo-libp2p/storage/postgreswithpubsub" + "github.com/matrix-org/dendrite/publicroomsapi/storage" + "github.com/matrix-org/dendrite/publicroomsapi/storage/sqlite3" +) + +const schemePostgres = "postgres" +const schemeFile = "file" + +// NewPublicRoomsServerDatabase opens a database connection. +func NewPublicRoomsServerDatabaseWithDHT(dataSourceName string, dht *dht.IpfsDHT) (storage.Database, error) { + uri, err := url.Parse(dataSourceName) + if err != nil { + return postgreswithdht.NewPublicRoomsServerDatabase(dataSourceName, dht) + } + switch uri.Scheme { + case schemePostgres: + return postgreswithdht.NewPublicRoomsServerDatabase(dataSourceName, dht) + case schemeFile: + return sqlite3.NewPublicRoomsServerDatabase(dataSourceName) + default: + return postgreswithdht.NewPublicRoomsServerDatabase(dataSourceName, dht) + } +} + +// NewPublicRoomsServerDatabase opens a database connection. +func NewPublicRoomsServerDatabaseWithPubSub(dataSourceName string, pubsub *pubsub.PubSub) (storage.Database, error) { + uri, err := url.Parse(dataSourceName) + if err != nil { + return postgreswithpubsub.NewPublicRoomsServerDatabase(dataSourceName, pubsub) + } + switch uri.Scheme { + case schemePostgres: + return postgreswithpubsub.NewPublicRoomsServerDatabase(dataSourceName, pubsub) + case schemeFile: + return sqlite3.NewPublicRoomsServerDatabase(dataSourceName) + default: + return postgreswithpubsub.NewPublicRoomsServerDatabase(dataSourceName, pubsub) + } +} diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index c71d956b2..d47d2e1bb 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -31,6 +31,7 @@ import ( "github.com/matrix-org/dendrite/federationsender" "github.com/matrix-org/dendrite/mediaapi" "github.com/matrix-org/dendrite/publicroomsapi" + "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/syncapi" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -71,7 +72,11 @@ func main() { eduProducer := producers.NewEDUServerProducer(eduInputAPI) federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fedSenderAPI, eduProducer) mediaapi.SetupMediaAPIComponent(base, deviceDB) - publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, query, federation, nil) + publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI)) + if err != nil { + logrus.WithError(err).Panicf("failed to connect to public rooms db") + } + publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, query, federation, nil) syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query, federation, cfg) httpHandler := common.WrapHandlerInCORS(base.APIMux) diff --git a/cmd/dendrite-public-rooms-api-server/main.go b/cmd/dendrite-public-rooms-api-server/main.go index 6b7eac7d5..f6a782f66 100644 --- a/cmd/dendrite-public-rooms-api-server/main.go +++ b/cmd/dendrite-public-rooms-api-server/main.go @@ -17,6 +17,8 @@ package main import ( "github.com/matrix-org/dendrite/common/basecomponent" "github.com/matrix-org/dendrite/publicroomsapi" + "github.com/matrix-org/dendrite/publicroomsapi/storage" + "github.com/sirupsen/logrus" ) func main() { @@ -27,8 +29,11 @@ func main() { deviceDB := base.CreateDeviceDB() _, _, query := base.CreateHTTPRoomserverAPIs() - - publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, query, nil, nil) + publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI)) + if err != nil { + logrus.WithError(err).Panicf("failed to connect to public rooms db") + } + publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, query, nil, nil) base.SetupAndServeHTTP(string(base.Cfg.Bind.PublicRoomsAPI), string(base.Cfg.Listen.PublicRoomsAPI)) diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 05802725d..9bd8f2ee2 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -34,6 +34,7 @@ import ( "github.com/matrix-org/dendrite/federationsender" "github.com/matrix-org/dendrite/mediaapi" "github.com/matrix-org/dendrite/publicroomsapi" + "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/syncapi" "github.com/matrix-org/go-http-js-libp2p/go_http_js_libp2p" @@ -137,7 +138,11 @@ func main() { eduProducer := producers.NewEDUServerProducer(eduInputAPI) federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fedSenderAPI, eduProducer) mediaapi.SetupMediaAPIComponent(base, deviceDB) - publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, query, federation, p2pPublicRoomProvider) + publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI)) + if err != nil { + logrus.WithError(err).Panicf("failed to connect to public rooms db") + } + publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, query, federation, p2pPublicRoomProvider) syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query, federation, cfg) httpHandler := common.WrapHandlerInCORS(base.APIMux) diff --git a/go.mod b/go.mod index 5854e552b..ec8f3eb69 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,16 @@ module github.com/matrix-org/dendrite require ( github.com/gorilla/mux v1.7.3 github.com/hashicorp/golang-lru v0.5.4 + github.com/kr/pretty v0.2.0 // indirect github.com/lib/pq v1.2.0 + github.com/libp2p/go-libp2p v0.6.0 + github.com/libp2p/go-libp2p-circuit v0.1.4 github.com/libp2p/go-libp2p-core v0.5.0 + github.com/libp2p/go-libp2p-gostream v0.2.1 + github.com/libp2p/go-libp2p-http v0.1.5 + github.com/libp2p/go-libp2p-kad-dht v0.5.0 + github.com/libp2p/go-libp2p-pubsub v0.2.5 + github.com/libp2p/go-libp2p-record v0.1.2 github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f github.com/matrix-org/go-sqlite3-js v0.0.0-20200325174927-327088cdef10 @@ -12,10 +20,11 @@ require ( github.com/matrix-org/gomatrixserverlib v0.0.0-20200409140603-8b9a51fe9b89 github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 - github.com/mattn/go-sqlite3 v2.0.2+incompatible + github.com/mattn/go-sqlite3 v2.0.3+incompatible github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 github.com/opentracing/opentracing-go v1.1.0 - github.com/pkg/errors v0.8.1 + github.com/pierrec/lz4 v2.5.0+incompatible // indirect + github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.4.1 github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect github.com/sirupsen/logrus v1.4.2 @@ -24,11 +33,11 @@ require ( github.com/uber/jaeger-client-go v2.22.1+incompatible github.com/uber/jaeger-lib v2.2.0+incompatible go.uber.org/atomic v1.6.0 - golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d + golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d golang.org/x/tools v0.0.0-20200402223321-bcf690261a44 // indirect gopkg.in/Shopify/sarama.v1 v1.20.1 gopkg.in/h2non/bimg.v1 v1.0.18 - gopkg.in/yaml.v2 v2.2.5 + gopkg.in/yaml.v2 v2.2.8 ) go 1.13 diff --git a/go.sum b/go.sum index b1f5ac2f4..0d03cd371 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,11 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= +github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/zstd v1.4.4 h1:+IawcoXhCBylN7ccwdwf8LOH2jKq7NavGpEPanrlTzE= github.com/DataDog/zstd v1.4.4/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= @@ -10,13 +13,18 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= +github.com/btcsuite/btcd v0.0.0-20190523000118-16327141da8c/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= +github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= +github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= github.com/btcsuite/btcutil v1.0.1 h1:GKOz8BnRjYrb/JTKgaOk+zh26NWNdSNvdvv0xoAZMSA= github.com/btcsuite/btcutil v1.0.1/go.mod h1:j9HUFwoQRsZL3V4n+qG+CUnEGHOarIxfC3Le2Yhbcts= @@ -31,13 +39,28 @@ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018 h1:6xT9KW8zLC5IlbaIF5Q7JNieBoACT7iW0YTxQHR0in0= +github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018/go.mod h1:rQYf4tfk5sSwFsnDg3qYaBxSjsD9S8+59vW0dKUgme4= +github.com/dgraph-io/badger v1.5.5-0.20190226225317-8115aed38f8f/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ= +github.com/dgraph-io/badger v1.6.0-rc1/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= +github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= +github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q= +github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q= +github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= @@ -48,40 +71,109 @@ github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3B github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJYCFOLkIBwI7xFExG03bbkOkCvUPI= +github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM= +github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= +github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/huin/goupnp v1.0.0 h1:wg75sLpL6DZqwHQN6E1Cfk6mtfzS45z8OV+ic+DtHRo= +github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= +github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= +github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= +github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= +github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M= github.com/ipfs/go-cid v0.0.5 h1:o0Ix8e/ql7Zb5UVUJEUfjsWCIY8t48++9lR8qi6oiJU= github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog= +github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= +github.com/ipfs/go-datastore v0.1.0/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= +github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= +github.com/ipfs/go-datastore v0.3.1 h1:SS1t869a6cctoSYmZXUk8eL6AzVXgASmKIWFNQkQ1jU= +github.com/ipfs/go-datastore v0.3.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= +github.com/ipfs/go-datastore v0.4.0/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= +github.com/ipfs/go-datastore v0.4.1/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= +github.com/ipfs/go-datastore v0.4.4 h1:rjvQ9+muFaJ+QZ7dN5B1MSDNQ0JVZKkkES/rMZmA8X8= +github.com/ipfs/go-datastore v0.4.4/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= +github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= +github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= +github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjvRcKeSGij8= +github.com/ipfs/go-ds-badger v0.0.5/go.mod h1:g5AuuCGmr7efyzQhLL8MzwqcauPojGPUaHzfGTzuE3s= +github.com/ipfs/go-ds-badger v0.0.7/go.mod h1:qt0/fWzZDoPW6jpQeqUjR5kBfhDNB65jd9YlmAvpQBk= +github.com/ipfs/go-ds-badger v0.2.1/go.mod h1:Tx7l3aTph3FMFrRS838dcSJh+jjA7cX9DrGVwx/NOwE= +github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIykR4L04tMOYc= +github.com/ipfs/go-ds-leveldb v0.1.0/go.mod h1:hqAW8y4bwX5LWcCtku2rFNX3vjDZCy5LZCg+cSZvYb8= +github.com/ipfs/go-ds-leveldb v0.4.1/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s= +github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= +github.com/ipfs/go-ipfs-util v0.0.1 h1:Wz9bL2wB2YBJqggkA4dD7oSmqB4cAnpNbGrlHJulv50= +github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= +github.com/ipfs/go-log v0.0.1 h1:9XTUN/rW64BCG1YhPK9Hoy3q8nr4gOmHHBpgFdfw6Lc= +github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= +github.com/ipfs/go-log v1.0.2 h1:s19ZwJxH8rPWzypjcDpqPLIyV7BnbLqvpli3iZoqYK0= +github.com/ipfs/go-log v1.0.2/go.mod h1:1MNjMxe0u6xvJZgeqbJ8vdo2TKaGwZ1a0Bpza+sr2Sk= +github.com/ipfs/go-log/v2 v2.0.2 h1:xguurydRdfKMJjKyxNXNU8lYP0VZH1NUwJRwUorjuEw= +github.com/ipfs/go-log/v2 v2.0.2/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0= +github.com/ipfs/go-todocounter v0.0.2 h1:9UBngSQhylg2UDcxSAtpkT+rEWFr26hDPXVStE8LFyc= +github.com/ipfs/go-todocounter v0.0.2/go.mod h1:l5aErvQc8qKE2r7NDMjmq5UNAvuZy0rC8BHOplkWvZ4= +github.com/jackpal/gateway v1.0.5 h1:qzXWUJfuMdlLMtt0a3Dgt+xkWQiA5itDEITVJtuSwMc= +github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= +github.com/jackpal/go-nat-pmp v1.0.1 h1:i0LektDkO1QlrTm/cSuP+PyBCDnYvjPLGl4LdWEMiaA= +github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= +github.com/jbenet/go-cienv v0.0.0-20150120210510-1bb1476777ec/go.mod h1:rGaEvXB4uRSZMmzKNLoXvTu1sfx+1kv/DojUlPrSZGs= +github.com/jbenet/go-cienv v0.1.0 h1:Vc/s0QbQtoxX8MwwSLWWh+xNNZvM3Lw7NsTcHrvvhMc= github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= +github.com/jbenet/go-temp-err-catcher v0.0.0-20150120210811-aac704a3f4f2 h1:vhC1OXXiT9R2pczegwz6moDvuRpggaroAXhPIseh57A= +github.com/jbenet/go-temp-err-catcher v0.0.0-20150120210811-aac704a3f4f2/go.mod h1:8GXXJV31xl8whumTzdZsTt3RnUIiPqzkyf7mxToRCMs= +github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY= github.com/jbenet/goprocess v0.1.3 h1:YKyIEECS/XvcfHtBzxtjBBbWK+MbvA6dG8ASiqwvr10= github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -92,31 +184,171 @@ github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d h1:68u9r4wEvL3gYg2jvAOgROwZ3H+Y3hIDk4tbbmIjcYQ= +github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/libp2p/go-addr-util v0.0.1 h1:TpTQm9cXVRVSKsYbgQ7GKc3KbbHVTnbostgGaDEP+88= +github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ= +github.com/libp2p/go-buffer-pool v0.0.1/go.mod h1:xtyIz9PMobb13WaxR6Zo1Pd1zXJKYg0a8KiIvDp3TzQ= github.com/libp2p/go-buffer-pool v0.0.2 h1:QNK2iAFa8gjAe1SPz6mHSMuCcjs+X1wlHzeOSqcmlfs= github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM= +github.com/libp2p/go-conn-security-multistream v0.1.0 h1:aqGmto+ttL/uJgX0JtQI0tD21CIEy5eYd1Hlp0juHY0= +github.com/libp2p/go-conn-security-multistream v0.1.0/go.mod h1:aw6eD7LOsHEX7+2hJkDxw1MteijaVcI+/eP2/x3J1xc= +github.com/libp2p/go-eventbus v0.1.0 h1:mlawomSAjjkk97QnYiEmHsLu7E136+2oCWSHRUvMfzQ= +github.com/libp2p/go-eventbus v0.1.0/go.mod h1:vROgu5cs5T7cv7POWlWxBaVLxfSegC5UGQf8A2eEmx4= +github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZxBdp967ls1g+k8= +github.com/libp2p/go-flow-metrics v0.0.2/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= +github.com/libp2p/go-flow-metrics v0.0.3 h1:8tAs/hSdNvUiLgtlSy3mxwxWP4I9y/jlkPFT7epKdeM= github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= +github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= +github.com/libp2p/go-libp2p v0.5.0 h1:/nnb5mc2TK6TwknECsWIkfCwMTHv0AXbvzxlnVivfeg= +github.com/libp2p/go-libp2p v0.5.0/go.mod h1:Os7a5Z3B+ErF4v7zgIJ7nBHNu2LYt8ZMLkTQUB3G/wA= +github.com/libp2p/go-libp2p v0.6.0 h1:EFArryT9N7AVA70LCcOh8zxsW+FeDnxwcpWQx9k7+GM= +github.com/libp2p/go-libp2p v0.6.0/go.mod h1:mfKWI7Soz3ABX+XEBR61lGbg+ewyMtJHVt043oWeqwg= +github.com/libp2p/go-libp2p-autonat v0.1.1 h1:WLBZcIRsjZlWdAZj9CiBSvU2wQXoUOiS1Zk1tM7DTJI= +github.com/libp2p/go-libp2p-autonat v0.1.1/go.mod h1:OXqkeGOY2xJVWKAGV2inNF5aKN/djNA3fdpCWloIudE= +github.com/libp2p/go-libp2p-blankhost v0.1.1/go.mod h1:pf2fvdLJPsC1FsVrNP3DUUvMzUts2dsLLBEpo1vW1ro= +github.com/libp2p/go-libp2p-blankhost v0.1.4 h1:I96SWjR4rK9irDHcHq3XHN6hawCRTPUADzkJacgZLvk= +github.com/libp2p/go-libp2p-blankhost v0.1.4/go.mod h1:oJF0saYsAXQCSfDq254GMNmLNz6ZTHTOvtF4ZydUvwU= +github.com/libp2p/go-libp2p-circuit v0.1.4 h1:Phzbmrg3BkVzbqd4ZZ149JxCuUWu2wZcXf/Kr6hZJj8= +github.com/libp2p/go-libp2p-circuit v0.1.4/go.mod h1:CY67BrEjKNDhdTk8UgBX1Y/H5c3xkAcs3gnksxY7osU= +github.com/libp2p/go-libp2p-core v0.0.1/go.mod h1:g/VxnTZ/1ygHxH3dKok7Vno1VfpvGcGip57wjTU4fco= +github.com/libp2p/go-libp2p-core v0.0.4/go.mod h1:jyuCQP356gzfCFtRKyvAbNkyeuxb7OlyhWZ3nls5d2I= +github.com/libp2p/go-libp2p-core v0.2.0/go.mod h1:X0eyB0Gy93v0DZtSYbEM7RnMChm9Uv3j7yRXjO77xSI= +github.com/libp2p/go-libp2p-core v0.2.2/go.mod h1:8fcwTbsG2B+lTgRJ1ICZtiM5GWCWZVoVrLaDRvIRng0= +github.com/libp2p/go-libp2p-core v0.2.4/go.mod h1:STh4fdfa5vDYr0/SzYYeqnt+E6KfEV5VxfIrm0bcI0g= +github.com/libp2p/go-libp2p-core v0.2.5/go.mod h1:6+5zJmKhsf7yHn1RbmYDu08qDUpIUxGdqHuEZckmZOA= +github.com/libp2p/go-libp2p-core v0.3.0 h1:F7PqduvrztDtFsAa/bcheQ3azmNo+Nq7m8hQY5GiUW8= +github.com/libp2p/go-libp2p-core v0.3.0/go.mod h1:ACp3DmS3/N64c2jDzcV429ukDpicbL6+TrrxANBjPGw= +github.com/libp2p/go-libp2p-core v0.3.1/go.mod h1:thvWy0hvaSBhnVBaW37BvzgVV68OUhgJJLAa6almrII= +github.com/libp2p/go-libp2p-core v0.4.0/go.mod h1:49XGI+kc38oGVwqSBhDEwytaAxgZasHhFfQKibzTls0= github.com/libp2p/go-libp2p-core v0.5.0 h1:FBQ1fpq2Fo/ClyjojVJ5AKXlKhvNc/B6U0O+7AN1ffE= github.com/libp2p/go-libp2p-core v0.5.0/go.mod h1:49XGI+kc38oGVwqSBhDEwytaAxgZasHhFfQKibzTls0= +github.com/libp2p/go-libp2p-crypto v0.1.0 h1:k9MFy+o2zGDNGsaoZl0MA3iZ75qXxr9OOoAZF+sD5OQ= +github.com/libp2p/go-libp2p-crypto v0.1.0/go.mod h1:sPUokVISZiy+nNuTTH/TY+leRSxnFj/2GLjtOTW90hI= +github.com/libp2p/go-libp2p-discovery v0.2.0 h1:1p3YSOq7VsgaL+xVHPi8XAmtGyas6D2J6rWBEfz/aiY= +github.com/libp2p/go-libp2p-discovery v0.2.0/go.mod h1:s4VGaxYMbw4+4+tsoQTqh7wfxg97AEdo4GYBt6BadWg= +github.com/libp2p/go-libp2p-gostream v0.2.1 h1:JjA9roGokaR2BgWmaI/3HQu1/+jSbVVDLatQGnVdGjI= +github.com/libp2p/go-libp2p-gostream v0.2.1/go.mod h1:1Mjp3LDmkqICe5tH9yLVNCqFaRTy6OwBvuJV6j1b9Nk= +github.com/libp2p/go-libp2p-http v0.1.5 h1:FfLnzjlEzV4/6UCXCpPXRYZNoGCfogqCFjd7eF0Jbm8= +github.com/libp2p/go-libp2p-http v0.1.5/go.mod h1:2YfPjsQxUlBGFQl2u461unkQ7ukwiSs7NX2eSslOJiU= +github.com/libp2p/go-libp2p-kad-dht v0.5.0 h1:kDMtCftpQOL2s84/dZmw5z4NmBe6ByeDLKpcn6TcyxU= +github.com/libp2p/go-libp2p-kad-dht v0.5.0/go.mod h1:42YDfiKXzIgaIexiEQ3rKZbVPVPziLOyHpXbOCVd814= +github.com/libp2p/go-libp2p-kbucket v0.2.3 h1:XtNfN4WUy0cfeJoJgWCf1lor4Pp3kBkFJ9vQ+Zs+VUM= +github.com/libp2p/go-libp2p-kbucket v0.2.3/go.mod h1:opWrBZSWnBYPc315q497huxY3sz1t488X6OiXUEYWKA= +github.com/libp2p/go-libp2p-loggables v0.1.0 h1:h3w8QFfCt2UJl/0/NW4K829HX/0S4KD31PQ7m8UXXO8= +github.com/libp2p/go-libp2p-loggables v0.1.0/go.mod h1:EyumB2Y6PrYjr55Q3/tiJ/o3xoDasoRYM7nOzEpoa90= +github.com/libp2p/go-libp2p-mplex v0.2.0/go.mod h1:Ejl9IyjvXJ0T9iqUTE1jpYATQ9NM3g+OtR+EMMODbKo= +github.com/libp2p/go-libp2p-mplex v0.2.1 h1:E1xaJBQnbSiTHGI1gaBKmKhu1TUKkErKJnE8iGvirYI= +github.com/libp2p/go-libp2p-mplex v0.2.1/go.mod h1:SC99Rxs8Vuzrf/6WhmH41kNn13TiYdAWNYHrwImKLnE= +github.com/libp2p/go-libp2p-mplex v0.2.2 h1:+Ld7YDAfVERQ0E+qqjE7o6fHwKuM0SqTzYiwN1lVVSA= +github.com/libp2p/go-libp2p-mplex v0.2.2/go.mod h1:74S9eum0tVQdAfFiKxAyKzNdSuLqw5oadDq7+L/FELo= +github.com/libp2p/go-libp2p-nat v0.0.5 h1:/mH8pXFVKleflDL1YwqMg27W9GD8kjEx7NY0P6eGc98= +github.com/libp2p/go-libp2p-nat v0.0.5/go.mod h1:1qubaE5bTZMJE+E/uu2URroMbzdubFz1ChgiN79yKPE= +github.com/libp2p/go-libp2p-netutil v0.1.0 h1:zscYDNVEcGxyUpMd0JReUZTrpMfia8PmLKcKF72EAMQ= +github.com/libp2p/go-libp2p-netutil v0.1.0/go.mod h1:3Qv/aDqtMLTUyQeundkKsA+YCThNdbQD54k3TqjpbFU= +github.com/libp2p/go-libp2p-peer v0.2.0 h1:EQ8kMjaCUwt/Y5uLgjT8iY2qg0mGUT0N1zUjer50DsY= +github.com/libp2p/go-libp2p-peer v0.2.0/go.mod h1:RCffaCvUyW2CJmG2gAWVqwePwW7JMgxjsHm7+J5kjWY= +github.com/libp2p/go-libp2p-peerstore v0.1.0/go.mod h1:2CeHkQsr8svp4fZ+Oi9ykN1HBb6u0MOvdJ7YIsmcwtY= +github.com/libp2p/go-libp2p-peerstore v0.1.3/go.mod h1:BJ9sHlm59/80oSkpWgr1MyY1ciXAXV397W6h1GH/uKI= +github.com/libp2p/go-libp2p-peerstore v0.1.4 h1:d23fvq5oYMJ/lkkbO4oTwBp/JP+I/1m5gZJobNXCE/k= +github.com/libp2p/go-libp2p-peerstore v0.1.4/go.mod h1:+4BDbDiiKf4PzpANZDAT+knVdLxvqh7hXOujessqdzs= +github.com/libp2p/go-libp2p-peerstore v0.2.0 h1:XcgJhI8WyUOCbHyRLNEX5542YNj8hnLSJ2G1InRjDhk= +github.com/libp2p/go-libp2p-peerstore v0.2.0/go.mod h1:N2l3eVIeAitSg3Pi2ipSrJYnqhVnMNQZo9nkSCuAbnQ= +github.com/libp2p/go-libp2p-pnet v0.2.0 h1:J6htxttBipJujEjz1y0a5+eYoiPcFHhSYHH6na5f0/k= +github.com/libp2p/go-libp2p-pnet v0.2.0/go.mod h1:Qqvq6JH/oMZGwqs3N1Fqhv8NVhrdYcO0BW4wssv21LA= +github.com/libp2p/go-libp2p-pubsub v0.2.5 h1:tPKbkjAUI0xLGN3KKTKKy9TQEviVfrP++zJgH5Muke4= +github.com/libp2p/go-libp2p-pubsub v0.2.5/go.mod h1:9Q2RRq8ofXkoewORcyVlgUFDKLKw7BuYSlJVWRcVk3Y= +github.com/libp2p/go-libp2p-record v0.1.2 h1:M50VKzWnmUrk/M5/Dz99qO9Xh4vs8ijsK+7HkJvRP+0= +github.com/libp2p/go-libp2p-record v0.1.2/go.mod h1:pal0eNcT5nqZaTV7UGhqeGqxFgGdsU/9W//C8dqjQDk= +github.com/libp2p/go-libp2p-routing v0.1.0 h1:hFnj3WR3E2tOcKaGpyzfP4gvFZ3t8JkQmbapN0Ct+oU= +github.com/libp2p/go-libp2p-routing v0.1.0/go.mod h1:zfLhI1RI8RLEzmEaaPwzonRvXeeSHddONWkcTcB54nE= +github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8= +github.com/libp2p/go-libp2p-secio v0.2.0/go.mod h1:2JdZepB8J5V9mBp79BmwsaPQhRPNN2NrnB2lKQcdy6g= +github.com/libp2p/go-libp2p-secio v0.2.1 h1:eNWbJTdyPA7NxhP7J3c5lT97DC5d+u+IldkgCYFTPVA= +github.com/libp2p/go-libp2p-secio v0.2.1/go.mod h1:cWtZpILJqkqrSkiYcDBh5lA3wbT2Q+hz3rJQq3iftD8= +github.com/libp2p/go-libp2p-swarm v0.1.0/go.mod h1:wQVsCdjsuZoc730CgOvh5ox6K8evllckjebkdiY5ta4= +github.com/libp2p/go-libp2p-swarm v0.2.2 h1:T4hUpgEs2r371PweU3DuH7EOmBIdTBCwWs+FLcgx3bQ= +github.com/libp2p/go-libp2p-swarm v0.2.2/go.mod h1:fvmtQ0T1nErXym1/aa1uJEyN7JzaTNyBcHImCxRpPKU= +github.com/libp2p/go-libp2p-testing v0.0.2/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= +github.com/libp2p/go-libp2p-testing v0.0.3/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= +github.com/libp2p/go-libp2p-testing v0.0.4/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= +github.com/libp2p/go-libp2p-testing v0.1.0/go.mod h1:xaZWMJrPUM5GlDBxCeGUi7kI4eqnjVyavGroI2nxEM0= +github.com/libp2p/go-libp2p-testing v0.1.1 h1:U03z3HnGI7Ni8Xx6ONVZvUFOAzWYmolWf5W5jAOPNmU= +github.com/libp2p/go-libp2p-testing v0.1.1/go.mod h1:xaZWMJrPUM5GlDBxCeGUi7kI4eqnjVyavGroI2nxEM0= +github.com/libp2p/go-libp2p-transport-upgrader v0.1.1 h1:PZMS9lhjK9VytzMCW3tWHAXtKXmlURSc3ZdvwEcKCzw= +github.com/libp2p/go-libp2p-transport-upgrader v0.1.1/go.mod h1:IEtA6or8JUbsV07qPW4r01GnTenLW4oi3lOPbUMGJJA= +github.com/libp2p/go-libp2p-transport-upgrader v0.2.0 h1:5EhPgQhXZNyfL22ERZTUoVp9UVVbNowWNVtELQaKCHk= +github.com/libp2p/go-libp2p-transport-upgrader v0.2.0/go.mod h1:mQcrHj4asu6ArfSoMuyojOdjx73Q47cYD7s5+gZOlns= +github.com/libp2p/go-libp2p-yamux v0.2.0/go.mod h1:Db2gU+XfLpm6E4rG5uGCFX6uXA8MEXOxFcRoXUODaK8= +github.com/libp2p/go-libp2p-yamux v0.2.1 h1:Q3XYNiKCC2vIxrvUJL+Jg1kiyeEaIDNKLjgEjo3VQdI= +github.com/libp2p/go-libp2p-yamux v0.2.1/go.mod h1:1FBXiHDk1VyRM1C0aez2bCfHQ4vMZKkAQzZbkSQt5fI= +github.com/libp2p/go-libp2p-yamux v0.2.2 h1:eGvbqWqWY9S5lrpe2gA0UCOLCdzCgYSAR3vo/xCsNQg= +github.com/libp2p/go-libp2p-yamux v0.2.2/go.mod h1:lIohaR0pT6mOt0AZ0L2dFze9hds9Req3OfS+B+dv4qw= +github.com/libp2p/go-maddr-filter v0.0.4/go.mod h1:6eT12kSQMA9x2pvFQa+xesMKUBlj9VImZbj3B9FBH/Q= +github.com/libp2p/go-maddr-filter v0.0.5 h1:CW3AgbMO6vUvT4kf87y4N+0P8KUl2aqLYhrGyDUbLSg= +github.com/libp2p/go-maddr-filter v0.0.5/go.mod h1:Jk+36PMfIqCJhAnaASRH83bdAvfDRp/w6ENFaC9bG+M= +github.com/libp2p/go-mplex v0.0.3/go.mod h1:pK5yMLmOoBR1pNCqDlA2GQrdAVTMkqFalaTWe7l4Yd0= +github.com/libp2p/go-mplex v0.1.0 h1:/nBTy5+1yRyY82YaO6HXQRnO5IAGsXTjEJaR3LdTPc0= +github.com/libp2p/go-mplex v0.1.0/go.mod h1:SXgmdki2kwCUlCCbfGLEgHjC4pFqhTp0ZoV6aiKgxDU= +github.com/libp2p/go-mplex v0.1.1 h1:huPH/GGRJzmsHR9IZJJsrSwIM5YE2gL4ssgl1YWb/ps= +github.com/libp2p/go-mplex v0.1.1/go.mod h1:Xgz2RDCi3co0LeZfgjm4OgUF15+sVR8SRcu3SFXI1lk= +github.com/libp2p/go-msgio v0.0.2/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= +github.com/libp2p/go-msgio v0.0.4 h1:agEFehY3zWJFUHK6SEMR7UYmk2z6kC3oeCM7ybLhguA= +github.com/libp2p/go-msgio v0.0.4/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= +github.com/libp2p/go-nat v0.0.4 h1:KbizNnq8YIf7+Hn7+VFL/xE0eDrkPru2zIO9NMwL8UQ= +github.com/libp2p/go-nat v0.0.4/go.mod h1:Nmw50VAvKuk38jUBcmNh6p9lUJLoODbJRvYAa/+KSDo= +github.com/libp2p/go-openssl v0.0.2/go.mod h1:v8Zw2ijCSWBQi8Pq5GAixw6DbFfa9u6VIYDXnvOXkc0= +github.com/libp2p/go-openssl v0.0.3/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= +github.com/libp2p/go-openssl v0.0.4 h1:d27YZvLoTyMhIN4njrkr8zMDOM4lfpHIp6A+TK9fovg= github.com/libp2p/go-openssl v0.0.4 h1:d27YZvLoTyMhIN4njrkr8zMDOM4lfpHIp6A+TK9fovg= github.com/libp2p/go-openssl v0.0.4/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= +github.com/libp2p/go-openssl v0.0.4/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= +github.com/libp2p/go-reuseport v0.0.1 h1:7PhkfH73VXfPJYKQ6JwS5I/eVcoyYi9IMNGc6FWpFLw= +github.com/libp2p/go-reuseport v0.0.1/go.mod h1:jn6RmB1ufnQwl0Q1f+YxAj8isJgDCQzaaxIFYDhcYEA= +github.com/libp2p/go-reuseport-transport v0.0.2 h1:WglMwyXyBu61CMkjCCtnmqNqnjib0GIEjMiHTwR/KN4= +github.com/libp2p/go-reuseport-transport v0.0.2/go.mod h1:YkbSDrvjUVDL6b8XqriyA20obEtsW9BLkuOUyQAOCbs= +github.com/libp2p/go-stream-muxer v0.0.1 h1:Ce6e2Pyu+b5MC1k3eeFtAax0pW4gc6MosYSLV05UeLw= +github.com/libp2p/go-stream-muxer v0.0.1/go.mod h1:bAo8x7YkSpadMTbtTaxGVHWUQsR/l5MEaHbKaliuT14= +github.com/libp2p/go-stream-muxer-multistream v0.2.0 h1:714bRJ4Zy9mdhyTLJ+ZKiROmAFwUHpeRidG+q7LTQOg= +github.com/libp2p/go-stream-muxer-multistream v0.2.0/go.mod h1:j9eyPol/LLRqT+GPLSxvimPhNph4sfYfMoDPd7HkzIc= +github.com/libp2p/go-tcp-transport v0.1.0/go.mod h1:oJ8I5VXryj493DEJ7OsBieu8fcg2nHGctwtInJVpipc= +github.com/libp2p/go-tcp-transport v0.1.1 h1:yGlqURmqgNA2fvzjSgZNlHcsd/IulAnKM8Ncu+vlqnw= +github.com/libp2p/go-tcp-transport v0.1.1/go.mod h1:3HzGvLbx6etZjnFlERyakbaYPdfjg2pWP97dFZworkY= +github.com/libp2p/go-ws-transport v0.2.0 h1:MJCw2OrPA9+76YNRvdo1wMnSOxb9Bivj6sVFY1Xrj6w= +github.com/libp2p/go-ws-transport v0.2.0/go.mod h1:9BHJz/4Q5A9ludYWKoGCFC5gUElzlHoKzu0yY9p/klM= +github.com/libp2p/go-yamux v1.2.2/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= +github.com/libp2p/go-yamux v1.2.3 h1:xX8A36vpXb59frIzWFdEgptLMsOANMFq2K7fPRlunYI= +github.com/libp2p/go-yamux v1.2.3/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= +github.com/libp2p/go-yamux v1.3.0 h1:FsYzT16Wq2XqUGJsBbOxoz9g+dFklvNi7jN6YFPfl7U= +github.com/libp2p/go-yamux v1.3.0/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/matrix-org/dendrite v0.0.0-20200220135450-0352f250b857/go.mod h1:DZ35IoR+ViBNVPe9umdlOSnjvKl7wfyRmZg4QfWGvTo= github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 h1:nMX2t7hbGF0NYDYySx0pCqEKGKAeZIiSqlWSspetlhY= +github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 h1:nMX2t7hbGF0NYDYySx0pCqEKGKAeZIiSqlWSspetlhY= +github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg= github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg= github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f h1:5TOte9uk/epk8L+Pbp6qwaV8YsKYXKjyECPHUhJTWQc= github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f/go.mod h1:qK3LUW7RCLhFM7gC3pabj3EXT9A1DsCK33MHstUhhbk= @@ -139,9 +371,15 @@ github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5/go.mod h1:lePuOiXL github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.5 h1:tHXDdz1cpzGaovsTB+TVB8q90WEokoVmfMqoVcrLUgw= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-sqlite3 v2.0.2+incompatible h1:qzw9c2GNT8UFrgWNDhCTqRqYUSmu/Dav/9Z58LGpk7U= github.com/mattn/go-sqlite3 v2.0.2+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U= +github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= @@ -151,24 +389,61 @@ github.com/miekg/dns v1.1.12 h1:WMhc1ik4LNkTg8U9l3hI1LvxKmIL+f1+WV/SZtCbDDA= github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= +github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= +github.com/minio/sha256-simd v0.0.0-20190328051042-05b4dd3047e5/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= +github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= +github.com/mr-tron/base58 v1.1.1/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= +github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc= github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI= github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= +github.com/multiformats/go-multiaddr v0.0.1/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= +github.com/multiformats/go-multiaddr v0.0.2/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= +github.com/multiformats/go-multiaddr v0.0.4/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= +github.com/multiformats/go-multiaddr v0.1.0/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= +github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo= +github.com/multiformats/go-multiaddr v0.2.0 h1:lR52sFwcTCuQb6bTfnXF6zA2XfyYvyd+5a9qECv/J90= +github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4= github.com/multiformats/go-multiaddr v0.2.1 h1:SgG/cw5vqyB5QQe5FPe2TqggU9WtrA9X4nZw7LlVqOI= github.com/multiformats/go-multiaddr v0.2.1/go.mod h1:s/Apk6IyxfvMjDafnhJgJ3/46z7tZ04iMk5wP4QMGGE= +github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= +github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= +github.com/multiformats/go-multiaddr-dns v0.2.0 h1:YWJoIDwLePniH7OU5hBnDZV6SWuvJqJ0YtN6pLeH9zA= +github.com/multiformats/go-multiaddr-dns v0.2.0/go.mod h1:TJ5pr5bBO7Y1B18djPuRsVkduhQH2YqYSbxWJzYGdK0= +github.com/multiformats/go-multiaddr-fmt v0.0.1/go.mod h1:aBYjqL4T/7j4Qx+R73XSv/8JsgnRFlf0w2KGLCmXl3Q= +github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E= +github.com/multiformats/go-multiaddr-fmt v0.1.0/go.mod h1:hGtDIW4PU4BqJ50gW2quDuPVjyWNZxToGUh/HwTZYJo= +github.com/multiformats/go-multiaddr-net v0.0.1/go.mod h1:nw6HSxNmCIQH27XPGBuX+d1tnvM7ihcFwHMSstNAVUU= +github.com/multiformats/go-multiaddr-net v0.1.0/go.mod h1:5JNbcfBOP4dnhoZOv10JJVkJO0pCCEf8mTnipAo2UZQ= +github.com/multiformats/go-multiaddr-net v0.1.1 h1:jFFKUuXTXv+3ARyHZi3XUqQO+YWMKgBdhEvuGRfnL6s= +github.com/multiformats/go-multiaddr-net v0.1.1/go.mod h1:5JNbcfBOP4dnhoZOv10JJVkJO0pCCEf8mTnipAo2UZQ= +github.com/multiformats/go-multiaddr-net v0.1.2 h1:P7zcBH9FRETdPkDrylcXVjQLQ2t1JQtNItZULWNWgeg= +github.com/multiformats/go-multiaddr-net v0.1.2/go.mod h1:QsWt3XK/3hwvNxZJp92iMQKME1qHfpYmyIjFVsSOY6Y= github.com/multiformats/go-multibase v0.0.1 h1:PN9/v21eLywrFWdFNsFKaU04kLJzuYzmrJR+ubhT9qA= github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= +github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= +github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= +github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= +github.com/multiformats/go-multihash v0.0.9/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= +github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.13 h1:06x+mk/zj1FoMsgNejLpy6QTvJqlSt/BhLEy87zidlc= github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= +github.com/multiformats/go-multistream v0.1.0 h1:UpO6jrsjqs46mqAK3n6wKRYFhugss9ArzbyUzU+4wkQ= +github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= +github.com/multiformats/go-multistream v0.1.1 h1:JlAdpIFhBhGRLxe9W6Om0w++Gd6KMWoFPZL/dEnm9nI= +github.com/multiformats/go-multistream v0.1.1/go.mod h1:KmHZ40hzVxiaiwlj3MEbYgK9JFk2/9UktWZAF54Du38= +github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.2/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.5 h1:XVZwSo04Cs3j/jS0uAEPpT3JY6DzMcVLLoWOSnCxOjg= github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= @@ -179,16 +454,30 @@ github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 h1:BvoENQQU+fZ9uukda/R github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.0 h1:Iw5WCbBcaAAd0fpRb1c9r5YCylv4XDoCSigm1zLevwU= +github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.9.0 h1:R1uwffexN6Pr340GtYRIdZmAiN4J+iw6WG4wog1DUXg= +github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pierrec/lz4 v2.4.1+incompatible h1:mFe7ttWaflA46Mhqh+jUfjp2qTbPYxLB2/OyBppH9dg= github.com/pierrec/lz4 v2.4.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4 v2.5.0+incompatible h1:MbdIZ43A//duwOjQqK3nP+up+65yraNFyX3Vp6Rwues= +github.com/pierrec/lz4 v2.5.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -213,17 +502,27 @@ github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+Gx github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ= +github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ= github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY= +github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a/go.mod h1:7AyxJNCJ7SBZ1MfVQCWD6Uqo2oubI2Eq2y2eqf+A5r0= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -231,6 +530,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tidwall/gjson v1.1.5 h1:QysILxBeUEY3GTLA0fQVgkQG1zme8NxGvhh2SSqWNwI= github.com/tidwall/gjson v1.1.5/go.mod h1:c/nTNbUr0E0OrXEhq1pwa8iEgc2DOt4ZZqAt1HtCkPA= github.com/tidwall/gjson v1.6.0 h1:9VEQWz6LLMUsUl6PueE49ir4Ka6CzLymOAZDxpFsTDc= @@ -252,25 +552,57 @@ github.com/uber/jaeger-lib v1.5.0 h1:OHbgr8l656Ub3Fw5k9SWnBfIEwvoHQ+W2y+Aa9D1Uyo github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw= github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k= +github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= +github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= +github.com/whyrusleeping/go-logging v0.0.1 h1:fwpzlmT0kRC/Fmd0MdmGgJG/CXIZ6gFq46FQZjprUcc= +github.com/whyrusleeping/go-logging v0.0.1/go.mod h1:lDPYj54zutzG1XYfHAhcc7oNXEburHQBn+Iqd4yS4vE= +github.com/whyrusleeping/mafmt v1.2.8 h1:TCghSl5kkwEE0j+sU/gudyhVMRlpBin8fMBBHg59EbA= +github.com/whyrusleeping/mafmt v1.2.8/go.mod h1:faQJFPbLSxzD9xpA02ttW/tS9vZykNvXwGvqIpk20FA= +github.com/whyrusleeping/mdns v0.0.0-20190826153040-b9b60ed33aa9 h1:Y1/FEOpaCpD21WxrmfeIYCFPuVPRCY2XZTWzTNHGw30= +github.com/whyrusleeping/mdns v0.0.0-20190826153040-b9b60ed33aa9/go.mod h1:j4l84WPFclQPj320J9gp0XwNKBb3U0zt5CBqjPp22G4= +github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 h1:E9S12nwJwEOXe2d6gT6qxdvqMnNq+VnSsKPgm2ZZNds= +github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7/go.mod h1:X2c0RVCI1eSUFI8eLcY3c0423ykwiUdxLJtkDvruhjI= +github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee h1:lYbXeSvJi5zk5GLKVuid9TVjS9a0OmLIDKTfoZBL6Ow= +github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee/go.mod h1:m2aV4LZI4Aez7dP5PMyVKEHhUyEJ/RjmPEDOpDvudHg= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.0 h1:vs7fgriifsPbGdK3bNuMWapNn3qnZhCRXc19NRdq010= go.uber.org/atomic v1.3.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613 h1:MQ/ZZiDsUapFFiMS+vzwXkCTeEKaum+Do5rINYJDmxc= golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d h1:2+ZP7EfsZV7Vvmx3TIqSlSzATMkTAKqM14YGFPoSKjI= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d h1:1ZiEyfaQIg3Qh0EoqpwAakHVhecoE5wlSg5GjnafJGw= +golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -282,8 +614,10 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190301231341-16b79f2e4e95 h1:fY7Dsw114eJN4boqzVSbpVHO6rTdhq6/GnXeu+PKnzU= golang.org/x/net v0.0.0-20190301231341-16b79f2e4e95/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -307,18 +641,28 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 h1:I6FyU15t786LL7oL/hn43zqTu golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 h1:ywK/j/KkyTHcdyYSZNXGjMwgmDSfjglYZ3vStQ/gSCU= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -337,6 +681,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -349,6 +695,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/h2non/bimg.v1 v1.0.18 h1:qn6/RpBHt+7WQqoBcK+aF2puc6nC78eZj5LexxoalT4= gopkg.in/h2non/bimg.v1 v1.0.18/go.mod h1:PgsZL7dLwUbsGm1NYps320GxGgvQNTnecMCZqxV11So= @@ -358,6 +706,7 @@ gopkg.in/macaroon.v2 v2.1.0 h1:HZcsjBCzq9t0eBPMKqTN/uSN6JOm78ZJ2INbqcBQOUI= gopkg.in/macaroon.v2 v2.1.0/go.mod h1:OUb+TQP/OP0WOerC2Jp/3CwhIKyIa9kQjuc7H24e6/o= gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= @@ -365,4 +714,6 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/publicroomsapi/publicroomsapi.go b/publicroomsapi/publicroomsapi.go index 4d8a26018..6efb54bd9 100644 --- a/publicroomsapi/publicroomsapi.go +++ b/publicroomsapi/publicroomsapi.go @@ -31,19 +31,15 @@ import ( func SetupPublicRoomsAPIComponent( base *basecomponent.BaseDendrite, deviceDB devices.Database, + publicRoomsDB storage.Database, rsQueryAPI roomserverAPI.RoomserverQueryAPI, fedClient *gomatrixserverlib.FederationClient, extRoomsProvider types.ExternalPublicRoomsProvider, ) { - publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI)) - if err != nil { - logrus.WithError(err).Panicf("failed to connect to public rooms db") - } - rsConsumer := consumers.NewOutputRoomEventConsumer( base.Cfg, base.KafkaConsumer, publicRoomsDB, rsQueryAPI, ) - if err = rsConsumer.Start(); err != nil { + if err := rsConsumer.Start(); err != nil { logrus.WithError(err).Panic("failed to start public rooms server consumer") } diff --git a/publicroomsapi/storage/storage.go b/publicroomsapi/storage/storage.go index 41dbd7819..e674514aa 100644 --- a/publicroomsapi/storage/storage.go +++ b/publicroomsapi/storage/storage.go @@ -23,6 +23,9 @@ import ( "github.com/matrix-org/dendrite/publicroomsapi/storage/sqlite3" ) +const schemePostgres = "postgres" +const schemeFile = "file" + // NewPublicRoomsServerDatabase opens a database connection. func NewPublicRoomsServerDatabase(dataSourceName string) (Database, error) { uri, err := url.Parse(dataSourceName) @@ -30,9 +33,9 @@ func NewPublicRoomsServerDatabase(dataSourceName string) (Database, error) { return postgres.NewPublicRoomsServerDatabase(dataSourceName) } switch uri.Scheme { - case "postgres": + case schemePostgres: return postgres.NewPublicRoomsServerDatabase(dataSourceName) - case "file": + case schemeFile: return sqlite3.NewPublicRoomsServerDatabase(dataSourceName) default: return postgres.NewPublicRoomsServerDatabase(dataSourceName) From 895a72b6eecf7a7e71770bfaede53cd51f7c91e1 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 14 Apr 2020 18:36:08 +0100 Subject: [PATCH 73/86] Move /room/{roomID}/state endpoints into client API (#606) (#962) * Move /room/{roomID}/state endpoints into client API (#606) * Update sytest-whitelist * Blacklist tests which rely on endpoints we don't implement --- clientapi/routing/routing.go | 26 ++++++++++++ {syncapi => clientapi}/routing/state.go | 54 ++++++++++++++++--------- common/events.go | 4 ++ federationapi/routing/threepid.go | 4 ++ roomserver/alias/alias.go | 4 ++ roomserver/query/query.go | 16 ++++++-- syncapi/routing/routing.go | 24 ----------- sytest-blacklist | 4 ++ sytest-whitelist | 6 ++- 9 files changed, 93 insertions(+), 49 deletions(-) rename {syncapi => clientapi}/routing/state.go (69%) diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 91a1588cb..5dc6d7db9 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -149,6 +149,31 @@ func Setup( return GetEvent(req, device, vars["roomID"], vars["eventID"], cfg, queryAPI, federation, keyRing) }), ).Methods(http.MethodGet, http.MethodOptions) + + r0mux.Handle("/rooms/{roomID}/state", common.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return OnIncomingStateRequest(req.Context(), queryAPI, vars["roomID"]) + })).Methods(http.MethodGet, http.MethodOptions) + + r0mux.Handle("/rooms/{roomID}/state/{type}", common.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return OnIncomingStateTypeRequest(req.Context(), queryAPI, vars["roomID"], vars["type"], "") + })).Methods(http.MethodGet, http.MethodOptions) + + r0mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", common.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return OnIncomingStateTypeRequest(req.Context(), queryAPI, vars["roomID"], vars["type"], vars["stateKey"]) + })).Methods(http.MethodGet, http.MethodOptions) + r0mux.Handle("/rooms/{roomID}/state/{eventType:[^/]+/?}", common.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { vars, err := common.URLDecodeMapValues(mux.Vars(req)) @@ -164,6 +189,7 @@ func Setup( return SendEvent(req, device, vars["roomID"], eventType, nil, &emptyString, cfg, queryAPI, producer, nil) }), ).Methods(http.MethodPut, http.MethodOptions) + r0mux.Handle("/rooms/{roomID}/state/{eventType}/{stateKey}", common.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { vars, err := common.URLDecodeMapValues(mux.Vars(req)) diff --git a/syncapi/routing/state.go b/clientapi/routing/state.go similarity index 69% rename from syncapi/routing/state.go rename to clientapi/routing/state.go index 87b6396ac..c243eec0f 100644 --- a/syncapi/routing/state.go +++ b/clientapi/routing/state.go @@ -15,11 +15,12 @@ package routing import ( + "context" "encoding/json" "net/http" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -39,22 +40,29 @@ type stateEventInStateResp struct { // TODO: Check if the user is in the room. If not, check if the room's history // is publicly visible. Current behaviour is returning an empty array if the // user cannot see the room's history. -func OnIncomingStateRequest(req *http.Request, db storage.Database, roomID string) util.JSONResponse { +func OnIncomingStateRequest(ctx context.Context, queryAPI api.RoomserverQueryAPI, roomID string) util.JSONResponse { // TODO(#287): Auth request and handle the case where the user has left (where // we should return the state at the poin they left) + stateReq := api.QueryLatestEventsAndStateRequest{ + RoomID: roomID, + } + stateRes := api.QueryLatestEventsAndStateResponse{} - stateFilter := gomatrixserverlib.DefaultStateFilter() - // TODO: stateFilter should not limit the number of state events (or only limits abusive number of events) - - stateEvents, err := db.GetStateEventsForRoom(req.Context(), roomID, &stateFilter) - if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("db.GetStateEventsForRoom failed") + if err := queryAPI.QueryLatestEventsAndState(ctx, &stateReq, &stateRes); err != nil { + util.GetLogger(ctx).WithError(err).Error("queryAPI.QueryLatestEventsAndState failed") return jsonerror.InternalServerError() } + if len(stateRes.StateEvents) == 0 { + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound("cannot find state"), + } + } + resp := []stateEventInStateResp{} // Fill the prev_content and replaces_state keys if necessary - for _, event := range stateEvents { + for _, event := range stateRes.StateEvents { stateEvent := stateEventInStateResp{ ClientEvent: gomatrixserverlib.HeaderedToClientEvents( []gomatrixserverlib.HeaderedEvent{event}, gomatrixserverlib.FormatAll, @@ -63,7 +71,7 @@ func OnIncomingStateRequest(req *http.Request, db storage.Database, roomID strin var prevEventRef types.PrevEventRef if len(event.Unsigned()) > 0 { if err := json.Unmarshal(event.Unsigned(), &prevEventRef); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("json.Unmarshal failed") + util.GetLogger(ctx).WithError(err).Error("json.Unmarshal failed") return jsonerror.InternalServerError() } // Fills the previous state event ID if the state event replaces another @@ -90,24 +98,32 @@ func OnIncomingStateRequest(req *http.Request, db storage.Database, roomID strin // /rooms/{roomID}/state/{type}/{statekey} request. It will look in current // state to see if there is an event with that type and state key, if there // is then (by default) we return the content, otherwise a 404. -func OnIncomingStateTypeRequest(req *http.Request, db storage.Database, roomID string, evType, stateKey string) util.JSONResponse { +func OnIncomingStateTypeRequest(ctx context.Context, queryAPI api.RoomserverQueryAPI, roomID string, evType, stateKey string) util.JSONResponse { // TODO(#287): Auth request and handle the case where the user has left (where // we should return the state at the poin they left) - - logger := util.GetLogger(req.Context()) - logger.WithFields(log.Fields{ + util.GetLogger(ctx).WithFields(log.Fields{ "roomID": roomID, "evType": evType, "stateKey": stateKey, }).Info("Fetching state") - event, err := db.GetStateEvent(req.Context(), roomID, evType, stateKey) - if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("db.GetStateEvent failed") + stateReq := api.QueryLatestEventsAndStateRequest{ + RoomID: roomID, + StateToFetch: []gomatrixserverlib.StateKeyTuple{ + gomatrixserverlib.StateKeyTuple{ + EventType: evType, + StateKey: stateKey, + }, + }, + } + stateRes := api.QueryLatestEventsAndStateResponse{} + + if err := queryAPI.QueryLatestEventsAndState(ctx, &stateReq, &stateRes); err != nil { + util.GetLogger(ctx).WithError(err).Error("queryAPI.QueryLatestEventsAndState failed") return jsonerror.InternalServerError() } - if event == nil { + if len(stateRes.StateEvents) == 0 { return util.JSONResponse{ Code: http.StatusNotFound, JSON: jsonerror.NotFound("cannot find state"), @@ -115,7 +131,7 @@ func OnIncomingStateTypeRequest(req *http.Request, db storage.Database, roomID s } stateEvent := stateEventInStateResp{ - ClientEvent: gomatrixserverlib.HeaderedToClientEvent(*event, gomatrixserverlib.FormatAll), + ClientEvent: gomatrixserverlib.HeaderedToClientEvent(stateRes.StateEvents[0], gomatrixserverlib.FormatAll), } return util.JSONResponse{ diff --git a/common/events.go b/common/events.go index adbdf3389..556b7b671 100644 --- a/common/events.go +++ b/common/events.go @@ -73,6 +73,10 @@ func AddPrevEventsToEvent( return fmt.Errorf("gomatrixserverlib.StateNeededForEventBuilder: %w", err) } + if len(eventsNeeded.Tuples()) == 0 { + return errors.New("expecting state tuples for event builder, got none") + } + // Ask the roomserver for information about this room queryReq := api.QueryLatestEventsAndStateRequest{ RoomID: builder.RoomID, diff --git a/federationapi/routing/threepid.go b/federationapi/routing/threepid.go index da7174730..f93d934ed 100644 --- a/federationapi/routing/threepid.go +++ b/federationapi/routing/threepid.go @@ -271,6 +271,10 @@ func buildMembershipEvent( return nil, err } + if len(eventsNeeded.Tuples()) == 0 { + return nil, errors.New("expecting state tuples for event builder, got none") + } + // Ask the roomserver for information about this room queryReq := roomserverAPI.QueryLatestEventsAndStateRequest{ RoomID: builder.RoomID, diff --git a/roomserver/alias/alias.go b/roomserver/alias/alias.go index 032b8da1c..eb606e5cd 100644 --- a/roomserver/alias/alias.go +++ b/roomserver/alias/alias.go @@ -17,6 +17,7 @@ package alias import ( "context" "encoding/json" + "errors" "net/http" "time" @@ -218,6 +219,9 @@ func (r *RoomserverAliasAPI) sendUpdatedAliasesEvent( if err != nil { return err } + if len(eventsNeeded.Tuples()) == 0 { + return errors.New("expecting state tuples for event builder, got none") + } req := roomserverAPI.QueryLatestEventsAndStateRequest{ RoomID: roomID, StateToFetch: eventsNeeded.Tuples(), diff --git a/roomserver/query/query.go b/roomserver/query/query.go index 7e05fe36f..12d8436ef 100644 --- a/roomserver/query/query.go +++ b/roomserver/query/query.go @@ -132,10 +132,18 @@ func (r *RoomserverQueryAPI) QueryLatestEventsAndState( return err } - // Look up the current state for the requested tuples. - stateEntries, err := roomState.LoadStateAtSnapshotForStringTuples( - ctx, currentStateSnapshotNID, request.StateToFetch, - ) + var stateEntries []types.StateEntry + if len(request.StateToFetch) == 0 { + // Look up all room state. + stateEntries, err = roomState.LoadStateAtSnapshot( + ctx, currentStateSnapshotNID, + ) + } else { + // Look up the current state for the requested tuples. + stateEntries, err = roomState.LoadStateAtSnapshotForStringTuples( + ctx, currentStateSnapshotNID, request.StateToFetch, + ) + } if err != nil { return err } diff --git a/syncapi/routing/routing.go b/syncapi/routing/routing.go index be90e0a07..9078b87ff 100644 --- a/syncapi/routing/routing.go +++ b/syncapi/routing/routing.go @@ -56,30 +56,6 @@ func Setup( return srp.OnIncomingSyncRequest(req, device) })).Methods(http.MethodGet, http.MethodOptions) - r0mux.Handle("/rooms/{roomID}/state", common.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(req)) - if err != nil { - return util.ErrorResponse(err) - } - return OnIncomingStateRequest(req, syncDB, vars["roomID"]) - })).Methods(http.MethodGet, http.MethodOptions) - - r0mux.Handle("/rooms/{roomID}/state/{type}", common.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(req)) - if err != nil { - return util.ErrorResponse(err) - } - return OnIncomingStateTypeRequest(req, syncDB, vars["roomID"], vars["type"], "") - })).Methods(http.MethodGet, http.MethodOptions) - - r0mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", common.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(req)) - if err != nil { - return util.ErrorResponse(err) - } - return OnIncomingStateTypeRequest(req, syncDB, vars["roomID"], vars["type"], vars["stateKey"]) - })).Methods(http.MethodGet, http.MethodOptions) - r0mux.Handle("/rooms/{roomID}/messages", common.MakeAuthAPI("room_messages", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { vars, err := common.URLDecodeMapValues(mux.Vars(req)) if err != nil { diff --git a/sytest-blacklist b/sytest-blacklist index da0667a1d..caad25455 100644 --- a/sytest-blacklist +++ b/sytest-blacklist @@ -35,3 +35,7 @@ Inbound federation rejects invites which are not signed by the sender # Blacklisted because we don't support ignores yet Ignore invite in incremental sync + +# Blacklisted because this test calls /r0/events which we don't implement +New room members see their own join event +Existing members see new members' join events diff --git a/sytest-whitelist b/sytest-whitelist index 97ded9bb8..3277cc59d 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -55,8 +55,9 @@ Request to logout with invalid an access token is rejected Request to logout without an access token is rejected Room creation reports m.room.create to myself Room creation reports m.room.member to myself -New room members see their own join event -Existing members see new members' join events +# Blacklisted because these tests call /r0/events which we don't implement +# New room members see their own join event +# Existing members see new members' join events setting 'm.room.power_levels' respects room powerlevel Unprivileged users can set m.room.topic if it only needs level 0 Users cannot set ban powerlevel higher than their own @@ -245,3 +246,4 @@ Remote user can backfill in a room with version 4 Outbound federation can send invites via v2 API User can invite local user to room with version 3 User can invite local user to room with version 4 +A pair of servers can establish a join in a v2 room From 7b3edf4622fa562a0d8b42c1ac845b88722374f0 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 15 Apr 2020 14:39:27 +0100 Subject: [PATCH 74/86] Tweaks to backfill (#964) --- federationapi/routing/backfill.go | 2 +- syncapi/storage/postgres/backward_extremities_table.go | 2 +- syncapi/storage/sqlite3/backward_extremities_table.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/federationapi/routing/backfill.go b/federationapi/routing/backfill.go index 72ce0c669..7092c4364 100644 --- a/federationapi/routing/backfill.go +++ b/federationapi/routing/backfill.go @@ -93,7 +93,7 @@ func Backfill( } var eventJSONs []json.RawMessage - for _, e := range evs { + for _, e := range gomatrixserverlib.ReverseTopologicalOrdering(evs) { eventJSONs = append(eventJSONs, e.JSON()) } diff --git a/syncapi/storage/postgres/backward_extremities_table.go b/syncapi/storage/postgres/backward_extremities_table.go index b3ee28e01..cb3629644 100644 --- a/syncapi/storage/postgres/backward_extremities_table.go +++ b/syncapi/storage/postgres/backward_extremities_table.go @@ -60,7 +60,7 @@ const insertBackwardExtremitySQL = "" + " ON CONFLICT DO NOTHING" const selectBackwardExtremitiesForRoomSQL = "" + - "SELECT event_id FROM syncapi_backward_extremities WHERE room_id = $1" + "SELECT DISTINCT event_id FROM syncapi_backward_extremities WHERE room_id = $1" const deleteBackwardExtremitySQL = "" + "DELETE FROM syncapi_backward_extremities WHERE room_id = $1 AND prev_event_id = $2" diff --git a/syncapi/storage/sqlite3/backward_extremities_table.go b/syncapi/storage/sqlite3/backward_extremities_table.go index 0663e2a12..3d8cb91fc 100644 --- a/syncapi/storage/sqlite3/backward_extremities_table.go +++ b/syncapi/storage/sqlite3/backward_extremities_table.go @@ -60,7 +60,7 @@ const insertBackwardExtremitySQL = "" + " ON CONFLICT (room_id, event_id, prev_event_id) DO NOTHING" const selectBackwardExtremitiesForRoomSQL = "" + - "SELECT event_id FROM syncapi_backward_extremities WHERE room_id = $1" + "SELECT DISTINCT event_id FROM syncapi_backward_extremities WHERE room_id = $1" const deleteBackwardExtremitySQL = "" + "DELETE FROM syncapi_backward_extremities WHERE room_id = $1 AND prev_event_id = $2" From dadb06f6ad3e367626706053945fb2c1f7f10f0c Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 15 Apr 2020 16:10:18 +0100 Subject: [PATCH 75/86] Use topological ordering for /messages response (#966) * Use topological ordering for /messages response * Update gomatrixserverlib --- go.mod | 2 +- go.sum | 4 ++-- syncapi/routing/messages.go | 25 +++++-------------------- 3 files changed, 8 insertions(+), 23 deletions(-) diff --git a/go.mod b/go.mod index ec8f3eb69..20235ae34 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f github.com/matrix-org/go-sqlite3-js v0.0.0-20200325174927-327088cdef10 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200409140603-8b9a51fe9b89 + github.com/matrix-org/gomatrixserverlib v0.0.0-20200415145257-d492cd4be836 github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.3+incompatible diff --git a/go.sum b/go.sum index 0d03cd371..5b4579bb0 100644 --- a/go.sum +++ b/go.sum @@ -362,8 +362,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:kmRjpmFOenVpOaV/DRlo9p6z/IbOKlUC+hhKsAAh8Qg= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200409140603-8b9a51fe9b89 h1:YAlUJK/Ty2ZrP/DL41CiR0Cp3pteshnyIS420KVs220= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200409140603-8b9a51fe9b89/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200415145257-d492cd4be836 h1:YiXBJ/0ZeBzuh9Ym0iYaJgDBlFdz7nIVKArqkkEgPzM= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200415145257-d492cd4be836/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 h1:W7l5CP4V7wPyPb4tYE11dbmeAOwtFQBTW0rf4OonOS8= diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index c9d62477d..873ee9366 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -209,18 +209,12 @@ func (r *messagesReq) retrieveEvents() ( return []gomatrixserverlib.ClientEvent{}, r.from, r.to, nil } - // Sort the events to ensure we send them in the right order. We currently - // do that based on the event's timestamp. + // Sort the events to ensure we send them in the right order. + events = gomatrixserverlib.HeaderedReverseTopologicalOrdering(events) if r.backwardOrdering { - sort.SliceStable(events, func(i int, j int) bool { - // Backward ordering is antichronological (latest event to oldest - // one). - return sortEvents(&(events[j]), &(events[i])) - }) - } else { - sort.SliceStable(events, func(i int, j int) bool { - // Forward ordering is chronological (oldest event to latest one). - return sortEvents(&(events[i]), &(events[j])) + // This reverses the array from old->new to new->old + sort.SliceStable(events, func(i, j int) bool { + return true }) } @@ -493,12 +487,3 @@ func setToDefault( return } - -// sortEvents is a function to give to sort.SliceStable, and compares the -// timestamp of two Matrix events. -// Returns true if the first event happened before the second one, false -// otherwise. -func sortEvents(e1 *gomatrixserverlib.HeaderedEvent, e2 *gomatrixserverlib.HeaderedEvent) bool { - t := e1.OriginServerTS().Time() - return e2.OriginServerTS().Time().After(t) -} From c2ea96190944d8e39b0c200e43b1eedfeeeb49a6 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Wed, 15 Apr 2020 17:48:40 +0100 Subject: [PATCH 76/86] Add HTTP trace logging (#965) * Dump all requests/response server-side * Wrap in DENDRITE_TRACE * DENDRITE_TRACE_HTTP is better * Bugfix for response body and linting * False is true and true is false * Linting * How did this get missed * More linting --- common/httpapi.go | 55 ++++++++++++++++++++++++++++++++++++++++++++++- go.mod | 1 + go.sum | 3 +++ 3 files changed, 58 insertions(+), 1 deletion(-) diff --git a/common/httpapi.go b/common/httpapi.go index 843336f5b..e5324bd17 100644 --- a/common/httpapi.go +++ b/common/httpapi.go @@ -1,7 +1,12 @@ package common import ( + "io" "net/http" + "net/http/httptest" + "net/http/httputil" + "os" + "strings" "time" "github.com/matrix-org/dendrite/clientapi/auth" @@ -46,12 +51,60 @@ func MakeAuthAPI( // MakeExternalAPI turns a util.JSONRequestHandler function into an http.Handler. // This is used for APIs that are called from the internet. func MakeExternalAPI(metricsName string, f func(*http.Request) util.JSONResponse) http.Handler { + // TODO: We shouldn't be directly reading env vars here, inject it in instead. + // Refactor this when we split out config structs. + verbose := false + if os.Getenv("DENDRITE_TRACE_HTTP") == "1" { + verbose = true + } h := util.MakeJSONAPI(util.NewJSONRequestHandler(f)) withSpan := func(w http.ResponseWriter, req *http.Request) { + nextWriter := w + if verbose { + logger := logrus.NewEntry(logrus.StandardLogger()) + // Log outgoing response + rec := httptest.NewRecorder() + nextWriter = rec + defer func() { + resp := rec.Result() + dump, err := httputil.DumpResponse(resp, true) + if err != nil { + logger.Debugf("Failed to dump outgoing response: %s", err) + } else { + strSlice := strings.Split(string(dump), "\n") + for _, s := range strSlice { + logger.Debug(s) + } + } + // copy the response to the client + for hdr, vals := range resp.Header { + for _, val := range vals { + w.Header().Add(hdr, val) + } + } + w.WriteHeader(resp.StatusCode) + // discard errors as this is for debugging + _, _ = io.Copy(w, resp.Body) + _ = resp.Body.Close() + }() + + // Log incoming request + dump, err := httputil.DumpRequest(req, true) + if err != nil { + logger.Debugf("Failed to dump incoming request: %s", err) + } else { + strSlice := strings.Split(string(dump), "\n") + for _, s := range strSlice { + logger.Debug(s) + } + } + } + span := opentracing.StartSpan(metricsName) defer span.Finish() req = req.WithContext(opentracing.ContextWithSpan(req.Context(), span)) - h.ServeHTTP(w, req) + h.ServeHTTP(nextWriter, req) + } return http.HandlerFunc(withSpan) diff --git a/go.mod b/go.mod index 20235ae34..a8f442d59 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,7 @@ require ( github.com/pierrec/lz4 v2.5.0+incompatible // indirect github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.4.1 + github.com/prometheus/common v0.9.1 github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect github.com/sirupsen/logrus v1.4.2 github.com/tidwall/gjson v1.6.0 // indirect diff --git a/go.sum b/go.sum index 5b4579bb0..5c4e809f3 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,10 @@ github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWso github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -688,6 +690,7 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= gopkg.in/Shopify/sarama.v1 v1.20.1 h1:Gi09A3fJXm0Jgt8kuKZ8YK+r60GfYn7MQuEmI3oq6hE= gopkg.in/Shopify/sarama.v1 v1.20.1/go.mod h1:AxnvoaevB2nBjNK17cG61A3LleFcWFwVBHBt+cot4Oc= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From c1bca95adbe04a17d8feb91aa24dde9c4cffb1fb Mon Sep 17 00:00:00 2001 From: Kegsay Date: Thu, 16 Apr 2020 10:06:55 +0100 Subject: [PATCH 77/86] Add SQL tracing via DENDRITE_TRACE_SQL (#968) * Add SQL tracing via DENDRITE_TRACE_SQL Add this to `internal/sqlutil` in preparation for #897 * Not entirely --- appservice/storage/postgres/storage.go | 3 +- appservice/storage/sqlite3/storage.go | 3 +- .../auth/storage/accounts/postgres/storage.go | 3 +- .../auth/storage/accounts/sqlite3/storage.go | 3 +- .../auth/storage/devices/postgres/storage.go | 3 +- .../auth/storage/devices/sqlite3/storage.go | 3 +- common/basecomponent/base.go | 5 +- common/keydb/postgres/keydb.go | 4 +- common/keydb/sqlite3/keydb.go | 4 +- federationsender/storage/postgres/storage.go | 3 +- federationsender/storage/sqlite3/storage.go | 3 +- go.mod | 1 + go.sum | 2 + internal/sqlutil/trace.go | 89 +++++++++++++++++++ internal/sqlutil/trace_driver.go | 35 ++++++++ internal/sqlutil/trace_driver_wasm.go | 33 +++++++ mediaapi/storage/postgres/storage.go | 3 +- mediaapi/storage/sqlite3/storage.go | 3 +- publicroomsapi/storage/postgres/storage.go | 3 +- publicroomsapi/storage/sqlite3/storage.go | 3 +- roomserver/storage/postgres/storage.go | 3 +- roomserver/storage/sqlite3/storage.go | 3 +- syncapi/storage/postgres/syncserver.go | 3 +- syncapi/storage/sqlite3/syncserver.go | 3 +- 24 files changed, 199 insertions(+), 22 deletions(-) create mode 100644 internal/sqlutil/trace.go create mode 100644 internal/sqlutil/trace_driver.go create mode 100644 internal/sqlutil/trace_driver_wasm.go diff --git a/appservice/storage/postgres/storage.go b/appservice/storage/postgres/storage.go index ef92db871..e145eeee2 100644 --- a/appservice/storage/postgres/storage.go +++ b/appservice/storage/postgres/storage.go @@ -21,6 +21,7 @@ import ( // Import postgres database driver _ "github.com/lib/pq" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) @@ -35,7 +36,7 @@ type Database struct { func NewDatabase(dataSourceName string) (*Database, error) { var result Database var err error - if result.db, err = sql.Open("postgres", dataSourceName); err != nil { + if result.db, err = sqlutil.Open("postgres", dataSourceName); err != nil { return nil, err } if err = result.prepare(); err != nil { diff --git a/appservice/storage/sqlite3/storage.go b/appservice/storage/sqlite3/storage.go index d0538e263..0cd1e4abc 100644 --- a/appservice/storage/sqlite3/storage.go +++ b/appservice/storage/sqlite3/storage.go @@ -21,6 +21,7 @@ import ( // Import SQLite database driver "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" _ "github.com/mattn/go-sqlite3" ) @@ -36,7 +37,7 @@ type Database struct { func NewDatabase(dataSourceName string) (*Database, error) { var result Database var err error - if result.db, err = sql.Open(common.SQLiteDriverName(), dataSourceName); err != nil { + if result.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil { return nil, err } if err = result.prepare(); err != nil { diff --git a/clientapi/auth/storage/accounts/postgres/storage.go b/clientapi/auth/storage/accounts/postgres/storage.go index 4a0a2060b..8ce367a3e 100644 --- a/clientapi/auth/storage/accounts/postgres/storage.go +++ b/clientapi/auth/storage/accounts/postgres/storage.go @@ -22,6 +22,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" "golang.org/x/crypto/bcrypt" @@ -46,7 +47,7 @@ type Database struct { func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) { var db *sql.DB var err error - if db, err = sql.Open("postgres", dataSourceName); err != nil { + if db, err = sqlutil.Open("postgres", dataSourceName); err != nil { return nil, err } partitions := common.PartitionOffsetStatements{} diff --git a/clientapi/auth/storage/accounts/sqlite3/storage.go b/clientapi/auth/storage/accounts/sqlite3/storage.go index bfb7b4ea1..e190ba6c2 100644 --- a/clientapi/auth/storage/accounts/sqlite3/storage.go +++ b/clientapi/auth/storage/accounts/sqlite3/storage.go @@ -23,6 +23,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" "golang.org/x/crypto/bcrypt" @@ -49,7 +50,7 @@ type Database struct { func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) { var db *sql.DB var err error - if db, err = sql.Open(common.SQLiteDriverName(), dataSourceName); err != nil { + if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil { return nil, err } partitions := common.PartitionOffsetStatements{} diff --git a/clientapi/auth/storage/devices/postgres/storage.go b/clientapi/auth/storage/devices/postgres/storage.go index 221c3998e..3f613cf32 100644 --- a/clientapi/auth/storage/devices/postgres/storage.go +++ b/clientapi/auth/storage/devices/postgres/storage.go @@ -22,6 +22,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) @@ -38,7 +39,7 @@ type Database struct { func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) { var db *sql.DB var err error - if db, err = sql.Open("postgres", dataSourceName); err != nil { + if db, err = sqlutil.Open("postgres", dataSourceName); err != nil { return nil, err } d := devicesStatements{} diff --git a/clientapi/auth/storage/devices/sqlite3/storage.go b/clientapi/auth/storage/devices/sqlite3/storage.go index 62c30322e..85a8def2c 100644 --- a/clientapi/auth/storage/devices/sqlite3/storage.go +++ b/clientapi/auth/storage/devices/sqlite3/storage.go @@ -22,6 +22,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" _ "github.com/mattn/go-sqlite3" @@ -40,7 +41,7 @@ type Database struct { func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) { var db *sql.DB var err error - if db, err = sql.Open(common.SQLiteDriverName(), dataSourceName); err != nil { + if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil { return nil, err } d := devicesStatements{} diff --git a/common/basecomponent/base.go b/common/basecomponent/base.go index dc27f5409..de3e04c21 100644 --- a/common/basecomponent/base.go +++ b/common/basecomponent/base.go @@ -24,6 +24,7 @@ import ( "golang.org/x/crypto/ed25519" "github.com/matrix-org/dendrite/common/keydb" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/naffka" @@ -243,7 +244,7 @@ func setupNaffka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) { uri, err := url.Parse(string(cfg.Database.Naffka)) if err != nil || uri.Scheme == "file" { - db, err = sql.Open(common.SQLiteDriverName(), string(cfg.Database.Naffka)) + db, err = sqlutil.Open(common.SQLiteDriverName(), string(cfg.Database.Naffka)) if err != nil { logrus.WithError(err).Panic("Failed to open naffka database") } @@ -253,7 +254,7 @@ func setupNaffka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) { logrus.WithError(err).Panic("Failed to setup naffka database") } } else { - db, err = sql.Open("postgres", string(cfg.Database.Naffka)) + db, err = sqlutil.Open("postgres", string(cfg.Database.Naffka)) if err != nil { logrus.WithError(err).Panic("Failed to open naffka database") } diff --git a/common/keydb/postgres/keydb.go b/common/keydb/postgres/keydb.go index 2dd8c866f..2879683e0 100644 --- a/common/keydb/postgres/keydb.go +++ b/common/keydb/postgres/keydb.go @@ -17,11 +17,11 @@ package postgres import ( "context" - "database/sql" "math" "golang.org/x/crypto/ed25519" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) @@ -41,7 +41,7 @@ func NewDatabase( serverKey ed25519.PublicKey, serverKeyID gomatrixserverlib.KeyID, ) (*Database, error) { - db, err := sql.Open("postgres", dataSourceName) + db, err := sqlutil.Open("postgres", dataSourceName) if err != nil { return nil, err } diff --git a/common/keydb/sqlite3/keydb.go b/common/keydb/sqlite3/keydb.go index 3c860d0c3..82d2a491f 100644 --- a/common/keydb/sqlite3/keydb.go +++ b/common/keydb/sqlite3/keydb.go @@ -17,12 +17,12 @@ package sqlite3 import ( "context" - "database/sql" "math" "golang.org/x/crypto/ed25519" "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" _ "github.com/mattn/go-sqlite3" @@ -44,7 +44,7 @@ func NewDatabase( serverKey ed25519.PublicKey, serverKeyID gomatrixserverlib.KeyID, ) (*Database, error) { - db, err := sql.Open(common.SQLiteDriverName(), dataSourceName) + db, err := sqlutil.Open(common.SQLiteDriverName(), dataSourceName) if err != nil { return nil, err } diff --git a/federationsender/storage/postgres/storage.go b/federationsender/storage/postgres/storage.go index d97b5d296..b909a189b 100644 --- a/federationsender/storage/postgres/storage.go +++ b/federationsender/storage/postgres/storage.go @@ -21,6 +21,7 @@ import ( "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/federationsender/types" + "github.com/matrix-org/dendrite/internal/sqlutil" ) // Database stores information needed by the federation sender @@ -35,7 +36,7 @@ type Database struct { func NewDatabase(dataSourceName string) (*Database, error) { var result Database var err error - if result.db, err = sql.Open("postgres", dataSourceName); err != nil { + if result.db, err = sqlutil.Open("postgres", dataSourceName); err != nil { return nil, err } if err = result.prepare(); err != nil { diff --git a/federationsender/storage/sqlite3/storage.go b/federationsender/storage/sqlite3/storage.go index 6ab9ae00a..458d7d7e5 100644 --- a/federationsender/storage/sqlite3/storage.go +++ b/federationsender/storage/sqlite3/storage.go @@ -23,6 +23,7 @@ import ( "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/federationsender/types" + "github.com/matrix-org/dendrite/internal/sqlutil" ) // Database stores information needed by the federation sender @@ -37,7 +38,7 @@ type Database struct { func NewDatabase(dataSourceName string) (*Database, error) { var result Database var err error - if result.db, err = sql.Open(common.SQLiteDriverName(), dataSourceName); err != nil { + if result.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil { return nil, err } if err = result.prepare(); err != nil { diff --git a/go.mod b/go.mod index a8f442d59..953b49d32 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.3+incompatible github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 + github.com/ngrok/sqlmw v0.0.0-20200129213757-d5c93a81bec6 github.com/opentracing/opentracing-go v1.1.0 github.com/pierrec/lz4 v2.5.0+incompatible // indirect github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index 5c4e809f3..f7295c38b 100644 --- a/go.sum +++ b/go.sum @@ -454,6 +454,8 @@ github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 h1:BvoENQQU+fZ9uukda/RzCAL/191HHwJA5b13R6diVlY= github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= +github.com/ngrok/sqlmw v0.0.0-20200129213757-d5c93a81bec6 h1:evlcQnJY+v8XRRchV3hXzpHDl6GcEZeLXAhlH9Csdww= +github.com/ngrok/sqlmw v0.0.0-20200129213757-d5c93a81bec6/go.mod h1:E26fwEtRNigBfFfHDWsklmo0T7Ixbg0XXgck+Hq4O9k= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w= diff --git a/internal/sqlutil/trace.go b/internal/sqlutil/trace.go new file mode 100644 index 000000000..3d5fa7dc7 --- /dev/null +++ b/internal/sqlutil/trace.go @@ -0,0 +1,89 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlutil + +import ( + "context" + "database/sql" + "database/sql/driver" + "fmt" + "io" + "os" + "strings" + "time" + + "github.com/ngrok/sqlmw" + "github.com/sirupsen/logrus" +) + +var tracingEnabled = os.Getenv("DENDRITE_TRACE_SQL") == "1" + +type traceInterceptor struct { + sqlmw.NullInterceptor +} + +func (in *traceInterceptor) StmtQueryContext(ctx context.Context, stmt driver.StmtQueryContext, query string, args []driver.NamedValue) (driver.Rows, error) { + startedAt := time.Now() + rows, err := stmt.QueryContext(ctx, args) + + logrus.WithField("duration", time.Since(startedAt)).WithField(logrus.ErrorKey, err).Debug("executed sql query ", query, " args: ", args) + + return rows, err +} + +func (in *traceInterceptor) StmtExecContext(ctx context.Context, stmt driver.StmtExecContext, query string, args []driver.NamedValue) (driver.Result, error) { + startedAt := time.Now() + result, err := stmt.ExecContext(ctx, args) + + logrus.WithField("duration", time.Since(startedAt)).WithField(logrus.ErrorKey, err).Debug("executed sql query ", query, " args: ", args) + + return result, err +} + +func (in *traceInterceptor) RowsNext(c context.Context, rows driver.Rows, dest []driver.Value) error { + err := rows.Next(dest) + if err == io.EOF { + // For all cases, we call Next() n+1 times, the first to populate the initial dest, then eventually + // it will io.EOF. If we log on each Next() call we log the last element twice, so don't. + return err + } + cols := rows.Columns() + logrus.Debug(strings.Join(cols, " | ")) + + b := strings.Builder{} + for i, val := range dest { + b.WriteString(fmt.Sprintf("%v", val)) + if i+1 <= len(dest)-1 { + b.WriteString(" | ") + } + } + logrus.Debug(b.String()) + return err +} + +// Open opens a database specified by its database driver name and a driver-specific data source name, +// usually consisting of at least a database name and connection information. Includes tracing driver +// if DENDRITE_TRACE_SQL=1 +func Open(driverName, dsn string) (*sql.DB, error) { + if tracingEnabled { + // install the wrapped driver + driverName += "-trace" + } + return sql.Open(driverName, dsn) +} + +func init() { + registerDrivers() +} diff --git a/internal/sqlutil/trace_driver.go b/internal/sqlutil/trace_driver.go new file mode 100644 index 000000000..f123b1e4d --- /dev/null +++ b/internal/sqlutil/trace_driver.go @@ -0,0 +1,35 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !wasm + +package sqlutil + +import ( + "database/sql" + + "github.com/lib/pq" + sqlite "github.com/mattn/go-sqlite3" + "github.com/ngrok/sqlmw" +) + +func registerDrivers() { + if !tracingEnabled { + return + } + // install the wrapped drivers + sql.Register("postgres-trace", sqlmw.Driver(&pq.Driver{}, new(traceInterceptor))) + sql.Register("sqlite3-trace", sqlmw.Driver(&sqlite.SQLiteDriver{}, new(traceInterceptor))) + +} diff --git a/internal/sqlutil/trace_driver_wasm.go b/internal/sqlutil/trace_driver_wasm.go new file mode 100644 index 000000000..a3c163f50 --- /dev/null +++ b/internal/sqlutil/trace_driver_wasm.go @@ -0,0 +1,33 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build wasm + +package sqlutil + +import ( + "database/sql" + + sqlitejs "github.com/matrix-org/go-sqlite3-js" + "github.com/ngrok/sqlmw" +) + +func registerDrivers() { + if !tracingEnabled { + return + } + // install the wrapped drivers + sql.Register("sqlite3_js-trace", sqlmw.Driver(&sqlitejs.SqliteJsDriver{}, new(traceInterceptor))) + +} diff --git a/mediaapi/storage/postgres/storage.go b/mediaapi/storage/postgres/storage.go index 6259f4a13..18126b151 100644 --- a/mediaapi/storage/postgres/storage.go +++ b/mediaapi/storage/postgres/storage.go @@ -21,6 +21,7 @@ import ( // Import the postgres database driver. _ "github.com/lib/pq" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/mediaapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -35,7 +36,7 @@ type Database struct { func Open(dataSourceName string) (*Database, error) { var d Database var err error - if d.db, err = sql.Open("postgres", dataSourceName); err != nil { + if d.db, err = sqlutil.Open("postgres", dataSourceName); err != nil { return nil, err } if err = d.statements.prepare(d.db); err != nil { diff --git a/mediaapi/storage/sqlite3/storage.go b/mediaapi/storage/sqlite3/storage.go index cfa5bb986..abafecf20 100644 --- a/mediaapi/storage/sqlite3/storage.go +++ b/mediaapi/storage/sqlite3/storage.go @@ -21,6 +21,7 @@ import ( // Import the postgres database driver. "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/mediaapi/types" "github.com/matrix-org/gomatrixserverlib" _ "github.com/mattn/go-sqlite3" @@ -36,7 +37,7 @@ type Database struct { func Open(dataSourceName string) (*Database, error) { var d Database var err error - if d.db, err = sql.Open(common.SQLiteDriverName(), dataSourceName); err != nil { + if d.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil { return nil, err } if err = d.statements.prepare(d.db); err != nil { diff --git a/publicroomsapi/storage/postgres/storage.go b/publicroomsapi/storage/postgres/storage.go index 5a4bc8f9a..8c4660cca 100644 --- a/publicroomsapi/storage/postgres/storage.go +++ b/publicroomsapi/storage/postgres/storage.go @@ -21,6 +21,7 @@ import ( "encoding/json" "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) @@ -38,7 +39,7 @@ type attributeValue interface{} func NewPublicRoomsServerDatabase(dataSourceName string) (*PublicRoomsServerDatabase, error) { var db *sql.DB var err error - if db, err = sql.Open("postgres", dataSourceName); err != nil { + if db, err = sqlutil.Open("postgres", dataSourceName); err != nil { return nil, err } storage := PublicRoomsServerDatabase{ diff --git a/publicroomsapi/storage/sqlite3/storage.go b/publicroomsapi/storage/sqlite3/storage.go index 80c04cab7..121601628 100644 --- a/publicroomsapi/storage/sqlite3/storage.go +++ b/publicroomsapi/storage/sqlite3/storage.go @@ -23,6 +23,7 @@ import ( _ "github.com/mattn/go-sqlite3" "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) @@ -40,7 +41,7 @@ type attributeValue interface{} func NewPublicRoomsServerDatabase(dataSourceName string) (*PublicRoomsServerDatabase, error) { var db *sql.DB var err error - if db, err = sql.Open(common.SQLiteDriverName(), dataSourceName); err != nil { + if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil { return nil, err } storage := PublicRoomsServerDatabase{ diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go index c91c59ebc..a33bed8f4 100644 --- a/roomserver/storage/postgres/storage.go +++ b/roomserver/storage/postgres/storage.go @@ -20,6 +20,7 @@ import ( "database/sql" "encoding/json" + "github.com/matrix-org/dendrite/internal/sqlutil" roomserverVersion "github.com/matrix-org/dendrite/roomserver/version" // Import the postgres database driver. @@ -39,7 +40,7 @@ type Database struct { func Open(dataSourceName string) (*Database, error) { var d Database var err error - if d.db, err = sql.Open("postgres", dataSourceName); err != nil { + if d.db, err = sqlutil.Open("postgres", dataSourceName); err != nil { return nil, err } if err = d.statements.prepare(d.db); err != nil { diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index f6c692fd1..6f594430a 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -22,6 +22,7 @@ import ( "errors" "net/url" + "github.com/matrix-org/dendrite/internal/sqlutil" roomserverVersion "github.com/matrix-org/dendrite/roomserver/version" "github.com/matrix-org/dendrite/common" @@ -52,7 +53,7 @@ func Open(dataSourceName string) (*Database, error) { } else { return nil, errors.New("no filename or path in connect string") } - if d.db, err = sql.Open(common.SQLiteDriverName(), cs); err != nil { + if d.db, err = sqlutil.Open(common.SQLiteDriverName(), cs); err != nil { return nil, err } //d.db.Exec("PRAGMA journal_mode=WAL;") diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go index ead1bf335..7fd75f066 100644 --- a/syncapi/storage/postgres/syncserver.go +++ b/syncapi/storage/postgres/syncserver.go @@ -25,6 +25,7 @@ import ( "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/api" // Import the postgres database driver. @@ -62,7 +63,7 @@ type SyncServerDatasource struct { func NewSyncServerDatasource(dbDataSourceName string) (*SyncServerDatasource, error) { var d SyncServerDatasource var err error - if d.db, err = sql.Open("postgres", dbDataSourceName); err != nil { + if d.db, err = sqlutil.Open("postgres", dbDataSourceName); err != nil { return nil, err } if err = d.PartitionOffsetStatements.Prepare(d.db, "syncapi"); err != nil { diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index 30f77e54d..29051cd06 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -27,6 +27,7 @@ import ( "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/api" // Import the sqlite3 package @@ -78,7 +79,7 @@ func NewSyncServerDatasource(dataSourceName string) (*SyncServerDatasource, erro } else { return nil, errors.New("no filename or path in connect string") } - if d.db, err = sql.Open(common.SQLiteDriverName(), cs); err != nil { + if d.db, err = sqlutil.Open(common.SQLiteDriverName(), cs); err != nil { return nil, err } if err = d.prepare(); err != nil { From 3110a819960dec0b4c1b70474186b7e40a2be59a Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 16 Apr 2020 12:53:27 +0100 Subject: [PATCH 78/86] Set default room version to 4 (#957) * Set default room version to 4 * Default to v1 when room_version key missing Signed-off-by: Alex Chen * Squashed commit of the following: commit c1bca95adbe04a17d8feb91aa24dde9c4cffb1fb Author: Kegsay Date: Thu Apr 16 10:06:55 2020 +0100 Add SQL tracing via DENDRITE_TRACE_SQL (#968) * Add SQL tracing via DENDRITE_TRACE_SQL Add this to `internal/sqlutil` in preparation for #897 * Not entirely commit c2ea96190944d8e39b0c200e43b1eedfeeeb49a6 Author: Kegsay Date: Wed Apr 15 17:48:40 2020 +0100 Add HTTP trace logging (#965) * Dump all requests/response server-side * Wrap in DENDRITE_TRACE * DENDRITE_TRACE_HTTP is better * Bugfix for response body and linting * False is true and true is false * Linting * How did this get missed * More linting * Update gomatrixserverlib * Remove unneeded imports Co-authored-by: Cnly --- go.mod | 2 +- go.sum | 4 ++-- roomserver/storage/postgres/storage.go | 3 +-- roomserver/storage/sqlite3/storage.go | 3 +-- roomserver/version/version.go | 2 +- 5 files changed, 6 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 953b49d32..849ad1f55 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f github.com/matrix-org/go-sqlite3-js v0.0.0-20200325174927-327088cdef10 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200415145257-d492cd4be836 + github.com/matrix-org/gomatrixserverlib v0.0.0-20200416113012-dafb32a889ea github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.3+incompatible diff --git a/go.sum b/go.sum index f7295c38b..f4b4e4ff8 100644 --- a/go.sum +++ b/go.sum @@ -364,8 +364,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:kmRjpmFOenVpOaV/DRlo9p6z/IbOKlUC+hhKsAAh8Qg= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200415145257-d492cd4be836 h1:YiXBJ/0ZeBzuh9Ym0iYaJgDBlFdz7nIVKArqkkEgPzM= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200415145257-d492cd4be836/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200416113012-dafb32a889ea h1:aiBD966UX0l4rTPgAu+u8CaiMfy+N8qWlZ2TIBp+0eM= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200416113012-dafb32a889ea/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 h1:W7l5CP4V7wPyPb4tYE11dbmeAOwtFQBTW0rf4OonOS8= diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go index a33bed8f4..6f2b96610 100644 --- a/roomserver/storage/postgres/storage.go +++ b/roomserver/storage/postgres/storage.go @@ -21,7 +21,6 @@ import ( "encoding/json" "github.com/matrix-org/dendrite/internal/sqlutil" - roomserverVersion "github.com/matrix-org/dendrite/roomserver/version" // Import the postgres database driver. _ "github.com/lib/pq" @@ -147,7 +146,7 @@ func extractRoomVersionFromCreateEvent(event gomatrixserverlib.Event) ( if event.Type() != gomatrixserverlib.MRoomCreate { return gomatrixserverlib.RoomVersion(""), nil } - roomVersion = roomserverVersion.DefaultRoomVersion() + roomVersion = gomatrixserverlib.RoomVersionV1 var createContent gomatrixserverlib.CreateContent // The m.room.create event contains an optional "room_version" key in // the event content, so we need to unmarshal that first. diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index 6f594430a..444a8fdd5 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -23,7 +23,6 @@ import ( "net/url" "github.com/matrix-org/dendrite/internal/sqlutil" - roomserverVersion "github.com/matrix-org/dendrite/roomserver/version" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/api" @@ -176,7 +175,7 @@ func extractRoomVersionFromCreateEvent(event gomatrixserverlib.Event) ( if event.Type() != gomatrixserverlib.MRoomCreate { return gomatrixserverlib.RoomVersion(""), nil } - roomVersion = roomserverVersion.DefaultRoomVersion() + roomVersion = gomatrixserverlib.RoomVersionV1 var createContent gomatrixserverlib.CreateContent // The m.room.create event contains an optional "room_version" key in // the event content, so we need to unmarshal that first. diff --git a/roomserver/version/version.go b/roomserver/version/version.go index e60b5ef7a..f2a67e74d 100644 --- a/roomserver/version/version.go +++ b/roomserver/version/version.go @@ -59,7 +59,7 @@ var roomVersions = map[gomatrixserverlib.RoomVersion]RoomVersionDescription{ // DefaultRoomVersion contains the room version that will, by // default, be used to create new rooms on this server. func DefaultRoomVersion() gomatrixserverlib.RoomVersion { - return gomatrixserverlib.RoomVersionV2 + return gomatrixserverlib.RoomVersionV4 } // RoomVersions returns a map of all known room versions to this From 3c2e6f967b44d895e2145d91b2f3414c9451f181 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 16 Apr 2020 17:59:55 +0100 Subject: [PATCH 79/86] Federation fixes and error handling (#970) * Improve error handling in federation /send endpoint a bit * Remove unknownRoomError, use unmarshalError when unable to get room ID * Swap out a couple more internal server errors * Update gomatrixserverlib * Update gomatrixserverlib * Update gomatrixserverlib * Update gomatrixserverlib * Update gomatrixserverlib * Update gomatrixserverlib * Return bad limit in error * Same with domain/userid --- federationapi/routing/backfill.go | 6 +++- federationapi/routing/profile.go | 12 +++++-- federationapi/routing/send.go | 52 +++++++++++++++++++++++-------- go.mod | 3 +- go.sum | 4 +-- 5 files changed, 56 insertions(+), 21 deletions(-) diff --git a/federationapi/routing/backfill.go b/federationapi/routing/backfill.go index 7092c4364..62471b8a9 100644 --- a/federationapi/routing/backfill.go +++ b/federationapi/routing/backfill.go @@ -16,6 +16,7 @@ package routing import ( "encoding/json" + "fmt" "net/http" "strconv" "time" @@ -73,7 +74,10 @@ func Backfill( } if req.Limit, err = strconv.Atoi(limit); err != nil { util.GetLogger(httpReq.Context()).WithError(err).Error("strconv.Atoi failed") - return jsonerror.InternalServerError() + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.InvalidArgumentValue(fmt.Sprintf("limit %q is invalid format", limit)), + } } // Query the roomserver. diff --git a/federationapi/routing/profile.go b/federationapi/routing/profile.go index 01a70c01b..9a81c1b33 100644 --- a/federationapi/routing/profile.go +++ b/federationapi/routing/profile.go @@ -15,6 +15,7 @@ package routing import ( + "fmt" "net/http" appserviceAPI "github.com/matrix-org/dendrite/appservice/api" @@ -46,12 +47,17 @@ func GetProfile( _, domain, err := gomatrixserverlib.SplitID('@', userID) if err != nil { util.GetLogger(httpReq.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") - return jsonerror.InternalServerError() + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.MissingArgument(fmt.Sprintf("Format of user ID %q is invalid", userID)), + } } if domain != cfg.Matrix.ServerName { - util.GetLogger(httpReq.Context()).WithError(err).Error("domain != cfg.Matrix.ServerName failed") - return jsonerror.InternalServerError() + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.InvalidArgumentValue(fmt.Sprintf("Domain %q does not match this server", domain)), + } } profile, err := appserviceAPI.RetrieveUserProfile(httpReq.Context(), userID, asAPI, accountDB) diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index 1013a44cf..5a9766f81 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -71,14 +71,28 @@ func Send( util.GetLogger(httpReq.Context()).Infof("Received transaction %q containing %d PDUs, %d EDUs", txnID, len(t.PDUs), len(t.EDUs)) resp, err := t.processTransaction() - if err != nil { + switch err.(type) { + // No error? Great! Send back a 200. + case nil: + return util.JSONResponse{ + Code: http.StatusOK, + JSON: resp, + } + // Handle known error cases as we will return a 400 error for these. + case roomNotFoundError: + case unmarshalError: + case verifySigError: + // Handle unknown error cases. Sending 500 errors back should be a last + // resort as this can make other homeservers back off sending federation + // events. + default: util.GetLogger(httpReq.Context()).WithError(err).Error("t.processTransaction failed") return jsonerror.InternalServerError() } - + // Return a 400 error for bad requests as fallen through from above. return util.JSONResponse{ - Code: http.StatusOK, - JSON: resp, + Code: http.StatusBadRequest, + JSON: jsonerror.BadJSON(err.Error()), } } @@ -93,6 +107,8 @@ type txnReq struct { } func (t *txnReq) processTransaction() (*gomatrixserverlib.RespSend, error) { + results := make(map[string]gomatrixserverlib.PDUResult) + var pdus []gomatrixserverlib.HeaderedEvent for _, pdu := range t.PDUs { var header struct { @@ -100,28 +116,27 @@ func (t *txnReq) processTransaction() (*gomatrixserverlib.RespSend, error) { } if err := json.Unmarshal(pdu, &header); err != nil { util.GetLogger(t.context).WithError(err).Warn("Transaction: Failed to extract room ID from event") - return nil, err + return nil, unmarshalError{err} } verReq := api.QueryRoomVersionForRoomRequest{RoomID: header.RoomID} verRes := api.QueryRoomVersionForRoomResponse{} if err := t.query.QueryRoomVersionForRoom(t.context, &verReq, &verRes); err != nil { util.GetLogger(t.context).WithError(err).Warn("Transaction: Failed to query room version for room", verReq.RoomID) - return nil, err + return nil, roomNotFoundError{verReq.RoomID} } event, err := gomatrixserverlib.NewEventFromUntrustedJSON(pdu, verRes.RoomVersion) if err != nil { util.GetLogger(t.context).WithError(err).Warnf("Transaction: Failed to parse event JSON of event %q", event.EventID()) - return nil, err + return nil, unmarshalError{err} } if err := gomatrixserverlib.VerifyAllEventSignatures(t.context, []gomatrixserverlib.Event{event}, t.keys); err != nil { util.GetLogger(t.context).WithError(err).Warnf("Transaction: Couldn't validate signature of event %q", event.EventID()) - return nil, err + return nil, verifySigError{event.EventID(), err} } pdus = append(pdus, event.Headered(verRes.RoomVersion)) } // Process the events. - results := map[string]gomatrixserverlib.PDUResult{} for _, e := range pdus { err := t.processEvent(e.Unwrap()) if err != nil { @@ -141,7 +156,7 @@ func (t *txnReq) processTransaction() (*gomatrixserverlib.RespSend, error) { // If we bail and stop processing then we risk wedging incoming // transactions from that server forever. switch err.(type) { - case unknownRoomError: + case roomNotFoundError: case *gomatrixserverlib.NotAllowed: default: // Any other error should be the result of a temporary error in @@ -162,11 +177,22 @@ func (t *txnReq) processTransaction() (*gomatrixserverlib.RespSend, error) { return &gomatrixserverlib.RespSend{PDUs: results}, nil } -type unknownRoomError struct { +type roomNotFoundError struct { roomID string } +type unmarshalError struct { + err error +} +type verifySigError struct { + eventID string + err error +} -func (e unknownRoomError) Error() string { return fmt.Sprintf("unknown room %q", e.roomID) } +func (e roomNotFoundError) Error() string { return fmt.Sprintf("room %q not found", e.roomID) } +func (e unmarshalError) Error() string { return fmt.Sprintf("unable to parse event: %s", e.err) } +func (e verifySigError) Error() string { + return fmt.Sprintf("unable to verify signature of event %q: %s", e.eventID, e.err) +} func (t *txnReq) processEDUs(edus []gomatrixserverlib.EDU) { for _, e := range edus { @@ -213,7 +239,7 @@ func (t *txnReq) processEvent(e gomatrixserverlib.Event) error { // that this server is unaware of. // However generally speaking we should reject events for rooms we // aren't a member of. - return unknownRoomError{e.RoomID()} + return roomNotFoundError{e.RoomID()} } if !stateResp.PrevEventsExist { diff --git a/go.mod b/go.mod index 849ad1f55..401492ae4 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f github.com/matrix-org/go-sqlite3-js v0.0.0-20200325174927-327088cdef10 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200416113012-dafb32a889ea + github.com/matrix-org/gomatrixserverlib v0.0.0-20200416165239-837ed63a0046 github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.3+incompatible @@ -27,7 +27,6 @@ require ( github.com/pierrec/lz4 v2.5.0+incompatible // indirect github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.4.1 - github.com/prometheus/common v0.9.1 github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect github.com/sirupsen/logrus v1.4.2 github.com/tidwall/gjson v1.6.0 // indirect diff --git a/go.sum b/go.sum index f4b4e4ff8..3bf623a9a 100644 --- a/go.sum +++ b/go.sum @@ -364,8 +364,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:kmRjpmFOenVpOaV/DRlo9p6z/IbOKlUC+hhKsAAh8Qg= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200416113012-dafb32a889ea h1:aiBD966UX0l4rTPgAu+u8CaiMfy+N8qWlZ2TIBp+0eM= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200416113012-dafb32a889ea/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200416165239-837ed63a0046 h1:R7iYuS8hhXdrqs5OSNF3Y3chaIFWU9KLpnzjjsdJkMU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200416165239-837ed63a0046/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 h1:W7l5CP4V7wPyPb4tYE11dbmeAOwtFQBTW0rf4OonOS8= From 0d116d92224040526094dd26dd054f91a8267c49 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Fri, 17 Apr 2020 12:17:20 +0100 Subject: [PATCH 80/86] Allow anyone to publish rooms (#971) * Allow anyone to publish rooms * Formatting and unused var * Review comments --- common/eventcontent.go | 1 + publicroomsapi/directory/directory.go | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/common/eventcontent.go b/common/eventcontent.go index c07c56276..f3817ba68 100644 --- a/common/eventcontent.go +++ b/common/eventcontent.go @@ -48,6 +48,7 @@ func InitialPowerLevelsContent(roomCreator string) (c gomatrixserverlib.PowerLev "m.room.history_visibility": 100, "m.room.canonical_alias": 50, "m.room.avatar": 50, + "m.room.aliases": 0, // anyone can publish aliases by default. Has to be 0 else state_default is used. } c.Users = map[string]int64{roomCreator: 100} return c diff --git a/publicroomsapi/directory/directory.go b/publicroomsapi/directory/directory.go index 1e305f3ca..837018e64 100644 --- a/publicroomsapi/directory/directory.go +++ b/publicroomsapi/directory/directory.go @@ -92,9 +92,9 @@ func SetVisibility( util.GetLogger(req.Context()).WithError(err).Error("could not query events from room") return jsonerror.InternalServerError() } - power, _ := gomatrixserverlib.NewPowerLevelContentFromEvent(queryEventsRes.StateEvents[0].Event) - // Check if the user's power is greater than power required to change m.room.aliases event + // NOTSPEC: Check if the user's power is greater than power required to change m.room.aliases event + power, _ := gomatrixserverlib.NewPowerLevelContentFromEvent(queryEventsRes.StateEvents[0].Event) if power.UserLevel(dev.UserID) < power.EventLevel(gomatrixserverlib.MRoomAliases, true) { return util.JSONResponse{ Code: http.StatusForbidden, From e1a4ada0e1325cbc72814f7ac00ee56f9986e014 Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Sat, 18 Apr 2020 18:01:48 +0800 Subject: [PATCH 81/86] Ensure room version tests can be grouped whether failed or passed (#972) Signed-off-by: Alex Chen --- are-we-synapse-yet.list | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/are-we-synapse-yet.list b/are-we-synapse-yet.list index 5d41c6ad5..5a900b3ef 100644 --- a/are-we-synapse-yet.list +++ b/are-we-synapse-yet.list @@ -405,6 +405,7 @@ syn Unnamed room comes with a name summary syn Named room comes with just joined member count summary syn Room summary only has 5 heroes syn Room summary counts change when membership changes +rmv User can create and send/receive messages in a room with version 1 rmv User can create and send/receive messages in a room with version 1 (2 subtests) rmv local user can join room with version 1 rmv User can invite local user to room with version 1 @@ -413,6 +414,7 @@ rmv User can invite remote user to room with version 1 rmv Remote user can backfill in a room with version 1 rmv Can reject invites over federation for rooms with version 1 rmv Can receive redactions from regular users over federation in room version 1 +rmv User can create and send/receive messages in a room with version 2 rmv User can create and send/receive messages in a room with version 2 (2 subtests) rmv local user can join room with version 2 rmv User can invite local user to room with version 2 @@ -422,6 +424,7 @@ rmv Remote user can backfill in a room with version 2 rmv Can reject invites over federation for rooms with version 2 rmv Can receive redactions from regular users over federation in room version 2 rmv User can create and send/receive messages in a room with version 3 +rmv User can create and send/receive messages in a room with version 3 (2 subtests) rmv local user can join room with version 3 rmv User can invite local user to room with version 3 rmv remote user can join room with version 3 @@ -430,6 +433,7 @@ rmv Remote user can backfill in a room with version 3 rmv Can reject invites over federation for rooms with version 3 rmv Can receive redactions from regular users over federation in room version 3 rmv User can create and send/receive messages in a room with version 4 +rmv User can create and send/receive messages in a room with version 4 (2 subtests) rmv local user can join room with version 4 rmv User can invite local user to room with version 4 rmv remote user can join room with version 4 @@ -438,6 +442,7 @@ rmv Remote user can backfill in a room with version 4 rmv Can reject invites over federation for rooms with version 4 rmv Can receive redactions from regular users over federation in room version 4 rmv User can create and send/receive messages in a room with version 5 +rmv User can create and send/receive messages in a room with version 5 (2 subtests) rmv local user can join room with version 5 rmv User can invite local user to room with version 5 rmv remote user can join room with version 5 From 9f5ece8045b820d8bbb9d8e23765cb6c83b60bcc Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Sun, 19 Apr 2020 18:18:38 +0800 Subject: [PATCH 82/86] Fix nil http clients provided to components in polylith mode (#973) Signed-off-by: Alex Chen --- common/basecomponent/base.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/common/basecomponent/base.go b/common/basecomponent/base.go index de3e04c21..5c6f64775 100644 --- a/common/basecomponent/base.go +++ b/common/basecomponent/base.go @@ -124,7 +124,7 @@ func (b *BaseDendrite) CreateHTTPRoomserverAPIs() ( if err != nil { logrus.WithError(err).Panic("NewRoomserverInputAPIHTTP failed", b.httpClient) } - query, err := roomserverAPI.NewRoomserverQueryAPIHTTP(b.Cfg.RoomServerURL(), nil) + query, err := roomserverAPI.NewRoomserverQueryAPIHTTP(b.Cfg.RoomServerURL(), b.httpClient) if err != nil { logrus.WithError(err).Panic("NewRoomserverQueryAPIHTTP failed", b.httpClient) } @@ -134,7 +134,7 @@ func (b *BaseDendrite) CreateHTTPRoomserverAPIs() ( // CreateHTTPEDUServerAPIs returns eduInputAPI for hitting the EDU // server over HTTP func (b *BaseDendrite) CreateHTTPEDUServerAPIs() eduServerAPI.EDUServerInputAPI { - e, err := eduServerAPI.NewEDUServerInputAPIHTTP(b.Cfg.EDUServerURL(), nil) + e, err := eduServerAPI.NewEDUServerInputAPIHTTP(b.Cfg.EDUServerURL(), b.httpClient) if err != nil { logrus.WithError(err).Panic("NewEDUServerInputAPIHTTP failed", b.httpClient) } @@ -144,7 +144,7 @@ func (b *BaseDendrite) CreateHTTPEDUServerAPIs() eduServerAPI.EDUServerInputAPI // CreateHTTPFederationSenderAPIs returns FederationSenderQueryAPI for hitting // the federation sender over HTTP func (b *BaseDendrite) CreateHTTPFederationSenderAPIs() federationSenderAPI.FederationSenderQueryAPI { - f, err := federationSenderAPI.NewFederationSenderQueryAPIHTTP(b.Cfg.FederationSenderURL(), nil) + f, err := federationSenderAPI.NewFederationSenderQueryAPIHTTP(b.Cfg.FederationSenderURL(), b.httpClient) if err != nil { logrus.WithError(err).Panic("NewFederationSenderQueryAPIHTTP failed", b.httpClient) } From c62844234bdecac3ad2facc7af011c704699e86d Mon Sep 17 00:00:00 2001 From: Kegsay Date: Mon, 20 Apr 2020 14:19:25 +0100 Subject: [PATCH 83/86] Add login sytests (#974) --- sytest-whitelist | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sytest-whitelist b/sytest-whitelist index 3277cc59d..d47bf1f60 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -17,6 +17,11 @@ POST /register rejects registration of usernames with 'é' POST /register rejects registration of usernames with '\n' POST /register rejects registration of usernames with ''' GET /login yields a set of flows +POST /login can log in as a user +POST /login returns the same device_id as that in the request +POST /login can log in as a user with just the local part of the id +POST /login as non-existing user is rejected +POST /login wrong password is rejected GET /events initially GET /initialSync initially Version responds 200 OK with valid structure From 9045b8e89fbe3fa1c441a59029365a98318180d8 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 20 Apr 2020 17:42:34 +0100 Subject: [PATCH 84/86] Perspective key fetching, some federation room join fixes (#975) * Update gomatrixserverlib * Test matrix.org as perspective key server * Base64 decode better * Optional strict validity checking in gmsl * Update gomatrixserverlib * Attempt to find missing auth events over federation (this shouldn't happen but I am guessing there is a synapse bug involved where we don't get all of the auth events) * Update gomatrixserverlib, debug logging * Remove debugging output * More verbose debugging * Print outliers * Increase timeouts for testing, observe contexts before trying to join over more servers * Don't block on roomserver (experimental) * Don't block on roomserver * Update gomatrixserverlib * Update gomatrixserverlib * Configurable perspective key fetchers * Output number of configured keys for perspective * Example perspective config included * Undo debug stack trace * Undo debug stack trace * Restore original HTTP listener in monolith * Fix lint * Review comments * Set default HTTP server timeout to 5 minutes now, block again when joining * Don't use HTTP address for HTTPS whoops * Update gomatrixserverlib * Update gomatrixserverlib * Update gomatrixserverlib * Actually add perspectives * Actually add perspectives * Update gomatrixserverlib --- clientapi/routing/joinroom.go | 69 ++++++++++++++++++++-- cmd/dendrite-client-api-server/main.go | 2 +- cmd/dendrite-demo-libp2p/main.go | 2 +- cmd/dendrite-federation-api-server/main.go | 2 +- cmd/dendrite-monolith-server/main.go | 28 ++++++--- common/basecomponent/base.go | 19 +++--- common/config/config.go | 18 ++++++ common/keydb/keyring.go | 52 ++++++++++++++-- dendrite-config.yaml | 8 +++ federationapi/routing/invite.go | 7 ++- federationapi/routing/join.go | 7 ++- federationapi/routing/leave.go | 7 ++- go.mod | 2 +- go.sum | 5 +- sytest-whitelist | 1 + 15 files changed, 189 insertions(+), 40 deletions(-) diff --git a/clientapi/routing/joinroom.go b/clientapi/routing/joinroom.go index 0f1a9ba4d..f72bb9162 100644 --- a/clientapi/routing/joinroom.go +++ b/clientapi/routing/joinroom.go @@ -32,6 +32,7 @@ import ( "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" + "github.com/sirupsen/logrus" ) // JoinRoomByIDOrAlias implements the "/join/{roomIDOrAlias}" API. @@ -290,7 +291,15 @@ func (r joinRoomReq) joinRoomUsingServers( // There was a problem talking to one of the servers. util.GetLogger(r.req.Context()).WithError(lastErr).WithField("server", server).Warn("Failed to join room using server") // Try the next server. - continue + if r.req.Context().Err() != nil { + // The request context has expired so don't bother trying any + // more servers - they will immediately fail due to the expired + // context. + break + } else { + // The request context hasn't expired yet so try the next server. + continue + } } return *response } @@ -365,16 +374,22 @@ func (r joinRoomReq) joinRoomUsingServer(roomID string, server gomatrixserverlib return nil, fmt.Errorf("r.federation.SendJoin: %w", err) } - if err = respSendJoin.Check(r.req.Context(), r.keyRing, event); err != nil { - return nil, fmt.Errorf("respSendJoin: %w", err) + if err = r.checkSendJoinResponse(event, server, respMakeJoin, respSendJoin); err != nil { + return nil, err } + util.GetLogger(r.req.Context()).WithFields(logrus.Fields{ + "room_id": roomID, + "num_auth_events": len(respSendJoin.AuthEvents), + "num_state_events": len(respSendJoin.StateEvents), + }).Info("Room join signature and auth verification passed") + if err = r.producer.SendEventWithState( r.req.Context(), gomatrixserverlib.RespState(respSendJoin.RespState), event.Headered(respMakeJoin.RoomVersion), ); err != nil { - return nil, fmt.Errorf("r.producer.SendEventWithState: %w", err) + util.GetLogger(r.req.Context()).WithError(err).Error("r.producer.SendEventWithState") } return &util.JSONResponse{ @@ -385,3 +400,49 @@ func (r joinRoomReq) joinRoomUsingServer(roomID string, server gomatrixserverlib }{roomID}, }, nil } + +// checkSendJoinResponse checks that all of the signatures are correct +// and that the join is allowed by the supplied state. +func (r joinRoomReq) checkSendJoinResponse( + event gomatrixserverlib.Event, + server gomatrixserverlib.ServerName, + respMakeJoin gomatrixserverlib.RespMakeJoin, + respSendJoin gomatrixserverlib.RespSendJoin, +) error { + // A list of events that we have retried, if they were not included in + // the auth events supplied in the send_join. + retries := map[string]bool{} + +retryCheck: + // TODO: Can we expand Check here to return a list of missing auth + // events rather than failing one at a time? + if err := respSendJoin.Check(r.req.Context(), r.keyRing, event); err != nil { + switch e := err.(type) { + case gomatrixserverlib.MissingAuthEventError: + // Check that we haven't already retried for this event, prevents + // us from ending up in endless loops + if !retries[e.AuthEventID] { + // Ask the server that we're talking to right now for the event + tx, txerr := r.federation.GetEvent(r.req.Context(), server, e.AuthEventID) + if txerr != nil { + return fmt.Errorf("r.federation.GetEvent: %w", txerr) + } + // For each event returned, add it to the auth events. + for _, pdu := range tx.PDUs { + ev, everr := gomatrixserverlib.NewEventFromUntrustedJSON(pdu, respMakeJoin.RoomVersion) + if everr != nil { + return fmt.Errorf("gomatrixserverlib.NewEventFromUntrustedJSON: %w", everr) + } + respSendJoin.AuthEvents = append(respSendJoin.AuthEvents, ev) + } + // Mark the event as retried and then give the check another go. + retries[e.AuthEventID] = true + goto retryCheck + } + return fmt.Errorf("respSendJoin (after retries): %w", e) + default: + return fmt.Errorf("respSendJoin: %w", err) + } + } + return nil +} diff --git a/cmd/dendrite-client-api-server/main.go b/cmd/dendrite-client-api-server/main.go index a7e241b13..815a978a8 100644 --- a/cmd/dendrite-client-api-server/main.go +++ b/cmd/dendrite-client-api-server/main.go @@ -33,7 +33,7 @@ func main() { deviceDB := base.CreateDeviceDB() keyDB := base.CreateKeyDB() federation := base.CreateFederationClient() - keyRing := keydb.CreateKeyRing(federation.Client, keyDB) + keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) asQuery := base.CreateHTTPAppServiceAPIs() alias, input, query := base.CreateHTTPRoomserverAPIs() diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index df3b48adf..f280c7483 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -146,7 +146,7 @@ func main() { deviceDB := base.Base.CreateDeviceDB() keyDB := createKeyDB(base) federation := createFederationClient(base) - keyRing := keydb.CreateKeyRing(federation.Client, keyDB) + keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) alias, input, query := roomserver.SetupRoomServerComponent(&base.Base) eduInputAPI := eduserver.SetupEDUServerComponent(&base.Base, cache.New()) diff --git a/cmd/dendrite-federation-api-server/main.go b/cmd/dendrite-federation-api-server/main.go index d18926a68..dd06cd3f9 100644 --- a/cmd/dendrite-federation-api-server/main.go +++ b/cmd/dendrite-federation-api-server/main.go @@ -33,7 +33,7 @@ func main() { keyDB := base.CreateKeyDB() federation := base.CreateFederationClient() federationSender := base.CreateHTTPFederationSenderAPIs() - keyRing := keydb.CreateKeyRing(federation.Client, keyDB) + keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) alias, input, query := base.CreateHTTPRoomserverAPIs() asQuery := base.CreateHTTPAppServiceAPIs() diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index d47d2e1bb..6b0d83ae1 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -55,7 +55,7 @@ func main() { deviceDB := base.CreateDeviceDB() keyDB := base.CreateKeyDB() federation := base.CreateFederationClient() - keyRing := keydb.CreateKeyRing(federation.Client, keyDB) + keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) alias, input, query := roomserver.SetupRoomServerComponent(base) eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New()) @@ -90,16 +90,26 @@ func main() { // Expose the matrix APIs directly rather than putting them under a /api path. go func() { - logrus.Info("Listening on ", *httpBindAddr) - logrus.Fatal(http.ListenAndServe(*httpBindAddr, nil)) + serv := http.Server{ + Addr: *httpBindAddr, + WriteTimeout: basecomponent.HTTPServerTimeout, + } + + logrus.Info("Listening on ", serv.Addr) + logrus.Fatal(serv.ListenAndServe()) }() // Handle HTTPS if certificate and key are provided - go func() { - if *certFile != "" && *keyFile != "" { - logrus.Info("Listening on ", *httpsBindAddr) - logrus.Fatal(http.ListenAndServeTLS(*httpsBindAddr, *certFile, *keyFile, nil)) - } - }() + if *certFile != "" && *keyFile != "" { + go func() { + serv := http.Server{ + Addr: *httpsBindAddr, + WriteTimeout: basecomponent.HTTPServerTimeout, + } + + logrus.Info("Listening on ", serv.Addr) + logrus.Fatal(serv.ListenAndServeTLS(*certFile, *keyFile)) + }() + } // We want to block forever to let the HTTP and HTTPS handler serve the APIs select {} diff --git a/common/basecomponent/base.go b/common/basecomponent/base.go index 5c6f64775..78894289e 100644 --- a/common/basecomponent/base.go +++ b/common/basecomponent/base.go @@ -60,6 +60,9 @@ type BaseDendrite struct { KafkaProducer sarama.SyncProducer } +const HTTPServerTimeout = time.Minute * 5 +const HTTPClientTimeout = time.Second * 30 + // NewBaseDendrite creates a new instance to be used by a component. // The componentName is used for logging purposes, and should be a friendly name // of the compontent running, e.g. "SyncAPI" @@ -80,14 +83,12 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string) *BaseDendrite { kafkaConsumer, kafkaProducer = setupKafka(cfg) } - const defaultHTTPTimeout = 30 * time.Second - return &BaseDendrite{ componentName: componentName, tracerCloser: closer, Cfg: cfg, APIMux: mux.NewRouter().UseEncodedPath(), - httpClient: &http.Client{Timeout: defaultHTTPTimeout}, + httpClient: &http.Client{Timeout: HTTPClientTimeout}, KafkaConsumer: kafkaConsumer, KafkaProducer: kafkaProducer, } @@ -209,16 +210,20 @@ func (b *BaseDendrite) SetupAndServeHTTP(bindaddr string, listenaddr string) { addr = listenaddr } + serv := http.Server{ + Addr: addr, + WriteTimeout: HTTPServerTimeout, + } + common.SetupHTTPAPI(http.DefaultServeMux, common.WrapHandlerInCORS(b.APIMux), b.Cfg) - logrus.Infof("Starting %s server on %s", b.componentName, addr) - - err := http.ListenAndServe(addr, nil) + logrus.Infof("Starting %s server on %s", b.componentName, serv.Addr) + err := serv.ListenAndServe() if err != nil { logrus.WithError(err).Fatal("failed to serve http") } - logrus.Infof("Stopped %s server on %s", b.componentName, addr) + logrus.Infof("Stopped %s server on %s", b.componentName, serv.Addr) } // setupKafka creates kafka consumer/producer pair from the config. diff --git a/common/config/config.go b/common/config/config.go index a1a844252..6b61fda7c 100644 --- a/common/config/config.go +++ b/common/config/config.go @@ -99,6 +99,9 @@ type Dendrite struct { // If set disables new users from registering (except via shared // secrets) RegistrationDisabled bool `yaml:"registration_disabled"` + // Perspective keyservers, to use as a backup when direct key fetch + // requests don't succeed + KeyPerspectives KeyPerspectives `yaml:"key_perspectives"` } `yaml:"matrix"` // The configuration specific to the media repostitory. @@ -285,6 +288,21 @@ type Dendrite struct { } `yaml:"-"` } +// KeyPerspectives are used to configure perspective key servers for +// retrieving server keys. +type KeyPerspectives []struct { + // The server name of the perspective key server + ServerName gomatrixserverlib.ServerName `yaml:"server_name"` + // Server keys for the perspective user, used to verify the + // keys have been signed by the perspective server + Keys []struct { + // The key ID, e.g. ed25519:auto + KeyID gomatrixserverlib.KeyID `yaml:"key_id"` + // The public key in base64 unpadded format + PublicKey string `yaml:"public_key"` + } `yaml:"keys"` +} + // A Path on the filesystem. type Path string diff --git a/common/keydb/keyring.go b/common/keydb/keyring.go index 1b20f7816..e9cc7903e 100644 --- a/common/keydb/keyring.go +++ b/common/keydb/keyring.go @@ -14,19 +14,61 @@ package keydb -import "github.com/matrix-org/gomatrixserverlib" +import ( + "encoding/base64" + + "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/gomatrixserverlib" + "github.com/sirupsen/logrus" + "golang.org/x/crypto/ed25519" +) // CreateKeyRing creates and configures a KeyRing object. // // It creates the necessary key fetchers and collects them into a KeyRing // backed by the given KeyDatabase. func CreateKeyRing(client gomatrixserverlib.Client, - keyDB gomatrixserverlib.KeyDatabase) gomatrixserverlib.KeyRing { - return gomatrixserverlib.KeyRing{ + keyDB gomatrixserverlib.KeyDatabase, + cfg config.KeyPerspectives) gomatrixserverlib.KeyRing { + + fetchers := gomatrixserverlib.KeyRing{ KeyFetchers: []gomatrixserverlib.KeyFetcher{ - // TODO: Use perspective key fetchers for production. - &gomatrixserverlib.DirectKeyFetcher{Client: client}, + &gomatrixserverlib.DirectKeyFetcher{ + Client: client, + }, }, KeyDatabase: keyDB, } + + logrus.Info("Enabled direct key fetcher") + + var b64e = base64.StdEncoding.WithPadding(base64.NoPadding) + for _, ps := range cfg { + perspective := &gomatrixserverlib.PerspectiveKeyFetcher{ + PerspectiveServerName: ps.ServerName, + PerspectiveServerKeys: map[gomatrixserverlib.KeyID]ed25519.PublicKey{}, + Client: client, + } + + for _, key := range ps.Keys { + rawkey, err := b64e.DecodeString(key.PublicKey) + if err != nil { + logrus.WithError(err).WithFields(logrus.Fields{ + "server_name": ps.ServerName, + "public_key": key.PublicKey, + }).Warn("Couldn't parse perspective key") + continue + } + perspective.PerspectiveServerKeys[key.KeyID] = rawkey + } + + fetchers.KeyFetchers = append(fetchers.KeyFetchers, perspective) + + logrus.WithFields(logrus.Fields{ + "server_name": ps.ServerName, + "num_public_keys": len(ps.Keys), + }).Info("Enabled perspective key fetcher") + } + + return fetchers } diff --git a/dendrite-config.yaml b/dendrite-config.yaml index 86a208d7f..bed78a5af 100644 --- a/dendrite-config.yaml +++ b/dendrite-config.yaml @@ -17,6 +17,14 @@ matrix: trusted_third_party_id_servers: - vector.im - matrix.org + # Perspective key servers which are used when direct key requests fail + #key_perspectives: + # - server_name: matrix.org + # keys: + # - key_id: ed25519:auto + # public_key: Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw + # - key_id: ed25519:a_RXGa + # public_key: l8Hft5qXKn1vfHrg3p4+W8gELQVo8N13JkluMfmn2sQ # The media repository config media: diff --git a/federationapi/routing/invite.go b/federationapi/routing/invite.go index 6c3e12e23..4b367e004 100644 --- a/federationapi/routing/invite.go +++ b/federationapi/routing/invite.go @@ -63,9 +63,10 @@ func Invite( // Check that the event is signed by the server sending the request. redacted := event.Redact() verifyRequests := []gomatrixserverlib.VerifyJSONRequest{{ - ServerName: event.Origin(), - Message: redacted.JSON(), - AtTS: event.OriginServerTS(), + ServerName: event.Origin(), + Message: redacted.JSON(), + AtTS: event.OriginServerTS(), + StrictValidityChecking: true, }} verifyResults, err := keys.VerifyJSONs(httpReq.Context(), verifyRequests) if err != nil { diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go index 0a7b23000..e06785954 100644 --- a/federationapi/routing/join.go +++ b/federationapi/routing/join.go @@ -196,9 +196,10 @@ func SendJoin( // Check that the event is signed by the server sending the request. redacted := event.Redact() verifyRequests := []gomatrixserverlib.VerifyJSONRequest{{ - ServerName: event.Origin(), - Message: redacted.JSON(), - AtTS: event.OriginServerTS(), + ServerName: event.Origin(), + Message: redacted.JSON(), + AtTS: event.OriginServerTS(), + StrictValidityChecking: true, }} verifyResults, err := keys.VerifyJSONs(httpReq.Context(), verifyRequests) if err != nil { diff --git a/federationapi/routing/leave.go b/federationapi/routing/leave.go index e0a142631..6fc3b12ed 100644 --- a/federationapi/routing/leave.go +++ b/federationapi/routing/leave.go @@ -145,9 +145,10 @@ func SendLeave( // Check that the event is signed by the server sending the request. redacted := event.Redact() verifyRequests := []gomatrixserverlib.VerifyJSONRequest{{ - ServerName: event.Origin(), - Message: redacted.JSON(), - AtTS: event.OriginServerTS(), + ServerName: event.Origin(), + Message: redacted.JSON(), + AtTS: event.OriginServerTS(), + StrictValidityChecking: true, }} verifyResults, err := keys.VerifyJSONs(httpReq.Context(), verifyRequests) if err != nil { diff --git a/go.mod b/go.mod index 401492ae4..1a3dddc2d 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f github.com/matrix-org/go-sqlite3-js v0.0.0-20200325174927-327088cdef10 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200416165239-837ed63a0046 + github.com/matrix-org/gomatrixserverlib v0.0.0-20200420162430-9662fc15e7e2 github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.3+incompatible diff --git a/go.sum b/go.sum index 3bf623a9a..627372be7 100644 --- a/go.sum +++ b/go.sum @@ -364,8 +364,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:kmRjpmFOenVpOaV/DRlo9p6z/IbOKlUC+hhKsAAh8Qg= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200416165239-837ed63a0046 h1:R7iYuS8hhXdrqs5OSNF3Y3chaIFWU9KLpnzjjsdJkMU= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200416165239-837ed63a0046/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200420162430-9662fc15e7e2 h1:XTg0bCZ+fQlWZHi1CinNecYfg2SFhyRNnGxKxXbLubg= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200420162430-9662fc15e7e2/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 h1:W7l5CP4V7wPyPb4tYE11dbmeAOwtFQBTW0rf4OonOS8= @@ -721,4 +721,5 @@ gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099 h1:XJP7lxbSxWLOMNdBE4B/STaqVy6L73o0knwj2vIlxnw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/sytest-whitelist b/sytest-whitelist index d47bf1f60..7bd2a63c4 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -252,3 +252,4 @@ Outbound federation can send invites via v2 API User can invite local user to room with version 3 User can invite local user to room with version 4 A pair of servers can establish a join in a v2 room +Can logout all devices From 71f9d35b7c2f1531b383f115ad3ab23e8d1ed0a5 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 21 Apr 2020 10:04:06 +0100 Subject: [PATCH 85/86] Update gomatrixserverlib --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 1a3dddc2d..8d91902d1 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f github.com/matrix-org/go-sqlite3-js v0.0.0-20200325174927-327088cdef10 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200420162430-9662fc15e7e2 + github.com/matrix-org/gomatrixserverlib v0.0.0-20200421090225-4ea81b29f5f7 github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.3+incompatible diff --git a/go.sum b/go.sum index 627372be7..e76f6d007 100644 --- a/go.sum +++ b/go.sum @@ -364,8 +364,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:kmRjpmFOenVpOaV/DRlo9p6z/IbOKlUC+hhKsAAh8Qg= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200420162430-9662fc15e7e2 h1:XTg0bCZ+fQlWZHi1CinNecYfg2SFhyRNnGxKxXbLubg= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200420162430-9662fc15e7e2/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200421090225-4ea81b29f5f7 h1:4vE84tE3r7BitCt2HQvT231JrhMjDfjDVDqVoiVPv0w= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200421090225-4ea81b29f5f7/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 h1:W7l5CP4V7wPyPb4tYE11dbmeAOwtFQBTW0rf4OonOS8= From a466e9e9cc86e04b4d28de45cd44199916664768 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 22 Apr 2020 13:00:05 +0100 Subject: [PATCH 86/86] LRU cache for room versions in RS query API (#976) * Experimental LRU cache for room versions * Don't accidentally try to type-assert nil * Also reduce hits on query API * Use hashicorp implementation which mutexes for us * Define const for max cache entries * Rename to be specifically immutable, panic if we try to mutate a cache entry * Review comments * Remove nil guards, give roomserver integration test a cache * go mod tidy --- cmd/roomserver-integration-tests/main.go | 8 ++++- common/basecomponent/base.go | 35 +++++++++++-------- common/caching/immutablecache.go | 12 +++++++ common/caching/immutableinmemorylru.go | 43 ++++++++++++++++++++++++ roomserver/api/query.go | 21 +++++++++--- roomserver/query/query.go | 10 +++++- roomserver/roomserver.go | 5 ++- 7 files changed, 112 insertions(+), 22 deletions(-) create mode 100644 common/caching/immutablecache.go create mode 100644 common/caching/immutableinmemorylru.go diff --git a/cmd/roomserver-integration-tests/main.go b/cmd/roomserver-integration-tests/main.go index df5607bcb..682fc6224 100644 --- a/cmd/roomserver-integration-tests/main.go +++ b/cmd/roomserver-integration-tests/main.go @@ -28,6 +28,7 @@ import ( "net/http" + "github.com/matrix-org/dendrite/common/caching" "github.com/matrix-org/dendrite/common/test" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" @@ -253,6 +254,11 @@ func testRoomserver(input []string, wantOutput []string, checkQueries func(api.R panic(err) } + cache, err := caching.NewImmutableInMemoryLRUCache() + if err != nil { + panic(err) + } + doInput := func() { fmt.Printf("Roomserver is ready to receive input, sending %d events\n", len(input)) if err = writeToRoomServer(input, cfg.RoomServerURL()); err != nil { @@ -270,7 +276,7 @@ func testRoomserver(input []string, wantOutput []string, checkQueries func(api.R cmd.Args = []string{"dendrite-room-server", "--config", filepath.Join(dir, test.ConfigFile)} gotOutput, err := runAndReadFromTopic(cmd, cfg.RoomServerURL()+"/metrics", doInput, outputTopic, len(wantOutput), func() { - queryAPI, _ := api.NewRoomserverQueryAPIHTTP("http://"+string(cfg.Listen.RoomServer), &http.Client{Timeout: timeoutHTTP}) + queryAPI, _ := api.NewRoomserverQueryAPIHTTP("http://"+string(cfg.Listen.RoomServer), &http.Client{Timeout: timeoutHTTP}, cache) checkQueries(queryAPI) }) if err != nil { diff --git a/common/basecomponent/base.go b/common/basecomponent/base.go index 78894289e..68a77cf99 100644 --- a/common/basecomponent/base.go +++ b/common/basecomponent/base.go @@ -23,6 +23,7 @@ import ( "golang.org/x/crypto/ed25519" + "github.com/matrix-org/dendrite/common/caching" "github.com/matrix-org/dendrite/common/keydb" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" @@ -53,11 +54,12 @@ type BaseDendrite struct { tracerCloser io.Closer // APIMux should be used to register new public matrix api endpoints - APIMux *mux.Router - httpClient *http.Client - Cfg *config.Dendrite - KafkaConsumer sarama.Consumer - KafkaProducer sarama.SyncProducer + APIMux *mux.Router + httpClient *http.Client + Cfg *config.Dendrite + ImmutableCache caching.ImmutableCache + KafkaConsumer sarama.Consumer + KafkaProducer sarama.SyncProducer } const HTTPServerTimeout = time.Minute * 5 @@ -83,14 +85,20 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string) *BaseDendrite { kafkaConsumer, kafkaProducer = setupKafka(cfg) } + cache, err := caching.NewImmutableInMemoryLRUCache() + if err != nil { + logrus.WithError(err).Warnf("Failed to create cache") + } + return &BaseDendrite{ - componentName: componentName, - tracerCloser: closer, - Cfg: cfg, - APIMux: mux.NewRouter().UseEncodedPath(), - httpClient: &http.Client{Timeout: HTTPClientTimeout}, - KafkaConsumer: kafkaConsumer, - KafkaProducer: kafkaProducer, + componentName: componentName, + tracerCloser: closer, + Cfg: cfg, + ImmutableCache: cache, + APIMux: mux.NewRouter().UseEncodedPath(), + httpClient: &http.Client{Timeout: HTTPClientTimeout}, + KafkaConsumer: kafkaConsumer, + KafkaProducer: kafkaProducer, } } @@ -116,7 +124,6 @@ func (b *BaseDendrite) CreateHTTPRoomserverAPIs() ( roomserverAPI.RoomserverInputAPI, roomserverAPI.RoomserverQueryAPI, ) { - alias, err := roomserverAPI.NewRoomserverAliasAPIHTTP(b.Cfg.RoomServerURL(), b.httpClient) if err != nil { logrus.WithError(err).Panic("NewRoomserverAliasAPIHTTP failed") @@ -125,7 +132,7 @@ func (b *BaseDendrite) CreateHTTPRoomserverAPIs() ( if err != nil { logrus.WithError(err).Panic("NewRoomserverInputAPIHTTP failed", b.httpClient) } - query, err := roomserverAPI.NewRoomserverQueryAPIHTTP(b.Cfg.RoomServerURL(), b.httpClient) + query, err := roomserverAPI.NewRoomserverQueryAPIHTTP(b.Cfg.RoomServerURL(), b.httpClient, b.ImmutableCache) if err != nil { logrus.WithError(err).Panic("NewRoomserverQueryAPIHTTP failed", b.httpClient) } diff --git a/common/caching/immutablecache.go b/common/caching/immutablecache.go new file mode 100644 index 000000000..9620667a2 --- /dev/null +++ b/common/caching/immutablecache.go @@ -0,0 +1,12 @@ +package caching + +import "github.com/matrix-org/gomatrixserverlib" + +const ( + RoomVersionMaxCacheEntries = 128 +) + +type ImmutableCache interface { + GetRoomVersion(roomId string) (gomatrixserverlib.RoomVersion, bool) + StoreRoomVersion(roomId string, roomVersion gomatrixserverlib.RoomVersion) +} diff --git a/common/caching/immutableinmemorylru.go b/common/caching/immutableinmemorylru.go new file mode 100644 index 000000000..3e8f4aadb --- /dev/null +++ b/common/caching/immutableinmemorylru.go @@ -0,0 +1,43 @@ +package caching + +import ( + "fmt" + + lru "github.com/hashicorp/golang-lru" + "github.com/matrix-org/gomatrixserverlib" +) + +type ImmutableInMemoryLRUCache struct { + roomVersions *lru.Cache +} + +func NewImmutableInMemoryLRUCache() (*ImmutableInMemoryLRUCache, error) { + roomVersionCache, rvErr := lru.New(RoomVersionMaxCacheEntries) + if rvErr != nil { + return nil, rvErr + } + return &ImmutableInMemoryLRUCache{ + roomVersions: roomVersionCache, + }, nil +} + +func checkForInvalidMutation(cache *lru.Cache, key string, value interface{}) { + if peek, ok := cache.Peek(key); ok && peek != value { + panic(fmt.Sprintf("invalid use of immutable cache tries to mutate existing value of %q", key)) + } +} + +func (c *ImmutableInMemoryLRUCache) GetRoomVersion(roomID string) (gomatrixserverlib.RoomVersion, bool) { + val, found := c.roomVersions.Get(roomID) + if found && val != nil { + if roomVersion, ok := val.(gomatrixserverlib.RoomVersion); ok { + return roomVersion, true + } + } + return "", false +} + +func (c *ImmutableInMemoryLRUCache) StoreRoomVersion(roomID string, roomVersion gomatrixserverlib.RoomVersion) { + checkForInvalidMutation(c.roomVersions, roomID, roomVersion) + c.roomVersions.Add(roomID, roomVersion) +} diff --git a/roomserver/api/query.go b/roomserver/api/query.go index 5f024d266..b272b1ebd 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -21,6 +21,7 @@ import ( "errors" "net/http" + "github.com/matrix-org/dendrite/common/caching" commonHTTP "github.com/matrix-org/dendrite/common/http" "github.com/matrix-org/gomatrixserverlib" opentracing "github.com/opentracing/opentracing-go" @@ -411,16 +412,17 @@ const RoomserverQueryRoomVersionForRoomPath = "/api/roomserver/queryRoomVersionF // NewRoomserverQueryAPIHTTP creates a RoomserverQueryAPI implemented by talking to a HTTP POST API. // If httpClient is nil an error is returned -func NewRoomserverQueryAPIHTTP(roomserverURL string, httpClient *http.Client) (RoomserverQueryAPI, error) { +func NewRoomserverQueryAPIHTTP(roomserverURL string, httpClient *http.Client, cache caching.ImmutableCache) (RoomserverQueryAPI, error) { if httpClient == nil { return nil, errors.New("NewRoomserverQueryAPIHTTP: httpClient is ") } - return &httpRoomserverQueryAPI{roomserverURL, httpClient}, nil + return &httpRoomserverQueryAPI{roomserverURL, httpClient, cache}, nil } type httpRoomserverQueryAPI struct { - roomserverURL string - httpClient *http.Client + roomserverURL string + httpClient *http.Client + immutableCache caching.ImmutableCache } // QueryLatestEventsAndState implements RoomserverQueryAPI @@ -585,9 +587,18 @@ func (h *httpRoomserverQueryAPI) QueryRoomVersionForRoom( request *QueryRoomVersionForRoomRequest, response *QueryRoomVersionForRoomResponse, ) error { + if roomVersion, ok := h.immutableCache.GetRoomVersion(request.RoomID); ok { + response.RoomVersion = roomVersion + return nil + } + span, ctx := opentracing.StartSpanFromContext(ctx, "QueryRoomVersionForRoom") defer span.Finish() apiURL := h.roomserverURL + RoomserverQueryRoomVersionForRoomPath - return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + err := commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + if err == nil { + h.immutableCache.StoreRoomVersion(request.RoomID, response.RoomVersion) + } + return err } diff --git a/roomserver/query/query.go b/roomserver/query/query.go index 12d8436ef..224d9fa22 100644 --- a/roomserver/query/query.go +++ b/roomserver/query/query.go @@ -22,6 +22,7 @@ import ( "net/http" "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/common/caching" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/auth" "github.com/matrix-org/dendrite/roomserver/state" @@ -97,7 +98,8 @@ type RoomserverQueryAPIDatabase interface { // RoomserverQueryAPI is an implementation of api.RoomserverQueryAPI type RoomserverQueryAPI struct { - DB RoomserverQueryAPIDatabase + DB RoomserverQueryAPIDatabase + ImmutableCache caching.ImmutableCache } // QueryLatestEventsAndState implements api.RoomserverQueryAPI @@ -896,11 +898,17 @@ func (r *RoomserverQueryAPI) QueryRoomVersionForRoom( request *api.QueryRoomVersionForRoomRequest, response *api.QueryRoomVersionForRoomResponse, ) error { + if roomVersion, ok := r.ImmutableCache.GetRoomVersion(request.RoomID); ok { + response.RoomVersion = roomVersion + return nil + } + roomVersion, err := r.DB.GetRoomVersionForRoom(ctx, request.RoomID) if err != nil { return err } response.RoomVersion = roomVersion + r.ImmutableCache.StoreRoomVersion(request.RoomID, response.RoomVersion) return nil } diff --git a/roomserver/roomserver.go b/roomserver/roomserver.go index 2ffbf67de..fa4f20626 100644 --- a/roomserver/roomserver.go +++ b/roomserver/roomserver.go @@ -48,7 +48,10 @@ func SetupRoomServerComponent( inputAPI.SetupHTTP(http.DefaultServeMux) - queryAPI := query.RoomserverQueryAPI{DB: roomserverDB} + queryAPI := query.RoomserverQueryAPI{ + DB: roomserverDB, + ImmutableCache: base.ImmutableCache, + } queryAPI.SetupHTTP(http.DefaultServeMux)