From 69f8d5a77e9392d5285fcc1c8d68659eac88c67a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Behouba=20Manass=C3=A9?= Date: Mon, 8 Jul 2019 16:06:17 +0300 Subject: [PATCH 01/49] Content-Disposition HTTP header in mediaapi's responses added (#685) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This should fix #628 I also Noticed that GET /_matrix/media/r0/download/{serverName}/{mediaId}/{fileName} is not yet implemented, but it should work for both. Signed-off-by: Kouamé Behouba Manassé behouba@gmail.com --- mediaapi/routing/download.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mediaapi/routing/download.go b/mediaapi/routing/download.go index 9c8f43c44..38c436367 100644 --- a/mediaapi/routing/download.go +++ b/mediaapi/routing/download.go @@ -305,6 +305,10 @@ func (r *downloadRequest) respondFromLocalFile( }).Info("Responding with file") responseFile = file responseMetadata = r.MediaMetadata + + if len(responseMetadata.UploadName) > 0 { + w.Header().Set("Content-Disposition", fmt.Sprintf(`inline; filename*=utf-8"%s"`, responseMetadata.UploadName)) + } } w.Header().Set("Content-Type", string(responseMetadata.ContentType)) From c72517687878267a021641a689fdf73fe6d1c127 Mon Sep 17 00:00:00 2001 From: Anant Prakash Date: Tue, 9 Jul 2019 18:51:33 +0530 Subject: [PATCH 02/49] Fix http responses in validateRecaptcha (#431) fixes #421 --- clientapi/routing/register.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index b1522e82b..243f9dd23 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -243,8 +243,8 @@ func validateRecaptcha( ) *util.JSONResponse { if !cfg.Matrix.RecaptchaEnabled { return &util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.BadJSON("Captcha registration is disabled"), + Code: http.StatusConflict, + JSON: jsonerror.Unknown("Captcha registration is disabled"), } } @@ -279,8 +279,8 @@ func validateRecaptcha( body, err := ioutil.ReadAll(resp.Body) if err != nil { return &util.JSONResponse{ - Code: http.StatusInternalServerError, - JSON: jsonerror.BadJSON("Error in contacting captcha server" + err.Error()), + Code: http.StatusGatewayTimeout, + JSON: jsonerror.Unknown("Error in contacting captcha server" + err.Error()), } } err = json.Unmarshal(body, &r) From 6106ec1399b4dd2dc90ca577cfcb1ec019f05507 Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Wed, 10 Jul 2019 00:13:43 +0800 Subject: [PATCH 03/49] Fix getAliasesForRoomID has no HTTP handler in aliasAPI (#705) This PR adds back the HTTP handler for internal API GetAliasesForRoomID in roomserver, which seemed to be missing. --- roomserver/alias/alias.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/roomserver/alias/alias.go b/roomserver/alias/alias.go index 6a34aacdd..f699e3362 100644 --- a/roomserver/alias/alias.go +++ b/roomserver/alias/alias.go @@ -277,6 +277,20 @@ func (r *RoomserverAliasAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) + servMux.Handle( + roomserverAPI.RoomserverGetAliasesForRoomIDPath, + common.MakeInternalAPI("getAliasesForRoomID", func(req *http.Request) util.JSONResponse { + var request roomserverAPI.GetAliasesForRoomIDRequest + var response roomserverAPI.GetAliasesForRoomIDResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.GetAliasesForRoomID(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) servMux.Handle( roomserverAPI.RoomserverRemoveRoomAliasPath, common.MakeInternalAPI("removeRoomAlias", func(req *http.Request) util.JSONResponse { From 86e65bb22d49ef2e6b5d5724e6e1fd0327f200c6 Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Wed, 10 Jul 2019 00:33:52 +0800 Subject: [PATCH 04/49] Add back missing returns for httputil.LogThenError calls (#730) Signed-off-by: Alex Chen --- clientapi/routing/filter.go | 2 +- clientapi/routing/login.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/clientapi/routing/filter.go b/clientapi/routing/filter.go index 109c55da1..1ed91cd2f 100644 --- a/clientapi/routing/filter.go +++ b/clientapi/routing/filter.go @@ -62,7 +62,7 @@ func GetFilter( filter := gomatrix.Filter{} err = json.Unmarshal(res, &filter) if err != nil { - httputil.LogThenError(req, err) + return httputil.LogThenError(req, err) } return util.JSONResponse{ diff --git a/clientapi/routing/login.go b/clientapi/routing/login.go index abcf7f569..2e2d409f6 100644 --- a/clientapi/routing/login.go +++ b/clientapi/routing/login.go @@ -107,7 +107,7 @@ func Login( token, err := auth.GenerateAccessToken() if err != nil { - httputil.LogThenError(req, err) + return httputil.LogThenError(req, err) } dev, err := getDevice(req.Context(), r, deviceDB, acc, localpart, token) From d4918b83c6f0fe0c6578d53a1c22d02a37c0d8b9 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Fri, 12 Jul 2019 14:23:27 +0100 Subject: [PATCH 05/49] Backup and restore go.mod & go.sum during linting (#735) Every time before sending a PR I like to run ./scripts/build-test-lint.sh to make sure the CI won't complain about anything. The problem is that this script attempts to install golangci-lint, which causes modifications to go.mod/go.sum. This PR backs up and restores those files before and after linting. Ideally instead of this hacky backing up/restoring we'd use go gets -mod=readonly option, but that still modifies go.sum. This will be fixed in go 1.13 apparently. golang/go#30667 --- scripts/find-lint.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/find-lint.sh b/scripts/find-lint.sh index 6511272b2..25b311f94 100755 --- a/scripts/find-lint.sh +++ b/scripts/find-lint.sh @@ -22,7 +22,15 @@ then args="--fast" fi echo "Installing golangci-lint..." + +# Make a backup of go.{mod,sum} first +# TODO: Once go 1.13 is out, use go get's -mod=readonly option +# https://github.com/golang/go/issues/30667 +cp go.mod go.mod.bak && cp go.sum go.sum.bak go get github.com/golangci/golangci-lint/cmd/golangci-lint echo "Looking for lint..." golangci-lint run $args + +# Restore go.{mod,sum} +mv go.mod.bak go.mod && mv go.sum.bak go.sum From 7edf197eccc89cb7368ed31df47c9d15f24d0923 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Fri, 12 Jul 2019 14:29:30 +0100 Subject: [PATCH 06/49] Fix response to /rooms/{roomId}/join v2 (#734) Continuation of #684 but merged-forward. Also did a little code cleanup and added a new, passing test to the testfile. --- clientapi/routing/membership.go | 72 ++++++++++++++++++++++++--------- testfile | 1 + 2 files changed, 53 insertions(+), 20 deletions(-) diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index b308de79a..22e66f452 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -58,27 +58,12 @@ func SendMembership( } } - inviteStored, err := threepid.CheckAndProcessInvite( - req.Context(), device, &body, cfg, queryAPI, accountDB, producer, + inviteStored, jsonErrResp := checkAndProcessThreepid( + req, device, &body, cfg, queryAPI, accountDB, producer, membership, roomID, evTime, ) - if err == threepid.ErrMissingParameter { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.BadJSON(err.Error()), - } - } else if err == threepid.ErrNotTrusted { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.NotTrusted(body.IDServer), - } - } else if err == common.ErrRoomNoExists { - return util.JSONResponse{ - Code: http.StatusNotFound, - JSON: jsonerror.NotFound(err.Error()), - } - } else if err != nil { - return httputil.LogThenError(req, err) + if jsonErrResp != nil { + return *jsonErrResp } // If an invite has been stored on an identity server, it means that a @@ -114,9 +99,18 @@ func SendMembership( return httputil.LogThenError(req, err) } + var returnData interface{} = struct{}{} + + // The join membership requires the room id to be sent in the response + if membership == "join" { + returnData = struct { + RoomID string `json:"room_id"` + }{roomID} + } + return util.JSONResponse{ Code: http.StatusOK, - JSON: struct{}{}, + JSON: returnData, } } @@ -215,3 +209,41 @@ func getMembershipStateKey( return } + +func checkAndProcessThreepid( + req *http.Request, + device *authtypes.Device, + body *threepid.MembershipRequest, + cfg config.Dendrite, + queryAPI roomserverAPI.RoomserverQueryAPI, + accountDB *accounts.Database, + producer *producers.RoomserverProducer, + membership, roomID string, + evTime time.Time, +) (inviteStored bool, errRes *util.JSONResponse) { + + inviteStored, err := threepid.CheckAndProcessInvite( + req.Context(), device, body, cfg, queryAPI, accountDB, producer, + membership, roomID, evTime, + ) + if err == threepid.ErrMissingParameter { + return inviteStored, &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.BadJSON(err.Error()), + } + } else if err == threepid.ErrNotTrusted { + return inviteStored, &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.NotTrusted(body.IDServer), + } + } else if err == common.ErrRoomNoExists { + return inviteStored, &util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound(err.Error()), + } + } else if err != nil { + er := httputil.LogThenError(req, err) + return inviteStored, &er + } + return +} diff --git a/testfile b/testfile index 362df4513..e869be4f0 100644 --- a/testfile +++ b/testfile @@ -42,6 +42,7 @@ POST /join/:room_alias can join a room POST /join/:room_id can join a room POST /join/:room_id can join a room with custom content POST /join/:room_alias can join a room with custom content +POST /rooms/:room_id/join can join a room POST /rooms/:room_id/leave can leave a room POST /rooms/:room_id/invite can send an invite POST /rooms/:room_id/ban can ban a user From f8463063ac4f45e61dfd0b27647bf00a8d05daa1 Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Fri, 12 Jul 2019 21:36:17 +0800 Subject: [PATCH 07/49] Fix #661 appservice can't set aliases in its own namespace (#731) Fixes #661. --- clientapi/routing/directory.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/clientapi/routing/directory.go b/clientapi/routing/directory.go index b23dfbfb6..ab85e86a9 100644 --- a/clientapi/routing/directory.go +++ b/clientapi/routing/directory.go @@ -117,12 +117,16 @@ func SetLocalAlias( // 1. The new method for checking for things matching an AS's namespace // 2. Using an overall Regex object for all AS's just like we did for usernames for _, appservice := range cfg.Derived.ApplicationServices { - if aliasNamespaces, ok := appservice.NamespaceMap["aliases"]; ok { - for _, namespace := range aliasNamespaces { - if namespace.Exclusive && namespace.RegexpObject.MatchString(alias) { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.ASExclusive("Alias is reserved by an application service"), + // Don't prevent AS from creating aliases in its own namespace + // Note that Dendrite uses SenderLocalpart as UserID for AS users + if device.UserID != appservice.SenderLocalpart { + if aliasNamespaces, ok := appservice.NamespaceMap["aliases"]; ok { + for _, namespace := range aliasNamespaces { + if namespace.Exclusive && namespace.RegexpObject.MatchString(alias) { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.ASExclusive("Alias is reserved by an application service"), + } } } } From 29841bed6b1a88787211368e6052a87a658c5714 Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Fri, 12 Jul 2019 22:59:53 +0800 Subject: [PATCH 08/49] Add typing notifications to /sync responses - fixes #635 (#718) This PR adds a new consumer for typing notifications in syncapi. It also brings changes to syncserver.go and some related files so EDUs can better fit in /sync responses. Fixes #635. Fixes #574. --- syncapi/consumers/clientapi.go | 9 +- syncapi/consumers/roomserver.go | 12 +- syncapi/consumers/typingserver.go | 96 ++++++ syncapi/routing/routing.go | 2 +- syncapi/routing/state.go | 4 +- syncapi/storage/account_data_table.go | 4 +- syncapi/storage/output_room_events_table.go | 11 +- syncapi/storage/syncserver.go | 342 ++++++++++++++------ syncapi/sync/notifier.go | 68 ++-- syncapi/sync/notifier_test.go | 130 +++++--- syncapi/sync/request.go | 44 ++- syncapi/sync/requestpool.go | 20 +- syncapi/sync/userstream.go | 18 +- syncapi/syncapi.go | 16 +- syncapi/types/types.go | 43 ++- testfile | 3 + typingserver/api/output.go | 7 +- typingserver/cache/cache.go | 123 +++++-- typingserver/input/input.go | 12 +- 19 files changed, 712 insertions(+), 252 deletions(-) create mode 100644 syncapi/consumers/typingserver.go diff --git a/syncapi/consumers/clientapi.go b/syncapi/consumers/clientapi.go index d05a76920..f0db56427 100644 --- a/syncapi/consumers/clientapi.go +++ b/syncapi/consumers/clientapi.go @@ -22,6 +22,7 @@ import ( "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/sync" + "github.com/matrix-org/dendrite/syncapi/types" log "github.com/sirupsen/logrus" sarama "gopkg.in/Shopify/sarama.v1" ) @@ -29,7 +30,7 @@ import ( // OutputClientDataConsumer consumes events that originated in the client API server. type OutputClientDataConsumer struct { clientAPIConsumer *common.ContinualConsumer - db *storage.SyncServerDatabase + db *storage.SyncServerDatasource notifier *sync.Notifier } @@ -38,7 +39,7 @@ func NewOutputClientDataConsumer( cfg *config.Dendrite, kafkaConsumer sarama.Consumer, n *sync.Notifier, - store *storage.SyncServerDatabase, + store *storage.SyncServerDatasource, ) *OutputClientDataConsumer { consumer := common.ContinualConsumer{ @@ -78,7 +79,7 @@ func (s *OutputClientDataConsumer) onMessage(msg *sarama.ConsumerMessage) error "room_id": output.RoomID, }).Info("received data from client API server") - syncStreamPos, err := s.db.UpsertAccountData( + pduPos, err := s.db.UpsertAccountData( context.TODO(), string(msg.Key), output.RoomID, output.Type, ) if err != nil { @@ -89,7 +90,7 @@ func (s *OutputClientDataConsumer) onMessage(msg *sarama.ConsumerMessage) error }).Panicf("could not save account data") } - s.notifier.OnNewEvent(nil, string(msg.Key), syncStreamPos) + s.notifier.OnNewEvent(nil, "", []string{string(msg.Key)}, types.SyncPosition{PDUPosition: pduPos}) return nil } diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index 1866a9667..e4f1ab460 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -33,7 +33,7 @@ import ( // OutputRoomEventConsumer consumes events that originated in the room server. type OutputRoomEventConsumer struct { roomServerConsumer *common.ContinualConsumer - db *storage.SyncServerDatabase + db *storage.SyncServerDatasource notifier *sync.Notifier query api.RoomserverQueryAPI } @@ -43,7 +43,7 @@ func NewOutputRoomEventConsumer( cfg *config.Dendrite, kafkaConsumer sarama.Consumer, n *sync.Notifier, - store *storage.SyncServerDatabase, + store *storage.SyncServerDatasource, queryAPI api.RoomserverQueryAPI, ) *OutputRoomEventConsumer { @@ -126,7 +126,7 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent( } } - syncStreamPos, err := s.db.WriteEvent( + pduPos, err := s.db.WriteEvent( ctx, &ev, addsStateEvents, @@ -144,7 +144,7 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent( }).Panicf("roomserver output log: write event failure") return nil } - s.notifier.OnNewEvent(&ev, "", types.StreamPosition(syncStreamPos)) + s.notifier.OnNewEvent(&ev, "", nil, types.SyncPosition{PDUPosition: pduPos}) return nil } @@ -152,7 +152,7 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent( func (s *OutputRoomEventConsumer) onNewInviteEvent( ctx context.Context, msg api.OutputNewInviteEvent, ) error { - syncStreamPos, err := s.db.AddInviteEvent(ctx, msg.Event) + pduPos, err := s.db.AddInviteEvent(ctx, msg.Event) if err != nil { // panic rather than continue with an inconsistent database log.WithFields(log.Fields{ @@ -161,7 +161,7 @@ func (s *OutputRoomEventConsumer) onNewInviteEvent( }).Panicf("roomserver output log: write invite failure") return nil } - s.notifier.OnNewEvent(&msg.Event, "", syncStreamPos) + s.notifier.OnNewEvent(&msg.Event, "", nil, types.SyncPosition{PDUPosition: pduPos}) return nil } diff --git a/syncapi/consumers/typingserver.go b/syncapi/consumers/typingserver.go new file mode 100644 index 000000000..5d998a18a --- /dev/null +++ b/syncapi/consumers/typingserver.go @@ -0,0 +1,96 @@ +// Copyright 2019 Alex Chen +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package consumers + +import ( + "encoding/json" + + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/dendrite/syncapi/sync" + "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/dendrite/typingserver/api" + log "github.com/sirupsen/logrus" + sarama "gopkg.in/Shopify/sarama.v1" +) + +// OutputTypingEventConsumer consumes events that originated in the typing server. +type OutputTypingEventConsumer struct { + typingConsumer *common.ContinualConsumer + db *storage.SyncServerDatasource + notifier *sync.Notifier +} + +// NewOutputTypingEventConsumer creates a new OutputTypingEventConsumer. +// Call Start() to begin consuming from the typing server. +func NewOutputTypingEventConsumer( + cfg *config.Dendrite, + kafkaConsumer sarama.Consumer, + n *sync.Notifier, + store *storage.SyncServerDatasource, +) *OutputTypingEventConsumer { + + consumer := common.ContinualConsumer{ + Topic: string(cfg.Kafka.Topics.OutputTypingEvent), + Consumer: kafkaConsumer, + PartitionStore: store, + } + + s := &OutputTypingEventConsumer{ + typingConsumer: &consumer, + db: store, + notifier: n, + } + + consumer.ProcessMessage = s.onMessage + + return s +} + +// Start consuming from typing api +func (s *OutputTypingEventConsumer) Start() error { + s.db.SetTypingTimeoutCallback(func(userID, roomID string, latestSyncPosition int64) { + s.notifier.OnNewEvent(nil, roomID, nil, types.SyncPosition{TypingPosition: latestSyncPosition}) + }) + + return s.typingConsumer.Start() +} + +func (s *OutputTypingEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { + var output api.OutputTypingEvent + if err := json.Unmarshal(msg.Value, &output); err != nil { + // If the message was invalid, log it and move on to the next message in the stream + log.WithError(err).Errorf("typing server output log: message parse failure") + return nil + } + + log.WithFields(log.Fields{ + "room_id": output.Event.RoomID, + "user_id": output.Event.UserID, + "typing": output.Event.Typing, + }).Debug("received data from typing server") + + var typingPos int64 + typingEvent := output.Event + if typingEvent.Typing { + typingPos = s.db.AddTypingUser(typingEvent.UserID, typingEvent.RoomID, output.ExpireTime) + } else { + typingPos = s.db.RemoveTypingUser(typingEvent.UserID, typingEvent.RoomID) + } + + s.notifier.OnNewEvent(nil, output.Event.RoomID, nil, types.SyncPosition{TypingPosition: typingPos}) + return nil +} diff --git a/syncapi/routing/routing.go b/syncapi/routing/routing.go index cbdcfb6bb..0f5019fc3 100644 --- a/syncapi/routing/routing.go +++ b/syncapi/routing/routing.go @@ -34,7 +34,7 @@ const pathPrefixR0 = "/_matrix/client/r0" // Due to Setup being used to call many other functions, a gocyclo nolint is // applied: // nolint: gocyclo -func Setup(apiMux *mux.Router, srp *sync.RequestPool, syncDB *storage.SyncServerDatabase, deviceDB *devices.Database) { +func Setup(apiMux *mux.Router, srp *sync.RequestPool, syncDB *storage.SyncServerDatasource, deviceDB *devices.Database) { r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter() authData := auth.Data{ diff --git a/syncapi/routing/state.go b/syncapi/routing/state.go index 6b98a0b7b..5571a0525 100644 --- a/syncapi/routing/state.go +++ b/syncapi/routing/state.go @@ -40,7 +40,7 @@ type stateEventInStateResp struct { // TODO: Check if the user is in the room. If not, check if the room's history // is publicly visible. Current behaviour is returning an empty array if the // user cannot see the room's history. -func OnIncomingStateRequest(req *http.Request, db *storage.SyncServerDatabase, roomID string) util.JSONResponse { +func OnIncomingStateRequest(req *http.Request, db *storage.SyncServerDatasource, roomID string) util.JSONResponse { // TODO(#287): Auth request and handle the case where the user has left (where // we should return the state at the poin they left) @@ -84,7 +84,7 @@ func OnIncomingStateRequest(req *http.Request, db *storage.SyncServerDatabase, r // /rooms/{roomID}/state/{type}/{statekey} request. It will look in current // state to see if there is an event with that type and state key, if there // is then (by default) we return the content, otherwise a 404. -func OnIncomingStateTypeRequest(req *http.Request, db *storage.SyncServerDatabase, roomID string, evType, stateKey string) util.JSONResponse { +func OnIncomingStateTypeRequest(req *http.Request, db *storage.SyncServerDatasource, roomID string, evType, stateKey string) util.JSONResponse { // TODO(#287): Auth request and handle the case where the user has left (where // we should return the state at the poin they left) diff --git a/syncapi/storage/account_data_table.go b/syncapi/storage/account_data_table.go index d4d74d158..9b73ce7d6 100644 --- a/syncapi/storage/account_data_table.go +++ b/syncapi/storage/account_data_table.go @@ -19,8 +19,6 @@ import ( "database/sql" "github.com/matrix-org/dendrite/common" - - "github.com/matrix-org/dendrite/syncapi/types" ) const accountDataSchema = ` @@ -94,7 +92,7 @@ func (s *accountDataStatements) insertAccountData( func (s *accountDataStatements) selectAccountDataInRange( ctx context.Context, userID string, - oldPos, newPos types.StreamPosition, + oldPos, newPos int64, ) (data map[string][]string, err error) { data = make(map[string][]string) diff --git a/syncapi/storage/output_room_events_table.go b/syncapi/storage/output_room_events_table.go index 035db9882..06df017cb 100644 --- a/syncapi/storage/output_room_events_table.go +++ b/syncapi/storage/output_room_events_table.go @@ -23,7 +23,6 @@ import ( "github.com/lib/pq" "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" log "github.com/sirupsen/logrus" ) @@ -109,11 +108,11 @@ func (s *outputRoomEventsStatements) prepare(db *sql.DB) (err error) { return } -// selectStateInRange returns the state events between the two given stream positions, exclusive of oldPos, inclusive of newPos. +// selectStateInRange returns the state events between the two given PDU stream positions, exclusive of oldPos, inclusive of newPos. // Results are bucketed based on the room ID. If the same state is overwritten multiple times between the // two positions, only the most recent state is returned. func (s *outputRoomEventsStatements) selectStateInRange( - ctx context.Context, txn *sql.Tx, oldPos, newPos types.StreamPosition, + ctx context.Context, txn *sql.Tx, oldPos, newPos int64, ) (map[string]map[string]bool, map[string]streamEvent, error) { stmt := common.TxStmt(txn, s.selectStateInRangeStmt) @@ -171,7 +170,7 @@ func (s *outputRoomEventsStatements) selectStateInRange( eventIDToEvent[ev.EventID()] = streamEvent{ Event: ev, - streamPosition: types.StreamPosition(streamPos), + streamPosition: streamPos, } } @@ -223,7 +222,7 @@ func (s *outputRoomEventsStatements) insertEvent( // RecentEventsInRoom returns the most recent events in the given room, up to a maximum of 'limit'. func (s *outputRoomEventsStatements) selectRecentEvents( ctx context.Context, txn *sql.Tx, - roomID string, fromPos, toPos types.StreamPosition, limit int, + roomID string, fromPos, toPos int64, limit int, ) ([]streamEvent, error) { stmt := common.TxStmt(txn, s.selectRecentEventsStmt) rows, err := stmt.QueryContext(ctx, roomID, fromPos, toPos, limit) @@ -286,7 +285,7 @@ func rowsToStreamEvents(rows *sql.Rows) ([]streamEvent, error) { result = append(result, streamEvent{ Event: ev, - streamPosition: types.StreamPosition(streamPos), + streamPosition: streamPos, transactionID: transactionID, }) } diff --git a/syncapi/storage/syncserver.go b/syncapi/storage/syncserver.go index b0655a0a8..b4d7ccbd2 100644 --- a/syncapi/storage/syncserver.go +++ b/syncapi/storage/syncserver.go @@ -17,7 +17,10 @@ package storage import ( "context" "database/sql" + "encoding/json" "fmt" + "strconv" + "time" "github.com/sirupsen/logrus" @@ -28,6 +31,7 @@ import ( _ "github.com/lib/pq" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/dendrite/typingserver/cache" "github.com/matrix-org/gomatrixserverlib" ) @@ -35,33 +39,35 @@ type stateDelta struct { roomID string stateEvents []gomatrixserverlib.Event membership string - // The stream position of the latest membership event for this user, if applicable. + // The PDU stream position of the latest membership event for this user, if applicable. // Can be 0 if there is no membership event in this delta. - membershipPos types.StreamPosition + membershipPos int64 } -// Same as gomatrixserverlib.Event but also has the stream position for this event. +// Same as gomatrixserverlib.Event but also has the PDU stream position for this event. type streamEvent struct { gomatrixserverlib.Event - streamPosition types.StreamPosition + streamPosition int64 transactionID *api.TransactionID } -// SyncServerDatabase represents a sync server database -type SyncServerDatabase struct { +// SyncServerDatabase represents a sync server datasource which manages +// both the database for PDUs and caches for EDUs. +type SyncServerDatasource struct { db *sql.DB common.PartitionOffsetStatements accountData accountDataStatements events outputRoomEventsStatements roomstate currentRoomStateStatements invites inviteEventsStatements + typingCache *cache.TypingCache } // NewSyncServerDatabase creates a new sync server database -func NewSyncServerDatabase(dataSourceName string) (*SyncServerDatabase, error) { - var d SyncServerDatabase +func NewSyncServerDatasource(dbDataSourceName string) (*SyncServerDatasource, error) { + var d SyncServerDatasource var err error - if d.db, err = sql.Open("postgres", dataSourceName); err != nil { + if d.db, err = sql.Open("postgres", dbDataSourceName); err != nil { return nil, err } if err = d.PartitionOffsetStatements.Prepare(d.db, "syncapi"); err != nil { @@ -79,11 +85,12 @@ func NewSyncServerDatabase(dataSourceName string) (*SyncServerDatabase, error) { if err := d.invites.prepare(d.db); err != nil { return nil, err } + d.typingCache = cache.NewTypingCache() return &d, nil } // AllJoinedUsersInRooms returns a map of room ID to a list of all joined user IDs. -func (d *SyncServerDatabase) AllJoinedUsersInRooms(ctx context.Context) (map[string][]string, error) { +func (d *SyncServerDatasource) AllJoinedUsersInRooms(ctx context.Context) (map[string][]string, error) { return d.roomstate.selectJoinedUsers(ctx) } @@ -92,7 +99,7 @@ func (d *SyncServerDatabase) AllJoinedUsersInRooms(ctx context.Context) (map[str // If an event is not found in the database then it will be omitted from the list. // Returns an error if there was a problem talking with the database. // Does not include any transaction IDs in the returned events. -func (d *SyncServerDatabase) Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.Event, error) { +func (d *SyncServerDatasource) Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.Event, error) { streamEvents, err := d.events.selectEvents(ctx, nil, eventIDs) if err != nil { return nil, err @@ -104,38 +111,38 @@ func (d *SyncServerDatabase) Events(ctx context.Context, eventIDs []string) ([]g } // WriteEvent into the database. It is not safe to call this function from multiple goroutines, as it would create races -// when generating the stream position for this event. Returns the sync stream position for the inserted event. +// when generating the sync stream position for this event. Returns the sync stream position for the inserted event. // Returns an error if there was a problem inserting this event. -func (d *SyncServerDatabase) WriteEvent( +func (d *SyncServerDatasource) WriteEvent( ctx context.Context, ev *gomatrixserverlib.Event, addStateEvents []gomatrixserverlib.Event, addStateEventIDs, removeStateEventIDs []string, transactionID *api.TransactionID, -) (streamPos types.StreamPosition, returnErr error) { +) (pduPosition int64, returnErr error) { returnErr = common.WithTransaction(d.db, func(txn *sql.Tx) error { var err error pos, err := d.events.insertEvent(ctx, txn, ev, addStateEventIDs, removeStateEventIDs, transactionID) if err != nil { return err } - streamPos = types.StreamPosition(pos) + pduPosition = pos if len(addStateEvents) == 0 && len(removeStateEventIDs) == 0 { // Nothing to do, the event may have just been a message event. return nil } - return d.updateRoomState(ctx, txn, removeStateEventIDs, addStateEvents, streamPos) + return d.updateRoomState(ctx, txn, removeStateEventIDs, addStateEvents, pduPosition) }) return } -func (d *SyncServerDatabase) updateRoomState( +func (d *SyncServerDatasource) updateRoomState( ctx context.Context, txn *sql.Tx, removedEventIDs []string, addedEvents []gomatrixserverlib.Event, - streamPos types.StreamPosition, + pduPosition int64, ) error { // remove first, then add, as we do not ever delete state, but do replace state which is a remove followed by an add. for _, eventID := range removedEventIDs { @@ -157,7 +164,7 @@ func (d *SyncServerDatabase) updateRoomState( } membership = &value } - if err := d.roomstate.upsertRoomState(ctx, txn, event, membership, int64(streamPos)); err != nil { + if err := d.roomstate.upsertRoomState(ctx, txn, event, membership, pduPosition); err != nil { return err } } @@ -168,7 +175,7 @@ func (d *SyncServerDatabase) updateRoomState( // GetStateEvent returns the Matrix state event of a given type for a given room with a given state key // If no event could be found, returns nil // If there was an issue during the retrieval, returns an error -func (d *SyncServerDatabase) GetStateEvent( +func (d *SyncServerDatasource) GetStateEvent( ctx context.Context, roomID, evType, stateKey string, ) (*gomatrixserverlib.Event, error) { return d.roomstate.selectStateEvent(ctx, roomID, evType, stateKey) @@ -177,7 +184,7 @@ func (d *SyncServerDatabase) GetStateEvent( // GetStateEventsForRoom fetches the state events for a given room. // Returns an empty slice if no state events could be found for this room. // Returns an error if there was an issue with the retrieval. -func (d *SyncServerDatabase) GetStateEventsForRoom( +func (d *SyncServerDatasource) GetStateEventsForRoom( ctx context.Context, roomID string, ) (stateEvents []gomatrixserverlib.Event, err error) { err = common.WithTransaction(d.db, func(txn *sql.Tx) error { @@ -187,46 +194,49 @@ func (d *SyncServerDatabase) GetStateEventsForRoom( return } -// SyncStreamPosition returns the latest position in the sync stream. Returns 0 if there are no events yet. -func (d *SyncServerDatabase) SyncStreamPosition(ctx context.Context) (types.StreamPosition, error) { - return d.syncStreamPositionTx(ctx, nil) +// SyncPosition returns the latest positions for syncing. +func (d *SyncServerDatasource) SyncPosition(ctx context.Context) (types.SyncPosition, error) { + return d.syncPositionTx(ctx, nil) } -func (d *SyncServerDatabase) syncStreamPositionTx( +func (d *SyncServerDatasource) syncPositionTx( ctx context.Context, txn *sql.Tx, -) (types.StreamPosition, error) { - maxID, err := d.events.selectMaxEventID(ctx, txn) +) (sp types.SyncPosition, err error) { + + maxEventID, err := d.events.selectMaxEventID(ctx, txn) if err != nil { - return 0, err + return sp, err } maxAccountDataID, err := d.accountData.selectMaxAccountDataID(ctx, txn) if err != nil { - return 0, err + return sp, err } - if maxAccountDataID > maxID { - maxID = maxAccountDataID + if maxAccountDataID > maxEventID { + maxEventID = maxAccountDataID } maxInviteID, err := d.invites.selectMaxInviteID(ctx, txn) if err != nil { - return 0, err + return sp, err } - if maxInviteID > maxID { - maxID = maxInviteID + if maxInviteID > maxEventID { + maxEventID = maxInviteID } - return types.StreamPosition(maxID), nil + sp.PDUPosition = maxEventID + + sp.TypingPosition = d.typingCache.GetLatestSyncPosition() + + return } -// IncrementalSync returns all the data needed in order to create an incremental -// sync response for the given user. Events returned will include any client -// transaction IDs associated with the given device. These transaction IDs come -// from when the device sent the event via an API that included a transaction -// ID. -func (d *SyncServerDatabase) IncrementalSync( +// addPDUDeltaToResponse adds all PDU deltas to a sync response. +// IDs of all rooms the user joined are returned so EDU deltas can be added for them. +func (d *SyncServerDatasource) addPDUDeltaToResponse( ctx context.Context, device authtypes.Device, - fromPos, toPos types.StreamPosition, + fromPos, toPos int64, numRecentEventsPerRoom int, -) (*types.Response, error) { + res *types.Response, +) ([]string, error) { txn, err := d.db.BeginTx(ctx, &txReadOnlySnapshot) if err != nil { return nil, err @@ -235,7 +245,7 @@ func (d *SyncServerDatabase) IncrementalSync( defer common.EndTransaction(txn, &succeeded) // Work out which rooms to return in the response. This is done by getting not only the currently - // joined rooms, but also which rooms have membership transitions for this user between the 2 stream positions. + // joined rooms, but also which rooms have membership transitions for this user between the 2 PDU stream positions. // This works out what the 'state' key should be for each room as well as which membership block // to put the room into. deltas, err := d.getStateDeltas(ctx, &device, txn, fromPos, toPos, device.UserID) @@ -243,8 +253,9 @@ func (d *SyncServerDatabase) IncrementalSync( return nil, err } - res := types.NewResponse(toPos) + joinedRoomIDs := make([]string, 0, len(deltas)) for _, delta := range deltas { + joinedRoomIDs = append(joinedRoomIDs, delta.roomID) err = d.addRoomDeltaToResponse(ctx, &device, txn, fromPos, toPos, delta, numRecentEventsPerRoom, res) if err != nil { return nil, err @@ -257,52 +268,151 @@ func (d *SyncServerDatabase) IncrementalSync( } succeeded = true + return joinedRoomIDs, nil +} + +// addTypingDeltaToResponse adds all typing notifications to a sync response +// since the specified position. +func (d *SyncServerDatasource) addTypingDeltaToResponse( + since int64, + joinedRoomIDs []string, + res *types.Response, +) error { + var jr types.JoinResponse + var ok bool + var err error + for _, roomID := range joinedRoomIDs { + if typingUsers, updated := d.typingCache.GetTypingUsersIfUpdatedAfter( + roomID, since, + ); updated { + ev := gomatrixserverlib.ClientEvent{ + Type: gomatrixserverlib.MTyping, + } + ev.Content, err = json.Marshal(map[string]interface{}{ + "user_ids": typingUsers, + }) + if err != nil { + return err + } + + if jr, ok = res.Rooms.Join[roomID]; !ok { + jr = *types.NewJoinResponse() + } + jr.Ephemeral.Events = append(jr.Ephemeral.Events, ev) + res.Rooms.Join[roomID] = jr + } + } + return nil +} + +// addEDUDeltaToResponse adds updates for EDUs of each type since fromPos if +// the positions of that type are not equal in fromPos and toPos. +func (d *SyncServerDatasource) addEDUDeltaToResponse( + fromPos, toPos types.SyncPosition, + joinedRoomIDs []string, + res *types.Response, +) (err error) { + + if fromPos.TypingPosition != toPos.TypingPosition { + err = d.addTypingDeltaToResponse( + fromPos.TypingPosition, joinedRoomIDs, res, + ) + } + + return +} + +// IncrementalSync returns all the data needed in order to create an incremental +// sync response for the given user. Events returned will include any client +// transaction IDs associated with the given device. These transaction IDs come +// from when the device sent the event via an API that included a transaction +// ID. +func (d *SyncServerDatasource) IncrementalSync( + ctx context.Context, + device authtypes.Device, + fromPos, toPos types.SyncPosition, + numRecentEventsPerRoom int, +) (*types.Response, error) { + nextBatchPos := fromPos.WithUpdates(toPos) + res := types.NewResponse(nextBatchPos) + + var joinedRoomIDs []string + var err error + if fromPos.PDUPosition != toPos.PDUPosition { + joinedRoomIDs, err = d.addPDUDeltaToResponse( + ctx, device, fromPos.PDUPosition, toPos.PDUPosition, numRecentEventsPerRoom, res, + ) + } else { + joinedRoomIDs, err = d.roomstate.selectRoomIDsWithMembership( + ctx, nil, device.UserID, "join", + ) + } + if err != nil { + return nil, err + } + + err = d.addEDUDeltaToResponse( + fromPos, toPos, joinedRoomIDs, res, + ) + if err != nil { + return nil, err + } + return res, nil } -// CompleteSync a complete /sync API response for the given user. -func (d *SyncServerDatabase) CompleteSync( - ctx context.Context, userID string, numRecentEventsPerRoom int, -) (*types.Response, error) { +// getResponseWithPDUsForCompleteSync creates a response and adds all PDUs needed +// to it. It returns toPos and joinedRoomIDs for use of adding EDUs. +func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( + ctx context.Context, + userID string, + numRecentEventsPerRoom int, +) ( + res *types.Response, + toPos types.SyncPosition, + joinedRoomIDs []string, + err error, +) { // This needs to be all done in a transaction as we need to do multiple SELECTs, and we need to have - // a consistent view of the database throughout. This includes extracting the sync stream position. + // a consistent view of the database throughout. This includes extracting the sync position. // This does have the unfortunate side-effect that all the matrixy logic resides in this function, // but it's better to not hide the fact that this is being done in a transaction. txn, err := d.db.BeginTx(ctx, &txReadOnlySnapshot) if err != nil { - return nil, err + return } var succeeded bool defer common.EndTransaction(txn, &succeeded) - // Get the current stream position which we will base the sync response on. - pos, err := d.syncStreamPositionTx(ctx, txn) + // Get the current sync position which we will base the sync response on. + toPos, err = d.syncPositionTx(ctx, txn) if err != nil { - return nil, err + return } + res = types.NewResponse(toPos) + // Extract room state and recent events for all rooms the user is joined to. - roomIDs, err := d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, "join") + joinedRoomIDs, err = d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, "join") if err != nil { - return nil, err + return } // Build up a /sync response. Add joined rooms. - res := types.NewResponse(pos) - for _, roomID := range roomIDs { + for _, roomID := range joinedRoomIDs { var stateEvents []gomatrixserverlib.Event stateEvents, err = d.roomstate.selectCurrentState(ctx, txn, roomID) if err != nil { - return nil, err + return } // TODO: When filters are added, we may need to call this multiple times to get enough events. // See: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L316 var recentStreamEvents []streamEvent recentStreamEvents, err = d.events.selectRecentEvents( - ctx, txn, roomID, types.StreamPosition(0), pos, numRecentEventsPerRoom, + ctx, txn, roomID, 0, toPos.PDUPosition, numRecentEventsPerRoom, ) if err != nil { - return nil, err + return } // We don't include a device here as we don't need to send down @@ -311,10 +421,12 @@ func (d *SyncServerDatabase) CompleteSync( stateEvents = removeDuplicates(stateEvents, recentEvents) jr := types.NewJoinResponse() - if prevBatch := recentStreamEvents[0].streamPosition - 1; prevBatch > 0 { - jr.Timeline.PrevBatch = types.StreamPosition(prevBatch).String() + if prevPDUPos := recentStreamEvents[0].streamPosition - 1; prevPDUPos > 0 { + // Use the short form of batch token for prev_batch + jr.Timeline.PrevBatch = strconv.FormatInt(prevPDUPos, 10) } else { - jr.Timeline.PrevBatch = types.StreamPosition(1).String() + // Use the short form of batch token for prev_batch + jr.Timeline.PrevBatch = "1" } jr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = true @@ -322,12 +434,34 @@ func (d *SyncServerDatabase) CompleteSync( res.Rooms.Join[roomID] = *jr } - if err = d.addInvitesToResponse(ctx, txn, userID, 0, pos, res); err != nil { - return nil, err + if err = d.addInvitesToResponse(ctx, txn, userID, 0, toPos.PDUPosition, res); err != nil { + return } succeeded = true - return res, err + return res, toPos, joinedRoomIDs, err +} + +// CompleteSync returns a complete /sync API response for the given user. +func (d *SyncServerDatasource) CompleteSync( + ctx context.Context, userID string, numRecentEventsPerRoom int, +) (*types.Response, error) { + res, toPos, joinedRoomIDs, err := d.getResponseWithPDUsForCompleteSync( + ctx, userID, numRecentEventsPerRoom, + ) + if err != nil { + return nil, err + } + + // Use a zero value SyncPosition for fromPos so all EDU states are added. + err = d.addEDUDeltaToResponse( + types.SyncPosition{}, toPos, joinedRoomIDs, res, + ) + if err != nil { + return nil, err + } + + return res, nil } var txReadOnlySnapshot = sql.TxOptions{ @@ -345,8 +479,8 @@ var txReadOnlySnapshot = sql.TxOptions{ // Returns a map following the format data[roomID] = []dataTypes // If no data is retrieved, returns an empty map // If there was an issue with the retrieval, returns an error -func (d *SyncServerDatabase) GetAccountDataInRange( - ctx context.Context, userID string, oldPos, newPos types.StreamPosition, +func (d *SyncServerDatasource) GetAccountDataInRange( + ctx context.Context, userID string, oldPos, newPos int64, ) (map[string][]string, error) { return d.accountData.selectAccountDataInRange(ctx, userID, oldPos, newPos) } @@ -357,26 +491,24 @@ func (d *SyncServerDatabase) GetAccountDataInRange( // If no data with the given type, user ID and room ID exists in the database, // creates a new row, else update the existing one // Returns an error if there was an issue with the upsert -func (d *SyncServerDatabase) UpsertAccountData( +func (d *SyncServerDatasource) UpsertAccountData( ctx context.Context, userID, roomID, dataType string, -) (types.StreamPosition, error) { - pos, err := d.accountData.insertAccountData(ctx, userID, roomID, dataType) - return types.StreamPosition(pos), err +) (int64, error) { + return d.accountData.insertAccountData(ctx, userID, roomID, dataType) } // AddInviteEvent stores a new invite event for a user. // If the invite was successfully stored this returns the stream ID it was stored at. // Returns an error if there was a problem communicating with the database. -func (d *SyncServerDatabase) AddInviteEvent( +func (d *SyncServerDatasource) AddInviteEvent( ctx context.Context, inviteEvent gomatrixserverlib.Event, -) (types.StreamPosition, error) { - pos, err := d.invites.insertInviteEvent(ctx, inviteEvent) - return types.StreamPosition(pos), err +) (int64, error) { + return d.invites.insertInviteEvent(ctx, inviteEvent) } // RetireInviteEvent removes an old invite event from the database. // Returns an error if there was a problem communicating with the database. -func (d *SyncServerDatabase) RetireInviteEvent( +func (d *SyncServerDatasource) RetireInviteEvent( ctx context.Context, inviteEventID string, ) error { // TODO: Record that invite has been retired in a stream so that we can @@ -385,10 +517,30 @@ func (d *SyncServerDatabase) RetireInviteEvent( return err } -func (d *SyncServerDatabase) addInvitesToResponse( +func (d *SyncServerDatasource) SetTypingTimeoutCallback(fn cache.TimeoutCallbackFn) { + d.typingCache.SetTimeoutCallback(fn) +} + +// AddTypingUser adds a typing user to the typing cache. +// Returns the newly calculated sync position for typing notifications. +func (d *SyncServerDatasource) AddTypingUser( + userID, roomID string, expireTime *time.Time, +) int64 { + return d.typingCache.AddTypingUser(userID, roomID, expireTime) +} + +// RemoveTypingUser removes a typing user from the typing cache. +// Returns the newly calculated sync position for typing notifications. +func (d *SyncServerDatasource) RemoveTypingUser( + userID, roomID string, +) int64 { + return d.typingCache.RemoveUser(userID, roomID) +} + +func (d *SyncServerDatasource) addInvitesToResponse( ctx context.Context, txn *sql.Tx, userID string, - fromPos, toPos types.StreamPosition, + fromPos, toPos int64, res *types.Response, ) error { invites, err := d.invites.selectInviteEventsInRange( @@ -409,11 +561,11 @@ func (d *SyncServerDatabase) addInvitesToResponse( } // addRoomDeltaToResponse adds a room state delta to a sync response -func (d *SyncServerDatabase) addRoomDeltaToResponse( +func (d *SyncServerDatasource) addRoomDeltaToResponse( ctx context.Context, device *authtypes.Device, txn *sql.Tx, - fromPos, toPos types.StreamPosition, + fromPos, toPos int64, delta stateDelta, numRecentEventsPerRoom int, res *types.Response, @@ -445,10 +597,12 @@ func (d *SyncServerDatabase) addRoomDeltaToResponse( switch delta.membership { case "join": jr := types.NewJoinResponse() - if prevBatch := recentStreamEvents[0].streamPosition - 1; prevBatch > 0 { - jr.Timeline.PrevBatch = types.StreamPosition(prevBatch).String() + if prevPDUPos := recentStreamEvents[0].streamPosition - 1; prevPDUPos > 0 { + // Use the short form of batch token for prev_batch + jr.Timeline.PrevBatch = strconv.FormatInt(prevPDUPos, 10) } else { - jr.Timeline.PrevBatch = types.StreamPosition(1).String() + // Use the short form of batch token for prev_batch + jr.Timeline.PrevBatch = "1" } jr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true @@ -460,10 +614,12 @@ func (d *SyncServerDatabase) addRoomDeltaToResponse( // TODO: recentEvents may contain events that this user is not allowed to see because they are // no longer in the room. lr := types.NewLeaveResponse() - if prevBatch := recentStreamEvents[0].streamPosition - 1; prevBatch > 0 { - lr.Timeline.PrevBatch = types.StreamPosition(prevBatch).String() + if prevPDUPos := recentStreamEvents[0].streamPosition - 1; prevPDUPos > 0 { + // Use the short form of batch token for prev_batch + lr.Timeline.PrevBatch = strconv.FormatInt(prevPDUPos, 10) } else { - lr.Timeline.PrevBatch = types.StreamPosition(1).String() + // Use the short form of batch token for prev_batch + lr.Timeline.PrevBatch = "1" } lr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync) lr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true @@ -476,7 +632,7 @@ func (d *SyncServerDatabase) addRoomDeltaToResponse( // fetchStateEvents converts the set of event IDs into a set of events. It will fetch any which are missing from the database. // Returns a map of room ID to list of events. -func (d *SyncServerDatabase) fetchStateEvents( +func (d *SyncServerDatasource) fetchStateEvents( ctx context.Context, txn *sql.Tx, roomIDToEventIDSet map[string]map[string]bool, eventIDToEvent map[string]streamEvent, @@ -521,7 +677,7 @@ func (d *SyncServerDatabase) fetchStateEvents( return stateBetween, nil } -func (d *SyncServerDatabase) fetchMissingStateEvents( +func (d *SyncServerDatasource) fetchMissingStateEvents( ctx context.Context, txn *sql.Tx, eventIDs []string, ) ([]streamEvent, error) { // Fetch from the events table first so we pick up the stream ID for the @@ -560,9 +716,9 @@ func (d *SyncServerDatabase) fetchMissingStateEvents( return events, nil } -func (d *SyncServerDatabase) getStateDeltas( +func (d *SyncServerDatasource) getStateDeltas( ctx context.Context, device *authtypes.Device, txn *sql.Tx, - fromPos, toPos types.StreamPosition, userID string, + fromPos, toPos int64, userID string, ) ([]stateDelta, error) { // Implement membership change algorithm: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L821 // - Get membership list changes for this user in this sync response @@ -601,7 +757,7 @@ func (d *SyncServerDatabase) getStateDeltas( } s := make([]streamEvent, len(allState)) for i := 0; i < len(s); i++ { - s[i] = streamEvent{Event: allState[i], streamPosition: types.StreamPosition(0)} + s[i] = streamEvent{Event: allState[i], streamPosition: 0} } state[roomID] = s continue // we'll add this room in when we do joined rooms diff --git a/syncapi/sync/notifier.go b/syncapi/sync/notifier.go index 5ed701d8e..30ac3a2e5 100644 --- a/syncapi/sync/notifier.go +++ b/syncapi/sync/notifier.go @@ -26,7 +26,7 @@ import ( ) // Notifier will wake up sleeping requests when there is some new data. -// It does not tell requests what that data is, only the stream position which +// It does not tell requests what that data is, only the sync position which // they can use to get at it. This is done to prevent races whereby we tell the caller // the event, but the token has already advanced by the time they fetch it, resulting // in missed events. @@ -35,18 +35,18 @@ type Notifier struct { roomIDToJoinedUsers map[string]userIDSet // Protects currPos and userStreams. streamLock *sync.Mutex - // The latest sync stream position - currPos types.StreamPosition + // The latest sync position + currPos types.SyncPosition // A map of user_id => UserStream which can be used to wake a given user's /sync request. userStreams map[string]*UserStream // The last time we cleaned out stale entries from the userStreams map lastCleanUpTime time.Time } -// NewNotifier creates a new notifier set to the given stream position. +// NewNotifier creates a new notifier set to the given sync position. // In order for this to be of any use, the Notifier needs to be told all rooms and // the joined users within each of them by calling Notifier.Load(*storage.SyncServerDatabase). -func NewNotifier(pos types.StreamPosition) *Notifier { +func NewNotifier(pos types.SyncPosition) *Notifier { return &Notifier{ currPos: pos, roomIDToJoinedUsers: make(map[string]userIDSet), @@ -58,20 +58,30 @@ func NewNotifier(pos types.StreamPosition) *Notifier { // OnNewEvent is called when a new event is received from the room server. Must only be // called from a single goroutine, to avoid races between updates which could set the -// current position in the stream incorrectly. -// Can be called either with a *gomatrixserverlib.Event, or with an user ID -func (n *Notifier) OnNewEvent(ev *gomatrixserverlib.Event, userID string, pos types.StreamPosition) { +// current sync position incorrectly. +// Chooses which user sync streams to update by a provided *gomatrixserverlib.Event +// (based on the users in the event's room), +// a roomID directly, or a list of user IDs, prioritised by parameter ordering. +// posUpdate contains the latest position(s) for one or more types of events. +// If a position in posUpdate is 0, it means no updates are available of that type. +// Typically a consumer supplies a posUpdate with the latest sync position for the +// event type it handles, leaving other fields as 0. +func (n *Notifier) OnNewEvent( + ev *gomatrixserverlib.Event, roomID string, userIDs []string, + posUpdate types.SyncPosition, +) { // update the current position then notify relevant /sync streams. // This needs to be done PRIOR to waking up users as they will read this value. n.streamLock.Lock() defer n.streamLock.Unlock() - n.currPos = pos + latestPos := n.currPos.WithUpdates(posUpdate) + n.currPos = latestPos n.removeEmptyUserStreams() if ev != nil { // Map this event's room_id to a list of joined users, and wake them up. - userIDs := n.joinedUsers(ev.RoomID()) + usersToNotify := n.joinedUsers(ev.RoomID()) // If this is an invite, also add in the invitee to this list. if ev.Type() == "m.room.member" && ev.StateKey() != nil { targetUserID := *ev.StateKey() @@ -84,11 +94,11 @@ func (n *Notifier) OnNewEvent(ev *gomatrixserverlib.Event, userID string, pos ty // Keep the joined user map up-to-date switch membership { case "invite": - userIDs = append(userIDs, targetUserID) + usersToNotify = append(usersToNotify, targetUserID) case "join": // Manually append the new user's ID so they get notified // along all members in the room - userIDs = append(userIDs, targetUserID) + usersToNotify = append(usersToNotify, targetUserID) n.addJoinedUser(ev.RoomID(), targetUserID) case "leave": fallthrough @@ -98,11 +108,15 @@ func (n *Notifier) OnNewEvent(ev *gomatrixserverlib.Event, userID string, pos ty } } - for _, toNotifyUserID := range userIDs { - n.wakeupUser(toNotifyUserID, pos) - } - } else if len(userID) > 0 { - n.wakeupUser(userID, pos) + n.wakeupUsers(usersToNotify, latestPos) + } else if roomID != "" { + n.wakeupUsers(n.joinedUsers(roomID), latestPos) + } else if len(userIDs) > 0 { + n.wakeupUsers(userIDs, latestPos) + } else { + log.WithFields(log.Fields{ + "posUpdate": posUpdate.String, + }).Warn("Notifier.OnNewEvent called but caller supplied no user to wake up") } } @@ -127,7 +141,7 @@ func (n *Notifier) GetListener(req syncRequest) UserStreamListener { } // Load the membership states required to notify users correctly. -func (n *Notifier) Load(ctx context.Context, db *storage.SyncServerDatabase) error { +func (n *Notifier) Load(ctx context.Context, db *storage.SyncServerDatasource) error { roomToUsers, err := db.AllJoinedUsersInRooms(ctx) if err != nil { return err @@ -136,8 +150,11 @@ func (n *Notifier) Load(ctx context.Context, db *storage.SyncServerDatabase) err return nil } -// CurrentPosition returns the current stream position -func (n *Notifier) CurrentPosition() types.StreamPosition { +// CurrentPosition returns the current sync position +func (n *Notifier) CurrentPosition() types.SyncPosition { + n.streamLock.Lock() + defer n.streamLock.Unlock() + return n.currPos } @@ -156,12 +173,13 @@ func (n *Notifier) setUsersJoinedToRooms(roomIDToUserIDs map[string][]string) { } } -func (n *Notifier) wakeupUser(userID string, newPos types.StreamPosition) { - stream := n.fetchUserStream(userID, false) - if stream == nil { - return +func (n *Notifier) wakeupUsers(userIDs []string, newPos types.SyncPosition) { + for _, userID := range userIDs { + stream := n.fetchUserStream(userID, false) + if stream != nil { + stream.Broadcast(newPos) // wake up all goroutines Wait()ing on this stream + } } - stream.Broadcast(newPos) // wakeup all goroutines Wait()ing on this stream } // fetchUserStream retrieves a stream unique to the given user. If makeIfNotExists is true, diff --git a/syncapi/sync/notifier_test.go b/syncapi/sync/notifier_test.go index 4fa543936..904315e9f 100644 --- a/syncapi/sync/notifier_test.go +++ b/syncapi/sync/notifier_test.go @@ -32,19 +32,40 @@ var ( randomMessageEvent gomatrixserverlib.Event aliceInviteBobEvent gomatrixserverlib.Event bobLeaveEvent gomatrixserverlib.Event + syncPositionVeryOld types.SyncPosition + syncPositionBefore types.SyncPosition + syncPositionAfter types.SyncPosition + syncPositionNewEDU types.SyncPosition + syncPositionAfter2 types.SyncPosition ) var ( - streamPositionVeryOld = types.StreamPosition(5) - streamPositionBefore = types.StreamPosition(11) - streamPositionAfter = types.StreamPosition(12) - streamPositionAfter2 = types.StreamPosition(13) - roomID = "!test:localhost" - alice = "@alice:localhost" - bob = "@bob:localhost" + roomID = "!test:localhost" + alice = "@alice:localhost" + bob = "@bob:localhost" ) func init() { + baseSyncPos := types.SyncPosition{ + PDUPosition: 0, + TypingPosition: 0, + } + + syncPositionVeryOld = baseSyncPos + syncPositionVeryOld.PDUPosition = 5 + + syncPositionBefore = baseSyncPos + syncPositionBefore.PDUPosition = 11 + + syncPositionAfter = baseSyncPos + syncPositionAfter.PDUPosition = 12 + + syncPositionNewEDU = syncPositionAfter + syncPositionNewEDU.TypingPosition = 1 + + syncPositionAfter2 = baseSyncPos + syncPositionAfter2.PDUPosition = 13 + var err error randomMessageEvent, err = gomatrixserverlib.NewEventFromTrustedJSON([]byte(`{ "type": "m.room.message", @@ -92,19 +113,19 @@ func init() { // Test that the current position is returned if a request is already behind. func TestImmediateNotification(t *testing.T) { - n := NewNotifier(streamPositionBefore) - pos, err := waitForEvents(n, newTestSyncRequest(alice, streamPositionVeryOld)) + n := NewNotifier(syncPositionBefore) + pos, err := waitForEvents(n, newTestSyncRequest(alice, syncPositionVeryOld)) if err != nil { t.Fatalf("TestImmediateNotification error: %s", err) } - if pos != streamPositionBefore { - t.Fatalf("TestImmediateNotification want %d, got %d", streamPositionBefore, pos) + if pos != syncPositionBefore { + t.Fatalf("TestImmediateNotification want %d, got %d", syncPositionBefore, pos) } } // Test that new events to a joined room unblocks the request. func TestNewEventAndJoinedToRoom(t *testing.T) { - n := NewNotifier(streamPositionBefore) + n := NewNotifier(syncPositionBefore) n.setUsersJoinedToRooms(map[string][]string{ roomID: {alice, bob}, }) @@ -112,12 +133,12 @@ func TestNewEventAndJoinedToRoom(t *testing.T) { var wg sync.WaitGroup wg.Add(1) go func() { - pos, err := waitForEvents(n, newTestSyncRequest(bob, streamPositionBefore)) + pos, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionBefore)) if err != nil { t.Errorf("TestNewEventAndJoinedToRoom error: %s", err) } - if pos != streamPositionAfter { - t.Errorf("TestNewEventAndJoinedToRoom want %d, got %d", streamPositionAfter, pos) + if pos != syncPositionAfter { + t.Errorf("TestNewEventAndJoinedToRoom want %d, got %d", syncPositionAfter, pos) } wg.Done() }() @@ -125,14 +146,14 @@ func TestNewEventAndJoinedToRoom(t *testing.T) { stream := n.fetchUserStream(bob, true) waitForBlocking(stream, 1) - n.OnNewEvent(&randomMessageEvent, "", streamPositionAfter) + n.OnNewEvent(&randomMessageEvent, "", nil, syncPositionAfter) wg.Wait() } // Test that an invite unblocks the request func TestNewInviteEventForUser(t *testing.T) { - n := NewNotifier(streamPositionBefore) + n := NewNotifier(syncPositionBefore) n.setUsersJoinedToRooms(map[string][]string{ roomID: {alice, bob}, }) @@ -140,12 +161,12 @@ func TestNewInviteEventForUser(t *testing.T) { var wg sync.WaitGroup wg.Add(1) go func() { - pos, err := waitForEvents(n, newTestSyncRequest(bob, streamPositionBefore)) + pos, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionBefore)) if err != nil { t.Errorf("TestNewInviteEventForUser error: %s", err) } - if pos != streamPositionAfter { - t.Errorf("TestNewInviteEventForUser want %d, got %d", streamPositionAfter, pos) + if pos != syncPositionAfter { + t.Errorf("TestNewInviteEventForUser want %d, got %d", syncPositionAfter, pos) } wg.Done() }() @@ -153,14 +174,42 @@ func TestNewInviteEventForUser(t *testing.T) { stream := n.fetchUserStream(bob, true) waitForBlocking(stream, 1) - n.OnNewEvent(&aliceInviteBobEvent, "", streamPositionAfter) + n.OnNewEvent(&aliceInviteBobEvent, "", nil, syncPositionAfter) + + wg.Wait() +} + +// Test an EDU-only update wakes up the request. +func TestEDUWakeup(t *testing.T) { + n := NewNotifier(syncPositionAfter) + n.setUsersJoinedToRooms(map[string][]string{ + roomID: {alice, bob}, + }) + + var wg sync.WaitGroup + wg.Add(1) + go func() { + pos, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionAfter)) + if err != nil { + t.Errorf("TestNewInviteEventForUser error: %s", err) + } + if pos != syncPositionNewEDU { + t.Errorf("TestNewInviteEventForUser want %d, got %d", syncPositionNewEDU, pos) + } + wg.Done() + }() + + stream := n.fetchUserStream(bob, true) + waitForBlocking(stream, 1) + + n.OnNewEvent(&aliceInviteBobEvent, "", nil, syncPositionNewEDU) wg.Wait() } // Test that all blocked requests get woken up on a new event. func TestMultipleRequestWakeup(t *testing.T) { - n := NewNotifier(streamPositionBefore) + n := NewNotifier(syncPositionBefore) n.setUsersJoinedToRooms(map[string][]string{ roomID: {alice, bob}, }) @@ -168,12 +217,12 @@ func TestMultipleRequestWakeup(t *testing.T) { var wg sync.WaitGroup wg.Add(3) poll := func() { - pos, err := waitForEvents(n, newTestSyncRequest(bob, streamPositionBefore)) + pos, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionBefore)) if err != nil { t.Errorf("TestMultipleRequestWakeup error: %s", err) } - if pos != streamPositionAfter { - t.Errorf("TestMultipleRequestWakeup want %d, got %d", streamPositionAfter, pos) + if pos != syncPositionAfter { + t.Errorf("TestMultipleRequestWakeup want %d, got %d", syncPositionAfter, pos) } wg.Done() } @@ -184,7 +233,7 @@ func TestMultipleRequestWakeup(t *testing.T) { stream := n.fetchUserStream(bob, true) waitForBlocking(stream, 3) - n.OnNewEvent(&randomMessageEvent, "", streamPositionAfter) + n.OnNewEvent(&randomMessageEvent, "", nil, syncPositionAfter) wg.Wait() @@ -198,7 +247,7 @@ func TestMultipleRequestWakeup(t *testing.T) { func TestNewEventAndWasPreviouslyJoinedToRoom(t *testing.T) { // listen as bob. Make bob leave room. Make alice send event to room. // Make sure alice gets woken up only and not bob as well. - n := NewNotifier(streamPositionBefore) + n := NewNotifier(syncPositionBefore) n.setUsersJoinedToRooms(map[string][]string{ roomID: {alice, bob}, }) @@ -208,18 +257,18 @@ func TestNewEventAndWasPreviouslyJoinedToRoom(t *testing.T) { // Make bob leave the room leaveWG.Add(1) go func() { - pos, err := waitForEvents(n, newTestSyncRequest(bob, streamPositionBefore)) + pos, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionBefore)) if err != nil { t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom error: %s", err) } - if pos != streamPositionAfter { - t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom want %d, got %d", streamPositionAfter, pos) + if pos != syncPositionAfter { + t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom want %d, got %d", syncPositionAfter, pos) } leaveWG.Done() }() bobStream := n.fetchUserStream(bob, true) waitForBlocking(bobStream, 1) - n.OnNewEvent(&bobLeaveEvent, "", streamPositionAfter) + n.OnNewEvent(&bobLeaveEvent, "", nil, syncPositionAfter) leaveWG.Wait() // send an event into the room. Make sure alice gets it. Bob should not. @@ -227,19 +276,19 @@ func TestNewEventAndWasPreviouslyJoinedToRoom(t *testing.T) { aliceStream := n.fetchUserStream(alice, true) aliceWG.Add(1) go func() { - pos, err := waitForEvents(n, newTestSyncRequest(alice, streamPositionAfter)) + pos, err := waitForEvents(n, newTestSyncRequest(alice, syncPositionAfter)) if err != nil { t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom error: %s", err) } - if pos != streamPositionAfter2 { - t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom want %d, got %d", streamPositionAfter2, pos) + if pos != syncPositionAfter2 { + t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom want %d, got %d", syncPositionAfter2, pos) } aliceWG.Done() }() go func() { // this should timeout with an error (but the main goroutine won't wait for the timeout explicitly) - _, err := waitForEvents(n, newTestSyncRequest(bob, streamPositionAfter)) + _, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionAfter)) if err == nil { t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom expect error but got nil") } @@ -248,7 +297,7 @@ func TestNewEventAndWasPreviouslyJoinedToRoom(t *testing.T) { waitForBlocking(aliceStream, 1) waitForBlocking(bobStream, 1) - n.OnNewEvent(&randomMessageEvent, "", streamPositionAfter2) + n.OnNewEvent(&randomMessageEvent, "", nil, syncPositionAfter2) aliceWG.Wait() // it's possible that at this point alice has been informed and bob is about to be informed, so wait @@ -256,18 +305,17 @@ func TestNewEventAndWasPreviouslyJoinedToRoom(t *testing.T) { time.Sleep(1 * time.Millisecond) } -// same as Notifier.WaitForEvents but with a timeout. -func waitForEvents(n *Notifier, req syncRequest) (types.StreamPosition, error) { +func waitForEvents(n *Notifier, req syncRequest) (types.SyncPosition, error) { listener := n.GetListener(req) defer listener.Close() select { case <-time.After(5 * time.Second): - return types.StreamPosition(0), fmt.Errorf( + return types.SyncPosition{}, fmt.Errorf( "waitForEvents timed out waiting for %s (pos=%d)", req.device.UserID, req.since, ) case <-listener.GetNotifyChannel(*req.since): - p := listener.GetStreamPosition() + p := listener.GetSyncPosition() return p, nil } } @@ -280,7 +328,7 @@ func waitForBlocking(s *UserStream, numBlocking uint) { } } -func newTestSyncRequest(userID string, since types.StreamPosition) syncRequest { +func newTestSyncRequest(userID string, since types.SyncPosition) syncRequest { return syncRequest{ device: authtypes.Device{UserID: userID}, timeout: 1 * time.Minute, diff --git a/syncapi/sync/request.go b/syncapi/sync/request.go index 35a15f6f9..a5d2f60f4 100644 --- a/syncapi/sync/request.go +++ b/syncapi/sync/request.go @@ -16,8 +16,10 @@ package sync import ( "context" + "errors" "net/http" "strconv" + "strings" "time" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" @@ -36,7 +38,7 @@ type syncRequest struct { device authtypes.Device limit int timeout time.Duration - since *types.StreamPosition // nil means that no since token was supplied + since *types.SyncPosition // nil means that no since token was supplied wantFullState bool log *log.Entry } @@ -73,15 +75,41 @@ func getTimeout(timeoutMS string) time.Duration { } // getSyncStreamPosition tries to parse a 'since' token taken from the API to a -// stream position. If the string is empty then (nil, nil) is returned. -func getSyncStreamPosition(since string) (*types.StreamPosition, error) { +// types.SyncPosition. If the string is empty then (nil, nil) is returned. +// There are two forms of tokens: The full length form containing all PDU and EDU +// positions separated by "_", and the short form containing only the PDU +// position. Short form can be used for, e.g., `prev_batch` tokens. +func getSyncStreamPosition(since string) (*types.SyncPosition, error) { if since == "" { return nil, nil } - i, err := strconv.Atoi(since) - if err != nil { - return nil, err + + posStrings := strings.Split(since, "_") + if len(posStrings) != 2 && len(posStrings) != 1 { + // A token can either be full length or short (PDU-only). + return nil, errors.New("malformed batch token") + } + + positions := make([]int64, len(posStrings)) + for i, posString := range posStrings { + pos, err := strconv.ParseInt(posString, 10, 64) + if err != nil { + return nil, err + } + positions[i] = pos + } + + if len(positions) == 2 { + // Full length token; construct SyncPosition with every entry in + // `positions`. These entries must have the same order with the fields + // in struct SyncPosition, so we disable the govet check below. + return &types.SyncPosition{ //nolint:govet + positions[0], positions[1], + }, nil + } else { + // Token with PDU position only + return &types.SyncPosition{ + PDUPosition: positions[0], + }, nil } - token := types.StreamPosition(i) - return &token, nil } diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index 89137eb59..a6ec6bd92 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -31,13 +31,13 @@ import ( // RequestPool manages HTTP long-poll connections for /sync type RequestPool struct { - db *storage.SyncServerDatabase + db *storage.SyncServerDatasource accountDB *accounts.Database notifier *Notifier } // NewRequestPool makes a new RequestPool -func NewRequestPool(db *storage.SyncServerDatabase, n *Notifier, adb *accounts.Database) *RequestPool { +func NewRequestPool(db *storage.SyncServerDatasource, n *Notifier, adb *accounts.Database) *RequestPool { return &RequestPool{db, adb, n} } @@ -92,11 +92,13 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *authtype // respond with, so we skip the return an go back to waiting for content to // be sent down or the request timing out. var hasTimedOut bool + sincePos := *syncReq.since for { select { // Wait for notifier to wake us up - case <-userStreamListener.GetNotifyChannel(currPos): - currPos = userStreamListener.GetStreamPosition() + case <-userStreamListener.GetNotifyChannel(sincePos): + currPos = userStreamListener.GetSyncPosition() + sincePos = currPos // Or for timeout to expire case <-timer.C: // We just need to ensure we get out of the select after reaching the @@ -128,24 +130,24 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *authtype } } -func (rp *RequestPool) currentSyncForUser(req syncRequest, currentPos types.StreamPosition) (res *types.Response, err error) { +func (rp *RequestPool) currentSyncForUser(req syncRequest, latestPos types.SyncPosition) (res *types.Response, err error) { // TODO: handle ignored users if req.since == nil { res, err = rp.db.CompleteSync(req.ctx, req.device.UserID, req.limit) } else { - res, err = rp.db.IncrementalSync(req.ctx, req.device, *req.since, currentPos, req.limit) + res, err = rp.db.IncrementalSync(req.ctx, req.device, *req.since, latestPos, req.limit) } if err != nil { return } - res, err = rp.appendAccountData(res, req.device.UserID, req, currentPos) + res, err = rp.appendAccountData(res, req.device.UserID, req, latestPos.PDUPosition) return } func (rp *RequestPool) appendAccountData( - data *types.Response, userID string, req syncRequest, currentPos types.StreamPosition, + data *types.Response, userID string, req syncRequest, currentPos int64, ) (*types.Response, error) { // TODO: Account data doesn't have a sync position of its own, meaning that // account data might be sent multiple time to the client if multiple account @@ -179,7 +181,7 @@ func (rp *RequestPool) appendAccountData( } // Sync is not initial, get all account data since the latest sync - dataTypes, err := rp.db.GetAccountDataInRange(req.ctx, userID, *req.since, currentPos) + dataTypes, err := rp.db.GetAccountDataInRange(req.ctx, userID, req.since.PDUPosition, currentPos) if err != nil { return nil, err } diff --git a/syncapi/sync/userstream.go b/syncapi/sync/userstream.go index 77d09c202..beb10e487 100644 --- a/syncapi/sync/userstream.go +++ b/syncapi/sync/userstream.go @@ -34,8 +34,8 @@ type UserStream struct { lock sync.Mutex // Closed when there is an update. signalChannel chan struct{} - // The last stream position that there may have been an update for the suser - pos types.StreamPosition + // The last sync position that there may have been an update for the user + pos types.SyncPosition // The last time when we had some listeners waiting timeOfLastChannel time.Time // The number of listeners waiting @@ -51,7 +51,7 @@ type UserStreamListener struct { } // NewUserStream creates a new user stream -func NewUserStream(userID string, currPos types.StreamPosition) *UserStream { +func NewUserStream(userID string, currPos types.SyncPosition) *UserStream { return &UserStream{ UserID: userID, timeOfLastChannel: time.Now(), @@ -84,8 +84,8 @@ func (s *UserStream) GetListener(ctx context.Context) UserStreamListener { return listener } -// Broadcast a new stream position for this user. -func (s *UserStream) Broadcast(pos types.StreamPosition) { +// Broadcast a new sync position for this user. +func (s *UserStream) Broadcast(pos types.SyncPosition) { s.lock.Lock() defer s.lock.Unlock() @@ -118,9 +118,9 @@ func (s *UserStream) TimeOfLastNonEmpty() time.Time { return s.timeOfLastChannel } -// GetStreamPosition returns last stream position which the UserStream was +// GetStreamPosition returns last sync position which the UserStream was // notified about -func (s *UserStreamListener) GetStreamPosition() types.StreamPosition { +func (s *UserStreamListener) GetSyncPosition() types.SyncPosition { s.userStream.lock.Lock() defer s.userStream.lock.Unlock() @@ -132,11 +132,11 @@ func (s *UserStreamListener) GetStreamPosition() types.StreamPosition { // sincePos specifies from which point we want to be notified about. If there // has already been an update after sincePos we'll return a closed channel // immediately. -func (s *UserStreamListener) GetNotifyChannel(sincePos types.StreamPosition) <-chan struct{} { +func (s *UserStreamListener) GetNotifyChannel(sincePos types.SyncPosition) <-chan struct{} { s.userStream.lock.Lock() defer s.userStream.lock.Unlock() - if sincePos < s.userStream.pos { + if s.userStream.pos.IsAfter(sincePos) { // If the listener is behind, i.e. missed a potential update, then we // want them to wake up immediately. We do this by returning a new // closed stream, which returns immediately when selected. diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go index 2db54c3ce..4738feea2 100644 --- a/syncapi/syncapi.go +++ b/syncapi/syncapi.go @@ -28,7 +28,6 @@ import ( "github.com/matrix-org/dendrite/syncapi/routing" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/sync" - "github.com/matrix-org/dendrite/syncapi/types" ) // SetupSyncAPIComponent sets up and registers HTTP handlers for the SyncAPI @@ -39,17 +38,17 @@ func SetupSyncAPIComponent( accountsDB *accounts.Database, queryAPI api.RoomserverQueryAPI, ) { - syncDB, err := storage.NewSyncServerDatabase(string(base.Cfg.Database.SyncAPI)) + syncDB, err := storage.NewSyncServerDatasource(string(base.Cfg.Database.SyncAPI)) if err != nil { logrus.WithError(err).Panicf("failed to connect to sync db") } - pos, err := syncDB.SyncStreamPosition(context.Background()) + pos, err := syncDB.SyncPosition(context.Background()) if err != nil { - logrus.WithError(err).Panicf("failed to get stream position") + logrus.WithError(err).Panicf("failed to get sync position") } - notifier := sync.NewNotifier(types.StreamPosition(pos)) + notifier := sync.NewNotifier(pos) err = notifier.Load(context.Background(), syncDB) if err != nil { logrus.WithError(err).Panicf("failed to start notifier") @@ -71,5 +70,12 @@ func SetupSyncAPIComponent( logrus.WithError(err).Panicf("failed to start client data consumer") } + typingConsumer := consumers.NewOutputTypingEventConsumer( + base.Cfg, base.KafkaConsumer, notifier, syncDB, + ) + if err = typingConsumer.Start(); err != nil { + logrus.WithError(err).Panicf("failed to start typing server consumer") + } + routing.Setup(base.APIMux, requestPool, syncDB, deviceDB) } diff --git a/syncapi/types/types.go b/syncapi/types/types.go index d0b1c38ab..af7ec865f 100644 --- a/syncapi/types/types.go +++ b/syncapi/types/types.go @@ -21,12 +21,38 @@ import ( "github.com/matrix-org/gomatrixserverlib" ) -// StreamPosition represents the offset in the sync stream a client is at. -type StreamPosition int64 +// SyncPosition contains the PDU and EDU stream sync positions for a client. +type SyncPosition struct { + // PDUPosition is the stream position for PDUs the client is at. + PDUPosition int64 + // TypingPosition is the client's position for typing notifications. + TypingPosition int64 +} // String implements the Stringer interface. -func (sp StreamPosition) String() string { - return strconv.FormatInt(int64(sp), 10) +func (sp SyncPosition) String() string { + return strconv.FormatInt(sp.PDUPosition, 10) + "_" + + strconv.FormatInt(sp.TypingPosition, 10) +} + +// IsAfter returns whether one SyncPosition refers to states newer than another SyncPosition. +func (sp SyncPosition) IsAfter(other SyncPosition) bool { + return sp.PDUPosition > other.PDUPosition || + sp.TypingPosition > other.TypingPosition +} + +// WithUpdates returns a copy of the SyncPosition with updates applied from another SyncPosition. +// If the latter SyncPosition contains a field that is not 0, it is considered an update, +// and its value will replace the corresponding value in the SyncPosition on which WithUpdates is called. +func (sp SyncPosition) WithUpdates(other SyncPosition) SyncPosition { + ret := sp + if other.PDUPosition != 0 { + ret.PDUPosition = other.PDUPosition + } + if other.TypingPosition != 0 { + ret.TypingPosition = other.TypingPosition + } + return ret } // PrevEventRef represents a reference to a previous event in a state event upgrade @@ -53,11 +79,10 @@ type Response struct { } // NewResponse creates an empty response with initialised maps. -func NewResponse(pos StreamPosition) *Response { - res := Response{} - // Make sure we send the next_batch as a string. We don't want to confuse clients by sending this - // as an integer even though (at the moment) it is. - res.NextBatch = pos.String() +func NewResponse(pos SyncPosition) *Response { + res := Response{ + NextBatch: pos.String(), + } // Pre-initialise the maps. Synapse will return {} even if there are no rooms under a specific section, // so let's do the same thing. Bonus: this means we can't get dreaded 'assignment to entry in nil map' errors. res.Rooms.Join = make(map[string]JoinResponse) diff --git a/testfile b/testfile index e869be4f0..bdd421e2d 100644 --- a/testfile +++ b/testfile @@ -144,3 +144,6 @@ Events come down the correct room local user can join room with version 5 User can invite local user to room with version 5 Inbound federation can receive room-join requests +Typing events appear in initial sync +Typing events appear in incremental sync +Typing events appear in gapped sync diff --git a/typingserver/api/output.go b/typingserver/api/output.go index 813b9b7c7..8696acf49 100644 --- a/typingserver/api/output.go +++ b/typingserver/api/output.go @@ -12,14 +12,17 @@ package api +import "time" + // OutputTypingEvent is an entry in typing server output kafka log. // This contains the event with extra fields used to create 'm.typing' event // in clientapi & federation. type OutputTypingEvent struct { // The Event for the typing edu event. Event TypingEvent `json:"event"` - // Users typing in the room when the event was generated. - TypingUsers []string `json:"typing_users"` + // ExpireTime is the interval after which the user should no longer be + // considered typing. Only available if Event.Typing is true. + ExpireTime *time.Time } // TypingEvent represents a matrix edu event of type 'm.typing'. diff --git a/typingserver/cache/cache.go b/typingserver/cache/cache.go index 85d74cd19..8d84f856c 100644 --- a/typingserver/cache/cache.go +++ b/typingserver/cache/cache.go @@ -22,25 +22,66 @@ const defaultTypingTimeout = 10 * time.Second // userSet is a map of user IDs to a timer, timer fires at expiry. type userSet map[string]*time.Timer +// TimeoutCallbackFn is a function called right after the removal of a user +// from the typing user list due to timeout. +// latestSyncPosition is the typing sync position after the removal. +type TimeoutCallbackFn func(userID, roomID string, latestSyncPosition int64) + +type roomData struct { + syncPosition int64 + userSet userSet +} + // TypingCache maintains a list of users typing in each room. type TypingCache struct { sync.RWMutex - data map[string]userSet + latestSyncPosition int64 + data map[string]*roomData + timeoutCallback TimeoutCallbackFn +} + +// Create a roomData with its sync position set to the latest sync position. +// Must only be called after locking the cache. +func (t *TypingCache) newRoomData() *roomData { + return &roomData{ + syncPosition: t.latestSyncPosition, + userSet: make(userSet), + } } // NewTypingCache returns a new TypingCache initialised for use. func NewTypingCache() *TypingCache { - return &TypingCache{data: make(map[string]userSet)} + return &TypingCache{data: make(map[string]*roomData)} +} + +// SetTimeoutCallback sets a callback function that is called right after +// a user is removed from the typing user list due to timeout. +func (t *TypingCache) SetTimeoutCallback(fn TimeoutCallbackFn) { + t.timeoutCallback = fn } // GetTypingUsers returns the list of users typing in a room. -func (t *TypingCache) GetTypingUsers(roomID string) (users []string) { +func (t *TypingCache) GetTypingUsers(roomID string) []string { + users, _ := t.GetTypingUsersIfUpdatedAfter(roomID, 0) + // 0 should work above because the first position used will be 1. + return users +} + +// GetTypingUsersIfUpdatedAfter returns all users typing in this room with +// updated == true if the typing sync position of the room is after the given +// position. Otherwise, returns an empty slice with updated == false. +func (t *TypingCache) GetTypingUsersIfUpdatedAfter( + roomID string, position int64, +) (users []string, updated bool) { t.RLock() - usersMap, ok := t.data[roomID] - t.RUnlock() - if ok { - users = make([]string, 0, len(usersMap)) - for userID := range usersMap { + defer t.RUnlock() + + roomData, ok := t.data[roomID] + if ok && roomData.syncPosition > position { + updated = true + userSet := roomData.userSet + users = make([]string, 0, len(userSet)) + for userID := range userSet { users = append(users, userID) } } @@ -51,25 +92,41 @@ func (t *TypingCache) GetTypingUsers(roomID string) (users []string) { // AddTypingUser sets an user as typing in a room. // expire is the time when the user typing should time out. // if expire is nil, defaultTypingTimeout is assumed. -func (t *TypingCache) AddTypingUser(userID, roomID string, expire *time.Time) { +// Returns the latest sync position for typing after update. +func (t *TypingCache) AddTypingUser( + userID, roomID string, expire *time.Time, +) int64 { expireTime := getExpireTime(expire) if until := time.Until(expireTime); until > 0 { - timer := time.AfterFunc(until, t.timeoutCallback(userID, roomID)) - t.addUser(userID, roomID, timer) + timer := time.AfterFunc(until, func() { + latestSyncPosition := t.RemoveUser(userID, roomID) + if t.timeoutCallback != nil { + t.timeoutCallback(userID, roomID, latestSyncPosition) + } + }) + return t.addUser(userID, roomID, timer) } + return t.GetLatestSyncPosition() } // addUser with mutex lock & replace the previous timer. -func (t *TypingCache) addUser(userID, roomID string, expiryTimer *time.Timer) { +// Returns the latest typing sync position after update. +func (t *TypingCache) addUser( + userID, roomID string, expiryTimer *time.Timer, +) int64 { t.Lock() defer t.Unlock() + t.latestSyncPosition++ + if t.data[roomID] == nil { - t.data[roomID] = make(userSet) + t.data[roomID] = t.newRoomData() + } else { + t.data[roomID].syncPosition = t.latestSyncPosition } // Stop the timer to cancel the call to timeoutCallback - if timer, ok := t.data[roomID][userID]; ok { + if timer, ok := t.data[roomID].userSet[userID]; ok { // It may happen that at this stage timer fires but now we have a lock on t. // Hence the execution of timeoutCallback will happen after we unlock. // So we may lose a typing state, though this event is highly unlikely. @@ -78,26 +135,40 @@ func (t *TypingCache) addUser(userID, roomID string, expiryTimer *time.Timer) { timer.Stop() } - t.data[roomID][userID] = expiryTimer -} + t.data[roomID].userSet[userID] = expiryTimer -// Returns a function which is called after timeout happens. -// This removes the user. -func (t *TypingCache) timeoutCallback(userID, roomID string) func() { - return func() { - t.RemoveUser(userID, roomID) - } + return t.latestSyncPosition } // RemoveUser with mutex lock & stop the timer. -func (t *TypingCache) RemoveUser(userID, roomID string) { +// Returns the latest sync position for typing after update. +func (t *TypingCache) RemoveUser(userID, roomID string) int64 { t.Lock() defer t.Unlock() - if timer, ok := t.data[roomID][userID]; ok { - timer.Stop() - delete(t.data[roomID], userID) + roomData, ok := t.data[roomID] + if !ok { + return t.latestSyncPosition } + + timer, ok := roomData.userSet[userID] + if !ok { + return t.latestSyncPosition + } + + timer.Stop() + delete(roomData.userSet, userID) + + t.latestSyncPosition++ + t.data[roomID].syncPosition = t.latestSyncPosition + + return t.latestSyncPosition +} + +func (t *TypingCache) GetLatestSyncPosition() int64 { + t.Lock() + defer t.Unlock() + return t.latestSyncPosition } func getExpireTime(expire *time.Time) time.Time { diff --git a/typingserver/input/input.go b/typingserver/input/input.go index b9968ce4c..0e2fbe51f 100644 --- a/typingserver/input/input.go +++ b/typingserver/input/input.go @@ -57,15 +57,21 @@ func (t *TypingServerInputAPI) InputTypingEvent( } func (t *TypingServerInputAPI) sendEvent(ite *api.InputTypingEvent) error { - userIDs := t.Cache.GetTypingUsers(ite.RoomID) ev := &api.TypingEvent{ Type: gomatrixserverlib.MTyping, RoomID: ite.RoomID, UserID: ite.UserID, + Typing: ite.Typing, } ote := &api.OutputTypingEvent{ - Event: *ev, - TypingUsers: userIDs, + Event: *ev, + } + + if ev.Typing { + expireTime := ite.OriginServerTS.Time().Add( + time.Duration(ite.Timeout) * time.Millisecond, + ) + ote.ExpireTime = &expireTime } eventJSON, err := json.Marshal(ote) From e2251199a49ab0bb846c02ba37e1cd437a7f725b Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Fri, 12 Jul 2019 16:43:01 +0100 Subject: [PATCH 09/49] Lots of small typo fixes (#737) --- appservice/api/query.go | 4 ++-- clientapi/routing/createroom.go | 2 +- clientapi/routing/membership.go | 2 +- clientapi/routing/profile.go | 6 +++--- federationapi/routing/profile.go | 2 +- federationapi/routing/threepid.go | 2 +- go.sum | 1 - syncapi/storage/output_room_events_table.go | 2 +- typingserver/cache/cache.go | 11 ++++++----- 9 files changed, 16 insertions(+), 16 deletions(-) diff --git a/appservice/api/query.go b/appservice/api/query.go index 9ec214486..8ce3b4e04 100644 --- a/appservice/api/query.go +++ b/appservice/api/query.go @@ -134,9 +134,9 @@ func (h *httpAppServiceQueryAPI) UserIDExists( return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } -// RetreiveUserProfile is a wrapper that queries both the local database and +// RetrieveUserProfile is a wrapper that queries both the local database and // application services for a given user's profile -func RetreiveUserProfile( +func RetrieveUserProfile( ctx context.Context, userID string, asAPI AppServiceQueryAPI, diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index a7187c495..220ba6ae8 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -163,7 +163,7 @@ func createRoom( "roomID": roomID, }).Info("Creating new room") - profile, err := appserviceAPI.RetreiveUserProfile(req.Context(), userID, asAPI, accountDB) + profile, err := appserviceAPI.RetrieveUserProfile(req.Context(), userID, asAPI, accountDB) if err != nil { return httputil.LogThenError(req, err) } diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index 22e66f452..61898fecd 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -176,7 +176,7 @@ func loadProfile( var profile *authtypes.Profile if serverName == cfg.Matrix.ServerName { - profile, err = appserviceAPI.RetreiveUserProfile(ctx, userID, asAPI, accountDB) + profile, err = appserviceAPI.RetrieveUserProfile(ctx, userID, asAPI, accountDB) } else { profile = &authtypes.Profile{} } diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index e57d16fbf..eb1acab74 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -43,7 +43,7 @@ func GetProfile( JSON: jsonerror.NotFound("Bad method"), } } - profile, err := appserviceAPI.RetreiveUserProfile(req.Context(), userID, asAPI, accountDB) + profile, err := appserviceAPI.RetrieveUserProfile(req.Context(), userID, asAPI, accountDB) if err != nil { return httputil.LogThenError(req, err) } @@ -62,7 +62,7 @@ func GetProfile( func GetAvatarURL( req *http.Request, accountDB *accounts.Database, userID string, asAPI appserviceAPI.AppServiceQueryAPI, ) util.JSONResponse { - profile, err := appserviceAPI.RetreiveUserProfile(req.Context(), userID, asAPI, accountDB) + profile, err := appserviceAPI.RetrieveUserProfile(req.Context(), userID, asAPI, accountDB) if err != nil { return httputil.LogThenError(req, err) } @@ -160,7 +160,7 @@ func SetAvatarURL( func GetDisplayName( req *http.Request, accountDB *accounts.Database, userID string, asAPI appserviceAPI.AppServiceQueryAPI, ) util.JSONResponse { - profile, err := appserviceAPI.RetreiveUserProfile(req.Context(), userID, asAPI, accountDB) + profile, err := appserviceAPI.RetrieveUserProfile(req.Context(), userID, asAPI, accountDB) if err != nil { return httputil.LogThenError(req, err) } diff --git a/federationapi/routing/profile.go b/federationapi/routing/profile.go index aa4fcdc42..2b478cfbf 100644 --- a/federationapi/routing/profile.go +++ b/federationapi/routing/profile.go @@ -53,7 +53,7 @@ func GetProfile( return httputil.LogThenError(httpReq, err) } - profile, err := appserviceAPI.RetreiveUserProfile(httpReq.Context(), userID, asAPI, accountDB) + profile, err := appserviceAPI.RetrieveUserProfile(httpReq.Context(), userID, asAPI, accountDB) if err != nil { return httputil.LogThenError(httpReq, err) } diff --git a/federationapi/routing/threepid.go b/federationapi/routing/threepid.go index 27796067b..05ca8892e 100644 --- a/federationapi/routing/threepid.go +++ b/federationapi/routing/threepid.go @@ -194,7 +194,7 @@ func createInviteFrom3PIDInvite( StateKey: &inv.MXID, } - profile, err := appserviceAPI.RetreiveUserProfile(ctx, inv.MXID, asAPI, accountDB) + profile, err := appserviceAPI.RetrieveUserProfile(ctx, inv.MXID, asAPI, accountDB) if err != nil { return nil, err } diff --git a/go.sum b/go.sum index ce3c07dd7..ef6e94e10 100644 --- a/go.sum +++ b/go.sum @@ -140,4 +140,3 @@ gopkg.in/yaml.v2 v2.0.0-20171116090243-287cf08546ab h1:yZ6iByf7GKeJ3gsd1Dr/xaj1D gopkg.in/yaml.v2 v2.0.0-20171116090243-287cf08546ab/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= - diff --git a/syncapi/storage/output_room_events_table.go b/syncapi/storage/output_room_events_table.go index 06df017cb..34632aedf 100644 --- a/syncapi/storage/output_room_events_table.go +++ b/syncapi/storage/output_room_events_table.go @@ -235,7 +235,7 @@ func (s *outputRoomEventsStatements) selectRecentEvents( return nil, err } // The events need to be returned from oldest to latest, which isn't - // necessary the way the SQL query returns them, so a sort is necessary to + // necessarily the way the SQL query returns them, so a sort is necessary to // ensure the events are in the right order in the slice. sort.SliceStable(events, func(i int, j int) bool { return events[i].streamPosition < events[j].streamPosition diff --git a/typingserver/cache/cache.go b/typingserver/cache/cache.go index 8d84f856c..3f05c938e 100644 --- a/typingserver/cache/cache.go +++ b/typingserver/cache/cache.go @@ -127,11 +127,12 @@ func (t *TypingCache) addUser( // Stop the timer to cancel the call to timeoutCallback if timer, ok := t.data[roomID].userSet[userID]; ok { - // It may happen that at this stage timer fires but now we have a lock on t. - // Hence the execution of timeoutCallback will happen after we unlock. - // So we may lose a typing state, though this event is highly unlikely. - // This can be mitigated by keeping another time.Time in the map and check against it - // before removing. This however is not required in most practical scenario. + // It may happen that at this stage the timer fires, but we now have a lock on + // it. Hence the execution of timeoutCallback will happen after we unlock. So + // we may lose a typing state, though this is highly unlikely. This can be + // mitigated by keeping another time.Time in the map and checking against it + // before removing, but its occurrence is so infrequent it does not seem + // worthwhile. timer.Stop() } From fd4ea78dbfcc9d2c98709f2170e593bff258e0ca Mon Sep 17 00:00:00 2001 From: Sumukha Pk Date: Sat, 13 Jul 2019 13:02:47 +0530 Subject: [PATCH 10/49] Moved to gomatrix latest in gomod (#738) Moved to gomatrix latest in gomod --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 072d9ef30..3b4b736a4 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6 github.com/lib/pq v0.0.0-20170918175043-23da1db4f16d github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 - github.com/matrix-org/gomatrix v0.0.0-20190130130140-385f072fe9af + github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 github.com/matrix-org/gomatrixserverlib v0.0.0-20190619132215-178ed5e3b8e2 github.com/matrix-org/naffka v0.0.0-20171115094957-662bfd0841d0 github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 diff --git a/go.sum b/go.sum index ef6e94e10..8026640b5 100644 --- a/go.sum +++ b/go.sum @@ -47,6 +47,8 @@ github.com/matrix-org/gomatrix v0.0.0-20171003113848-a7fc80c8060c h1:aZap604NyBG github.com/matrix-org/gomatrix v0.0.0-20171003113848-a7fc80c8060c/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20190130130140-385f072fe9af h1:piaIBNQGIHnni27xRB7VKkEwoWCgAmeuYf8pxAyG0bI= github.com/matrix-org/gomatrix v0.0.0-20190130130140-385f072fe9af/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= +github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bhrnp3Ky1qgx/fzCtCALOoGYylh2tpS9K4= +github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrixserverlib v0.0.0-20181109104322-1c2cbc0872f0 h1:3UzhmERBbis4ZaB3imEbZwtDjGz/oVRC2cLLEajCzJA= github.com/matrix-org/gomatrixserverlib v0.0.0-20181109104322-1c2cbc0872f0/go.mod h1:YHyhIQUmuXyKtoVfDUMk/DyU93Taamlu6nPZkij/JtA= github.com/matrix-org/gomatrixserverlib v0.0.0-20190619132215-178ed5e3b8e2 h1:pYajAEdi3sowj4iSunqctchhcMNW3rDjeeH0T4uDkMY= From 9bef5d52f4952270720198ee97b163dcdf8f9bcc Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Tue, 16 Jul 2019 23:16:43 +0800 Subject: [PATCH 11/49] Fix broken Kafka download URL in INSTALL.md (#740) Signed-off-by: Alex Chen --- INSTALL.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 82f7f00af..0fb0c08e5 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -35,10 +35,10 @@ cd dendrite If using Kafka, install and start it (c.f. [scripts/install-local-kafka.sh](scripts/install-local-kafka.sh)): ```bash -MIRROR=http://apache.mirror.anlx.net/kafka/0.10.2.0/kafka_2.11-0.10.2.0.tgz +KAFKA_URL=http://archive.apache.org/dist/kafka/2.1.0/kafka_2.11-2.1.0.tgz # Only download the kafka if it isn't already downloaded. -test -f kafka.tgz || wget $MIRROR -O kafka.tgz +test -f kafka.tgz || wget $KAFKA_URL -O kafka.tgz # Unpack the kafka over the top of any existing installation mkdir -p kafka && tar xzf kafka.tgz -C kafka --strip-components 1 From c494d81235725957e7c7b55a2f798cf345266b47 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 16 Jul 2019 18:01:16 +0100 Subject: [PATCH 12/49] Small clarification for method (#742) --- publicroomsapi/directory/public_rooms.go | 1 + 1 file changed, 1 insertion(+) diff --git a/publicroomsapi/directory/public_rooms.go b/publicroomsapi/directory/public_rooms.go index 100e28e9b..afa1220ec 100644 --- a/publicroomsapi/directory/public_rooms.go +++ b/publicroomsapi/directory/public_rooms.go @@ -89,6 +89,7 @@ func GetPublicRooms( // fillPublicRoomsReq fills the Limit, Since and Filter attributes of a GET or POST request // on /publicRooms by parsing the incoming HTTP request +// Filter is only filled for POST requests func fillPublicRoomsReq(httpReq *http.Request, request *publicRoomReq) *util.JSONResponse { if httpReq.Method == http.MethodGet { limit, err := strconv.Atoi(httpReq.FormValue("limit")) From bff60953f36f87706546e49cd38fd7fc93727efa Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Wed, 17 Jul 2019 04:55:25 +0100 Subject: [PATCH 13/49] Prevent duplicate entries in the completed registration flows (#741) --- clientapi/routing/register.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index 243f9dd23..fa15f4fc0 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -85,6 +85,12 @@ func (d sessionsDict) GetCompletedStages(sessionID string) []authtypes.LoginType // AddCompletedStage records that a session has completed an auth stage. func (d *sessionsDict) AddCompletedStage(sessionID string, stage authtypes.LoginType) { + // Return if the stage is already present + for _, completedStage := range d.GetCompletedStages(sessionID) { + if completedStage == stage { + return + } + } d.sessions[sessionID] = append(d.GetCompletedStages(sessionID), stage) } From 504d23f468caf6193431f2670c13315b43a84790 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Thu, 18 Jul 2019 08:40:10 +0100 Subject: [PATCH 14/49] Remove unnecessary http method checks (#747) Closes #523 There were a lot of unnecessary checks for HTTP methods of requests. gorilla/mux makes sure that these methods will only be called if certain HTTP methods are used, thus there's no reason to have these extra checks. --- clientapi/routing/account_data.go | 7 ------- clientapi/routing/device.go | 7 ------- clientapi/routing/filter.go | 12 ------------ clientapi/routing/logout.go | 8 -------- clientapi/routing/profile.go | 6 ------ mediaapi/routing/download.go | 10 +--------- mediaapi/routing/upload.go | 9 +-------- 7 files changed, 2 insertions(+), 57 deletions(-) diff --git a/clientapi/routing/account_data.go b/clientapi/routing/account_data.go index 30e00f723..d57a6d370 100644 --- a/clientapi/routing/account_data.go +++ b/clientapi/routing/account_data.go @@ -33,13 +33,6 @@ func SaveAccountData( req *http.Request, accountDB *accounts.Database, device *authtypes.Device, userID string, roomID string, dataType string, syncProducer *producers.SyncAPIProducer, ) util.JSONResponse { - if req.Method != http.MethodPut { - return util.JSONResponse{ - Code: http.StatusMethodNotAllowed, - JSON: jsonerror.NotFound("Bad method"), - } - } - if userID != device.UserID { return util.JSONResponse{ Code: http.StatusForbidden, diff --git a/clientapi/routing/device.go b/clientapi/routing/device.go index cf6f24a7d..c858e88aa 100644 --- a/clientapi/routing/device.go +++ b/clientapi/routing/device.go @@ -106,13 +106,6 @@ func UpdateDeviceByID( req *http.Request, deviceDB *devices.Database, device *authtypes.Device, deviceID string, ) util.JSONResponse { - if req.Method != http.MethodPut { - return util.JSONResponse{ - Code: http.StatusMethodNotAllowed, - JSON: jsonerror.NotFound("Bad Method"), - } - } - localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) if err != nil { return httputil.LogThenError(req, err) diff --git a/clientapi/routing/filter.go b/clientapi/routing/filter.go index 1ed91cd2f..291a165b7 100644 --- a/clientapi/routing/filter.go +++ b/clientapi/routing/filter.go @@ -32,12 +32,6 @@ import ( func GetFilter( req *http.Request, device *authtypes.Device, accountDB *accounts.Database, userID string, filterID string, ) util.JSONResponse { - if req.Method != http.MethodGet { - return util.JSONResponse{ - Code: http.StatusMethodNotAllowed, - JSON: jsonerror.NotFound("Bad method"), - } - } if userID != device.UserID { return util.JSONResponse{ Code: http.StatusForbidden, @@ -79,12 +73,6 @@ type filterResponse struct { func PutFilter( req *http.Request, device *authtypes.Device, accountDB *accounts.Database, userID string, ) util.JSONResponse { - if req.Method != http.MethodPost { - return util.JSONResponse{ - Code: http.StatusMethodNotAllowed, - JSON: jsonerror.NotFound("Bad method"), - } - } if userID != device.UserID { return util.JSONResponse{ Code: http.StatusForbidden, diff --git a/clientapi/routing/logout.go b/clientapi/routing/logout.go index d20138534..3294fbcdc 100644 --- a/clientapi/routing/logout.go +++ b/clientapi/routing/logout.go @@ -20,7 +20,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/clientapi/httputil" - "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -29,13 +28,6 @@ import ( func Logout( req *http.Request, deviceDB *devices.Database, device *authtypes.Device, ) util.JSONResponse { - if req.Method != http.MethodPost { - return util.JSONResponse{ - Code: http.StatusMethodNotAllowed, - JSON: jsonerror.NotFound("Bad method"), - } - } - localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) if err != nil { return httputil.LogThenError(req, err) diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index eb1acab74..034b9ac84 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -37,12 +37,6 @@ import ( func GetProfile( req *http.Request, accountDB *accounts.Database, userID string, asAPI appserviceAPI.AppServiceQueryAPI, ) util.JSONResponse { - if req.Method != http.MethodGet { - return util.JSONResponse{ - Code: http.StatusMethodNotAllowed, - JSON: jsonerror.NotFound("Bad method"), - } - } profile, err := appserviceAPI.RetrieveUserProfile(req.Context(), userID, asAPI, accountDB) if err != nil { return httputil.LogThenError(req, err) diff --git a/mediaapi/routing/download.go b/mediaapi/routing/download.go index 38c436367..80ad8418d 100644 --- a/mediaapi/routing/download.go +++ b/mediaapi/routing/download.go @@ -55,7 +55,7 @@ type downloadRequest struct { Logger *log.Entry } -// Download implements /download amd /thumbnail +// Download implements GET /download and GET /thumbnail // Files from this server (i.e. origin == cfg.ServerName) are served directly // Files from remote servers (i.e. origin != cfg.ServerName) are cached locally. // If they are present in the cache, they are served directly. @@ -107,14 +107,6 @@ func Download( } // request validation - if req.Method != http.MethodGet { - dReq.jsonErrorResponse(w, util.JSONResponse{ - Code: http.StatusMethodNotAllowed, - JSON: jsonerror.Unknown("request method must be GET"), - }) - return - } - if resErr := dReq.Validate(); resErr != nil { dReq.jsonErrorResponse(w, *resErr) return diff --git a/mediaapi/routing/upload.go b/mediaapi/routing/upload.go index 1051e0e03..2cb0d8757 100644 --- a/mediaapi/routing/upload.go +++ b/mediaapi/routing/upload.go @@ -48,7 +48,7 @@ type uploadResponse struct { ContentURI string `json:"content_uri"` } -// Upload implements /upload +// Upload implements POST /upload // This endpoint involves uploading potentially significant amounts of data to the homeserver. // This implementation supports a configurable maximum file size limit in bytes. If a user tries to upload more than this, they will receive an error that their upload is too large. // Uploaded files are processed piece-wise to avoid DoS attacks which would starve the server of memory. @@ -75,13 +75,6 @@ func Upload(req *http.Request, cfg *config.Dendrite, db *storage.Database, activ // all the metadata about the media being uploaded. // Returns either an uploadRequest or an error formatted as a util.JSONResponse func parseAndValidateRequest(req *http.Request, cfg *config.Dendrite) (*uploadRequest, *util.JSONResponse) { - if req.Method != http.MethodPost { - return nil, &util.JSONResponse{ - Code: http.StatusMethodNotAllowed, - JSON: jsonerror.Unknown("HTTP request method must be POST."), - } - } - r := &uploadRequest{ MediaMetadata: &types.MediaMetadata{ Origin: cfg.Matrix.ServerName, From e56d6e41fdb1cc86ffcfdb2a2812d66b98662a1a Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Thu, 18 Jul 2019 08:41:29 +0100 Subject: [PATCH 15/49] Remove the trailing slash from /send/{txnID} and /backfill/{roomID} (#746) In conjunction with matrix-org/sytest#651, /send/{txnID} and /backfill/{roomID} should not have trailing slashes according to the spec. --- federationapi/routing/routing.go | 4 ++-- testfile | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index 16704e0b2..ed32c8904 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -65,7 +65,7 @@ func Setup( v2keysmux.Handle("/server/{keyID}", localKeys).Methods(http.MethodGet) v2keysmux.Handle("/server/", localKeys).Methods(http.MethodGet) - v1fedmux.Handle("/send/{txnID}/", common.MakeFedAPI( + v1fedmux.Handle("/send/{txnID}", common.MakeFedAPI( "federation_send", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) @@ -260,7 +260,7 @@ func Setup( }, )).Methods(http.MethodPost) - v1fedmux.Handle("/backfill/{roomID}/", common.MakeFedAPI( + v1fedmux.Handle("/backfill/{roomID}", common.MakeFedAPI( "federation_backfill", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) diff --git a/testfile b/testfile index bdd421e2d..8a8225a83 100644 --- a/testfile +++ b/testfile @@ -147,3 +147,5 @@ Inbound federation can receive room-join requests Typing events appear in initial sync Typing events appear in incremental sync Typing events appear in gapped sync +Inbound federation of state requires event_id as a mandatory paramater +Inbound federation of state_ids requires event_id as a mandatory paramater From 6ff7b6a72a39786854bcd3107d94adba7e8b7cb0 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Thu, 18 Jul 2019 17:21:06 +0100 Subject: [PATCH 16/49] Clarify that POST publicRooms is implemented (#743) As a response to #638, it seems that POST /publicRooms is already implemented. It is, however, unclear from the code that it is. Add some comments and change a method name to make this more clear. --- publicroomsapi/directory/public_rooms.go | 4 ++-- publicroomsapi/routing/routing.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/publicroomsapi/directory/public_rooms.go b/publicroomsapi/directory/public_rooms.go index afa1220ec..ef7b2662e 100644 --- a/publicroomsapi/directory/public_rooms.go +++ b/publicroomsapi/directory/public_rooms.go @@ -42,8 +42,8 @@ type publicRoomRes struct { Estimate int64 `json:"total_room_count_estimate,omitempty"` } -// GetPublicRooms implements GET /publicRooms -func GetPublicRooms( +// GetPostPublicRooms implements GET and POST /publicRooms +func GetPostPublicRooms( req *http.Request, publicRoomDatabase *storage.PublicRoomsServerDatabase, ) util.JSONResponse { var limit int16 diff --git a/publicroomsapi/routing/routing.go b/publicroomsapi/routing/routing.go index 3a1c9eb58..422414bc2 100644 --- a/publicroomsapi/routing/routing.go +++ b/publicroomsapi/routing/routing.go @@ -64,7 +64,7 @@ func Setup(apiMux *mux.Router, deviceDB *devices.Database, publicRoomsDB *storag ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/publicRooms", common.MakeExternalAPI("public_rooms", func(req *http.Request) util.JSONResponse { - return directory.GetPublicRooms(req, publicRoomsDB) + return directory.GetPostPublicRooms(req, publicRoomsDB) }), ).Methods(http.MethodGet, http.MethodPost, http.MethodOptions) } From bdd1a87d4ddab4b937049f55ef9ceda943cff218 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Fri, 19 Jul 2019 07:04:06 +0100 Subject: [PATCH 17/49] Add appservice API to config unit test (#744) Fixes #558 --- common/config/config_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/common/config/config_test.go b/common/config/config_test.go index acc4dbd12..110c8b84c 100644 --- a/common/config/config_test.go +++ b/common/config/config_test.go @@ -54,12 +54,14 @@ database: server_key: "postgresql:///server_keys" sync_api: "postgresql:///syn_api" room_server: "postgresql:///room_server" + appservice: "postgresql:///appservice" listen: room_server: "localhost:7770" client_api: "localhost:7771" federation_api: "localhost:7772" sync_api: "localhost:7773" media_api: "localhost:7774" + appservice_api: "localhost:7777" typing_server: "localhost:7778" logging: - type: "file" From 78032b3f4c0ca2d272180afb09142fce82313109 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Mon, 22 Jul 2019 15:05:38 +0100 Subject: [PATCH 18/49] Correctly create new device when device_id is passed to /login (#753) Fixes https://github.com/matrix-org/dendrite/issues/401 Currently when passing a `device_id` parameter to `/login`, which is [supposed](https://matrix.org/docs/spec/client_server/unstable#post-matrix-client-r0-login) to return a device with that ID set, it instead just generates a random `device_id` and hands that back to you. The code was already there to do this correctly, it looks like it had just been broken during some change. Hopefully sytest will prevent this from becoming broken again. --- .../auth/storage/devices/devices_table.go | 2 ++ clientapi/auth/storage/devices/storage.go | 2 +- clientapi/routing/login.go | 25 ++++++++--------- clientapi/routing/register.go | 28 ++++++++++++------- testfile | 2 ++ 5 files changed, 34 insertions(+), 25 deletions(-) diff --git a/clientapi/auth/storage/devices/devices_table.go b/clientapi/auth/storage/devices/devices_table.go index 96d6521d8..60aa563a2 100644 --- a/clientapi/auth/storage/devices/devices_table.go +++ b/clientapi/auth/storage/devices/devices_table.go @@ -169,6 +169,8 @@ func (s *devicesStatements) selectDeviceByToken( return &dev, err } +// selectDeviceByID retrieves a device from the database with the given user +// localpart and deviceID func (s *devicesStatements) selectDeviceByID( ctx context.Context, localpart, deviceID string, ) (*authtypes.Device, error) { diff --git a/clientapi/auth/storage/devices/storage.go b/clientapi/auth/storage/devices/storage.go index 7032fe7bf..82c8e97a2 100644 --- a/clientapi/auth/storage/devices/storage.go +++ b/clientapi/auth/storage/devices/storage.go @@ -84,7 +84,7 @@ func (d *Database) CreateDevice( if deviceID != nil { returnErr = common.WithTransaction(d.db, func(txn *sql.Tx) error { var err error - // Revoke existing token for this device + // Revoke existing tokens for this device if err = d.devices.deleteDevice(ctx, txn, *deviceID, localpart); err != nil { return err } diff --git a/clientapi/routing/login.go b/clientapi/routing/login.go index 2e2d409f6..02d958152 100644 --- a/clientapi/routing/login.go +++ b/clientapi/routing/login.go @@ -18,7 +18,6 @@ import ( "net/http" "context" - "database/sql" "github.com/matrix-org/dendrite/clientapi/auth" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" @@ -42,10 +41,12 @@ type flow struct { } type passwordRequest struct { - User string `json:"user"` - Password string `json:"password"` + User string `json:"user"` + Password string `json:"password"` + // Both DeviceID and InitialDisplayName can be omitted, or empty strings ("") + // Thus a pointer is needed to differentiate between the two InitialDisplayName *string `json:"initial_device_display_name"` - DeviceID string `json:"device_id"` + DeviceID *string `json:"device_id"` } type loginResponse struct { @@ -110,7 +111,7 @@ func Login( return httputil.LogThenError(req, err) } - dev, err := getDevice(req.Context(), r, deviceDB, acc, localpart, token) + dev, err := getDevice(req.Context(), r, deviceDB, acc, token) if err != nil { return util.JSONResponse{ Code: http.StatusInternalServerError, @@ -134,20 +135,16 @@ func Login( } } -// check if device exists else create one +// getDevice returns a new or existing device func getDevice( ctx context.Context, r passwordRequest, deviceDB *devices.Database, acc *authtypes.Account, - localpart, token string, + token string, ) (dev *authtypes.Device, err error) { - dev, err = deviceDB.GetDeviceByID(ctx, localpart, r.DeviceID) - if err == sql.ErrNoRows { - // device doesn't exist, create one - dev, err = deviceDB.CreateDevice( - ctx, acc.Localpart, nil, token, r.InitialDisplayName, - ) - } + dev, err = deviceDB.CreateDevice( + ctx, acc.Localpart, r.DeviceID, token, r.InitialDisplayName, + ) return } diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index fa15f4fc0..c5a3d3018 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -121,7 +121,10 @@ type registerRequest struct { // user-interactive auth params Auth authDict `json:"auth"` + // Both DeviceID and InitialDisplayName can be omitted, or empty strings ("") + // Thus a pointer is needed to differentiate between the two InitialDisplayName *string `json:"initial_device_display_name"` + DeviceID *string `json:"device_id"` // Prevent this user from logging in InhibitLogin common.WeakBoolean `json:"inhibit_login"` @@ -626,7 +629,7 @@ func handleApplicationServiceRegistration( // application service registration is entirely separate. return completeRegistration( req.Context(), accountDB, deviceDB, r.Username, "", appserviceID, - r.InhibitLogin, r.InitialDisplayName, + r.InhibitLogin, r.InitialDisplayName, r.DeviceID, ) } @@ -646,7 +649,7 @@ func checkAndCompleteFlow( // This flow was completed, registration can continue return completeRegistration( req.Context(), accountDB, deviceDB, r.Username, r.Password, "", - r.InhibitLogin, r.InitialDisplayName, + r.InhibitLogin, r.InitialDisplayName, r.DeviceID, ) } @@ -697,10 +700,10 @@ func LegacyRegister( return util.MessageResponse(http.StatusForbidden, "HMAC incorrect") } - return completeRegistration(req.Context(), accountDB, deviceDB, r.Username, r.Password, "", false, nil) + return completeRegistration(req.Context(), accountDB, deviceDB, r.Username, r.Password, "", false, nil, nil) case authtypes.LoginTypeDummy: // there is nothing to do - return completeRegistration(req.Context(), accountDB, deviceDB, r.Username, r.Password, "", false, nil) + return completeRegistration(req.Context(), accountDB, deviceDB, r.Username, r.Password, "", false, nil, nil) default: return util.JSONResponse{ Code: http.StatusNotImplemented, @@ -738,13 +741,19 @@ func parseAndValidateLegacyLogin(req *http.Request, r *legacyRegisterRequest) *u return nil } +// completeRegistration runs some rudimentary checks against the submitted +// input, then if successful creates an account and a newly associated device +// We pass in each individual part of the request here instead of just passing a +// registerRequest, as this function serves requests encoded as both +// registerRequests and legacyRegisterRequests, which share some attributes but +// not all func completeRegistration( ctx context.Context, accountDB *accounts.Database, deviceDB *devices.Database, username, password, appserviceID string, inhibitLogin common.WeakBoolean, - displayName *string, + displayName, deviceID *string, ) util.JSONResponse { if username == "" { return util.JSONResponse{ @@ -773,6 +782,9 @@ func completeRegistration( } } + // Increment prometheus counter for created users + amtRegUsers.Inc() + // Check whether inhibit_login option is set. If so, don't create an access // token or a device for this user if inhibitLogin { @@ -793,8 +805,7 @@ func completeRegistration( } } - // TODO: Use the device ID in the request. - dev, err := deviceDB.CreateDevice(ctx, username, nil, token, displayName) + dev, err := deviceDB.CreateDevice(ctx, username, deviceID, token, displayName) if err != nil { return util.JSONResponse{ Code: http.StatusInternalServerError, @@ -802,9 +813,6 @@ func completeRegistration( } } - // Increment prometheus counter for created users - amtRegUsers.Inc() - return util.JSONResponse{ Code: http.StatusOK, JSON: registerResponse{ diff --git a/testfile b/testfile index 8a8225a83..1b2ad9e69 100644 --- a/testfile +++ b/testfile @@ -149,3 +149,5 @@ Typing events appear in incremental sync Typing events appear in gapped sync Inbound federation of state requires event_id as a mandatory paramater Inbound federation of state_ids requires event_id as a mandatory paramater +POST /register returns the same device_id as that in the request +POST /login returns the same device_id as that in the request From 4410acc673b27e747e8ba757e9b271ada55c0269 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Wed, 24 Jul 2019 05:44:05 +0100 Subject: [PATCH 19/49] Add filepath and function name to log output (#755) Adds detailed logging, describing which file/line a log message came from, as well as the name of the function that it was contained within. --- common/log.go | 28 ++++++++++++++++++++++++++-- go.mod | 8 +++++--- go.sum | 9 +++++++++ 3 files changed, 40 insertions(+), 5 deletions(-) diff --git a/common/log.go b/common/log.go index 89a705822..f9ed84edb 100644 --- a/common/log.go +++ b/common/log.go @@ -15,9 +15,12 @@ package common import ( + "fmt" "os" "path" "path/filepath" + "runtime" + "strings" "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dugong" @@ -54,15 +57,35 @@ func (h *logLevelHook) Levels() []logrus.Level { return levels } +// callerPrettyfier is a function that given a runtime.Frame object, will +// extract the calling function's name and file, and return them in a nicely +// formatted way +func callerPrettyfier(f *runtime.Frame) (string, string) { + // Retrieve just the function name + s := strings.Split(f.Function, ".") + funcname := s[len(s)-1] + + // Append a newline + tab to it to move the actual log content to its own line + funcname += "\n\t" + + // Surround the filepath in brackets and append line number so IDEs can quickly + // navigate + filename := fmt.Sprintf(" [%s:%d]", f.File, f.Line) + + return funcname, filename +} + // SetupStdLogging configures the logging format to standard output. Typically, it is called when the config is not yet loaded. func SetupStdLogging() { + logrus.SetReportCaller(true) logrus.SetFormatter(&utcFormatter{ &logrus.TextFormatter{ TimestampFormat: "2006-01-02T15:04:05.000000000Z07:00", FullTimestamp: true, DisableColors: false, DisableTimestamp: false, - DisableSorting: false, + QuoteEmptyFields: true, + CallerPrettyfier: callerPrettyfier, }, }) } @@ -71,8 +94,8 @@ func SetupStdLogging() { // If something fails here it means that the logging was improperly configured, // so we just exit with the error func SetupHookLogging(hooks []config.LogrusHook, componentName string) { + logrus.SetReportCaller(true) for _, hook := range hooks { - // Check we received a proper logging level level, err := logrus.ParseLevel(hook.Level) if err != nil { @@ -126,6 +149,7 @@ func setupFileHook(hook config.LogrusHook, level logrus.Level, componentName str DisableColors: true, DisableTimestamp: false, DisableSorting: false, + QuoteEmptyFields: true, }, }, &dugong.DailyRotationSchedule{GZip: true}, diff --git a/go.mod b/go.mod index 3b4b736a4..4e5123730 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/jaegertracing/jaeger-client-go v0.0.0-20170921145708-3ad49a1d839b github.com/jaegertracing/jaeger-lib v0.0.0-20170920222118-21a3da6d66fe github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6 + github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect github.com/lib/pq v0.0.0-20170918175043-23da1db4f16d github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 @@ -40,8 +41,9 @@ require ( github.com/prometheus/common v0.0.0-20170108231212-dd2f054febf4 github.com/prometheus/procfs v0.0.0-20170128160123-1878d9fbb537 github.com/rcrowley/go-metrics v0.0.0-20161128210544-1f30fe9094a5 - github.com/sirupsen/logrus v1.3.0 - github.com/stretchr/testify v1.2.2 + github.com/sirupsen/logrus v1.4.2 + github.com/stretchr/objx v0.2.0 // indirect + github.com/stretchr/testify v1.3.0 github.com/tidwall/gjson v1.1.5 github.com/tidwall/match v1.0.1 github.com/tidwall/sjson v1.0.3 @@ -54,7 +56,7 @@ require ( go.uber.org/zap v1.7.1 golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613 golang.org/x/net v0.0.0-20190301231341-16b79f2e4e95 - golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 + golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 gopkg.in/Shopify/sarama.v1 v1.11.0 gopkg.in/airbrake/gobrake.v2 v2.0.9 gopkg.in/alecthomas/kingpin.v3-unstable v3.0.0-20170727041045-23bcc3c4eae3 diff --git a/go.sum b/go.sum index 8026640b5..5fd3dc5b4 100644 --- a/go.sum +++ b/go.sum @@ -36,6 +36,7 @@ github.com/jaegertracing/jaeger-lib v0.0.0-20170920222118-21a3da6d66fe/go.mod h1 github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6 h1:KAZ1BW2TCmT6PRihDPpocIy1QTtsAsrx6TneU/4+CMg= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -90,9 +91,14 @@ github.com/sirupsen/logrus v0.0.0-20170822132746-89742aefa4b2 h1:+8J/sCAVv2Y9Ct1 github.com/sirupsen/logrus v0.0.0-20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v0.0.0-20170809224252-890a5c3458b4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/tidwall/gjson v1.0.2 h1:5BsM7kyEAHAUGEGDkEKO9Mdyiuw6QQ6TSDdarP0Nnmk= github.com/tidwall/gjson v1.0.2/go.mod h1:c/nTNbUr0E0OrXEhq1pwa8iEgc2DOt4ZZqAt1HtCkPA= github.com/tidwall/gjson v1.1.5 h1:QysILxBeUEY3GTLA0fQVgkQG1zme8NxGvhh2SSqWNwI= @@ -128,6 +134,9 @@ golang.org/x/sys v0.0.0-20171012164349-43eea11bc926 h1:PY6OU86NqbyZiOzaPnDw6oOjA golang.org/x/sys v0.0.0-20171012164349-43eea11bc926/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 h1:I6FyU15t786LL7oL/hn43zqTuEGr4PN7F4XJ1p4E3Y8= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 h1:LepdCS8Gf/MVejFIt8lsiexZATdoGVyp5bcyS+rYoUI= +golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= gopkg.in/Shopify/sarama.v1 v1.11.0 h1:/3kaCyeYaPbr59IBjeqhIcUOB1vXlIVqXAYa5g5C5F0= gopkg.in/Shopify/sarama.v1 v1.11.0/go.mod h1:AxnvoaevB2nBjNK17cG61A3LleFcWFwVBHBt+cot4Oc= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= From 6773572907a7748ce7f4ccd5467ee2e1d5d06f77 Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Wed, 24 Jul 2019 23:27:40 +0800 Subject: [PATCH 20/49] Update gomatrixserverlib to v0.0.0-20190724145009-a6df10ef35d6 (#762) Signed-off-by: Alex Chen --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 4e5123730..5d01012c9 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/lib/pq v0.0.0-20170918175043-23da1db4f16d github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20190619132215-178ed5e3b8e2 + github.com/matrix-org/gomatrixserverlib v0.0.0-20190724145009-a6df10ef35d6 github.com/matrix-org/naffka v0.0.0-20171115094957-662bfd0841d0 github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 github.com/matttproud/golang_protobuf_extensions v1.0.1 diff --git a/go.sum b/go.sum index 5fd3dc5b4..53151121b 100644 --- a/go.sum +++ b/go.sum @@ -54,6 +54,8 @@ github.com/matrix-org/gomatrixserverlib v0.0.0-20181109104322-1c2cbc0872f0 h1:3U github.com/matrix-org/gomatrixserverlib v0.0.0-20181109104322-1c2cbc0872f0/go.mod h1:YHyhIQUmuXyKtoVfDUMk/DyU93Taamlu6nPZkij/JtA= github.com/matrix-org/gomatrixserverlib v0.0.0-20190619132215-178ed5e3b8e2 h1:pYajAEdi3sowj4iSunqctchhcMNW3rDjeeH0T4uDkMY= github.com/matrix-org/gomatrixserverlib v0.0.0-20190619132215-178ed5e3b8e2/go.mod h1:sf0RcKOdiwJeTti7A313xsaejNUGYDq02MQZ4JD4w/E= +github.com/matrix-org/gomatrixserverlib v0.0.0-20190724145009-a6df10ef35d6 h1:B8n1H5Wb1B5jwLzTylBpY0kJCMRqrofT7PmOw4aJFJA= +github.com/matrix-org/gomatrixserverlib v0.0.0-20190724145009-a6df10ef35d6/go.mod h1:sf0RcKOdiwJeTti7A313xsaejNUGYDq02MQZ4JD4w/E= github.com/matrix-org/naffka v0.0.0-20171115094957-662bfd0841d0 h1:p7WTwG+aXM86+yVrYAiCMW3ZHSmotVvuRbjtt3jC+4A= github.com/matrix-org/naffka v0.0.0-20171115094957-662bfd0841d0/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/util v0.0.0-20171013132526-8b1c8ab81986 h1:TiWl4hLvezAhRPM8tPcPDFTysZ7k4T/1J4GPp/iqlZo= From b729a10366f9cb6f8b34db58c7bc1b9b69e67b5f Mon Sep 17 00:00:00 2001 From: Thibaut CHARLES Date: Wed, 24 Jul 2019 18:08:51 +0200 Subject: [PATCH 21/49] Store & retrieve filters as structs rather than []byte (#436) Manipulate filters as gomatrix.Filter structures, instead of their []byte JSON representation. This lays ground work for using filters in dendrite for /sync requests. --- .../auth/storage/accounts/filter_table.go | 38 ++++++++++++++----- clientapi/auth/storage/accounts/storage.go | 8 ++-- clientapi/routing/filter.go | 20 +++------- 3 files changed, 39 insertions(+), 27 deletions(-) diff --git a/clientapi/auth/storage/accounts/filter_table.go b/clientapi/auth/storage/accounts/filter_table.go index 81bae4545..2b07ef17e 100644 --- a/clientapi/auth/storage/accounts/filter_table.go +++ b/clientapi/auth/storage/accounts/filter_table.go @@ -17,6 +17,7 @@ package accounts import ( "context" "database/sql" + "encoding/json" "github.com/matrix-org/gomatrixserverlib" ) @@ -71,25 +72,44 @@ func (s *filterStatements) prepare(db *sql.DB) (err error) { func (s *filterStatements) selectFilter( ctx context.Context, localpart string, filterID string, -) (filter []byte, err error) { - err = s.selectFilterStmt.QueryRowContext(ctx, localpart, filterID).Scan(&filter) - return +) (*gomatrixserverlib.Filter, error) { + // Retrieve filter from database (stored as canonical JSON) + var filterData []byte + err := s.selectFilterStmt.QueryRowContext(ctx, localpart, filterID).Scan(&filterData) + if err != nil { + return nil, err + } + + // Unmarshal JSON into Filter struct + var filter gomatrixserverlib.Filter + if err = json.Unmarshal(filterData, &filter); err != nil { + return nil, err + } + return &filter, nil } func (s *filterStatements) insertFilter( - ctx context.Context, filter []byte, localpart string, + ctx context.Context, filter *gomatrixserverlib.Filter, localpart string, ) (filterID string, err error) { var existingFilterID string - // This can result in a race condition when two clients try to insert the - // same filter and localpart at the same time, however this is not a - // problem as both calls will result in the same filterID - filterJSON, err := gomatrixserverlib.CanonicalJSON(filter) + // Serialise json + filterJSON, err := json.Marshal(filter) + if err != nil { + return "", err + } + // Remove whitespaces and sort JSON data + // needed to prevent from inserting the same filter multiple times + filterJSON, err = gomatrixserverlib.CanonicalJSON(filterJSON) if err != nil { return "", err } - // Check if filter already exists in the database + // Check if filter already exists in the database using its localpart and content + // + // This can result in a race condition when two clients try to insert the + // same filter and localpart at the same time, however this is not a + // problem as both calls will result in the same filterID err = s.selectFilterIDByContentStmt.QueryRowContext(ctx, localpart, filterJSON).Scan(&existingFilterID) if err != nil && err != sql.ErrNoRows { diff --git a/clientapi/auth/storage/accounts/storage.go b/clientapi/auth/storage/accounts/storage.go index 27c0a176a..5c8ffffeb 100644 --- a/clientapi/auth/storage/accounts/storage.go +++ b/clientapi/auth/storage/accounts/storage.go @@ -344,11 +344,11 @@ func (d *Database) GetThreePIDsForLocalpart( } // GetFilter looks up the filter associated with a given local user and filter ID. -// Returns a filter represented as a byte slice. Otherwise returns an error if -// no such filter exists or if there was an error talking to the database. +// Returns a filter structure. Otherwise returns an error if no such filter exists +// or if there was an error talking to the database. func (d *Database) GetFilter( ctx context.Context, localpart string, filterID string, -) ([]byte, error) { +) (*gomatrixserverlib.Filter, error) { return d.filter.selectFilter(ctx, localpart, filterID) } @@ -356,7 +356,7 @@ func (d *Database) GetFilter( // Returns the filterID as a string. Otherwise returns an error if something // goes wrong. func (d *Database) PutFilter( - ctx context.Context, localpart string, filter []byte, + ctx context.Context, localpart string, filter *gomatrixserverlib.Filter, ) (string, error) { return d.filter.insertFilter(ctx, filter, localpart) } diff --git a/clientapi/routing/filter.go b/clientapi/routing/filter.go index 291a165b7..eec501ff7 100644 --- a/clientapi/routing/filter.go +++ b/clientapi/routing/filter.go @@ -17,13 +17,10 @@ package routing import ( "net/http" - "encoding/json" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -43,7 +40,7 @@ func GetFilter( return httputil.LogThenError(req, err) } - res, err := accountDB.GetFilter(req.Context(), localpart, filterID) + filter, err := accountDB.GetFilter(req.Context(), localpart, filterID) if err != nil { //TODO better error handling. This error message is *probably* right, // but if there are obscure db errors, this will also be returned, @@ -53,11 +50,6 @@ func GetFilter( JSON: jsonerror.NotFound("No such filter"), } } - filter := gomatrix.Filter{} - err = json.Unmarshal(res, &filter) - if err != nil { - return httputil.LogThenError(req, err) - } return util.JSONResponse{ Code: http.StatusOK, @@ -85,21 +77,21 @@ func PutFilter( return httputil.LogThenError(req, err) } - var filter gomatrix.Filter + var filter gomatrixserverlib.Filter if reqErr := httputil.UnmarshalJSONRequest(req, &filter); reqErr != nil { return *reqErr } - filterArray, err := json.Marshal(filter) - if err != nil { + // Validate generates a user-friendly error + if err = filter.Validate(); err != nil { return util.JSONResponse{ Code: http.StatusBadRequest, - JSON: jsonerror.BadJSON("Filter is malformed"), + JSON: jsonerror.BadJSON("Invalid filter: " + err.Error()), } } - filterID, err := accountDB.PutFilter(req.Context(), localpart, filterArray) + filterID, err := accountDB.PutFilter(req.Context(), localpart, &filter) if err != nil { return httputil.LogThenError(req, err) } From 604685c5035d836d7069be92963191d6d9f49f84 Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Thu, 25 Jul 2019 00:15:36 +0800 Subject: [PATCH 22/49] Implement room creation content (#754) Fixes #660. Signed-off-by: Alex Chen minecnly@gmail.com --- clientapi/routing/createroom.go | 36 +++++++++++++++++++++++++++++++-- common/eventcontent.go | 12 +++++++++-- testfile | 3 +++ 3 files changed, 47 insertions(+), 4 deletions(-) diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index 220ba6ae8..8c5ee975c 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -15,6 +15,7 @@ package routing import ( + "encoding/json" "fmt" "net/http" "strings" @@ -97,6 +98,27 @@ func (r createRoomRequest) Validate() *util.JSONResponse { } } + // Validate creation_content fields defined in the spec by marshalling the + // creation_content map into bytes and then unmarshalling the bytes into + // common.CreateContent. + + creationContentBytes, err := json.Marshal(r.CreationContent) + if err != nil { + return &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.BadJSON("malformed creation_content"), + } + } + + var CreationContent common.CreateContent + err = json.Unmarshal(creationContentBytes, &CreationContent) + if err != nil { + return &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.BadJSON("malformed creation_content"), + } + } + return nil } @@ -154,7 +176,17 @@ func createRoom( JSON: jsonerror.InvalidArgumentValue(err.Error()), } } - // TODO: visibility/presets/raw initial state/creation content + + // Clobber keys: creator, room_version + + if r.CreationContent == nil { + r.CreationContent = make(map[string]interface{}, 2) + } + + r.CreationContent["creator"] = userID + r.CreationContent["room_version"] = "1" // TODO: We set this to 1 before we support Room versioning + + // TODO: visibility/presets/raw initial state // TODO: Create room alias association // Make sure this doesn't fall into an application service's namespace though! @@ -214,7 +246,7 @@ func createRoom( // harder to reason about, hence sticking to a strict static ordering. // TODO: Synapse has txn/token ID on each event. Do we need to do this here? eventsToMake := []fledglingEvent{ - {"m.room.create", "", common.CreateContent{Creator: userID}}, + {"m.room.create", "", r.CreationContent}, {"m.room.member", userID, membershipContent}, {"m.room.power_levels", "", common.InitialPowerLevelsContent(userID)}, // TODO: m.room.canonical_alias diff --git a/common/eventcontent.go b/common/eventcontent.go index 971c4f0a7..c45724fcd 100644 --- a/common/eventcontent.go +++ b/common/eventcontent.go @@ -16,8 +16,16 @@ package common // CreateContent is the event content for http://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-create type CreateContent struct { - Creator string `json:"creator"` - Federate *bool `json:"m.federate,omitempty"` + Creator string `json:"creator"` + Federate *bool `json:"m.federate,omitempty"` + RoomVersion string `json:"room_version,omitempty"` + Predecessor PreviousRoom `json:"predecessor,omitempty"` +} + +// PreviousRoom is the "Previous Room" structure defined at https://matrix.org/docs/spec/client_server/r0.5.0#m-room-create +type PreviousRoom struct { + RoomID string `json:"room_id"` + EventID string `json:"event_id"` } // MemberContent is the event content for http://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-member diff --git a/testfile b/testfile index 1b2ad9e69..4003638f1 100644 --- a/testfile +++ b/testfile @@ -151,3 +151,6 @@ Inbound federation of state requires event_id as a mandatory paramater Inbound federation of state_ids requires event_id as a mandatory paramater POST /register returns the same device_id as that in the request POST /login returns the same device_id as that in the request +POST /createRoom with creation content +User can create and send/receive messages in a room with version 1 +POST /createRoom ignores attempts to set the room version via creation_content From 45d24d3fb5b84b89bcd8eec7058896b3b4a6f2e3 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Wed, 24 Jul 2019 18:41:39 +0100 Subject: [PATCH 23/49] Remove the buildkite pipeline (#763) New repo: https://github.com/matrix-org/pipelines/ --- .buildkite/pipeline.yaml | 49 ---------------------------------------- 1 file changed, 49 deletions(-) delete mode 100644 .buildkite/pipeline.yaml diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml deleted file mode 100644 index 9d755a244..000000000 --- a/.buildkite/pipeline.yaml +++ /dev/null @@ -1,49 +0,0 @@ -steps: - - command: - # https://github.com/golangci/golangci-lint#memory-usage-of-golangci-lint - - "GOGC=20 ./scripts/find-lint.sh" - label: "\U0001F9F9 Lint / :go: 1.12" - agents: - # Use a larger instance as linting takes a looot of memory - queue: "medium" - plugins: - - docker#v3.0.1: - image: "golang:1.12" - - - wait - - - command: - - "go build ./cmd/..." - label: "\U0001F528 Build / :go: 1.11" - plugins: - - docker#v3.0.1: - image: "golang:1.11" - retry: - automatic: - - exit_status: 128 - limit: 3 - - - command: - - "go build ./cmd/..." - label: "\U0001F528 Build / :go: 1.12" - plugins: - - docker#v3.0.1: - image: "golang:1.12" - retry: - automatic: - - exit_status: 128 - limit: 3 - - - command: - - "go test ./..." - label: "\U0001F9EA Unit tests / :go: 1.11" - plugins: - - docker#v3.0.1: - image: "golang:1.11" - - - command: - - "go test ./..." - label: "\U0001F9EA Unit tests / :go: 1.12" - plugins: - - docker#v3.0.1: - image: "golang:1.12" From e66933b108a4656d01bfa6285c782bdbfb249756 Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Fri, 26 Jul 2019 00:00:22 +0800 Subject: [PATCH 24/49] Fix data races reported by go test -race ./... (#748) --- syncapi/sync/notifier.go | 1 + syncapi/sync/notifier_test.go | 21 +++++++++++++++------ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/syncapi/sync/notifier.go b/syncapi/sync/notifier.go index 30ac3a2e5..14bc2efb6 100644 --- a/syncapi/sync/notifier.go +++ b/syncapi/sync/notifier.go @@ -185,6 +185,7 @@ func (n *Notifier) wakeupUsers(userIDs []string, newPos types.SyncPosition) { // fetchUserStream retrieves a stream unique to the given user. If makeIfNotExists is true, // a stream will be made for this user if one doesn't exist and it will be returned. This // function does not wait for data to be available on the stream. +// NB: Callers should have locked the mutex before calling this function. func (n *Notifier) fetchUserStream(userID string, makeIfNotExists bool) *UserStream { stream, ok := n.userStreams[userID] if !ok && makeIfNotExists { diff --git a/syncapi/sync/notifier_test.go b/syncapi/sync/notifier_test.go index 904315e9f..808e07cc7 100644 --- a/syncapi/sync/notifier_test.go +++ b/syncapi/sync/notifier_test.go @@ -143,7 +143,7 @@ func TestNewEventAndJoinedToRoom(t *testing.T) { wg.Done() }() - stream := n.fetchUserStream(bob, true) + stream := lockedFetchUserStream(n, bob) waitForBlocking(stream, 1) n.OnNewEvent(&randomMessageEvent, "", nil, syncPositionAfter) @@ -171,7 +171,7 @@ func TestNewInviteEventForUser(t *testing.T) { wg.Done() }() - stream := n.fetchUserStream(bob, true) + stream := lockedFetchUserStream(n, bob) waitForBlocking(stream, 1) n.OnNewEvent(&aliceInviteBobEvent, "", nil, syncPositionAfter) @@ -199,7 +199,7 @@ func TestEDUWakeup(t *testing.T) { wg.Done() }() - stream := n.fetchUserStream(bob, true) + stream := lockedFetchUserStream(n, bob) waitForBlocking(stream, 1) n.OnNewEvent(&aliceInviteBobEvent, "", nil, syncPositionNewEDU) @@ -230,7 +230,7 @@ func TestMultipleRequestWakeup(t *testing.T) { go poll() go poll() - stream := n.fetchUserStream(bob, true) + stream := lockedFetchUserStream(n, bob) waitForBlocking(stream, 3) n.OnNewEvent(&randomMessageEvent, "", nil, syncPositionAfter) @@ -266,14 +266,14 @@ func TestNewEventAndWasPreviouslyJoinedToRoom(t *testing.T) { } leaveWG.Done() }() - bobStream := n.fetchUserStream(bob, true) + bobStream := lockedFetchUserStream(n, bob) waitForBlocking(bobStream, 1) n.OnNewEvent(&bobLeaveEvent, "", nil, syncPositionAfter) leaveWG.Wait() // send an event into the room. Make sure alice gets it. Bob should not. var aliceWG sync.WaitGroup - aliceStream := n.fetchUserStream(alice, true) + aliceStream := lockedFetchUserStream(n, alice) aliceWG.Add(1) go func() { pos, err := waitForEvents(n, newTestSyncRequest(alice, syncPositionAfter)) @@ -328,6 +328,15 @@ func waitForBlocking(s *UserStream, numBlocking uint) { } } +// lockedFetchUserStream invokes Notifier.fetchUserStream, respecting Notifier.streamLock. +// A new stream is made if it doesn't exist already. +func lockedFetchUserStream(n *Notifier, userID string) *UserStream { + n.streamLock.Lock() + defer n.streamLock.Unlock() + + return n.fetchUserStream(userID, true) +} + func newTestSyncRequest(userID string, since types.SyncPosition) syncRequest { return syncRequest{ device: authtypes.Device{UserID: userID}, From 3e6d0a6246ed1bc10760c214fce762e6880bc5be Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Mon, 29 Jul 2019 15:18:21 +0800 Subject: [PATCH 25/49] Add newly passing tests from matrix-org/sytest 56de891 (#769) --- testfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testfile b/testfile index 4003638f1..2f9229e87 100644 --- a/testfile +++ b/testfile @@ -154,3 +154,5 @@ POST /login returns the same device_id as that in the request POST /createRoom with creation content User can create and send/receive messages in a room with version 1 POST /createRoom ignores attempts to set the room version via creation_content +Inbound federation rejects remote attempts to join local users to rooms +Inbound federation rejects remote attempts to kick local users to rooms From 40e44c5f3b445b2e5bb7dc98d991a9177d0f2abb Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Wed, 31 Jul 2019 20:45:45 +0800 Subject: [PATCH 26/49] Add newly passing tests from matrix-org/sytest (#771) Signed-off-by: Alex Chen --- testfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testfile b/testfile index 2f9229e87..c23dda662 100644 --- a/testfile +++ b/testfile @@ -156,3 +156,5 @@ User can create and send/receive messages in a room with version 1 POST /createRoom ignores attempts to set the room version via creation_content Inbound federation rejects remote attempts to join local users to rooms Inbound federation rejects remote attempts to kick local users to rooms +An event which redacts itself should be ignored +A pair of events which redact each other should be ignored From 3e1abe9ad3464a6c47dfcf003bb4bf6422c820af Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Wed, 31 Jul 2019 21:20:11 +0800 Subject: [PATCH 27/49] Fix /sync may contain duplicate EDUs and EDUs for left rooms (#752) In 29841be (#718), EDUs are added to /sync responses for rooms listed in joinedRoomIDs returned by addPDUDeltaToResponse. However this list may contain rooms other than those currently joined. Some variable renamings are done to make golangci-lint pass. Signed-off-by: Alex Chen minecnly@gmail.com --- syncapi/storage/syncserver.go | 44 +++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/syncapi/storage/syncserver.go b/syncapi/storage/syncserver.go index b4d7ccbd2..20fa8a4e0 100644 --- a/syncapi/storage/syncserver.go +++ b/syncapi/storage/syncserver.go @@ -35,6 +35,12 @@ import ( "github.com/matrix-org/gomatrixserverlib" ) +const ( + membershipJoin = "join" + membershipLeave = "leave" + membershipBan = "ban" +) + type stateDelta struct { roomID string stateEvents []gomatrixserverlib.Event @@ -248,14 +254,12 @@ func (d *SyncServerDatasource) addPDUDeltaToResponse( // joined rooms, but also which rooms have membership transitions for this user between the 2 PDU stream positions. // This works out what the 'state' key should be for each room as well as which membership block // to put the room into. - deltas, err := d.getStateDeltas(ctx, &device, txn, fromPos, toPos, device.UserID) + deltas, joinedRoomIDs, err := d.getStateDeltas(ctx, &device, txn, fromPos, toPos, device.UserID) if err != nil { return nil, err } - joinedRoomIDs := make([]string, 0, len(deltas)) for _, delta := range deltas { - joinedRoomIDs = append(joinedRoomIDs, delta.roomID) err = d.addRoomDeltaToResponse(ctx, &device, txn, fromPos, toPos, delta, numRecentEventsPerRoom, res) if err != nil { return nil, err @@ -344,7 +348,7 @@ func (d *SyncServerDatasource) IncrementalSync( ) } else { joinedRoomIDs, err = d.roomstate.selectRoomIDsWithMembership( - ctx, nil, device.UserID, "join", + ctx, nil, device.UserID, membershipJoin, ) } if err != nil { @@ -393,7 +397,7 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( res = types.NewResponse(toPos) // Extract room state and recent events for all rooms the user is joined to. - joinedRoomIDs, err = d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, "join") + joinedRoomIDs, err = d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, membershipJoin) if err != nil { return } @@ -571,7 +575,7 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( res *types.Response, ) error { endPos := toPos - if delta.membershipPos > 0 && delta.membership == "leave" { + if delta.membershipPos > 0 && delta.membership == membershipLeave { // make sure we don't leak recent events after the leave event. // TODO: History visibility makes this somewhat complex to handle correctly. For example: // TODO: This doesn't work for join -> leave in a single /sync request (see events prior to join). @@ -595,7 +599,7 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( } switch delta.membership { - case "join": + case membershipJoin: jr := types.NewJoinResponse() if prevPDUPos := recentStreamEvents[0].streamPosition - 1; prevPDUPos > 0 { // Use the short form of batch token for prev_batch @@ -608,9 +612,9 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( jr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true jr.State.Events = gomatrixserverlib.ToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) res.Rooms.Join[delta.roomID] = *jr - case "leave": + case membershipLeave: fallthrough // transitions to leave are the same as ban - case "ban": + case membershipBan: // TODO: recentEvents may contain events that this user is not allowed to see because they are // no longer in the room. lr := types.NewLeaveResponse() @@ -716,10 +720,14 @@ func (d *SyncServerDatasource) fetchMissingStateEvents( return events, nil } +// getStateDeltas returns the state deltas between fromPos and toPos, +// exclusive of oldPos, inclusive of newPos, for the rooms in which +// the user has new membership events. +// A list of joined room IDs is also returned in case the caller needs it. func (d *SyncServerDatasource) getStateDeltas( ctx context.Context, device *authtypes.Device, txn *sql.Tx, fromPos, toPos int64, userID string, -) ([]stateDelta, error) { +) ([]stateDelta, []string, error) { // Implement membership change algorithm: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L821 // - Get membership list changes for this user in this sync response // - For each room which has membership list changes: @@ -733,11 +741,11 @@ func (d *SyncServerDatasource) getStateDeltas( // get all the state events ever between these two positions stateNeeded, eventMap, err := d.events.selectStateInRange(ctx, txn, fromPos, toPos) if err != nil { - return nil, err + return nil, nil, err } state, err := d.fetchStateEvents(ctx, txn, stateNeeded, eventMap) if err != nil { - return nil, err + return nil, nil, err } for roomID, stateStreamEvents := range state { @@ -748,12 +756,12 @@ func (d *SyncServerDatasource) getStateDeltas( // the 'state' part of the response though, so is transparent modulo bandwidth concerns as it is not added to // the timeline. if membership := getMembershipFromEvent(&ev.Event, userID); membership != "" { - if membership == "join" { + if membership == membershipJoin { // send full room state down instead of a delta var allState []gomatrixserverlib.Event allState, err = d.roomstate.selectCurrentState(ctx, txn, roomID) if err != nil { - return nil, err + return nil, nil, err } s := make([]streamEvent, len(allState)) for i := 0; i < len(s); i++ { @@ -775,19 +783,19 @@ func (d *SyncServerDatasource) getStateDeltas( } // Add in currently joined rooms - joinedRoomIDs, err := d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, "join") + joinedRoomIDs, err := d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, membershipJoin) if err != nil { - return nil, err + return nil, nil, err } for _, joinedRoomID := range joinedRoomIDs { deltas = append(deltas, stateDelta{ - membership: "join", + membership: membershipJoin, stateEvents: streamEventsToEvents(device, state[joinedRoomID]), roomID: joinedRoomID, }) } - return deltas, nil + return deltas, joinedRoomIDs, nil } // streamEventsToEvents converts streamEvent to Event. If device is non-nil and From 92db6cd0eaecbd2c27f83488a176ef76ecf16f36 Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Wed, 31 Jul 2019 21:36:21 +0800 Subject: [PATCH 28/49] Fix index in invites_table.go (#770) This PR fixes a possible typo in an index created in invites_table.go. Signed-off-by: Alex Chen minecnly@gmail.com --- syncapi/storage/invites_table.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncapi/storage/invites_table.go b/syncapi/storage/invites_table.go index 88c98f7e3..9f52087f6 100644 --- a/syncapi/storage/invites_table.go +++ b/syncapi/storage/invites_table.go @@ -23,7 +23,7 @@ CREATE INDEX IF NOT EXISTS syncapi_invites_target_user_id_idx -- For deleting old invites CREATE INDEX IF NOT EXISTS syncapi_invites_event_id_idx - ON syncapi_invite_events(target_user_id, id); + ON syncapi_invite_events (event_id); ` const insertInviteEventSQL = "" + From 3578d77d259c852a16dc430725d348e2c62c4ff5 Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Thu, 1 Aug 2019 12:36:13 +0800 Subject: [PATCH 30/49] Implement "full_state" query parameter for /sync (#751) Closes #637. --- syncapi/storage/syncserver.go | 133 +++++++++++++++++++++++++++------- syncapi/sync/requestpool.go | 12 ++- testfile | 1 + 3 files changed, 117 insertions(+), 29 deletions(-) diff --git a/syncapi/storage/syncserver.go b/syncapi/storage/syncserver.go index 20fa8a4e0..e914bddfe 100644 --- a/syncapi/storage/syncserver.go +++ b/syncapi/storage/syncserver.go @@ -241,6 +241,7 @@ func (d *SyncServerDatasource) addPDUDeltaToResponse( device authtypes.Device, fromPos, toPos int64, numRecentEventsPerRoom int, + wantFullState bool, res *types.Response, ) ([]string, error) { txn, err := d.db.BeginTx(ctx, &txReadOnlySnapshot) @@ -254,7 +255,13 @@ func (d *SyncServerDatasource) addPDUDeltaToResponse( // joined rooms, but also which rooms have membership transitions for this user between the 2 PDU stream positions. // This works out what the 'state' key should be for each room as well as which membership block // to put the room into. - deltas, joinedRoomIDs, err := d.getStateDeltas(ctx, &device, txn, fromPos, toPos, device.UserID) + var deltas []stateDelta + var joinedRoomIDs []string + if !wantFullState { + deltas, joinedRoomIDs, err = d.getStateDeltas(ctx, &device, txn, fromPos, toPos, device.UserID) + } else { + deltas, joinedRoomIDs, err = d.getStateDeltasForFullStateSync(ctx, &device, txn, fromPos, toPos, device.UserID) + } if err != nil { return nil, err } @@ -336,15 +343,16 @@ func (d *SyncServerDatasource) IncrementalSync( device authtypes.Device, fromPos, toPos types.SyncPosition, numRecentEventsPerRoom int, + wantFullState bool, ) (*types.Response, error) { nextBatchPos := fromPos.WithUpdates(toPos) res := types.NewResponse(nextBatchPos) var joinedRoomIDs []string var err error - if fromPos.PDUPosition != toPos.PDUPosition { + if fromPos.PDUPosition != toPos.PDUPosition || wantFullState { joinedRoomIDs, err = d.addPDUDeltaToResponse( - ctx, device, fromPos.PDUPosition, toPos.PDUPosition, numRecentEventsPerRoom, res, + ctx, device, fromPos.PDUPosition, toPos.PDUPosition, numRecentEventsPerRoom, wantFullState, res, ) } else { joinedRoomIDs, err = d.roomstate.selectRoomIDsWithMembership( @@ -593,21 +601,30 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( recentEvents := streamEventsToEvents(device, recentStreamEvents) delta.stateEvents = removeDuplicates(delta.stateEvents, recentEvents) // roll back - // Don't bother appending empty room entries - if len(recentEvents) == 0 && len(delta.stateEvents) == 0 { - return nil + var prevPDUPos int64 + + if len(recentEvents) == 0 { + if len(delta.stateEvents) == 0 { + // Don't bother appending empty room entries + return nil + } + + // If full_state=true and since is already up to date, then we'll have + // state events but no recent events. + prevPDUPos = toPos - 1 + } else { + prevPDUPos = recentStreamEvents[0].streamPosition - 1 + } + + if prevPDUPos <= 0 { + prevPDUPos = 1 } switch delta.membership { case membershipJoin: jr := types.NewJoinResponse() - if prevPDUPos := recentStreamEvents[0].streamPosition - 1; prevPDUPos > 0 { - // Use the short form of batch token for prev_batch - jr.Timeline.PrevBatch = strconv.FormatInt(prevPDUPos, 10) - } else { - // Use the short form of batch token for prev_batch - jr.Timeline.PrevBatch = "1" - } + // Use the short form of batch token for prev_batch + jr.Timeline.PrevBatch = strconv.FormatInt(prevPDUPos, 10) jr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true jr.State.Events = gomatrixserverlib.ToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) @@ -618,13 +635,8 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( // TODO: recentEvents may contain events that this user is not allowed to see because they are // no longer in the room. lr := types.NewLeaveResponse() - if prevPDUPos := recentStreamEvents[0].streamPosition - 1; prevPDUPos > 0 { - // Use the short form of batch token for prev_batch - lr.Timeline.PrevBatch = strconv.FormatInt(prevPDUPos, 10) - } else { - // Use the short form of batch token for prev_batch - lr.Timeline.PrevBatch = "1" - } + // Use the short form of batch token for prev_batch + lr.Timeline.PrevBatch = strconv.FormatInt(prevPDUPos, 10) lr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync) lr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true lr.State.Events = gomatrixserverlib.ToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) @@ -758,15 +770,11 @@ func (d *SyncServerDatasource) getStateDeltas( if membership := getMembershipFromEvent(&ev.Event, userID); membership != "" { if membership == membershipJoin { // send full room state down instead of a delta - var allState []gomatrixserverlib.Event - allState, err = d.roomstate.selectCurrentState(ctx, txn, roomID) + var s []streamEvent + s, err = d.currentStateStreamEventsForRoom(ctx, txn, roomID) if err != nil { return nil, nil, err } - s := make([]streamEvent, len(allState)) - for i := 0; i < len(s); i++ { - s[i] = streamEvent{Event: allState[i], streamPosition: 0} - } state[roomID] = s continue // we'll add this room in when we do joined rooms } @@ -798,6 +806,79 @@ func (d *SyncServerDatasource) getStateDeltas( return deltas, joinedRoomIDs, nil } +// getStateDeltasForFullStateSync is a variant of getStateDeltas used for /sync +// requests with full_state=true. +// Fetches full state for all joined rooms and uses selectStateInRange to get +// updates for other rooms. +func (d *SyncServerDatasource) getStateDeltasForFullStateSync( + ctx context.Context, device *authtypes.Device, txn *sql.Tx, + fromPos, toPos int64, userID string, +) ([]stateDelta, []string, error) { + joinedRoomIDs, err := d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, "join") + if err != nil { + return nil, nil, err + } + + // Use a reasonable initial capacity + deltas := make([]stateDelta, 0, len(joinedRoomIDs)) + + // Add full states for all joined rooms + for _, joinedRoomID := range joinedRoomIDs { + s, stateErr := d.currentStateStreamEventsForRoom(ctx, txn, joinedRoomID) + if stateErr != nil { + return nil, nil, stateErr + } + deltas = append(deltas, stateDelta{ + membership: "join", + stateEvents: streamEventsToEvents(device, s), + roomID: joinedRoomID, + }) + } + + // Get all the state events ever between these two positions + stateNeeded, eventMap, err := d.events.selectStateInRange(ctx, txn, fromPos, toPos) + if err != nil { + return nil, nil, err + } + state, err := d.fetchStateEvents(ctx, txn, stateNeeded, eventMap) + if err != nil { + return nil, nil, err + } + + for roomID, stateStreamEvents := range state { + for _, ev := range stateStreamEvents { + if membership := getMembershipFromEvent(&ev.Event, userID); membership != "" { + if membership != "join" { // We've already added full state for all joined rooms above. + deltas = append(deltas, stateDelta{ + membership: membership, + membershipPos: ev.streamPosition, + stateEvents: streamEventsToEvents(device, stateStreamEvents), + roomID: roomID, + }) + } + + break + } + } + } + + return deltas, joinedRoomIDs, nil +} + +func (d *SyncServerDatasource) currentStateStreamEventsForRoom( + ctx context.Context, txn *sql.Tx, roomID string, +) ([]streamEvent, error) { + allState, err := d.roomstate.selectCurrentState(ctx, txn, roomID) + if err != nil { + return nil, err + } + s := make([]streamEvent, len(allState)) + for i := 0; i < len(s); i++ { + s[i] = streamEvent{Event: allState[i], streamPosition: 0} + } + return s, nil +} + // streamEventsToEvents converts streamEvent to Event. If device is non-nil and // matches the streamevent.transactionID device then the transaction ID gets // added to the unsigned section of the output event. diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index a6ec6bd92..d773a4606 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -65,8 +65,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *authtype currPos := rp.notifier.CurrentPosition() - // If this is an initial sync or timeout=0 we return immediately - if syncReq.since == nil || syncReq.timeout == 0 { + if shouldReturnImmediately(syncReq) { syncData, err = rp.currentSyncForUser(*syncReq, currPos) if err != nil { return httputil.LogThenError(req, err) @@ -135,7 +134,7 @@ func (rp *RequestPool) currentSyncForUser(req syncRequest, latestPos types.SyncP if req.since == nil { res, err = rp.db.CompleteSync(req.ctx, req.device.UserID, req.limit) } else { - res, err = rp.db.IncrementalSync(req.ctx, req.device, *req.since, latestPos, req.limit) + res, err = rp.db.IncrementalSync(req.ctx, req.device, *req.since, latestPos, req.limit, req.wantFullState) } if err != nil { @@ -216,3 +215,10 @@ func (rp *RequestPool) appendAccountData( return data, nil } + +// shouldReturnImmediately returns whether the /sync request is an initial sync, +// or timeout=0, or full_state=true, in any of the cases the request should +// return immediately. +func shouldReturnImmediately(syncReq *syncRequest) bool { + return syncReq.since == nil || syncReq.timeout == 0 || syncReq.wantFullState +} diff --git a/testfile b/testfile index c23dda662..4c5163e5e 100644 --- a/testfile +++ b/testfile @@ -158,3 +158,4 @@ Inbound federation rejects remote attempts to join local users to rooms Inbound federation rejects remote attempts to kick local users to rooms An event which redacts itself should be ignored A pair of events which redact each other should be ignored +Full state sync includes joined rooms From d283676b9ac5137e745dcc61587929a296359265 Mon Sep 17 00:00:00 2001 From: Sumukha Pk Date: Fri, 2 Aug 2019 16:47:51 +0530 Subject: [PATCH 31/49] Implements room tagging. (#694) --- clientapi/routing/room_tagging.go | 234 ++++++++++++++++++++++++++++++ clientapi/routing/routing.go | 30 ++++ testfile | 6 + 3 files changed, 270 insertions(+) create mode 100644 clientapi/routing/room_tagging.go diff --git a/clientapi/routing/room_tagging.go b/clientapi/routing/room_tagging.go new file mode 100644 index 000000000..6e7324cd8 --- /dev/null +++ b/clientapi/routing/room_tagging.go @@ -0,0 +1,234 @@ +// Copyright 2019 Sumukha PK +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routing + +import ( + "encoding/json" + "net/http" + + "github.com/sirupsen/logrus" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" + "github.com/matrix-org/dendrite/clientapi/httputil" + "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/clientapi/producers" + "github.com/matrix-org/gomatrix" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" +) + +// newTag creates and returns a new gomatrix.TagContent +func newTag() gomatrix.TagContent { + return gomatrix.TagContent{ + Tags: make(map[string]gomatrix.TagProperties), + } +} + +// GetTags implements GET /_matrix/client/r0/user/{userID}/rooms/{roomID}/tags +func GetTags( + req *http.Request, + accountDB *accounts.Database, + device *authtypes.Device, + userID string, + roomID string, + syncProducer *producers.SyncAPIProducer, +) util.JSONResponse { + + if device.UserID != userID { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("Cannot retrieve another user's tags"), + } + } + + _, data, err := obtainSavedTags(req, userID, roomID, accountDB) + if err != nil { + return httputil.LogThenError(req, err) + } + + if len(data) == 0 { + return util.JSONResponse{ + Code: http.StatusOK, + JSON: struct{}{}, + } + } + + return util.JSONResponse{ + Code: http.StatusOK, + JSON: data[0].Content, + } +} + +// PutTag implements PUT /_matrix/client/r0/user/{userID}/rooms/{roomID}/tags/{tag} +// Put functionality works by getting existing data from the DB (if any), adding +// the tag to the "map" and saving the new "map" to the DB +func PutTag( + req *http.Request, + accountDB *accounts.Database, + device *authtypes.Device, + userID string, + roomID string, + tag string, + syncProducer *producers.SyncAPIProducer, +) util.JSONResponse { + + if device.UserID != userID { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("Cannot modify another user's tags"), + } + } + + var properties gomatrix.TagProperties + if reqErr := httputil.UnmarshalJSONRequest(req, &properties); reqErr != nil { + return *reqErr + } + + localpart, data, err := obtainSavedTags(req, userID, roomID, accountDB) + if err != nil { + return httputil.LogThenError(req, err) + } + + var tagContent gomatrix.TagContent + if len(data) > 0 { + if err = json.Unmarshal(data[0].Content, &tagContent); err != nil { + return httputil.LogThenError(req, err) + } + } else { + tagContent = newTag() + } + tagContent.Tags[tag] = properties + if err = saveTagData(req, localpart, roomID, accountDB, tagContent); err != nil { + return httputil.LogThenError(req, err) + } + + // Send data to syncProducer in order to inform clients of changes + // Run in a goroutine in order to prevent blocking the tag request response + go func() { + if err := syncProducer.SendData(userID, roomID, "m.tag"); err != nil { + logrus.WithError(err).Error("Failed to send m.tag account data update to syncapi") + } + }() + + return util.JSONResponse{ + Code: http.StatusOK, + JSON: struct{}{}, + } +} + +// DeleteTag implements DELETE /_matrix/client/r0/user/{userID}/rooms/{roomID}/tags/{tag} +// Delete functionality works by obtaining the saved tags, removing the intended tag from +// the "map" and then saving the new "map" in the DB +func DeleteTag( + req *http.Request, + accountDB *accounts.Database, + device *authtypes.Device, + userID string, + roomID string, + tag string, + syncProducer *producers.SyncAPIProducer, +) util.JSONResponse { + + if device.UserID != userID { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("Cannot modify another user's tags"), + } + } + + localpart, data, err := obtainSavedTags(req, userID, roomID, accountDB) + if err != nil { + return httputil.LogThenError(req, err) + } + + // If there are no tags in the database, exit + if len(data) == 0 { + // Spec only defines 200 responses for this endpoint so we don't return anything else. + return util.JSONResponse{ + Code: http.StatusOK, + JSON: struct{}{}, + } + } + + var tagContent gomatrix.TagContent + err = json.Unmarshal(data[0].Content, &tagContent) + if err != nil { + return httputil.LogThenError(req, err) + } + + // Check whether the tag to be deleted exists + if _, ok := tagContent.Tags[tag]; ok { + delete(tagContent.Tags, tag) + } else { + // Spec only defines 200 responses for this endpoint so we don't return anything else. + return util.JSONResponse{ + Code: http.StatusOK, + JSON: struct{}{}, + } + } + if err = saveTagData(req, localpart, roomID, accountDB, tagContent); err != nil { + return httputil.LogThenError(req, err) + } + + // Send data to syncProducer in order to inform clients of changes + // Run in a goroutine in order to prevent blocking the tag request response + go func() { + if err := syncProducer.SendData(userID, roomID, "m.tag"); err != nil { + logrus.WithError(err).Error("Failed to send m.tag account data update to syncapi") + } + }() + + return util.JSONResponse{ + Code: http.StatusOK, + JSON: struct{}{}, + } +} + +// obtainSavedTags gets all tags scoped to a userID and roomID +// from the database +func obtainSavedTags( + req *http.Request, + userID string, + roomID string, + accountDB *accounts.Database, +) (string, []gomatrixserverlib.ClientEvent, error) { + localpart, _, err := gomatrixserverlib.SplitID('@', userID) + if err != nil { + return "", nil, err + } + + data, err := accountDB.GetAccountDataByType( + req.Context(), localpart, roomID, "m.tag", + ) + + return localpart, data, err +} + +// saveTagData saves the provided tag data into the database +func saveTagData( + req *http.Request, + localpart string, + roomID string, + accountDB *accounts.Database, + Tag gomatrix.TagContent, +) error { + newTagData, err := json.Marshal(Tag) + if err != nil { + return err + } + + return accountDB.SaveAccountData(req.Context(), localpart, roomID, "m.tag", string(newTagData)) +} diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 8135e49af..ab8f89731 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -483,4 +483,34 @@ func Setup( }} }), ).Methods(http.MethodGet, http.MethodOptions) + + r0mux.Handle("/user/{userId}/rooms/{roomId}/tags", + common.MakeAuthAPI("get_tags", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return GetTags(req, accountDB, device, vars["userId"], vars["roomId"], syncProducer) + }), + ).Methods(http.MethodGet, http.MethodOptions) + + r0mux.Handle("/user/{userId}/rooms/{roomId}/tags/{tag}", + common.MakeAuthAPI("put_tag", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return PutTag(req, accountDB, device, vars["userId"], vars["roomId"], vars["tag"], syncProducer) + }), + ).Methods(http.MethodPut, http.MethodOptions) + + r0mux.Handle("/user/{userId}/rooms/{roomId}/tags/{tag}", + common.MakeAuthAPI("delete_tag", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return DeleteTag(req, accountDB, device, vars["userId"], vars["roomId"], vars["tag"], syncProducer) + }), + ).Methods(http.MethodDelete, http.MethodOptions) } diff --git a/testfile b/testfile index 4c5163e5e..1d97eb37e 100644 --- a/testfile +++ b/testfile @@ -159,3 +159,9 @@ Inbound federation rejects remote attempts to kick local users to rooms An event which redacts itself should be ignored A pair of events which redact each other should be ignored Full state sync includes joined rooms +Can add tag +Can remove tag +Can list tags for a room +Tags appear in an initial v2 /sync +Newly updated tags appear in an incremental v2 /sync +Deleted tags appear in an incremental v2 /sync From f8d2860765325c29d516f766002b46515b3e0a8c Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Tue, 6 Aug 2019 22:07:36 +0800 Subject: [PATCH 32/49] Replace membership and visibility values with constants (#774) Signed-off-by: Alex Chen --- clientapi/auth/storage/accounts/storage.go | 2 +- clientapi/routing/createroom.go | 14 ++++------ clientapi/routing/joinroom.go | 2 +- clientapi/routing/membership.go | 6 ++--- clientapi/routing/profile.go | 2 +- clientapi/routing/routing.go | 2 +- clientapi/threepid/invites.go | 2 +- cmd/create-room-events/main.go | 2 +- federationapi/routing/join.go | 2 +- federationapi/routing/leave.go | 4 +-- federationapi/routing/threepid.go | 2 +- federationsender/consumers/roomserver.go | 2 +- go.mod | 2 +- go.sum | 2 ++ publicroomsapi/directory/directory.go | 5 ++-- publicroomsapi/storage/storage.go | 2 +- roomserver/auth/auth.go | 2 +- roomserver/input/membership.go | 21 +++++---------- roomserver/query/query.go | 2 +- syncapi/storage/syncserver.go | 30 +++++++++------------- syncapi/sync/notifier.go | 8 +++--- 21 files changed, 51 insertions(+), 65 deletions(-) diff --git a/clientapi/auth/storage/accounts/storage.go b/clientapi/auth/storage/accounts/storage.go index 5c8ffffeb..41d75daad 100644 --- a/clientapi/auth/storage/accounts/storage.go +++ b/clientapi/auth/storage/accounts/storage.go @@ -230,7 +230,7 @@ func (d *Database) newMembership( } // Only "join" membership events can be considered as new memberships - if membership == "join" { + if membership == gomatrixserverlib.Join { if err := d.saveMembership(ctx, txn, localpart, roomID, eventID); err != nil { return err } diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index 8c5ee975c..4a76e1b06 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -55,10 +55,6 @@ const ( presetPublicChat = "public_chat" ) -const ( - joinRulePublic = "public" - joinRuleInvite = "invite" -) const ( historyVisibilityShared = "shared" // TODO: These should be implemented once history visibility is implemented @@ -201,7 +197,7 @@ func createRoom( } membershipContent := common.MemberContent{ - Membership: "join", + Membership: gomatrixserverlib.Join, DisplayName: profile.DisplayName, AvatarURL: profile.AvatarURL, } @@ -209,19 +205,19 @@ func createRoom( var joinRules, historyVisibility string switch r.Preset { case presetPrivateChat: - joinRules = joinRuleInvite + joinRules = gomatrixserverlib.Invite historyVisibility = historyVisibilityShared case presetTrustedPrivateChat: - joinRules = joinRuleInvite + joinRules = gomatrixserverlib.Invite historyVisibility = historyVisibilityShared // TODO If trusted_private_chat, all invitees are given the same power level as the room creator. case presetPublicChat: - joinRules = joinRulePublic + joinRules = gomatrixserverlib.Public historyVisibility = historyVisibilityShared default: // Default room rules, r.Preset was previously checked for valid values so // only a request with no preset should end up here. - joinRules = joinRuleInvite + joinRules = gomatrixserverlib.Invite historyVisibility = historyVisibilityShared } diff --git a/clientapi/routing/joinroom.go b/clientapi/routing/joinroom.go index 9c02a93ca..432c982b4 100644 --- a/clientapi/routing/joinroom.go +++ b/clientapi/routing/joinroom.go @@ -70,7 +70,7 @@ func JoinRoomByIDOrAlias( return httputil.LogThenError(req, err) } - content["membership"] = "join" + content["membership"] = gomatrixserverlib.Join content["displayname"] = profile.DisplayName content["avatar_url"] = profile.AvatarURL diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index 61898fecd..5e183fa0f 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -102,7 +102,7 @@ func SendMembership( var returnData interface{} = struct{}{} // The join membership requires the room id to be sent in the response - if membership == "join" { + if membership == gomatrixserverlib.Join { returnData = struct { RoomID string `json:"room_id"` }{roomID} @@ -141,7 +141,7 @@ func buildMembershipEvent( // "unban" or "kick" isn't a valid membership value, change it to "leave" if membership == "unban" || membership == "kick" { - membership = "leave" + membership = gomatrixserverlib.Leave } content := common.MemberContent{ @@ -192,7 +192,7 @@ func loadProfile( func getMembershipStateKey( body threepid.MembershipRequest, device *authtypes.Device, membership string, ) (stateKey string, reason string, err error) { - if membership == "ban" || membership == "unban" || membership == "kick" || membership == "invite" { + if membership == gomatrixserverlib.Ban || membership == "unban" || membership == "kick" || membership == gomatrixserverlib.Invite { // If we're in this case, the state key is contained in the request body, // possibly along with a reason (for "kick" and "ban") so we need to parse // it diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index 034b9ac84..8d28b3660 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -264,7 +264,7 @@ func buildMembershipEvents( } content := common.MemberContent{ - Membership: "join", + Membership: gomatrixserverlib.Join, } content.DisplayName = newProfile.DisplayName diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index ab8f89731..c262db3d8 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -93,7 +93,7 @@ func Setup( }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/join/{roomIDOrAlias}", - common.MakeAuthAPI("join", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + common.MakeAuthAPI(gomatrixserverlib.Join, authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { vars, err := common.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) diff --git a/clientapi/threepid/invites.go b/clientapi/threepid/invites.go index 2538577fd..251afb0d3 100644 --- a/clientapi/threepid/invites.go +++ b/clientapi/threepid/invites.go @@ -91,7 +91,7 @@ func CheckAndProcessInvite( producer *producers.RoomserverProducer, membership string, roomID string, evTime time.Time, ) (inviteStoredOnIDServer bool, err error) { - if membership != "invite" || (body.Address == "" && body.IDServer == "" && body.Medium == "") { + if membership != gomatrixserverlib.Invite || (body.Address == "" && body.IDServer == "" && body.Medium == "") { // If none of the 3PID-specific fields are supplied, it's a standard invite // so return nil for it to be processed as such return diff --git a/cmd/create-room-events/main.go b/cmd/create-room-events/main.go index 1d05b2a12..8475914f0 100644 --- a/cmd/create-room-events/main.go +++ b/cmd/create-room-events/main.go @@ -86,7 +86,7 @@ func main() { // Build a m.room.member event. b.Type = "m.room.member" b.StateKey = userID - b.SetContent(map[string]string{"membership": "join"}) // nolint: errcheck + b.SetContent(map[string]string{"membership": gomatrixserverlib.Join}) // nolint: errcheck b.AuthEvents = []gomatrixserverlib.EventReference{create} member := buildAndOutput() diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go index 0b60408f7..6f6574dd7 100644 --- a/federationapi/routing/join.go +++ b/federationapi/routing/join.go @@ -58,7 +58,7 @@ func MakeJoin( Type: "m.room.member", StateKey: &userID, } - err = builder.SetContent(map[string]interface{}{"membership": "join"}) + err = builder.SetContent(map[string]interface{}{"membership": gomatrixserverlib.Join}) if err != nil { return httputil.LogThenError(httpReq, err) } diff --git a/federationapi/routing/leave.go b/federationapi/routing/leave.go index 3c57d39d1..a982b87f8 100644 --- a/federationapi/routing/leave.go +++ b/federationapi/routing/leave.go @@ -56,7 +56,7 @@ func MakeLeave( Type: "m.room.member", StateKey: &userID, } - err = builder.SetContent(map[string]interface{}{"membership": "leave"}) + err = builder.SetContent(map[string]interface{}{"membership": gomatrixserverlib.Leave}) if err != nil { return httputil.LogThenError(httpReq, err) } @@ -153,7 +153,7 @@ func SendLeave( mem, err := event.Membership() if err != nil { return httputil.LogThenError(httpReq, err) - } else if mem != "leave" { + } else if mem != gomatrixserverlib.Leave { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.BadJSON("The membership in the event content must be set to leave"), diff --git a/federationapi/routing/threepid.go b/federationapi/routing/threepid.go index 05ca8892e..cff311cc4 100644 --- a/federationapi/routing/threepid.go +++ b/federationapi/routing/threepid.go @@ -202,7 +202,7 @@ func createInviteFrom3PIDInvite( content := common.MemberContent{ AvatarURL: profile.AvatarURL, DisplayName: profile.DisplayName, - Membership: "invite", + Membership: gomatrixserverlib.Invite, ThirdPartyInvite: &common.TPInvite{ Signed: inv.Signed, }, diff --git a/federationsender/consumers/roomserver.go b/federationsender/consumers/roomserver.go index 45e48f166..3ba978b1d 100644 --- a/federationsender/consumers/roomserver.go +++ b/federationsender/consumers/roomserver.go @@ -233,7 +233,7 @@ func joinedHostsFromEvents(evs []gomatrixserverlib.Event) ([]types.JoinedHost, e if err != nil { return nil, err } - if membership != "join" { + if membership != gomatrixserverlib.Join { continue } _, serverName, err := gomatrixserverlib.SplitID('@', *ev.StateKey()) diff --git a/go.mod b/go.mod index 5d01012c9..8e14253ca 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/lib/pq v0.0.0-20170918175043-23da1db4f16d github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20190724145009-a6df10ef35d6 + github.com/matrix-org/gomatrixserverlib v0.0.0-20190805173246-3a2199d5ecd6 github.com/matrix-org/naffka v0.0.0-20171115094957-662bfd0841d0 github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 github.com/matttproud/golang_protobuf_extensions v1.0.1 diff --git a/go.sum b/go.sum index 53151121b..0d59d1dd6 100644 --- a/go.sum +++ b/go.sum @@ -56,6 +56,8 @@ github.com/matrix-org/gomatrixserverlib v0.0.0-20190619132215-178ed5e3b8e2 h1:pY github.com/matrix-org/gomatrixserverlib v0.0.0-20190619132215-178ed5e3b8e2/go.mod h1:sf0RcKOdiwJeTti7A313xsaejNUGYDq02MQZ4JD4w/E= github.com/matrix-org/gomatrixserverlib v0.0.0-20190724145009-a6df10ef35d6 h1:B8n1H5Wb1B5jwLzTylBpY0kJCMRqrofT7PmOw4aJFJA= github.com/matrix-org/gomatrixserverlib v0.0.0-20190724145009-a6df10ef35d6/go.mod h1:sf0RcKOdiwJeTti7A313xsaejNUGYDq02MQZ4JD4w/E= +github.com/matrix-org/gomatrixserverlib v0.0.0-20190805173246-3a2199d5ecd6 h1:xr69Hk6QM3RIN6JSvx3RpDowBGpHpDDqhqXCeySwYow= +github.com/matrix-org/gomatrixserverlib v0.0.0-20190805173246-3a2199d5ecd6/go.mod h1:sf0RcKOdiwJeTti7A313xsaejNUGYDq02MQZ4JD4w/E= github.com/matrix-org/naffka v0.0.0-20171115094957-662bfd0841d0 h1:p7WTwG+aXM86+yVrYAiCMW3ZHSmotVvuRbjtt3jC+4A= github.com/matrix-org/naffka v0.0.0-20171115094957-662bfd0841d0/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/util v0.0.0-20171013132526-8b1c8ab81986 h1:TiWl4hLvezAhRPM8tPcPDFTysZ7k4T/1J4GPp/iqlZo= diff --git a/publicroomsapi/directory/directory.go b/publicroomsapi/directory/directory.go index bb0153850..626a1c153 100644 --- a/publicroomsapi/directory/directory.go +++ b/publicroomsapi/directory/directory.go @@ -19,6 +19,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/publicroomsapi/storage" + "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -39,7 +40,7 @@ func GetVisibility( var v roomVisibility if isPublic { - v.Visibility = "public" + v.Visibility = gomatrixserverlib.Public } else { v.Visibility = "private" } @@ -61,7 +62,7 @@ func SetVisibility( return *reqErr } - isPublic := v.Visibility == "public" + isPublic := v.Visibility == gomatrixserverlib.Public if err := publicRoomsDatabase.SetRoomVisibility(req.Context(), isPublic, roomID); err != nil { return httputil.LogThenError(req, err) } diff --git a/publicroomsapi/storage/storage.go b/publicroomsapi/storage/storage.go index eab27041b..aa9806945 100644 --- a/publicroomsapi/storage/storage.go +++ b/publicroomsapi/storage/storage.go @@ -185,7 +185,7 @@ func (d *PublicRoomsServerDatabase) updateNumJoinedUsers( return err } - if membership != "join" { + if membership != gomatrixserverlib.Join { return nil } diff --git a/roomserver/auth/auth.go b/roomserver/auth/auth.go index 2dce6f6dc..5ff1fadad 100644 --- a/roomserver/auth/auth.go +++ b/roomserver/auth/auth.go @@ -23,7 +23,7 @@ func IsServerAllowed( ) bool { for _, ev := range authEvents { membership, err := ev.Membership() - if err != nil || membership != "join" { + if err != nil || membership != gomatrixserverlib.Join { continue } diff --git a/roomserver/input/membership.go b/roomserver/input/membership.go index 0c3fbb80a..841c5fec6 100644 --- a/roomserver/input/membership.go +++ b/roomserver/input/membership.go @@ -23,13 +23,6 @@ import ( "github.com/matrix-org/gomatrixserverlib" ) -// Membership values -// TODO: Factor these out somewhere sensible? -const join = "join" -const leave = "leave" -const invite = "invite" -const ban = "ban" - // updateMembership updates the current membership and the invites for each // user affected by a change in the current state of the room. // Returns a list of output events to write to the kafka log to inform the @@ -91,8 +84,8 @@ func updateMembership( ) ([]api.OutputEvent, error) { var err error // Default the membership to Leave if no event was added or removed. - oldMembership := leave - newMembership := leave + oldMembership := gomatrixserverlib.Leave + newMembership := gomatrixserverlib.Leave if remove != nil { oldMembership, err = remove.Membership() @@ -106,7 +99,7 @@ func updateMembership( return nil, err } } - if oldMembership == newMembership && newMembership != join { + if oldMembership == newMembership && newMembership != gomatrixserverlib.Join { // If the membership is the same then nothing changed and we can return // immediately, unless it's a Join update (e.g. profile update). return updates, nil @@ -118,11 +111,11 @@ func updateMembership( } switch newMembership { - case invite: + case gomatrixserverlib.Invite: return updateToInviteMembership(mu, add, updates) - case join: + case gomatrixserverlib.Join: return updateToJoinMembership(mu, add, updates) - case leave, ban: + case gomatrixserverlib.Leave, gomatrixserverlib.Ban: return updateToLeaveMembership(mu, add, newMembership, updates) default: panic(fmt.Errorf( @@ -183,7 +176,7 @@ func updateToJoinMembership( for _, eventID := range retired { orie := api.OutputRetireInviteEvent{ EventID: eventID, - Membership: join, + Membership: gomatrixserverlib.Join, RetiredByEventID: add.EventID(), TargetUserID: *add.StateKey(), } diff --git a/roomserver/query/query.go b/roomserver/query/query.go index b97d50b17..a62a1f706 100644 --- a/roomserver/query/query.go +++ b/roomserver/query/query.go @@ -359,7 +359,7 @@ func (r *RoomserverQueryAPI) getMembershipsBeforeEventNID( return nil, err } - if membership == "join" { + if membership == gomatrixserverlib.Join { events = append(events, event) } } diff --git a/syncapi/storage/syncserver.go b/syncapi/storage/syncserver.go index e914bddfe..ebec6c3e1 100644 --- a/syncapi/storage/syncserver.go +++ b/syncapi/storage/syncserver.go @@ -35,12 +35,6 @@ import ( "github.com/matrix-org/gomatrixserverlib" ) -const ( - membershipJoin = "join" - membershipLeave = "leave" - membershipBan = "ban" -) - type stateDelta struct { roomID string stateEvents []gomatrixserverlib.Event @@ -356,7 +350,7 @@ func (d *SyncServerDatasource) IncrementalSync( ) } else { joinedRoomIDs, err = d.roomstate.selectRoomIDsWithMembership( - ctx, nil, device.UserID, membershipJoin, + ctx, nil, device.UserID, gomatrixserverlib.Join, ) } if err != nil { @@ -405,7 +399,7 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( res = types.NewResponse(toPos) // Extract room state and recent events for all rooms the user is joined to. - joinedRoomIDs, err = d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, membershipJoin) + joinedRoomIDs, err = d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, gomatrixserverlib.Join) if err != nil { return } @@ -583,7 +577,7 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( res *types.Response, ) error { endPos := toPos - if delta.membershipPos > 0 && delta.membership == membershipLeave { + if delta.membershipPos > 0 && delta.membership == gomatrixserverlib.Leave { // make sure we don't leak recent events after the leave event. // TODO: History visibility makes this somewhat complex to handle correctly. For example: // TODO: This doesn't work for join -> leave in a single /sync request (see events prior to join). @@ -621,7 +615,7 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( } switch delta.membership { - case membershipJoin: + case gomatrixserverlib.Join: jr := types.NewJoinResponse() // Use the short form of batch token for prev_batch jr.Timeline.PrevBatch = strconv.FormatInt(prevPDUPos, 10) @@ -629,9 +623,9 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( jr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true jr.State.Events = gomatrixserverlib.ToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) res.Rooms.Join[delta.roomID] = *jr - case membershipLeave: + case gomatrixserverlib.Leave: fallthrough // transitions to leave are the same as ban - case membershipBan: + case gomatrixserverlib.Ban: // TODO: recentEvents may contain events that this user is not allowed to see because they are // no longer in the room. lr := types.NewLeaveResponse() @@ -768,7 +762,7 @@ func (d *SyncServerDatasource) getStateDeltas( // the 'state' part of the response though, so is transparent modulo bandwidth concerns as it is not added to // the timeline. if membership := getMembershipFromEvent(&ev.Event, userID); membership != "" { - if membership == membershipJoin { + if membership == gomatrixserverlib.Join { // send full room state down instead of a delta var s []streamEvent s, err = d.currentStateStreamEventsForRoom(ctx, txn, roomID) @@ -791,13 +785,13 @@ func (d *SyncServerDatasource) getStateDeltas( } // Add in currently joined rooms - joinedRoomIDs, err := d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, membershipJoin) + joinedRoomIDs, err := d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, gomatrixserverlib.Join) if err != nil { return nil, nil, err } for _, joinedRoomID := range joinedRoomIDs { deltas = append(deltas, stateDelta{ - membership: membershipJoin, + membership: gomatrixserverlib.Join, stateEvents: streamEventsToEvents(device, state[joinedRoomID]), roomID: joinedRoomID, }) @@ -814,7 +808,7 @@ func (d *SyncServerDatasource) getStateDeltasForFullStateSync( ctx context.Context, device *authtypes.Device, txn *sql.Tx, fromPos, toPos int64, userID string, ) ([]stateDelta, []string, error) { - joinedRoomIDs, err := d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, "join") + joinedRoomIDs, err := d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, gomatrixserverlib.Join) if err != nil { return nil, nil, err } @@ -829,7 +823,7 @@ func (d *SyncServerDatasource) getStateDeltasForFullStateSync( return nil, nil, stateErr } deltas = append(deltas, stateDelta{ - membership: "join", + membership: gomatrixserverlib.Join, stateEvents: streamEventsToEvents(device, s), roomID: joinedRoomID, }) @@ -848,7 +842,7 @@ func (d *SyncServerDatasource) getStateDeltasForFullStateSync( for roomID, stateStreamEvents := range state { for _, ev := range stateStreamEvents { if membership := getMembershipFromEvent(&ev.Event, userID); membership != "" { - if membership != "join" { // We've already added full state for all joined rooms above. + if membership != gomatrixserverlib.Join { // We've already added full state for all joined rooms above. deltas = append(deltas, stateDelta{ membership: membership, membershipPos: ev.streamPosition, diff --git a/syncapi/sync/notifier.go b/syncapi/sync/notifier.go index 14bc2efb6..15d6b070c 100644 --- a/syncapi/sync/notifier.go +++ b/syncapi/sync/notifier.go @@ -93,16 +93,16 @@ func (n *Notifier) OnNewEvent( } else { // Keep the joined user map up-to-date switch membership { - case "invite": + case gomatrixserverlib.Invite: usersToNotify = append(usersToNotify, targetUserID) - case "join": + case gomatrixserverlib.Join: // Manually append the new user's ID so they get notified // along all members in the room usersToNotify = append(usersToNotify, targetUserID) n.addJoinedUser(ev.RoomID(), targetUserID) - case "leave": + case gomatrixserverlib.Leave: fallthrough - case "ban": + case gomatrixserverlib.Ban: n.removeJoinedUser(ev.RoomID(), targetUserID) } } From 83f8e05032b82e2dd85d93e77643139bb3231ba2 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 6 Aug 2019 15:52:04 +0100 Subject: [PATCH 33/49] Add /event/ on non world readable room does not work (#777) --- testfile | 1 + 1 file changed, 1 insertion(+) diff --git a/testfile b/testfile index 1d97eb37e..81e47780f 100644 --- a/testfile +++ b/testfile @@ -165,3 +165,4 @@ Can list tags for a room Tags appear in an initial v2 /sync Newly updated tags appear in an incremental v2 /sync Deleted tags appear in an incremental v2 /sync +/event/ on non world readable room does not work From 66bf615360cfe6eac11e901ffe34f70f97330a22 Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Tue, 6 Aug 2019 23:33:53 +0800 Subject: [PATCH 34/49] Fix transaction IDs in transaction cache have global scope (#772) --- clientapi/routing/sendevent.go | 4 +-- common/transactions/transactions.go | 21 +++++++++----- common/transactions/transactions_test.go | 35 +++++++++++++++++++++--- testfile | 1 + 4 files changed, 48 insertions(+), 13 deletions(-) diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go index e916e451e..9696b360e 100644 --- a/clientapi/routing/sendevent.go +++ b/clientapi/routing/sendevent.go @@ -50,7 +50,7 @@ func SendEvent( ) util.JSONResponse { if txnID != nil { // Try to fetch response from transactionsCache - if res, ok := txnCache.FetchTransaction(*txnID); ok { + if res, ok := txnCache.FetchTransaction(device.AccessToken, *txnID); ok { return *res } } @@ -83,7 +83,7 @@ func SendEvent( } // Add response to transactionsCache if txnID != nil { - txnCache.AddTransaction(*txnID, &res) + txnCache.AddTransaction(device.AccessToken, *txnID, &res) } return res diff --git a/common/transactions/transactions.go b/common/transactions/transactions.go index febcb9a75..80b403a98 100644 --- a/common/transactions/transactions.go +++ b/common/transactions/transactions.go @@ -22,7 +22,14 @@ import ( // DefaultCleanupPeriod represents the default time duration after which cacheCleanService runs. const DefaultCleanupPeriod time.Duration = 30 * time.Minute -type txnsMap map[string]*util.JSONResponse +type txnsMap map[CacheKey]*util.JSONResponse + +// CacheKey is the type for the key in a transactions cache. +// This is needed because the spec requires transaction IDs to have a per-access token scope. +type CacheKey struct { + AccessToken string + TxnID string +} // Cache represents a temporary store for response entries. // Entries are evicted after a certain period, defined by cleanupPeriod. @@ -50,14 +57,14 @@ func NewWithCleanupPeriod(cleanupPeriod time.Duration) *Cache { return &t } -// FetchTransaction looks up an entry for txnID in Cache. +// FetchTransaction looks up an entry for the (accessToken, txnID) tuple in Cache. // Looks in both the txnMaps. // Returns (JSON response, true) if txnID is found, else the returned bool is false. -func (t *Cache) FetchTransaction(txnID string) (*util.JSONResponse, bool) { +func (t *Cache) FetchTransaction(accessToken, txnID string) (*util.JSONResponse, bool) { t.RLock() defer t.RUnlock() for _, txns := range t.txnsMaps { - res, ok := txns[txnID] + res, ok := txns[CacheKey{accessToken, txnID}] if ok { return res, true } @@ -65,13 +72,13 @@ func (t *Cache) FetchTransaction(txnID string) (*util.JSONResponse, bool) { return nil, false } -// AddTransaction adds an entry for txnID in Cache for later access. +// AddTransaction adds an entry for the (accessToken, txnID) tuple in Cache. // Adds to the front txnMap. -func (t *Cache) AddTransaction(txnID string, res *util.JSONResponse) { +func (t *Cache) AddTransaction(accessToken, txnID string, res *util.JSONResponse) { t.Lock() defer t.Unlock() - t.txnsMaps[0][txnID] = res + t.txnsMaps[0][CacheKey{accessToken, txnID}] = res } // cacheCleanService is responsible for cleaning up entries after cleanupPeriod. diff --git a/common/transactions/transactions_test.go b/common/transactions/transactions_test.go index 0cdb776cc..f565e4846 100644 --- a/common/transactions/transactions_test.go +++ b/common/transactions/transactions_test.go @@ -24,27 +24,54 @@ type fakeType struct { } var ( - fakeTxnID = "aRandomTxnID" - fakeResponse = &util.JSONResponse{Code: http.StatusOK, JSON: fakeType{ID: "0"}} + fakeAccessToken = "aRandomAccessToken" + fakeAccessToken2 = "anotherRandomAccessToken" + fakeTxnID = "aRandomTxnID" + fakeResponse = &util.JSONResponse{ + Code: http.StatusOK, JSON: fakeType{ID: "0"}, + } + fakeResponse2 = &util.JSONResponse{ + Code: http.StatusOK, JSON: fakeType{ID: "1"}, + } ) // TestCache creates a New Cache and tests AddTransaction & FetchTransaction func TestCache(t *testing.T) { fakeTxnCache := New() - fakeTxnCache.AddTransaction(fakeTxnID, fakeResponse) + fakeTxnCache.AddTransaction(fakeAccessToken, fakeTxnID, fakeResponse) // Add entries for noise. for i := 1; i <= 100; i++ { fakeTxnCache.AddTransaction( + fakeAccessToken, fakeTxnID+string(i), &util.JSONResponse{Code: http.StatusOK, JSON: fakeType{ID: string(i)}}, ) } - testResponse, ok := fakeTxnCache.FetchTransaction(fakeTxnID) + testResponse, ok := fakeTxnCache.FetchTransaction(fakeAccessToken, fakeTxnID) if !ok { t.Error("Failed to retrieve entry for txnID: ", fakeTxnID) } else if testResponse.JSON != fakeResponse.JSON { t.Error("Fetched response incorrect. Expected: ", fakeResponse.JSON, " got: ", testResponse.JSON) } } + +// TestCacheScope ensures transactions with the same transaction ID are not shared +// across multiple access tokens. +func TestCacheScope(t *testing.T) { + cache := New() + cache.AddTransaction(fakeAccessToken, fakeTxnID, fakeResponse) + cache.AddTransaction(fakeAccessToken2, fakeTxnID, fakeResponse2) + + if res, ok := cache.FetchTransaction(fakeAccessToken, fakeTxnID); !ok { + t.Errorf("failed to retrieve entry for (%s, %s)", fakeAccessToken, fakeTxnID) + } else if res.JSON != fakeResponse.JSON { + t.Errorf("Wrong cache entry for (%s, %s). Expected: %v; got: %v", fakeAccessToken, fakeTxnID, fakeResponse.JSON, res.JSON) + } + if res, ok := cache.FetchTransaction(fakeAccessToken2, fakeTxnID); !ok { + t.Errorf("failed to retrieve entry for (%s, %s)", fakeAccessToken, fakeTxnID) + } else if res.JSON != fakeResponse2.JSON { + t.Errorf("Wrong cache entry for (%s, %s). Expected: %v; got: %v", fakeAccessToken, fakeTxnID, fakeResponse2.JSON, res.JSON) + } +} diff --git a/testfile b/testfile index 81e47780f..5791938f7 100644 --- a/testfile +++ b/testfile @@ -159,6 +159,7 @@ Inbound federation rejects remote attempts to kick local users to rooms An event which redacts itself should be ignored A pair of events which redact each other should be ignored Full state sync includes joined rooms +A message sent after an initial sync appears in the timeline of an incremental sync. Can add tag Can remove tag Can list tags for a room From 324ca22b358d53339b2971ec85a02de7282604e6 Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Wed, 7 Aug 2019 00:02:12 +0800 Subject: [PATCH 35/49] Implement profile retrieval over federation (#726) --- appservice/api/query.go | 4 +- clientapi/auth/authtypes/profile.go | 2 +- clientapi/routing/profile.go | 107 +++++++++++++++++++++++----- clientapi/routing/routing.go | 6 +- common/types.go | 5 ++ testfile | 1 + 6 files changed, 100 insertions(+), 25 deletions(-) diff --git a/appservice/api/query.go b/appservice/api/query.go index 8ce3b4e04..9542df565 100644 --- a/appservice/api/query.go +++ b/appservice/api/query.go @@ -20,13 +20,13 @@ package api import ( "context" "database/sql" - "errors" "net/http" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/dendrite/common" commonHTTP "github.com/matrix-org/dendrite/common/http" opentracing "github.com/opentracing/opentracing-go" ) @@ -164,7 +164,7 @@ func RetrieveUserProfile( // If no user exists, return if !userResp.UserIDExists { - return nil, errors.New("no known profile for given user ID") + return nil, common.ErrProfileNoExists } // Try to query the user from the local database again diff --git a/clientapi/auth/authtypes/profile.go b/clientapi/auth/authtypes/profile.go index 6cf508f4f..0bc49658b 100644 --- a/clientapi/auth/authtypes/profile.go +++ b/clientapi/auth/authtypes/profile.go @@ -14,7 +14,7 @@ package authtypes -// Profile represents the profile for a Matrix account on this home server. +// Profile represents the profile for a Matrix account. type Profile struct { Localpart string DisplayName string diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index 8d28b3660..e8ea6cf13 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -30,43 +30,61 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/gomatrix" "github.com/matrix-org/util" ) // GetProfile implements GET /profile/{userID} func GetProfile( - req *http.Request, accountDB *accounts.Database, userID string, asAPI appserviceAPI.AppServiceQueryAPI, + req *http.Request, accountDB *accounts.Database, cfg *config.Dendrite, + userID string, + asAPI appserviceAPI.AppServiceQueryAPI, + federation *gomatrixserverlib.FederationClient, ) util.JSONResponse { - profile, err := appserviceAPI.RetrieveUserProfile(req.Context(), userID, asAPI, accountDB) + profile, err := getProfile(req.Context(), accountDB, cfg, userID, asAPI, federation) if err != nil { + if err == common.ErrProfileNoExists { + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound("The user does not exist or does not have a profile"), + } + } + return httputil.LogThenError(req, err) } - res := common.ProfileResponse{ - AvatarURL: profile.AvatarURL, - DisplayName: profile.DisplayName, - } return util.JSONResponse{ Code: http.StatusOK, - JSON: res, + JSON: common.ProfileResponse{ + AvatarURL: profile.AvatarURL, + DisplayName: profile.DisplayName, + }, } } // GetAvatarURL implements GET /profile/{userID}/avatar_url func GetAvatarURL( - req *http.Request, accountDB *accounts.Database, userID string, asAPI appserviceAPI.AppServiceQueryAPI, + req *http.Request, accountDB *accounts.Database, cfg *config.Dendrite, + userID string, asAPI appserviceAPI.AppServiceQueryAPI, + federation *gomatrixserverlib.FederationClient, ) util.JSONResponse { - profile, err := appserviceAPI.RetrieveUserProfile(req.Context(), userID, asAPI, accountDB) + profile, err := getProfile(req.Context(), accountDB, cfg, userID, asAPI, federation) if err != nil { + if err == common.ErrProfileNoExists { + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound("The user does not exist or does not have a profile"), + } + } + return httputil.LogThenError(req, err) } - res := common.AvatarURL{ - AvatarURL: profile.AvatarURL, - } return util.JSONResponse{ Code: http.StatusOK, - JSON: res, + JSON: common.AvatarURL{ + AvatarURL: profile.AvatarURL, + }, } } @@ -152,18 +170,27 @@ func SetAvatarURL( // GetDisplayName implements GET /profile/{userID}/displayname func GetDisplayName( - req *http.Request, accountDB *accounts.Database, userID string, asAPI appserviceAPI.AppServiceQueryAPI, + req *http.Request, accountDB *accounts.Database, cfg *config.Dendrite, + userID string, asAPI appserviceAPI.AppServiceQueryAPI, + federation *gomatrixserverlib.FederationClient, ) util.JSONResponse { - profile, err := appserviceAPI.RetrieveUserProfile(req.Context(), userID, asAPI, accountDB) + profile, err := getProfile(req.Context(), accountDB, cfg, userID, asAPI, federation) if err != nil { + if err == common.ErrProfileNoExists { + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound("The user does not exist or does not have a profile"), + } + } + return httputil.LogThenError(req, err) } - res := common.DisplayName{ - DisplayName: profile.DisplayName, - } + return util.JSONResponse{ Code: http.StatusOK, - JSON: res, + JSON: common.DisplayName{ + DisplayName: profile.DisplayName, + }, } } @@ -247,6 +274,48 @@ func SetDisplayName( } } +// getProfile gets the full profile of a user by querying the database or a +// remote homeserver. +// Returns an error when something goes wrong or specifically +// common.ErrProfileNoExists when the profile doesn't exist. +func getProfile( + ctx context.Context, accountDB *accounts.Database, cfg *config.Dendrite, + userID string, + asAPI appserviceAPI.AppServiceQueryAPI, + federation *gomatrixserverlib.FederationClient, +) (*authtypes.Profile, error) { + localpart, domain, err := gomatrixserverlib.SplitID('@', userID) + if err != nil { + return nil, err + } + + if domain != cfg.Matrix.ServerName { + profile, fedErr := federation.LookupProfile(ctx, domain, userID, "") + if fedErr != nil { + if x, ok := fedErr.(gomatrix.HTTPError); ok { + if x.Code == http.StatusNotFound { + return nil, common.ErrProfileNoExists + } + } + + return nil, fedErr + } + + return &authtypes.Profile{ + Localpart: localpart, + DisplayName: profile.DisplayName, + AvatarURL: profile.AvatarURL, + }, nil + } + + profile, err := appserviceAPI.RetrieveUserProfile(ctx, userID, asAPI, accountDB) + if err != nil { + return nil, err + } + + return profile, nil +} + func buildMembershipEvents( ctx context.Context, memberships []authtypes.Membership, diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index c262db3d8..825dd97aa 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -283,7 +283,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return GetProfile(req, accountDB, vars["userID"], asAPI) + return GetProfile(req, accountDB, &cfg, vars["userID"], asAPI, federation) }), ).Methods(http.MethodGet, http.MethodOptions) @@ -293,7 +293,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return GetAvatarURL(req, accountDB, vars["userID"], asAPI) + return GetAvatarURL(req, accountDB, &cfg, vars["userID"], asAPI, federation) }), ).Methods(http.MethodGet, http.MethodOptions) @@ -315,7 +315,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return GetDisplayName(req, accountDB, vars["userID"], asAPI) + return GetDisplayName(req, accountDB, &cfg, vars["userID"], asAPI, federation) }), ).Methods(http.MethodGet, http.MethodOptions) diff --git a/common/types.go b/common/types.go index 6888d3806..91765be00 100644 --- a/common/types.go +++ b/common/types.go @@ -15,9 +15,14 @@ package common import ( + "errors" "strconv" ) +// ErrProfileNoExists is returned when trying to lookup a user's profile that +// doesn't exist locally. +var ErrProfileNoExists = errors.New("no known profile for given user ID") + // AccountData represents account data sent from the client API server to the // sync API server type AccountData struct { diff --git a/testfile b/testfile index 5791938f7..74c9d9e4f 100644 --- a/testfile +++ b/testfile @@ -167,3 +167,4 @@ Tags appear in an initial v2 /sync Newly updated tags appear in an incremental v2 /sync Deleted tags appear in an incremental v2 /sync /event/ on non world readable room does not work +Outbound federation can query profile data From 8c721b555ed51b15010feada9f9f7ed8854ff42f Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 6 Aug 2019 20:26:15 +0100 Subject: [PATCH 36/49] Scope the buildkite build badge only to tests running on master (#779) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8eadaf431..4e628c0ff 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Dendrite [![Build Status](https://badge.buildkite.com/4be40938ab19f2bbc4a6c6724517353ee3ec1422e279faf374.svg)](https://buildkite.com/matrix-dot-org/dendrite) [![CircleCI](https://circleci.com/gh/matrix-org/dendrite.svg?style=svg)](https://circleci.com/gh/matrix-org/dendrite) [![Dendrite Dev on Matrix](https://img.shields.io/matrix/dendrite-dev:matrix.org.svg?label=%23dendrite-dev%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite-dev:matrix.org) [![Dendrite on Matrix](https://img.shields.io/matrix/dendrite:matrix.org.svg?label=%23dendrite%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite:matrix.org) +# Dendrite [![Build Status](https://badge.buildkite.com/4be40938ab19f2bbc4a6c6724517353ee3ec1422e279faf374.svg?branch=master)](https://buildkite.com/matrix-dot-org/dendrite) [![CircleCI](https://circleci.com/gh/matrix-org/dendrite.svg?style=svg)](https://circleci.com/gh/matrix-org/dendrite) [![Dendrite Dev on Matrix](https://img.shields.io/matrix/dendrite-dev:matrix.org.svg?label=%23dendrite-dev%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite-dev:matrix.org) [![Dendrite on Matrix](https://img.shields.io/matrix/dendrite:matrix.org.svg?label=%23dendrite%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite:matrix.org) Dendrite will be a matrix homeserver written in go. From 94ea325c93d03362795ce85c49008797d37d36dd Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Wed, 7 Aug 2019 11:00:58 +0800 Subject: [PATCH 37/49] Fix permission and 404 response for alias deletion - #654 (#706) --- clientapi/routing/directory.go | 25 +++++++++++++++- roomserver/alias/alias.go | 37 ++++++++++++++++++++++-- roomserver/alias/alias_test.go | 8 ++++- roomserver/api/alias.go | 35 ++++++++++++++++++++++ roomserver/storage/room_aliases_table.go | 33 ++++++++++++++++----- roomserver/storage/storage.go | 11 +++++-- 6 files changed, 135 insertions(+), 14 deletions(-) diff --git a/clientapi/routing/directory.go b/clientapi/routing/directory.go index ab85e86a9..0d91d0426 100644 --- a/clientapi/routing/directory.go +++ b/clientapi/routing/directory.go @@ -164,13 +164,36 @@ func SetLocalAlias( } // RemoveLocalAlias implements DELETE /directory/room/{roomAlias} -// TODO: Check if the user has the power level to remove an alias func RemoveLocalAlias( req *http.Request, device *authtypes.Device, alias string, aliasAPI roomserverAPI.RoomserverAliasAPI, ) util.JSONResponse { + + creatorQueryReq := roomserverAPI.GetCreatorIDForAliasRequest{ + Alias: alias, + } + var creatorQueryRes roomserverAPI.GetCreatorIDForAliasResponse + if err := aliasAPI.GetCreatorIDForAlias(req.Context(), &creatorQueryReq, &creatorQueryRes); err != nil { + return httputil.LogThenError(req, err) + } + + if creatorQueryRes.UserID == "" { + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound("Alias does not exist"), + } + } + + if creatorQueryRes.UserID != device.UserID { + // TODO: Still allow deletion if user is admin + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("You do not have permission to delete this alias"), + } + } + queryReq := roomserverAPI.RemoveRoomAliasRequest{ Alias: alias, UserID: device.UserID, diff --git a/roomserver/alias/alias.go b/roomserver/alias/alias.go index f699e3362..aeaf5ae94 100644 --- a/roomserver/alias/alias.go +++ b/roomserver/alias/alias.go @@ -33,13 +33,16 @@ import ( type RoomserverAliasAPIDatabase interface { // Save a given room alias with the room ID it refers to. // Returns an error if there was a problem talking to the database. - SetRoomAlias(ctx context.Context, alias string, roomID string) error + SetRoomAlias(ctx context.Context, alias string, roomID string, creatorUserID string) error // Look up the room ID a given alias refers to. // Returns an error if there was a problem talking to the database. GetRoomIDForAlias(ctx context.Context, alias string) (string, error) // Look up all aliases referring to a given room ID. // Returns an error if there was a problem talking to the database. GetAliasesForRoomID(ctx context.Context, roomID string) ([]string, error) + // Get the user ID of the creator of an alias. + // Returns an error if there was a problem talking to the database. + GetCreatorIDForAlias(ctx context.Context, alias string) (string, error) // Remove a given room alias. // Returns an error if there was a problem talking to the database. RemoveRoomAlias(ctx context.Context, alias string) error @@ -73,7 +76,7 @@ func (r *RoomserverAliasAPI) SetRoomAlias( response.AliasExists = false // Save the new alias - if err := r.DB.SetRoomAlias(ctx, request.Alias, request.RoomID); err != nil { + if err := r.DB.SetRoomAlias(ctx, request.Alias, request.RoomID, request.UserID); err != nil { return err } @@ -133,6 +136,22 @@ func (r *RoomserverAliasAPI) GetAliasesForRoomID( return nil } +// GetCreatorIDForAlias implements alias.RoomserverAliasAPI +func (r *RoomserverAliasAPI) GetCreatorIDForAlias( + ctx context.Context, + request *roomserverAPI.GetCreatorIDForAliasRequest, + response *roomserverAPI.GetCreatorIDForAliasResponse, +) error { + // Look up the aliases in the database for the given RoomID + creatorID, err := r.DB.GetCreatorIDForAlias(ctx, request.Alias) + if err != nil { + return err + } + + response.UserID = creatorID + return nil +} + // RemoveRoomAlias implements alias.RoomserverAliasAPI func (r *RoomserverAliasAPI) RemoveRoomAlias( ctx context.Context, @@ -277,6 +296,20 @@ func (r *RoomserverAliasAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) + servMux.Handle( + roomserverAPI.RoomserverGetCreatorIDForAliasPath, + common.MakeInternalAPI("GetCreatorIDForAlias", func(req *http.Request) util.JSONResponse { + var request roomserverAPI.GetCreatorIDForAliasRequest + var response roomserverAPI.GetCreatorIDForAliasResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.GetCreatorIDForAlias(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) servMux.Handle( roomserverAPI.RoomserverGetAliasesForRoomIDPath, common.MakeInternalAPI("getAliasesForRoomID", func(req *http.Request) util.JSONResponse { diff --git a/roomserver/alias/alias_test.go b/roomserver/alias/alias_test.go index 4b9ca022d..6ddb63a73 100644 --- a/roomserver/alias/alias_test.go +++ b/roomserver/alias/alias_test.go @@ -30,7 +30,7 @@ type MockRoomserverAliasAPIDatabase struct { } // These methods can be essentially noop -func (db MockRoomserverAliasAPIDatabase) SetRoomAlias(ctx context.Context, alias string, roomID string) error { +func (db MockRoomserverAliasAPIDatabase) SetRoomAlias(ctx context.Context, alias string, roomID string, creatorUserID string) error { return nil } @@ -43,6 +43,12 @@ func (db MockRoomserverAliasAPIDatabase) RemoveRoomAlias(ctx context.Context, al return nil } +func (db *MockRoomserverAliasAPIDatabase) GetCreatorIDForAlias( + ctx context.Context, alias string, +) (string, error) { + return "", nil +} + // This method needs to change depending on test case func (db *MockRoomserverAliasAPIDatabase) GetRoomIDForAlias( ctx context.Context, diff --git a/roomserver/api/alias.go b/roomserver/api/alias.go index 576710713..cb78f726a 100644 --- a/roomserver/api/alias.go +++ b/roomserver/api/alias.go @@ -62,6 +62,18 @@ type GetAliasesForRoomIDResponse struct { Aliases []string `json:"aliases"` } +// GetCreatorIDForAliasRequest is a request to GetCreatorIDForAlias +type GetCreatorIDForAliasRequest struct { + // The alias we want to find the creator of + Alias string `json:"alias"` +} + +// GetCreatorIDForAliasResponse is a response to GetCreatorIDForAlias +type GetCreatorIDForAliasResponse struct { + // The user ID of the alias creator + UserID string `json:"user_id"` +} + // RemoveRoomAliasRequest is a request to RemoveRoomAlias type RemoveRoomAliasRequest struct { // ID of the user removing the alias @@ -96,6 +108,13 @@ type RoomserverAliasAPI interface { response *GetAliasesForRoomIDResponse, ) error + // Get the user ID of the creator of an alias + GetCreatorIDForAlias( + ctx context.Context, + req *GetCreatorIDForAliasRequest, + response *GetCreatorIDForAliasResponse, + ) error + // Remove a room alias RemoveRoomAlias( ctx context.Context, @@ -113,6 +132,9 @@ const RoomserverGetRoomIDForAliasPath = "/api/roomserver/GetRoomIDForAlias" // RoomserverGetAliasesForRoomIDPath is the HTTP path for the GetAliasesForRoomID API. const RoomserverGetAliasesForRoomIDPath = "/api/roomserver/GetAliasesForRoomID" +// RoomserverGetCreatorIDForAliasPath is the HTTP path for the GetCreatorIDForAlias API. +const RoomserverGetCreatorIDForAliasPath = "/api/roomserver/GetCreatorIDForAlias" + // RoomserverRemoveRoomAliasPath is the HTTP path for the RemoveRoomAlias API. const RoomserverRemoveRoomAliasPath = "/api/roomserver/removeRoomAlias" @@ -169,6 +191,19 @@ func (h *httpRoomserverAliasAPI) GetAliasesForRoomID( return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } +// GetCreatorIDForAlias implements RoomserverAliasAPI +func (h *httpRoomserverAliasAPI) GetCreatorIDForAlias( + ctx context.Context, + request *GetCreatorIDForAliasRequest, + response *GetCreatorIDForAliasResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "GetCreatorIDForAlias") + defer span.Finish() + + apiURL := h.roomserverURL + RoomserverGetCreatorIDForAliasPath + return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + // RemoveRoomAlias implements RoomserverAliasAPI func (h *httpRoomserverAliasAPI) RemoveRoomAlias( ctx context.Context, diff --git a/roomserver/storage/room_aliases_table.go b/roomserver/storage/room_aliases_table.go index f640c37fe..3ed20e8e3 100644 --- a/roomserver/storage/room_aliases_table.go +++ b/roomserver/storage/room_aliases_table.go @@ -25,14 +25,16 @@ CREATE TABLE IF NOT EXISTS roomserver_room_aliases ( -- Alias of the room alias TEXT NOT NULL PRIMARY KEY, -- Room ID the alias refers to - room_id TEXT NOT NULL + room_id TEXT NOT NULL, + -- User ID of the creator of this alias + creator_id TEXT NOT NULL ); CREATE INDEX IF NOT EXISTS roomserver_room_id_idx ON roomserver_room_aliases(room_id); ` const insertRoomAliasSQL = "" + - "INSERT INTO roomserver_room_aliases (alias, room_id) VALUES ($1, $2)" + "INSERT INTO roomserver_room_aliases (alias, room_id, creator_id) VALUES ($1, $2, $3)" const selectRoomIDFromAliasSQL = "" + "SELECT room_id FROM roomserver_room_aliases WHERE alias = $1" @@ -40,14 +42,18 @@ const selectRoomIDFromAliasSQL = "" + const selectAliasesFromRoomIDSQL = "" + "SELECT alias FROM roomserver_room_aliases WHERE room_id = $1" +const selectCreatorIDFromAliasSQL = "" + + "SELECT creator_id FROM roomserver_room_aliases WHERE alias = $1" + const deleteRoomAliasSQL = "" + "DELETE FROM roomserver_room_aliases WHERE alias = $1" type roomAliasesStatements struct { - insertRoomAliasStmt *sql.Stmt - selectRoomIDFromAliasStmt *sql.Stmt - selectAliasesFromRoomIDStmt *sql.Stmt - deleteRoomAliasStmt *sql.Stmt + insertRoomAliasStmt *sql.Stmt + selectRoomIDFromAliasStmt *sql.Stmt + selectAliasesFromRoomIDStmt *sql.Stmt + selectCreatorIDFromAliasStmt *sql.Stmt + deleteRoomAliasStmt *sql.Stmt } func (s *roomAliasesStatements) prepare(db *sql.DB) (err error) { @@ -59,14 +65,15 @@ func (s *roomAliasesStatements) prepare(db *sql.DB) (err error) { {&s.insertRoomAliasStmt, insertRoomAliasSQL}, {&s.selectRoomIDFromAliasStmt, selectRoomIDFromAliasSQL}, {&s.selectAliasesFromRoomIDStmt, selectAliasesFromRoomIDSQL}, + {&s.selectCreatorIDFromAliasStmt, selectCreatorIDFromAliasSQL}, {&s.deleteRoomAliasStmt, deleteRoomAliasSQL}, }.prepare(db) } func (s *roomAliasesStatements) insertRoomAlias( - ctx context.Context, alias string, roomID string, + ctx context.Context, alias string, roomID string, creatorUserID string, ) (err error) { - _, err = s.insertRoomAliasStmt.ExecContext(ctx, alias, roomID) + _, err = s.insertRoomAliasStmt.ExecContext(ctx, alias, roomID, creatorUserID) return } @@ -101,6 +108,16 @@ func (s *roomAliasesStatements) selectAliasesFromRoomID( return } +func (s *roomAliasesStatements) selectCreatorIDFromAlias( + ctx context.Context, alias string, +) (creatorID string, err error) { + err = s.selectCreatorIDFromAliasStmt.QueryRowContext(ctx, alias).Scan(&creatorID) + if err == sql.ErrNoRows { + return "", nil + } + return +} + func (s *roomAliasesStatements) deleteRoomAlias( ctx context.Context, alias string, ) (err error) { diff --git a/roomserver/storage/storage.go b/roomserver/storage/storage.go index f6c2fccd4..71c13b7ca 100644 --- a/roomserver/storage/storage.go +++ b/roomserver/storage/storage.go @@ -441,8 +441,8 @@ func (d *Database) GetInvitesForUser( } // SetRoomAlias implements alias.RoomserverAliasAPIDB -func (d *Database) SetRoomAlias(ctx context.Context, alias string, roomID string) error { - return d.statements.insertRoomAlias(ctx, alias, roomID) +func (d *Database) SetRoomAlias(ctx context.Context, alias string, roomID string, creatorUserID string) error { + return d.statements.insertRoomAlias(ctx, alias, roomID, creatorUserID) } // GetRoomIDForAlias implements alias.RoomserverAliasAPIDB @@ -455,6 +455,13 @@ func (d *Database) GetAliasesForRoomID(ctx context.Context, roomID string) ([]st return d.statements.selectAliasesFromRoomID(ctx, roomID) } +// GetCreatorIDForAlias implements alias.RoomserverAliasAPIDB +func (d *Database) GetCreatorIDForAlias( + ctx context.Context, alias string, +) (string, error) { + return d.statements.selectCreatorIDFromAlias(ctx, alias) +} + // RemoveRoomAlias implements alias.RoomserverAliasAPIDB func (d *Database) RemoveRoomAlias(ctx context.Context, alias string) error { return d.statements.deleteRoomAlias(ctx, alias) From 76e4ebaf78a698eb775ccb358fb5c253084747b9 Mon Sep 17 00:00:00 2001 From: Thibaut CHARLES Date: Wed, 7 Aug 2019 12:12:09 +0200 Subject: [PATCH 38/49] State events filtering database api (#438) This PR adds a gomatrixserverlib.Filter parameter to functions handling the syncapi_current_room_state table. It does not implement any filtering logic inside the syncapi IncrementalSync/CompleteSync functions, just the APIs for future use. Default filters are provided as placeholders in IncrementalSync/CompleteSync, so behaviour should be unchanged (except the default 20 event limit) SQL table will be changed. You can upgrade an existing database using: ``` ALTER TABLE syncapi_current_room_state ADD COLUMN IF NOT EXISTS sender text; UPDATE syncapi_current_room_state SET sender=(event_json::json->>'sender'); ALTER TABLE syncapi_current_room_state ALTER COLUMN sender SET NOT NULL; ALTER TABLE syncapi_current_room_state ADD COLUMN IF NOT EXISTS contains_url bool; UPDATE syncapi_current_room_state SET contains_url=(event_json::json->>'content')::json->>'url' IS NOT NULL; ALTER TABLE syncapi_current_room_state ALTER COLUMN contains_url SET NOT NULL; ``` Note: This depends on #436 (and includes all its commits). I'm not sure if Github will remove the duplicated commits once #436 is merged. --- syncapi/routing/state.go | 5 ++- syncapi/storage/current_room_state_table.go | 42 ++++++++++++++++++--- syncapi/storage/filtering.go | 36 ++++++++++++++++++ syncapi/storage/output_room_events_table.go | 41 ++++++++++++++++++-- syncapi/storage/syncserver.go | 31 ++++++++++----- 5 files changed, 134 insertions(+), 21 deletions(-) create mode 100644 syncapi/storage/filtering.go diff --git a/syncapi/routing/state.go b/syncapi/routing/state.go index 5571a0525..87a93d194 100644 --- a/syncapi/routing/state.go +++ b/syncapi/routing/state.go @@ -44,7 +44,10 @@ func OnIncomingStateRequest(req *http.Request, db *storage.SyncServerDatasource, // TODO(#287): Auth request and handle the case where the user has left (where // we should return the state at the poin they left) - stateEvents, err := db.GetStateEventsForRoom(req.Context(), roomID) + stateFilterPart := gomatrixserverlib.DefaultFilterPart() + // TODO: stateFilterPart should not limit the number of state events (or only limits abusive number of events) + + stateEvents, err := db.GetStateEventsForRoom(req.Context(), roomID, &stateFilterPart) if err != nil { return httputil.LogThenError(req, err) } diff --git a/syncapi/storage/current_room_state_table.go b/syncapi/storage/current_room_state_table.go index 852bfd760..88e7a76c3 100644 --- a/syncapi/storage/current_room_state_table.go +++ b/syncapi/storage/current_room_state_table.go @@ -17,6 +17,7 @@ package storage import ( "context" "database/sql" + "encoding/json" "github.com/lib/pq" "github.com/matrix-org/dendrite/common" @@ -32,6 +33,10 @@ CREATE TABLE IF NOT EXISTS syncapi_current_room_state ( event_id TEXT NOT NULL, -- The state event type e.g 'm.room.member' type TEXT NOT NULL, + -- The 'sender' property of the event. + sender TEXT NOT NULL, + -- true if the event content contains a url key + contains_url BOOL NOT NULL, -- The state_key value for this state event e.g '' state_key TEXT NOT NULL, -- The JSON for the event. Stored as TEXT because this should be valid UTF-8. @@ -46,16 +51,16 @@ CREATE TABLE IF NOT EXISTS syncapi_current_room_state ( CONSTRAINT syncapi_room_state_unique UNIQUE (room_id, type, state_key) ); -- for event deletion -CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_id_idx ON syncapi_current_room_state(event_id); +CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_id_idx ON syncapi_current_room_state(event_id, room_id, type, sender, contains_url); -- for querying membership states of users CREATE INDEX IF NOT EXISTS syncapi_membership_idx ON syncapi_current_room_state(type, state_key, membership) WHERE membership IS NOT NULL AND membership != 'leave'; ` const upsertRoomStateSQL = "" + - "INSERT INTO syncapi_current_room_state (room_id, event_id, type, state_key, event_json, membership, added_at)" + - " VALUES ($1, $2, $3, $4, $5, $6, $7)" + + "INSERT INTO syncapi_current_room_state (room_id, event_id, type, sender, contains_url, state_key, event_json, membership, added_at)" + + " VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)" + " ON CONFLICT ON CONSTRAINT syncapi_room_state_unique" + - " DO UPDATE SET event_id = $2, event_json = $5, membership = $6, added_at = $7" + " DO UPDATE SET event_id = $2, sender=$4, contains_url=$5, event_json = $7, membership = $8, added_at = $9" const deleteRoomStateByEventIDSQL = "" + "DELETE FROM syncapi_current_room_state WHERE event_id = $1" @@ -64,7 +69,13 @@ const selectRoomIDsWithMembershipSQL = "" + "SELECT room_id FROM syncapi_current_room_state WHERE type = 'm.room.member' AND state_key = $1 AND membership = $2" const selectCurrentStateSQL = "" + - "SELECT event_json FROM syncapi_current_room_state WHERE room_id = $1" + "SELECT event_json FROM syncapi_current_room_state WHERE room_id = $1" + + " AND ( $2::text[] IS NULL OR sender = ANY($2) )" + + " AND ( $3::text[] IS NULL OR NOT(sender = ANY($3)) )" + + " AND ( $4::text[] IS NULL OR type LIKE ANY($4) )" + + " AND ( $5::text[] IS NULL OR NOT(type LIKE ANY($5)) )" + + " AND ( $6::bool IS NULL OR contains_url = $6 )" + + " LIMIT $7" const selectJoinedUsersSQL = "" + "SELECT room_id, state_key FROM syncapi_current_room_state WHERE type = 'm.room.member' AND membership = 'join'" @@ -166,9 +177,17 @@ func (s *currentRoomStateStatements) selectRoomIDsWithMembership( // CurrentState returns all the current state events for the given room. func (s *currentRoomStateStatements) selectCurrentState( ctx context.Context, txn *sql.Tx, roomID string, + stateFilterPart *gomatrixserverlib.FilterPart, ) ([]gomatrixserverlib.Event, error) { stmt := common.TxStmt(txn, s.selectCurrentStateStmt) - rows, err := stmt.QueryContext(ctx, roomID) + rows, err := stmt.QueryContext(ctx, roomID, + pq.StringArray(stateFilterPart.Senders), + pq.StringArray(stateFilterPart.NotSenders), + pq.StringArray(filterConvertTypeWildcardToSQL(stateFilterPart.Types)), + pq.StringArray(filterConvertTypeWildcardToSQL(stateFilterPart.NotTypes)), + stateFilterPart.ContainsURL, + stateFilterPart.Limit, + ) if err != nil { return nil, err } @@ -189,12 +208,23 @@ func (s *currentRoomStateStatements) upsertRoomState( ctx context.Context, txn *sql.Tx, event gomatrixserverlib.Event, membership *string, addedAt int64, ) error { + // Parse content as JSON and search for an "url" key + containsURL := false + var content map[string]interface{} + if json.Unmarshal(event.Content(), &content) != nil { + // Set containsURL to true if url is present + _, containsURL = content["url"] + } + + // upsert state event stmt := common.TxStmt(txn, s.upsertRoomStateStmt) _, err := stmt.ExecContext( ctx, event.RoomID(), event.EventID(), event.Type(), + event.Sender(), + containsURL, *event.StateKey(), event.JSON(), membership, diff --git a/syncapi/storage/filtering.go b/syncapi/storage/filtering.go new file mode 100644 index 000000000..27b0b888a --- /dev/null +++ b/syncapi/storage/filtering.go @@ -0,0 +1,36 @@ +// Copyright 2017 Thibaut CHARLES +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "strings" +) + +// filterConvertWildcardToSQL converts wildcards as defined in +// https://matrix.org/docs/spec/client_server/r0.3.0.html#post-matrix-client-r0-user-userid-filter +// to SQL wildcards that can be used with LIKE() +func filterConvertTypeWildcardToSQL(values []string) []string { + if values == nil { + // Return nil instead of []string{} so IS NULL can work correctly when + // the return value is passed into SQL queries + return nil + } + + ret := make([]string, len(values)) + for i := range values { + ret[i] = strings.Replace(values[i], "*", "%", -1) + } + return ret +} diff --git a/syncapi/storage/output_room_events_table.go b/syncapi/storage/output_room_events_table.go index 34632aedf..8fbeb18c9 100644 --- a/syncapi/storage/output_room_events_table.go +++ b/syncapi/storage/output_room_events_table.go @@ -17,6 +17,7 @@ package storage import ( "context" "database/sql" + "encoding/json" "sort" "github.com/matrix-org/dendrite/roomserver/api" @@ -43,6 +44,12 @@ CREATE TABLE IF NOT EXISTS syncapi_output_room_events ( room_id TEXT NOT NULL, -- The JSON for the event. Stored as TEXT because this should be valid UTF-8. event_json TEXT NOT NULL, + -- The event type e.g 'm.room.member'. + type TEXT NOT NULL, + -- The 'sender' property of the event. + sender TEXT NOT NULL, + -- true if the event content contains a url key. + contains_url BOOL NOT NULL, -- A list of event IDs which represent a delta of added/removed room state. This can be NULL -- if there is no delta. add_state_ids TEXT[], @@ -56,8 +63,8 @@ CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_id_idx ON syncapi_output_room_ev const insertEventSQL = "" + "INSERT INTO syncapi_output_room_events (" + - " room_id, event_id, event_json, add_state_ids, remove_state_ids, device_id, transaction_id" + - ") VALUES ($1, $2, $3, $4, $5, $6, $7) RETURNING id" + "room_id, event_id, event_json, type, sender, contains_url, add_state_ids, remove_state_ids, device_id, transaction_id" + + ") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) RETURNING id" const selectEventsSQL = "" + "SELECT id, event_json FROM syncapi_output_room_events WHERE event_id = ANY($1)" @@ -75,7 +82,13 @@ const selectStateInRangeSQL = "" + "SELECT id, event_json, add_state_ids, remove_state_ids" + " FROM syncapi_output_room_events" + " WHERE (id > $1 AND id <= $2) AND (add_state_ids IS NOT NULL OR remove_state_ids IS NOT NULL)" + - " ORDER BY id ASC" + " AND ( $3::text[] IS NULL OR sender = ANY($3) )" + + " AND ( $4::text[] IS NULL OR NOT(sender = ANY($4)) )" + + " AND ( $5::text[] IS NULL OR type LIKE ANY($5) )" + + " AND ( $6::text[] IS NULL OR NOT(type LIKE ANY($6)) )" + + " AND ( $7::bool IS NULL OR contains_url = $7 )" + + " ORDER BY id ASC" + + " LIMIT $8" type outputRoomEventsStatements struct { insertEventStmt *sql.Stmt @@ -113,10 +126,19 @@ func (s *outputRoomEventsStatements) prepare(db *sql.DB) (err error) { // two positions, only the most recent state is returned. func (s *outputRoomEventsStatements) selectStateInRange( ctx context.Context, txn *sql.Tx, oldPos, newPos int64, + stateFilterPart *gomatrixserverlib.FilterPart, ) (map[string]map[string]bool, map[string]streamEvent, error) { stmt := common.TxStmt(txn, s.selectStateInRangeStmt) - rows, err := stmt.QueryContext(ctx, oldPos, newPos) + rows, err := stmt.QueryContext( + ctx, oldPos, newPos, + pq.StringArray(stateFilterPart.Senders), + pq.StringArray(stateFilterPart.NotSenders), + pq.StringArray(filterConvertTypeWildcardToSQL(stateFilterPart.Types)), + pq.StringArray(filterConvertTypeWildcardToSQL(stateFilterPart.NotTypes)), + stateFilterPart.ContainsURL, + stateFilterPart.Limit, + ) if err != nil { return nil, nil, err } @@ -205,12 +227,23 @@ func (s *outputRoomEventsStatements) insertEvent( txnID = &transactionID.TransactionID } + // Parse content as JSON and search for an "url" key + containsURL := false + var content map[string]interface{} + if json.Unmarshal(event.Content(), &content) != nil { + // Set containsURL to true if url is present + _, containsURL = content["url"] + } + stmt := common.TxStmt(txn, s.insertEventStmt) err = stmt.QueryRowContext( ctx, event.RoomID(), event.EventID(), event.JSON(), + event.Type(), + event.Sender(), + containsURL, pq.StringArray(addState), pq.StringArray(removeState), deviceID, diff --git a/syncapi/storage/syncserver.go b/syncapi/storage/syncserver.go index ebec6c3e1..c57a90256 100644 --- a/syncapi/storage/syncserver.go +++ b/syncapi/storage/syncserver.go @@ -185,10 +185,10 @@ func (d *SyncServerDatasource) GetStateEvent( // Returns an empty slice if no state events could be found for this room. // Returns an error if there was an issue with the retrieval. func (d *SyncServerDatasource) GetStateEventsForRoom( - ctx context.Context, roomID string, + ctx context.Context, roomID string, stateFilterPart *gomatrixserverlib.FilterPart, ) (stateEvents []gomatrixserverlib.Event, err error) { err = common.WithTransaction(d.db, func(txn *sql.Tx) error { - stateEvents, err = d.roomstate.selectCurrentState(ctx, txn, roomID) + stateEvents, err = d.roomstate.selectCurrentState(ctx, txn, roomID, stateFilterPart) return err }) return @@ -245,6 +245,8 @@ func (d *SyncServerDatasource) addPDUDeltaToResponse( var succeeded bool defer common.EndTransaction(txn, &succeeded) + stateFilterPart := gomatrixserverlib.DefaultFilterPart() // TODO: use filter provided in request + // Work out which rooms to return in the response. This is done by getting not only the currently // joined rooms, but also which rooms have membership transitions for this user between the 2 PDU stream positions. // This works out what the 'state' key should be for each room as well as which membership block @@ -252,9 +254,13 @@ func (d *SyncServerDatasource) addPDUDeltaToResponse( var deltas []stateDelta var joinedRoomIDs []string if !wantFullState { - deltas, joinedRoomIDs, err = d.getStateDeltas(ctx, &device, txn, fromPos, toPos, device.UserID) + deltas, joinedRoomIDs, err = d.getStateDeltas( + ctx, &device, txn, fromPos, toPos, device.UserID, &stateFilterPart, + ) } else { - deltas, joinedRoomIDs, err = d.getStateDeltasForFullStateSync(ctx, &device, txn, fromPos, toPos, device.UserID) + deltas, joinedRoomIDs, err = d.getStateDeltasForFullStateSync( + ctx, &device, txn, fromPos, toPos, device.UserID, &stateFilterPart, + ) } if err != nil { return nil, err @@ -404,10 +410,12 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( return } + stateFilterPart := gomatrixserverlib.DefaultFilterPart() // TODO: use filter provided in request + // Build up a /sync response. Add joined rooms. for _, roomID := range joinedRoomIDs { var stateEvents []gomatrixserverlib.Event - stateEvents, err = d.roomstate.selectCurrentState(ctx, txn, roomID) + stateEvents, err = d.roomstate.selectCurrentState(ctx, txn, roomID, &stateFilterPart) if err != nil { return } @@ -733,6 +741,7 @@ func (d *SyncServerDatasource) fetchMissingStateEvents( func (d *SyncServerDatasource) getStateDeltas( ctx context.Context, device *authtypes.Device, txn *sql.Tx, fromPos, toPos int64, userID string, + stateFilterPart *gomatrixserverlib.FilterPart, ) ([]stateDelta, []string, error) { // Implement membership change algorithm: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L821 // - Get membership list changes for this user in this sync response @@ -745,7 +754,7 @@ func (d *SyncServerDatasource) getStateDeltas( var deltas []stateDelta // get all the state events ever between these two positions - stateNeeded, eventMap, err := d.events.selectStateInRange(ctx, txn, fromPos, toPos) + stateNeeded, eventMap, err := d.events.selectStateInRange(ctx, txn, fromPos, toPos, stateFilterPart) if err != nil { return nil, nil, err } @@ -765,7 +774,7 @@ func (d *SyncServerDatasource) getStateDeltas( if membership == gomatrixserverlib.Join { // send full room state down instead of a delta var s []streamEvent - s, err = d.currentStateStreamEventsForRoom(ctx, txn, roomID) + s, err = d.currentStateStreamEventsForRoom(ctx, txn, roomID, stateFilterPart) if err != nil { return nil, nil, err } @@ -807,6 +816,7 @@ func (d *SyncServerDatasource) getStateDeltas( func (d *SyncServerDatasource) getStateDeltasForFullStateSync( ctx context.Context, device *authtypes.Device, txn *sql.Tx, fromPos, toPos int64, userID string, + stateFilterPart *gomatrixserverlib.FilterPart, ) ([]stateDelta, []string, error) { joinedRoomIDs, err := d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, gomatrixserverlib.Join) if err != nil { @@ -818,7 +828,7 @@ func (d *SyncServerDatasource) getStateDeltasForFullStateSync( // Add full states for all joined rooms for _, joinedRoomID := range joinedRoomIDs { - s, stateErr := d.currentStateStreamEventsForRoom(ctx, txn, joinedRoomID) + s, stateErr := d.currentStateStreamEventsForRoom(ctx, txn, joinedRoomID, stateFilterPart) if stateErr != nil { return nil, nil, stateErr } @@ -830,7 +840,7 @@ func (d *SyncServerDatasource) getStateDeltasForFullStateSync( } // Get all the state events ever between these two positions - stateNeeded, eventMap, err := d.events.selectStateInRange(ctx, txn, fromPos, toPos) + stateNeeded, eventMap, err := d.events.selectStateInRange(ctx, txn, fromPos, toPos, stateFilterPart) if err != nil { return nil, nil, err } @@ -861,8 +871,9 @@ func (d *SyncServerDatasource) getStateDeltasForFullStateSync( func (d *SyncServerDatasource) currentStateStreamEventsForRoom( ctx context.Context, txn *sql.Tx, roomID string, + stateFilterPart *gomatrixserverlib.FilterPart, ) ([]streamEvent, error) { - allState, err := d.roomstate.selectCurrentState(ctx, txn, roomID) + allState, err := d.roomstate.selectCurrentState(ctx, txn, roomID, stateFilterPart) if err != nil { return nil, err } From 76040bfa87e4ed9bb8c97deb0d596942825799a7 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Wed, 7 Aug 2019 11:46:36 +0100 Subject: [PATCH 39/49] Add CI information to CONTRIBUTING.md (#778) Add information about how the continuous integration is set up in Dendrite and how to run the tests locally so that people don't need to wait around for things to churn. --- CONTRIBUTING.md | 33 ++++++++++++++++++++++++ docs/images/details-button-location.jpg | Bin 0 -> 25460 bytes 2 files changed, 33 insertions(+) create mode 100644 docs/images/details-button-location.jpg diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 22ad0586f..dc962fee7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,6 +20,39 @@ should pick up any unit test and run it). There are also [scripts](scripts) for [linting](scripts/find-lint.sh) and doing a [build/test/lint run](scripts/build-test-lint.sh). +## Continuous Integration + +When a Pull Request is submitted, continuous integration jobs are run +automatically to ensure the code builds and is relatively well-written. Checks +are run on [Buildkite](https://buildkite.com/matrix-dot-org/dendrite/) and +[CircleCI](https://circleci.com/gh/matrix-org/dendrite/). + +If a job fails, click the "details" button and you should be taken to the job's +logs. + +![Click the details button on the failing build step](docs/images/details-button-location.jpg) + +Scroll down to the failing step and you should see some log output. Scan +the logs until you find what it's complaining about, fix it, submit a new +commit, then rinse and repeat until CI passes. + +### Running CI Tests Locally + +To save waiting for CI to finish after every commit, it is ideal to run the +checks locally before pushing, fixing errors first. This also saves other +people time as only so many PRs can be tested at a given time. + +To execute what Buildkite tests, simply run `./scripts/build-test-lint.sh`. +This script will build the code, lint it, and run `go test ./...` with race +condition checking enabled. If something needs to be changed, fix it and then +run the script again until it no longer complains. Be warned that the linting +can take a significant amount of CPU and RAM. + +CircleCI simply runs [Sytest](https://github.com/matrix-org/sytest) with a test +whitelist. See +[docs/sytest.md](https://github.com/matrix-org/dendrite/blob/master/docs/sytest.md#using-a-sytest-docker-image) +for instructions on setting it up to run locally. + ## Picking Things To Do diff --git a/docs/images/details-button-location.jpg b/docs/images/details-button-location.jpg new file mode 100644 index 0000000000000000000000000000000000000000..53129a6e1bfbd3c991c3483894ee05c5e4bac608 GIT binary patch literal 25460 zcmeFYbx<8!w=deg6nA%rg}b}EyW7Iu2_Z;u2oN-QAh-pGgb*Z1kN^P!!8IX-U`YsU z65z4FecpG@d3A5yy7m5eRj=+`y}C!wp*_}bc6ZG=*6qjJEr38%RYMg3?;1DA4FI=0 z%t;zbN_P5&da4@QDt8P3U=H(g@(zR$0D!koP=KMDB7?bwB?EF3zy`1Y9DoQAcW?^y z|AK{nCw?b8x<+x9(`}fV%(z0E+W>zvDkN!{4;)KeX`Qbbzs;(w&St05IUV{ukQuztB!@ z0p54lZ+s`u@=pwR6adf!-X(zt?d65$F%{$q3~+S}3UK0ece;!5-;EXs1PlQ+KoQ^p z(16$72YpAG0)lt@Kp+5c1ss8(yQ34p3%K9y|Euo5jr_kMg3w`ixd8wrU%&7GcUQL{ z20?xS5pD*RyRaFIf_#0@3>=|o$3XWWv?PODP>`P_AD@rA584^+8h}OzI-ITSC0v737!!yS4U`~rMk-0$4noqD-Dp?w0;&J4jm z&gcLJ2L`18v_p`42%163*W26ICy+rQC@8?)F*wNG*N4Z*%^?7-;N|X#W)R}#XE5>c z^9=~PQ`Y{+3IoUAfd<}1>hvF0?gISV2(NE|E8l;(3gmMP=LvM+yEDM&g?4rDa`JUX zqj>+$#oxI7`v?I4iN`<}A;EujfjEi)0N=R1{lN+V$ZP<(&b+<-U4DCeU2%t#H2@g! z`A>b{rvM;+e%GJ*pM5N40DvC{0A0iX+2@!K0NsfIK(Xu;91!v^Kfv8Ux&to&0bl?e zKmbSp^KKkr-;E?Z0RN6j2oM7#04YEQPymzw6+jKp0CWI7zyL4;%m53(3a|kj01Ds) zxbCpT3-AH_cc>BqgaHvi3=jt-07*a!kO5=?IY9mne@cKdpmK*qbwC5q1hfGiKo`&h z4DQfr1Q-J*fEi#8SOAuQHDCkS0(OAI9WtE&XTaqSqi%QD^t{8aH{b*K0{(Y+4ZK5b z2oMT{0pUOda36>S9stom3=j)E1mb}NAQ4Ccl7Unp4R{2k1DU{MAPdL_a)CS`A9w;3 z0!2VEPy&6fh0U0JFe*U>;ZiJ^)L=M_?IP0oH(ZU<23!wt*dB z5BLOp2EG6Xz*pcKa0HwHr@$HT1NaI20xp0{;0pK+Tmv`2?cH4v0>VHzhy*b}ED#&S z1@S<9kPsvSNkCGN9HaoLKx&W{qz4&5CXg9q1=&Cj5Cw99JRmQ~4+?-npfD&3ih~lM z6etbKg7TmOs06Bjs-QZk32K2lpdP3X8iK~431|jdfR>;&Xbakd4xkf=23(>1HiFGy3)lvBfL&k@_!4{tz6J-tA#fNR1>b@b;1oCw&VqB`0=Ni%1XsW{ za0A>1cfn8KK6n5gf=A#9cn1Ce&%sOZ3j71!KmZ5?0)rqSm=J6TE(9M!1R;TtK`0

wg?xjYK)yrHAs3J<$Tj2^3W36*7*H%IE|dUD3?+q9K&hd0PzES7lnsi4azlBc z0#IS77*rA}1C@g+LRFyZP%Wq~)BtJ>HHBJ0t)X^MM<^QV2K9vcK>eXX&`{_-Xe2Zm z8VikwCP7o7>Cnf}9Ox5h5wsLq4y}aNKHyLFe{iX%n^o$xxu_(zOX=82rL{H35$ls!4hGq zuyj}!EEiS)D~3IVJ%iQ2>S4{WR#+#j2lfiq4|@X}g-yVwVeerdU>{*?ur1gwY#;U& zb`1LtJBR%RyM_aB7#stR4abKQ!^z-Oa5^{>oE46O^S}k*!fs7<>{w1D}U4!B^p%@Ll*m{1AQu{|Uc@|3Lr<7y<)y1IdFFK#C$Ik#a~Sq&iXuX@E3ES|aU`PDnSTH_{&&f{Z{$A>)wA$VbR5WInP8 z`4m}&tV1><+mYSKSI9x+C~^WhgIqu^BiE5T$bIBD4NEr>5mzL8G#vtnShy!`4}?~vk0>svl_Dj zvlX)o^A+YG<|yVA=6lQ~%r(p%%zexw%G}mL`@ymMNAMmIIaxmKRn4Rwz~^);tXEirSYuf4 zu;#HoVr^jUVSU9q!8*tKjSXTWv2n18uqm+VuvxG9OP#jeI~#BRgx!G4WBj6H!pi@k`whP{J*fPI2}j{O@4 z#6jZV;t=Cd;xOQ_;c(*!;)vtO;;7(g;TYhU;n?6f<9Ois;e_Bs;ylDj#>v3R!zsq8 zz^TP~j?;Qy~Z2Co5GvNTfy7L`+|3Z_Y3bDABK;OPlQj2&w$U4&xx{LlDD_~-b42%rR5 z1VjXs1Plc11bhUd1TqB51lj~f1eOGj1nvZW1YrbG1PKJ`1i1ty1eF901nmSb35E#9 z31$gC5^NIe6C4x#BDf}m6XFn(5KB^)B0AesL}^4hM8!mvM2$opM6Za3iKd7ah*pVqi4KW=5d9{G5@Qn+6H^m2 z6LS#@5lay(6KfM26I&BI6MGQ{5#J|{BTggEAub`VB5oq?B<>>~A$~`^NW4M(nfRFa z7x66#k_4ZGoP>ddgM^<%oJ4^{gT#QulEjh3gCvk7g5)7d8c7aGF-a9k6G<1zYm!lt z8ImQEEs`%JXC!}-f}~iaM5NTD%%t3;!lW{!Dx|ukW~6qcZlr#s;iNI7DWut?MWmIa zjijBVuSrKqXGlMiZjm04ekZ*mgOcHpk&w}nv61nSiIK^ZX^{k}>QYf-1iYcloo>TNt3{gx{EKqDv>{FalTv0+PaVW_s=_yf^f|OE}DwMjE z=9CVUo|Hk94=58UA5#`mR#G-mc2f>ePEgKMu2X)dJf*y%f>Pm9kx?;FaZ(9U$xx|M z=~G!!IZ=62g;K>(rBLNil~UDEwNmv`jZjTfeWco<`bKq5bxVy&O-xNo%}&ivElI6R ztxIiA?MUrK9ZVfXolKogT|!+$-Adg{JwiQ0y-dAJeMEgh1JYpAkkZi8plF0>WN6fA z3}~!q&@_HD_h{m1(rKR1RM0fibkPjbOwxRy*`zt3`AKs_i$P07OGC>}D?lqnt3sq{F>`;a!B_6cnTZ4+%b?I7(G?IP_q?IG$-?hieZo`{}?o}FHhUYcHw-jLp!-jzOp z{yu#oeHMKQeGPpZeINZ>`Z@Y_`Y-f9=x-P>7>F6@7*Gtt3~~&b48{z03?2-@4ABg! z40#OY42=xk3_}d>7(O!WG8{Af#Ry}>XQX0eW#ng+VpL@`V6mVq#_DXOd=8V=`p2WpZZJZcF*7|gC$k8%0<$)=IkOY9FY`U-1m?%gCCs(V9nAgAlgx|EJIu$- zf3d(>2v}%XI9P;O zW1ZuG;};5o!bee~I8eeU1(XiT66J~tL`9?0Pz9)JR2!-vHHG?!`h+?|-Ed-aQgE_x z3UJDDYI2%!I&=DSMslWb=5tnZwsQ7yPH--9?s1-R-f&@Yk#n(d32@1BX>plzp}7LM zqPWty3b?Ac+PM0;rnr{5K6CxJo7}=6hG39aQ@#lHKlgd-TQ_a)PGsrW|v&wV8^NSb8OUO&d z%grmntHx{0>%i;Fd!ILjH=nnPw~cpz_Z{yl?-$-*d{90@J~}>bK1n_`J`+AiK0m$( ze5rf|d^LO>d_#P*eCvFNe1GvH_(}Mg`1$x{__g>g_+9yf_+$Aq`Ahj5`Cssl@qggo zj~QmdkIGfCksCjt`_bTej_|5yd``hd@X__LM4I{ z5ff1rF&1$W2@r`E$q*?KX%y)d85j8|vM+Kj3KJz3WfbKXl@rw!wHEahy(gM1nlD-{ z+9^6LIxo5{dMbJ=h9^cV#w{i#rXgl7<|-B<7B7|~_DrlzY*1`gY*Xx5>{=X0oLZby zTtZx3+)Ug>JXkzVJX^d%yj6Tqd{%r@{8;>20!M;cf=fbDLPNq_!c`(vB0(ZoqEezm z;*G?d#J0qlBp`_|NiWGODI=*PX)Wm~86lY>Stwa2*(3Q@a!GPu@|P4sid2eKN?1x+ z%1FvdDnKe$>akS0REyMr)U4E|)QQxMG@dk_G>^25w2rj3w3qaK=~U?==?3YS(i74v z(qE-lG&HJkVVRp%d*Rg$*Rek$-2sh z$|lO@%ht$t%Z|w|$?nTu$RXv(<=ExKlJ$yCl%Kej})(!@RaD4_>|<843r#{ z0+b#q&PODStB+bH`eM=572mn*j^4=K+p?uGxh-mN}idfVqmfxw)5lw0X99jrmLSY4aWPa|=uhS_^&)Weal)Pm3sv zY>OI;mlo3&I~KnzF)e8=1uRu8EiAn(qb+kRYb{?{&RXtSURq&W(OU^wsaaWB`C2`+ z%C~B;>bIJ=+PC^`jc?6tEoQA{ZEqc9on&2X-C{j#{n7fH4QNAZgR+saF|a|~+_Ooy zsj%s?nY7uo`C*G;OJmD#t88mw>unorn`c{Z+iyE>yKno)j=+x9PQp&t&e1N^F4gX- zU5DMc-MZa(dxSl;J)ga@y}7-&eXM<+eS>|!{eu00{j~#;1G|HigT4dW;hsaf!!w5- zhj$L!4!<0+9T^-&95ozm9RnSc97`PA9LF429Z#L$PE<~OPRdRePCiZ#ot`)~ISn~2 zIel{moynZJoaLQOoIRYQoO7M)o%@{^oDZCD(8OpIS{7}Dc0)&^v(dHaKJ*;=3;NoH z$c5cS#>LRZ)g{s;%caJp&t=Z#i_5hukt>I*jH{8Wo9hGDY}Y#1*RBh$2d=knByOB; za&9JW9&XWYd2S7EgKmp%-`pYYK-;8 zfgZ^oWgeX#6CPV0zdUg~nLNckbv>Ou?|Ei=R(rnkob&wRdE-Um#pxyQW$NYS73=lH ztJ!PVYsKr-8{tjkE#$4?ZRZ{Ao$6ig-Qzv&z2|-9L+HcqBjaP_e+{RjM){Eq`*0n`D40qOyE0U-g80xAPu2D}gW5^x(x7RVi_6lfXf zAD9$a7T6W|E^sgKDu^(MBS#fBAzwT6v{ZG~Ni|Gr0jkMo}5JaS?RXjHU9v}JTabV~HI=-%js=x;Hw7}^+-7~L3`n5dZi zn3kBgG21bJ#S+DG#wx{H#RkQu#a6|>j$MpBd5C<-@KEBR!9$OS4<8mi?0ER@;irc; zapZCQaq4jnarfe~;~L|JkCP*e2C3q#o zCzK|1C(I@sBtjCY6NM9X5?vFc6AKdC5+@V)60eiUllYS~k{pvFl5&%plg5&^lCF}8 zlev>slkJkjk{>5GBo8NVBwwTuq;RGvrP!o|q-3Pjr3|I4rJSeYr*fn!rdp*2r>3XY zrVgd9rk zuAFY09+sY!-k3g`zLkEJL7c&pp_bv05s{Ib@jPQZV>jbElRQ%(Q!5jl8I@U>*^xP& z`QG0kJq$NG;wAICo~d;H?@{Nv*+j4Y-s=`6D>|E$!k>a2mRm8_rH_}QpzCRD2Ok3TF_gtSa4Q|Q^;PZSZG@qUYJw(yl|rMQxQ-^QzTYoSma%l zR8(2iU$jzmUQAfbU948@RQ#a0pt!Smw)n6FS;ABzQ({pPRFYZJP%>JwQ*vEOSt?wr zU+P(!SXxo~x^$)Vyo|7nyG*^zsVu6jsI04OuI%_J=2O3%u1 zoW5ML+_XHPJiWZWe6)PK{JMg&LZrf=!n-25qN-xBV!h(8XQa;rp6NVue-{6&{8`_# zdMdIIGmEoT{R#imH057OGCGajH?(D%B3v52_2R zyQ=4_PinAhIBJw@>}w)x3TnD)=4wuAv1>VMm1`YpBWnw5yK3iaPwH^$P<1MGj&%>} zit2jm7V6IG@#;D2)#{z=qw7oRU)C?x|7aj+;AzloaBXqLs8&uvNd+yEUb?wsoX+w+(2cYm;iT zXbWk}X=`npY5UfW)z03o((c$E-Co+>+rHfXtAnINutTrIyCb!uwqvwouM^Tq-zn2+ z)fwKI*V)lI*Lm87*TvnX(dF6|-}S6(uxqpHx|_ONqT8%HxI4SMwR@)fs0X_T)uYyf z?s?c#-qYW+-t+qfvH- zyf>sbx3|6beedZjyjMK0v|f3lg$IoW z0|p-twhYb;9uMITaSv$?c?=~F)eemfeSQOf!}3PyjpLh`H&5RTyxDwnGfX=yJ!~}` zF2X8Una=cZ0>-sj~ZS~ucx1Yx0<1FLKN%ou2`+yUnyPbU)fp( zSD99oSJA5ptF^1+t6$f!*Lc=+*L>F=ueGf$tevlut&6W)tVgUDufJa3+yFP2Hk3Ee z8;KjW8xtGfHgPxkHuX0HHgh&RHye3(bJ<2(4B<-7WK&+qBqTfcw! ze))s?hujaxAMroxeoXy1{Ym^&^rz*|$e-muhkt%K$3EvfH#`qHFF5Zz-}(jn#r{j{ zm+!BvUtPbJe_dZNUMOFu7t15ukK$xy&Asy@*C$j|8L{pVZV!i5B%QygYk#^kN%(FKLvmK{%l{vuTj@J z*8$gg*DtR(ZXh@8H(EEoH#s*wH*2@RE$gkut@mx#ZP)GU?d?BxC3jT-(7#Ioo-F}@ z@qGZ$eg^;rg#f_jdRIqaeY<+Ma*u$)5pWm+0Y@N_2n;L&EG$e+ED}6?90GC@3JP)( zGBPS!R(dLGW*Raw25tssHg*&zijtm(pNE5=l>^1`w-FGDMBXjA5MyBxbKFgta{O<% z+W~+O1Bim;K|v+}LI^?$!Q0`xqJ+B=2I$?2`QMlDPlF;La3l=GxZ~;*03Z|!!ePj} z4TiwJD|moF@9G^05k$m{0`HMb3Wg*GMh=g&vPqc*{bR_416#>i6w&3CJ3?ZPg8{J* z+qX|y*_2eAf^#d{hUV`qQvL1f&fov-?C$utNjTz;CqM}P?G6IFTOj>cO8>6;Mg=}48}>Tlu2S1E@Ua}RMp{*X8) zT`X$-9Qc*1$<+e3=Mu~F-}-b~PoBq%Q$9`oU#M#0h&fH|FPK^Sj1|n(Hg}eA{BL{z zKPLY3m-zoA7mE$XwuJUAlP`MaO0___5IoUjef8b#;S}7we-NYd4>Vqg{T#m@xe2@` zxhatA>7S0OXgkQ6lAQiu2$cxB(7OIN7+DuxIp6Hu0unbVQQBo|cKdSgbqj>>+W!IU z{6DhII@f|x{!zuZfJ#s#Uzp0x8=}vvnrOcG$YqITMiusq3<3{rZOTdXf75C_FfjiI ziXH!q0#8YOhyRC!Lc%{0Yfk473|>#jozDNM_ej7$Hn;^6%03zCbCr7j{&r0n=5oCD zTqlO#huqUY%+NSf-O5@bykBmT7OxX?epb$%=CycyuZ#{t-`aG}QhmQguD$r|78u^g z+HnyLcIRE~wEhqxM6ugnegF0S&XQ8j`;(H*7gB{TFZ>yQc8pI7WT(NH$C$@9WWw1# zSf=8Y#W;opG$s32SM|O&l8Oq6cS1TyXP3G@1-!q$CnijGsB13Dcc~XI6&>sDhI^hY zDY++1ZZ~~3-cn5PkBJSuA#G{QiIaS?|;k0IMw9pNtHEln%sIqg1#6zgInX|&_zN&z1sW-bP z{qc;^E%5!K{|~Nyh3D#Z?C6svjJ&GagUwFwul7v}jF@`b^*-On+yt;i5Xx?F93u8ByAuT@2N+JpDKP{PkRqpA9~O|*qWot(Mkmu zp2o%+;)!O*#Dg8uxM=kIE^3y%%2n^}I$U4=uO*g?nnQ{yDLIe0%tm4a5DN$Rg zxECyo=)^)jTP~s*f4NX`p|f>{Iy+hhRa1nJK3m{HgT1b59eZ4xomnwvulj_@{Yg`KUag& z92tksvH=y|kk{%U%-1wj%D&3_!^T_oGnZGU8`B+CZJ*x_!OOc?+f1u-@~z)Q*jFi_ zRt8`kr%)O<4g(RDyIY>?Y#orYv`_L9|J9F5_r0pkCT|9WN_>0``CZFpT*tadBErqA7C5b+H z`-l`)%P1kivX`Ytq;bYBa#H)8_T|pVoA4gxN-rm=%jVYN{k#>`@UVbp=bx-1h=V|# zY6!uH6{U0=>cFdcH|<<+4t9H2!0GJ!>1lJjrLlXeAxqCG`p2tay}ByRU>(vrE9K*f zE;+&(-e5=yP468^X`T1R23}vJ^zhCilh=`WyQ7tO+dZz4FtSxKqmaD_aNe3#k!U)3lGhTLB zCj1Dnt+y7>E=4~*#9>8szcq_+*X2twhV(Y2GZz~BM`JcrwT(kot~Qe<-WxzF} zGf>|Xtk6Y9pVmE?Ogwm<|Kg^#Z`1pTeyn$WpK@$_`|M2kZIfWKuAM&51!mSKF_eo_ zmF7>=q7#Rv*pQF}eW|{0u^KcD?=S^TjcdtYJV<92U+XQ-&E>C2hZxTp`|*eL;}O60 z7wLD>vfOQ~%XufyX_x}L_$64O+1nc-t-+rEh{@eb+ccF&gjJg?RVv>_`Y;0K z8kJc?J>HvlxLk?-ggY{&9n`V1?BV?A>cfV)QL?EnkzeV71jU|^WjCAl!&<)du0OQD z+o{r*1BB9fIB?+GD8-cgv>U>htSZ-`ro2gIE{`>9)wK{=Qs6fC{Bj_xKxg-BaqrdP*_Ep`VG89)81I^^H^c3=FnkX774 zs-i`3&hQ&u(L+w%^RSOWl|4v0N)CuKxf#LVV(4ZtI`|e&yb5_UOH7{t)K5Nu5 zZb?R>zT)maG>5T+4=cNRP5BOys8fEv?$k{3*L+2JZnamF5E=mi^HKye~B3 zrAA-5K8?bP`pYXA9oOM6Qt+x&biGk)l?3uM)LVXj_wf@ZKGb;ss-ZR}aq zsv%r+_ozGP)un>QiSw@W<<@@EXD(5`B_?L|VE${;gYS$!bMf%#cuAtZrpBmmmuX@9 z%(=dmmJaHfyE8%^hbd@HDGy2>?xucvI8o${M1?nh);v#D`mP{33hHi6V_;zT;xe}( zkykDkZh8?_dYqfIi;YeuZhZdEDp|8FzN+iP+E5IA+ScCI)}=qY(b_!Y=ng zg>LwFTNaikcRw`7=n^a|FrCGv`~AcG_pgW;4 zJ`H^KT=Gg{kaywt*F}tq=4h~IG-f?uxd}3pv~9I|zEM5$Qq5np%eO+=dyg+jG54Ls zpe=tDKAv@g$4oU&QocLC%atXq5k@3Oh(k`6pn~appvq#@Q0i6duGLc0d%DaYCxW>> zKV;KCFRB}bvpc7o?>rD8ZJt3a){x>>vHAeHJY%LiU8;V?wOPRUz@CuyGu@-_5tiBS z?X=xaQxDL6hq~x3L#gpFG-#y;!XWa);P`785 z4N15bjF_9$9M$&}j7oNY8aSi=a6B0ekXcQSn}K7L+jdqhR#s&P$!Vdh9)6IWRCmkv zj;FC9is`!Qlsl0@R8ieNt=xw%W;~#2FYxCq+;dT!is{n?6&fqF+quyK>DDR^vcE&S zUe^W~fq!`jB6Ega0*3DvwHl4|MFir`@I z&I+877MLuf8j)1Qe6jlM1nsk@l65sb7eKOXt}FGb_nBOYPtdzEo0Vp5o=e{sI1@z9 z46;KD&q~kWT3hN{eyaC9;g%6fp7*9Wrp0iY=2l3)1sLXLX$BbUihrLL(!RFWgV4>~jXkP)Yzq2BjgS$JY?l1ATUIgik7F(Eq{&=D!13M^1e-1w2{6GV%q5cfX zA-+`5H+>HsAaXzE@3UwkI})n3j7?n6#d8u^{Fygj9NU;fAPAmE#0NEx-vaB`4L3_S z=fN|)ro8WY{ql)|&{DK$PD`!UK?6M|bqV&(T^l8ND^`n+N3U^y!Zh5tn-~?$S1fbW z(rJ?Fq`3ncS2O?c5PC1*TKi}fLV2b0t9pL;!ns*BI4l^PiDLyGrmVC!Q)UNhyS;m3 zX%p#RaPz9<%wI~Lv;({=(HadaW%#z#wpmG$GNnGf)@-;nY-l=Vw(!x5q_{>3L&Fpg zs(&hGoOADZc#swCPdJ*@rIxnhfca3_DJ#k zE&oq5XZ|-s?-?;xpw4nE@$Atv1jRI27^gIWFPt>S(vz5EhPLL)+iKr6ar(NoRp1Ru z(ImVbq%gw^O0KjYL3d^T>16B}Q54O|jvEWs6_vwpGM8lmceCt>btvnZ@88@4P8EN4 z*h0pRF7CH|MX#OkDH2k-D0AV>5Z{faA2g(b#W)zE&z2j5jVlR4o~m*?8}_((@~ClBle60XgYj8LqTjK<(c9 zQf?2H{dLs)yBXrKZ_V>fG)_|ij>KQXd2fN(%lY~U`uQf4>->X@;XRX%JC)HlSQ&V^-P{0)IVZhQlJr>bPqsnz;!%f3 ztHGA_ZVgYkzQDde&xIYF9ewNAtIA>Y{D?K`7gj2V)!wcXJnvIb|6*9_eOsh4Lwayc zS^c-uQuo|Rt@IvMqNOOhAL%9?k9d+FU$ok2HGQAZWBw{zU*P%7$mLU@Ps)x*21i~{ z{oS+CjMB6OH?qfSeQoLy?viDoGZ$Kcc^Ky!c3Qv-&PPrUB&NZ*HFBCjr&VCoh50O=g?3Rx13p3u> z{c7D(5(b7&vZB9i+_5EgPX_k3+COaf-NY#BorropMWwRb@n%VSh7zo^cQ{tZCu6okgkrp8o zO0iflnO9@N#-yLzRGw8?*GW?h$G_AgeEd-1L!I^e*Y7JfpUh_+*?Fy2=)7pB;Q7dt zGrpo#;Onlrm1!{IHjV^b)0x^rYC1-i%_gn8Ha$*{x=7sKvFggT3AL%7KI(|crHMPE zV0&0=Set0sc~3If6j|c^!qComSbP>wbRmf2(8g%bG4RHo!By0HZO(ULc@mk<{J{3wCxj0d16g-8r=4x z@q0HdwfNiuy9;q26_8VeR#(1Y+h&u}mPq)qfvHu*u{>%r)0^hJryrbR0=a%ZPi%hB z5Zo%TZ|Ot#_`rzLk_Pv9mULa;9O_J4wMTbAF}`hQAJR}$#dtxhGQ^CLvbSiHH9Fzk zmN8s?sv(PnK_5=UUTc1OncZo)nB23$9pV|wsZNFG~nJwZ=JwXZ60?TRTBXng^0{Tstu46!zN-hQCdUaN@GlBXb`)M)FFc>Vv)6OnzwQK{tG;+O1!okh zR?FxbHhg8zV===d6zyNQKGR#kt@)_>lO=k?l9F&UH?AfL_S_z&M0p-Kuw97zfiroz zx#{Q4jMs&?L14nva{QL**VzeE!|!ttHE;lsD{&XGb9WeDd{P)Jw5oXW{%CsI*!aoJ zqrQAg#p$r$rxG+&cuks>&pL_(n-7wh*ZN3}WM_2Vj@{HdyHbPUGqRMULc6w+gwNwY z+ToRJN604z-6IKP92Tb0^k0|o`!swDc+zg1j)u+J)fsbn|IlyftTUA=d)n=-Qy0iA zLaTzddq9vYUd2Mt2wkUyO20!teV+e2q4mS!y0QbqY9E2%l%Vmd!!Szib!C+%dv9?% z#r6sII!Y7H+0e+#H{5H=6$&)?6QtW?5>X)j*+4AXF@wt~8UOD0W0ApAFkCgHWwmTZKEHsZw z&BvOJzkXzF`kBC{nqFOsz^j^|Okd*NoK z7LGW*CW6S)a473(U-o6-Kr!%CUVO@=F!z~hg|`@N#N_q8&~))hygE!xI)6;V6r)_| zFsnHWtT?by_}#UcdPq61mcas(O=aozdW^VFckuot#j5 zpC7MkrgB&?vMPD9KmNmL-t?`T)|Q@oC5-P!w#fyI!n3i^lW?&Q+~DS zEOl7XmhV)K{;ni^8@d#$b`W!ADx%A{zP)lUn0%-37U&Wy%VC%=G(B0tnd$1}uCa`+ zid;cLTEEKg<@`Qu{`ifomzui4n(bMdeJxI%U10{X6&aLAdsifgbh$JA$_Nz0xjwWlhT ztV~!<6!B{MFy7k>G#v%s-@? z%*^tU)bLE@X}(M+lV=GhouUurn|Pkrr}%E#qacIGXWx5to})`l%aF9KJX1rDz`q0G z|3>6x6FHBLP|CB=kZhZP;d6@ypAEy$k;Ne~2Td~G3+Ze!Lman&$4O`yIEY**zY(r? z^$Gdx;L+tg24X!K@E?#i#-zNKCuyP$)SY?uZ19_>`y&_4UvHUI;bj|94G6 zx2$n}?k%A4qK!7z3V!d&KTNgu&%^x%-B$e@1l0xc*6lyLackUXfq}@zPts zXW}NKeWq3AF~s;x6zU1VG?!Xj9ow?l^M4zW_=TIKib;SXLCnV_q^`Ny`MJ!*I^8~j z0E*?AlzipZ!?BFZrHa?_1f0`HVm$K!1rurGhS;iv z*y+I=mjLM+{oQFv8`DugL10u**2I2Q_UomwdZXi$MA8iGdH3UyRpT8;fj{SebxJnR z@?ClCyYZ31J{~&VBkpYN5GZzb-8HVg&Tsm>sq*UZA1?_idrplsYX8anEgS#ctN;KtEVfv9xT?AInItm*iiF?0R} zNF=cD3PIm=iiSuQHw~2p-g)BhyS5be9AhOyDI&CvAT~~`@!Be6o19qI&^6s3OBQA* z#$55q;*TE-tC^H&>!cK!Hl}%t6K&pd$IL_)tX#`@N>g*wUuXWw1@g5tv@M^byYMNY z&8tI;z2|kJg*a9(G0Ttxwu}R>8l8{|qbzzb7%?;^j*0eS3mt=IvNN+UFGkii6Y0T-bLAyOpBfBh0vBf<=6v^N_TXf61=n<>w^~9PJD$j~ zhoH|5QBY^smxt@V2GMTa7KGLo!f15wH4N}fhsGQq2${~8 z-_nFQb!SbE7j|!Cw@>!G3wq5Ks4w$E{Y5kvRm?gN57W8bfvs_7t<8s(WK>=&w^|>U ziRR=wmSQ)*Z^;flb<-pw35d9*jpw6NWtFL}M_G$fHde~DQ{LIz*ac8N?ee!{ z8R0mc?9daY(R=51c( zMaAKfcz}4nzdmLHJ7oh*nv^6bQLL_o*FMr?6ClB78LpmS2ir9}$$RH^24PdvjzDdt zZM8W31y%4-7AhKZqHwI|I7@#jL3nBoiq%_5t4ONwNGu(>P0dA`P#P9s>s5OVqw#ODAVrHc`aexhNA25OW(QjRA`-987w}Tx9N8S7suc zJ!il(PIHpmNk?T;?P%xd$Uad7^EOQaNfB-$2eiwPQz@C7^dgx~HvVF0Y>C3jMY?b%yRp zoK}+w$>8qG1~%U1aje?lz96^`+jF~_-MXd2x&x?Toz0ZxMC{#PSXe`-x>)iQf%%Z6 zbPzWDOyo1+wcf72!tUjBvo+0AX9cTZMV|te<-Y*7*+V7pI zDCw=sNnqq_LB+nUek{78Dldem#WQv9mpjmkVV++^Li1-Iwg}!T+gL-EA`vDI0hu&N zuZ}#%59sE)1X1}7P?|g~YC#)nQTG7ZDF}H!TU%y9rek(`{*OeF+>dT(%6F16@a^r0 z@4jBDJ*Q6s?N6Dk&D<0~x8Z z&b#(6L)*-y#j4|~5ZZwlenxhH@=yTArr|`nqj8aYo+DSk#Xt_h%#DywZ{B|&ENt+H zWLs&dr1`QM*{rl5)UvSe2Vx?)uGM`9E7Bgk84@0vSEvmLkx%|9ZJhlVP?smx@<#+( z?IB`)gnNMBt^}@0(KWTAn440XGpb`$CIOQoBB+GqTVLWtS*&6V0==Q_N^ZbI2cluk9VO&iltR zgnP$)Yu+mRsKlP*+f*Xxsu=&S_N8+wQHB=m+3PM! z8oTuua6BPs;m`86C;$6}z1HgsgWVFRkbks~f7y-T!{y|hni``auMUM=7u9f49ra$D z`T_o0T<#&>g?E-ImVFA;C01DBbI8Ad^pjfGpyL?EK(}GC0|QP9tF?j8-#`XG%ra%) z&cFR-@7>H^9qD-+td@=o{|g{~xEDIqWZi2zx)B(ZsmO!SvMC7yU+ldyoU_hpM-^6; zjkdHl+)=Qj&VrB@aRLf0Mj;q~rH77@wk@r<@KXphlJ5-uDbDCj8vBe!Fw4|N&U^LT zb(<#^FL|zJ86|t`!1-i$n0Nl?+{$1u|8L`khRADl=vMhpllg|Nck3d{54H=M{-bH; z7vlZjf&X2P|1aUtywmnR&fDG3tF@J%v)nM+{@Z9>h){#n<2`=Ww##Q_yOl6criv=% z_gl#kaai89B&$({obOYLvT695UXN$|}>bG@36oBN}d<*18Z6|84+V|pdATo1eG1q~wW1Ef% zr8yHedSRyfTW0J8@AM4~>#*KK2ndafRyy|As-NE@m`#34_tpz{`jQ3T&gU;}?~Ag7 z^nGQWuxHE(>@Z;He)SjdzTL?sE@3D%;9GFBBjGai@tW9mvi7;qQK;X|Yp6!4giXTt z_Z3+Af>~r!PCtQ&!V%pK=7BNqx9e+reeO(BJR2)HzO^QNT724a6l}NbUEDFNElhzo z(g8Spy@&N~>n_|V*Gk)1N~y8JLOKoBdZOd%DyW3MeACzTu7xVgG#qv8-T-|xR22GJ z>100>)vWgjVwP;{vbV`HI4M*i?9O2m?g-1)SwC%@0)0#Gxvu#mE!yXe8-peQ@#2U`afx?kK@nn_3z%p>wuB~5Xe`S?Uy1eUQ>$)CkC!914g_$ z4eq*Qeh>zR5|ickf-IZ*5|IRrCoj6oR3p6#1{*3C&3G)N+n;zW0^1W^s=2F?T-r&SaKv zXJ)D7K&EMTYEv4;dw>HC=?|jxEekRR=|dUj&Vf8Y)es z8E++o`SeFt_AM+xTd$9crP$pbWpl&Ljw)c}yj^e43*H-B(J^pU&_fvdwm2ypQo&$k*kMB#Tt)D8MR?wK?2&)Geb!yX@v2=_u)6EWXbArRt%bff z8Pd^3RBiRhu04Eb4y~yC!UcZ%-7G3dfBFrZ_#MP5+ zeSN61(ZUYO;9cHv{h;k*X*ylgvJF#efgq`E>YLBMd~ntIJ)B`{R&3$&jK)UZX*0Pi z9vw4j5hXO{14$Oc9bI^P^=Dh_=T6s;KSZphR(inlK4#82GB=fC0$9u_F3!f10YdWV z33E!%B~b=M7LzV|g%a*b&5MeTrfWJ=aPII*9XK22HC$>wDd-yP)Nvwk#9l7D#a=hw zLxB0bpD1W}h$M|dCTXiGGC4?)EeE0`tkt^rt!>vVvwVfDf>psZi5MZye`Ld0_y{*C zc?XG6JCAoxYfb&+yI1>omRAr-3qOdC=2tMc_Aq}b&CkKxRvm6aXXcJ{SH_IKsU3)+ zb#|h9{Hj&1I#(F_RXfOiY5s8Gn;p0s%^z2+3D(_L_?nu*ggmuHLMBBDT&VAQwT#(G zJ@3846wqkWe-AyoOFQC)pzTxx^jqB~g0RmJDIzg59ad&I5%qz#Wwt~{gA#ZB(Z@rp z>!)mR!=lsysuZP{0pV|GaEyi)pOxCe{aVLTJJ)0Wb7umBf0_nu;5nLK98~x}&`L!1 z@DEC_1}18|cBiR~hb0`-M}3LRa0oD!f}ui@O|8p>N*s>Ov0I94dO)PNf zu#?~73!ilrn5-6cR|S1jaAhBq>&=ozx8)v`zs+9=4ob}bWf?*#YF{vQyYttgeDA(h z_1%2Q;}*xsli0DhuXjJC;R}=QIf@qBrsQErnQf7u6ZWcpx#JW*&}9xBJq|k$w)(1L zI;7yCv%<2h?V7(wyG=IeaPJu#O?l`d9yL+Ux1KmAy`Q?O4x=kX%rjSP^7)5-l1lid zqs-i@UxX93cB$-KNyrh>O_1b!hnTBy6o&a^5JMuMi*zq=$0q-v=cbfLU5k07owiC~ zgdnxKl}aNWWDx_Lft>7#;!DWF(4TjHpl*k1Ol^ORl9+}oO^zgQFNieivn$^Vx+Vgv zM+&k(61VC}hfOsV*cDTRjB?#|kk6c=xJn~)**RP~m}d)`r>53T4scQ*$)4TBa2KWb z0n~5>fh@y`#uq(dd7bVy(ZWAN{iKZf!5LmIP?o3);wej6@>3+N3-BI`%AWpJc7IKR z2hqW(8#DhA+PPLtE{0LSZX2rbYk%}|VBow1p(857;NeU=XWi7*X!?b<%O%90w)Lsw$d&UmHB)lFo9fZ&6-tMl z_48BV@p9{yr#q{l9vgwI=|+24h_D?l#XBl0f|k}|Q;%utZK8ZDtiQXJ)yPPwy>Tq7 z3C^&cI=k6iA6Z@yq^!#QiWX+?Z-fWfS7pCxoxp zM}g(ON~3T;Mn=cAnuPYCS6gh48m(LyEd5zvF-)41pLlL>Z@r}?Rq`IdA`}ebefKmV zB5!wcsKG_`S;XRa?Xeke{iRLh2gRd7jNkQD{Cp4f&dn+ zIf4fAkKaM6Yr7mC*Sc4nWh>D4oL_Z-xb^DVjzy4{cbMgB;We%;C_e~u)PkF>E7|3y z^w5)fVkN=0m-!YUEDZQMx~=nBb*|EKIrG3^#46FolRl-!t}Nv}EFb_OYtS2aR>Ru- z5wW+>c`-VNQOR6b^~#_%`k=CT9V^4kad;%jKSs*_Qh=`)ELQhG`#y%7U6~s=iR!jt z78aWrR2S+|deJ)Wr@>vQ3#7cTtpxL3H`s{$N$u?ZHMKMHSU2Lsi`OKhu(S*jd*_b8 zZ_3Q2bNE;Gv4zB+JQ3Ljs^=&wlX^T6zXPVfxQQ{tft1E!=S@Xpd$3A-12vUU#z&J! z6PA{Ee&(s zo9ED8zfByjKPi>>ZP49jZUOa?BR% zo^qQ6!{=@Ls)F~Tuu#DD&?9p@E6;6f*9WS8dd<-CS65zqmF!vUMiw4XLex!!qy?fY zL(7SEe2)H*9Qku~un4l`8)_;=QipW}V1lxmV;^8?wCjoBYwlKebX@`1g2D`VRFq5H zY6ojiFXy|jim*b7NWUu*$EYU5`fb$I?Uw~Vgz;S8`xJ$(G(@lsog`&Tgz7k=kq5F_ zxn#opjTt-DWP+#=0`Kc+@%x9JEAX}v(|#7~6iSK*)B7T-@V{gQ#H=XDM3g5>Tm7gWZ@-T5g%}CO@W2V1QycryAs^0SG}$|Mn-cx0(KU)*}lIip?+DR)anBM=^=( zYz>?NKwgSxmdWm2IQRNt+0XFj>el8aGw-tCJ^R0P`EJcI=q!_2w*lYGm)CxLA3Hk@ zyfZGFX?j%wmDmy0ryowy0dm$OKk*d4#<5mcK?%wh*Z0`IY)kB}%${^Lct`wRv}=%+ zO2cmx$o=?Te$UW63OY=;YqlN&##OvbX%Xnr5p|sA-T!($qC}PZr9|psb+h-g zRy}47W2i5_wu&I53H+a!Xoc-RyZ3&${$nabjK=(5YlPK{yR#WOnb1l+V=B@lvxkx0 z&=UVI=3e^bSkhB;l@Gt!t<>eA<1Np0h#Uv<@)E4r{#%UyFIE1gBSMXrD_)B_RYR$r u4RhA~Y6DLGj}T!2H;KAfreuh+f{j?gpGcBTo&QbN|94OS)err<{C@y{Rpo;K literal 0 HcmV?d00001 From 5716cd60b5b343e62f1ccdfbd764d4803515d4c7 Mon Sep 17 00:00:00 2001 From: Thibaut CHARLES Date: Thu, 8 Aug 2019 07:10:42 +0200 Subject: [PATCH 40/49] Add filtering database API to syncapi account data table (#513) --- syncapi/storage/account_data_table.go | 15 ++++++++++++--- syncapi/storage/syncserver.go | 3 ++- syncapi/sync/requestpool.go | 6 ++++-- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/syncapi/storage/account_data_table.go b/syncapi/storage/account_data_table.go index 9b73ce7d6..7b4803e3d 100644 --- a/syncapi/storage/account_data_table.go +++ b/syncapi/storage/account_data_table.go @@ -18,7 +18,9 @@ import ( "context" "database/sql" + "github.com/lib/pq" "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/gomatrixserverlib" ) const accountDataSchema = ` @@ -41,7 +43,7 @@ CREATE TABLE IF NOT EXISTS syncapi_account_data_type ( CONSTRAINT syncapi_account_data_unique UNIQUE (user_id, room_id, type) ); -CREATE UNIQUE INDEX IF NOT EXISTS syncapi_account_data_id_idx ON syncapi_account_data_type(id); +CREATE UNIQUE INDEX IF NOT EXISTS syncapi_account_data_id_idx ON syncapi_account_data_type(id, type); ` const insertAccountDataSQL = "" + @@ -53,7 +55,9 @@ const insertAccountDataSQL = "" + const selectAccountDataInRangeSQL = "" + "SELECT room_id, type FROM syncapi_account_data_type" + " WHERE user_id = $1 AND id > $2 AND id <= $3" + - " ORDER BY id ASC" + " AND ( $4::text[] IS NULL OR type LIKE ANY($4) )" + + " AND ( $5::text[] IS NULL OR NOT(type LIKE ANY($5)) )" + + " ORDER BY id ASC LIMIT $6" const selectMaxAccountDataIDSQL = "" + "SELECT MAX(id) FROM syncapi_account_data_type" @@ -93,6 +97,7 @@ func (s *accountDataStatements) selectAccountDataInRange( ctx context.Context, userID string, oldPos, newPos int64, + accountDataFilterPart *gomatrixserverlib.FilterPart, ) (data map[string][]string, err error) { data = make(map[string][]string) @@ -103,7 +108,11 @@ func (s *accountDataStatements) selectAccountDataInRange( oldPos-- } - rows, err := s.selectAccountDataInRangeStmt.QueryContext(ctx, userID, oldPos, newPos) + rows, err := s.selectAccountDataInRangeStmt.QueryContext(ctx, userID, oldPos, newPos, + pq.StringArray(filterConvertTypeWildcardToSQL(accountDataFilterPart.Types)), + pq.StringArray(filterConvertTypeWildcardToSQL(accountDataFilterPart.NotTypes)), + accountDataFilterPart.Limit, + ) if err != nil { return } diff --git a/syncapi/storage/syncserver.go b/syncapi/storage/syncserver.go index c57a90256..fb883702c 100644 --- a/syncapi/storage/syncserver.go +++ b/syncapi/storage/syncserver.go @@ -495,8 +495,9 @@ var txReadOnlySnapshot = sql.TxOptions{ // If there was an issue with the retrieval, returns an error func (d *SyncServerDatasource) GetAccountDataInRange( ctx context.Context, userID string, oldPos, newPos int64, + accountDataFilterPart *gomatrixserverlib.FilterPart, ) (map[string][]string, error) { - return d.accountData.selectAccountDataInRange(ctx, userID, oldPos, newPos) + return d.accountData.selectAccountDataInRange(ctx, userID, oldPos, newPos, accountDataFilterPart) } // UpsertAccountData keeps track of new or updated account data, by saving the type diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index d773a4606..6b95f4698 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -141,12 +141,14 @@ func (rp *RequestPool) currentSyncForUser(req syncRequest, latestPos types.SyncP return } - res, err = rp.appendAccountData(res, req.device.UserID, req, latestPos.PDUPosition) + accountDataFilter := gomatrixserverlib.DefaultFilterPart() // TODO: use filter provided in req instead + res, err = rp.appendAccountData(res, req.device.UserID, req, latestPos.PDUPosition, &accountDataFilter) return } func (rp *RequestPool) appendAccountData( data *types.Response, userID string, req syncRequest, currentPos int64, + accountDataFilter *gomatrixserverlib.FilterPart, ) (*types.Response, error) { // TODO: Account data doesn't have a sync position of its own, meaning that // account data might be sent multiple time to the client if multiple account @@ -180,7 +182,7 @@ func (rp *RequestPool) appendAccountData( } // Sync is not initial, get all account data since the latest sync - dataTypes, err := rp.db.GetAccountDataInRange(req.ctx, userID, req.since.PDUPosition, currentPos) + dataTypes, err := rp.db.GetAccountDataInRange(req.ctx, userID, req.since.PDUPosition, currentPos, accountDataFilter) if err != nil { return nil, err } From aa0d22bf50da7e3d4637ee5d46a1151342eda8d3 Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Fri, 9 Aug 2019 17:45:54 +0800 Subject: [PATCH 41/49] Implement client single event retrieval (#693) --- clientapi/routing/getevent.go | 127 ++++++++++++++++++++++++++++++++++ clientapi/routing/routing.go | 9 +++ testfile | 2 + 3 files changed, 138 insertions(+) create mode 100644 clientapi/routing/getevent.go diff --git a/clientapi/routing/getevent.go b/clientapi/routing/getevent.go new file mode 100644 index 000000000..7071d16f0 --- /dev/null +++ b/clientapi/routing/getevent.go @@ -0,0 +1,127 @@ +// Copyright 2019 Alex Chen +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routing + +import ( + "net/http" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/clientapi/httputil" + "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" +) + +type getEventRequest struct { + req *http.Request + device *authtypes.Device + roomID string + eventID string + cfg config.Dendrite + federation *gomatrixserverlib.FederationClient + keyRing gomatrixserverlib.KeyRing + requestedEvent gomatrixserverlib.Event +} + +// GetEvent implements GET /_matrix/client/r0/rooms/{roomId}/event/{eventId} +// https://matrix.org/docs/spec/client_server/r0.4.0.html#get-matrix-client-r0-rooms-roomid-event-eventid +func GetEvent( + req *http.Request, + device *authtypes.Device, + roomID string, + eventID string, + cfg config.Dendrite, + queryAPI api.RoomserverQueryAPI, + federation *gomatrixserverlib.FederationClient, + keyRing gomatrixserverlib.KeyRing, +) util.JSONResponse { + eventsReq := api.QueryEventsByIDRequest{ + EventIDs: []string{eventID}, + } + var eventsResp api.QueryEventsByIDResponse + err := queryAPI.QueryEventsByID(req.Context(), &eventsReq, &eventsResp) + if err != nil { + return httputil.LogThenError(req, err) + } + + if len(eventsResp.Events) == 0 { + // Event not found locally + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound("The event was not found or you do not have permission to read this event"), + } + } + + requestedEvent := eventsResp.Events[0] + + r := getEventRequest{ + req: req, + device: device, + roomID: roomID, + eventID: eventID, + cfg: cfg, + federation: federation, + keyRing: keyRing, + requestedEvent: requestedEvent, + } + + stateReq := api.QueryStateAfterEventsRequest{ + RoomID: r.requestedEvent.RoomID(), + PrevEventIDs: r.requestedEvent.PrevEventIDs(), + StateToFetch: []gomatrixserverlib.StateKeyTuple{{ + EventType: gomatrixserverlib.MRoomMember, + StateKey: device.UserID, + }}, + } + var stateResp api.QueryStateAfterEventsResponse + if err := queryAPI.QueryStateAfterEvents(req.Context(), &stateReq, &stateResp); err != nil { + return httputil.LogThenError(req, err) + } + + if !stateResp.RoomExists { + util.GetLogger(req.Context()).Errorf("Expected to find room for event %s but failed", r.requestedEvent.EventID()) + return jsonerror.InternalServerError() + } + + if !stateResp.PrevEventsExist { + // Missing some events locally; stateResp.StateEvents unavailable. + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound("The event was not found or you do not have permission to read this event"), + } + } + + for _, stateEvent := range stateResp.StateEvents { + if stateEvent.StateKeyEquals(r.device.UserID) { + membership, err := stateEvent.Membership() + if err != nil { + return httputil.LogThenError(req, err) + } + if membership == gomatrixserverlib.Join { + return util.JSONResponse{ + Code: http.StatusOK, + JSON: gomatrixserverlib.ToClientEvent(r.requestedEvent, gomatrixserverlib.FormatAll), + } + } + } + } + + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound("The event was not found or you do not have permission to read this event"), + } +} diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 825dd97aa..d36ed6957 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -132,6 +132,15 @@ func Setup( nil, cfg, queryAPI, producer, transactionsCache) }), ).Methods(http.MethodPut, http.MethodOptions) + r0mux.Handle("/rooms/{roomID}/event/{eventID}", + common.MakeAuthAPI("rooms_get_event", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return GetEvent(req, device, vars["roomID"], vars["eventID"], cfg, queryAPI, federation, keyRing) + }), + ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/state/{eventType:[^/]+/?}", common.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { vars, err := common.URLDecodeMapValues(mux.Vars(req)) diff --git a/testfile b/testfile index 74c9d9e4f..d04ab731c 100644 --- a/testfile +++ b/testfile @@ -168,3 +168,5 @@ Newly updated tags appear in an incremental v2 /sync Deleted tags appear in an incremental v2 /sync /event/ on non world readable room does not work Outbound federation can query profile data +/event/ on joined room works +/event/ does not allow access to events before the user joined From 386cc975f08b44cd8f83e6d2fbcb08ee55fdb868 Mon Sep 17 00:00:00 2001 From: Victor Cuadrado Juan Date: Fri, 9 Aug 2019 12:30:38 +0200 Subject: [PATCH 42/49] Add typingserver service to docker-compose (#692) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Víctor Cuadrado Juan --- docker/README.md | 2 +- docker/dendrite-docker.yml | 1 + docker/docker-compose.yml | 10 ++++++++++ docker/services/typing-server.sh | 5 +++++ 4 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 docker/services/typing-server.sh diff --git a/docker/README.md b/docker/README.md index 7d18ce605..ff88c0818 100644 --- a/docker/README.md +++ b/docker/README.md @@ -58,7 +58,7 @@ docker-compose up kafka zookeeper postgres and the following dendrite components ``` -docker-compose up client_api media_api sync_api room_server public_rooms_api +docker-compose up client_api media_api sync_api room_server public_rooms_api typing_server docker-compose up client_api_proxy ``` diff --git a/docker/dendrite-docker.yml b/docker/dendrite-docker.yml index c2e7682eb..abb8c3307 100644 --- a/docker/dendrite-docker.yml +++ b/docker/dendrite-docker.yml @@ -114,6 +114,7 @@ listen: media_api: "media_api:7774" public_rooms_api: "public_rooms_api:7775" federation_sender: "federation_sender:7776" + typing_server: "typing_server:7777" # The configuration for tracing the dendrite components. tracing: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 763e5b0f0..9cf67457c 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -95,6 +95,16 @@ services: networks: - internal + typing_server: + container_name: dendrite_typing_server + hostname: typing_server + entrypoint: ["bash", "./docker/services/typing-server.sh"] + build: ./ + volumes: + - ..:/build + networks: + - internal + federation_api_proxy: container_name: dendrite_federation_api_proxy hostname: federation_api_proxy diff --git a/docker/services/typing-server.sh b/docker/services/typing-server.sh new file mode 100644 index 000000000..16ee0fa62 --- /dev/null +++ b/docker/services/typing-server.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +bash ./docker/build.sh + +./bin/dendrite-typing-server --config=dendrite.yaml From 28f60bec4e8a7bc86c2a6564d28f3afa925b26d8 Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Wed, 14 Aug 2019 21:44:34 +0800 Subject: [PATCH 43/49] Update name of a passing test in testfile (#784) --- testfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testfile b/testfile index d04ab731c..cea6a4f46 100644 --- a/testfile +++ b/testfile @@ -143,7 +143,7 @@ Trying to get push rules with unknown rule_id fails with 404 Events come down the correct room local user can join room with version 5 User can invite local user to room with version 5 -Inbound federation can receive room-join requests +Inbound federation can receive v1 room-join requests Typing events appear in initial sync Typing events appear in incremental sync Typing events appear in gapped sync From d21a2fb152143b2ce7600213abe18013519ccedf Mon Sep 17 00:00:00 2001 From: Parminder Singh Date: Wed, 14 Aug 2019 23:04:49 +0530 Subject: [PATCH 44/49] Add auth fallback endpoint (#405) Also adds support for the recaptcha auth type. --- clientapi/routing/auth_fallback.go | 210 +++++++++++++++++++++++++++++ clientapi/routing/register.go | 27 ++-- clientapi/routing/routing.go | 7 + common/httpapi.go | 19 +++ 4 files changed, 249 insertions(+), 14 deletions(-) create mode 100644 clientapi/routing/auth_fallback.go diff --git a/clientapi/routing/auth_fallback.go b/clientapi/routing/auth_fallback.go new file mode 100644 index 000000000..cd4530d1b --- /dev/null +++ b/clientapi/routing/auth_fallback.go @@ -0,0 +1,210 @@ +// Copyright 2019 Parminder Singh +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routing + +import ( + "html/template" + "net/http" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/clientapi/httputil" + "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/util" +) + +// recaptchaTemplate is an HTML webpage template for recaptcha auth +const recaptchaTemplate = ` + + +Authentication + + + + + + +

+
+

+ Hello! We need to prevent computer programs and other automated + things from creating accounts on this server. +

+

+ Please verify that you're not a robot. +

+ +
+
+ +
+ + + + +` + +// successTemplate is an HTML template presented to the user after successful +// recaptcha completion +const successTemplate = ` + + +Success! + + + + +
+

Thank you!

+

You may now close this window and return to the application.

+
+ + +` + +// serveTemplate fills template data and serves it using http.ResponseWriter +func serveTemplate(w http.ResponseWriter, templateHTML string, data map[string]string) { + t := template.Must(template.New("response").Parse(templateHTML)) + if err := t.Execute(w, data); err != nil { + panic(err) + } +} + +// AuthFallback implements GET and POST /auth/{authType}/fallback/web?session={sessionID} +func AuthFallback( + w http.ResponseWriter, req *http.Request, authType string, + cfg config.Dendrite, +) *util.JSONResponse { + sessionID := req.URL.Query().Get("session") + + if sessionID == "" { + return writeHTTPMessage(w, req, + "Session ID not provided", + http.StatusBadRequest, + ) + } + + serveRecaptcha := func() { + data := map[string]string{ + "myUrl": req.URL.String(), + "session": sessionID, + "siteKey": cfg.Matrix.RecaptchaPublicKey, + } + serveTemplate(w, recaptchaTemplate, data) + } + + serveSuccess := func() { + data := map[string]string{} + serveTemplate(w, successTemplate, data) + } + + if req.Method == http.MethodGet { + // Handle Recaptcha + if authType == authtypes.LoginTypeRecaptcha { + if err := checkRecaptchaEnabled(&cfg, w, req); err != nil { + return err + } + + serveRecaptcha() + return nil + } + return &util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound("Unknown auth stage type"), + } + } else if req.Method == http.MethodPost { + // Handle Recaptcha + if authType == authtypes.LoginTypeRecaptcha { + if err := checkRecaptchaEnabled(&cfg, w, req); err != nil { + return err + } + + clientIP := req.RemoteAddr + err := req.ParseForm() + if err != nil { + res := httputil.LogThenError(req, err) + return &res + } + + response := req.Form.Get("g-recaptcha-response") + if err := validateRecaptcha(&cfg, response, clientIP); err != nil { + util.GetLogger(req.Context()).Error(err) + return err + } + + // Success. Add recaptcha as a completed login flow + AddCompletedSessionStage(sessionID, authtypes.LoginTypeRecaptcha) + + serveSuccess() + return nil + } + + return &util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound("Unknown auth stage type"), + } + } + return &util.JSONResponse{ + Code: http.StatusMethodNotAllowed, + JSON: jsonerror.NotFound("Bad method"), + } +} + +// checkRecaptchaEnabled creates an error response if recaptcha is not usable on homeserver. +func checkRecaptchaEnabled( + cfg *config.Dendrite, + w http.ResponseWriter, + req *http.Request, +) *util.JSONResponse { + if !cfg.Matrix.RecaptchaEnabled { + return writeHTTPMessage(w, req, + "Recaptcha login is disabled on this Homeserver", + http.StatusBadRequest, + ) + } + return nil +} + +// writeHTTPMessage writes the given header and message to the HTTP response writer. +// Returns an error JSONResponse obtained through httputil.LogThenError if the writing failed, otherwise nil. +func writeHTTPMessage( + w http.ResponseWriter, req *http.Request, + message string, header int, +) *util.JSONResponse { + w.WriteHeader(header) + _, err := w.Write([]byte(message)) + if err != nil { + res := httputil.LogThenError(req, err) + return &res + } + return nil +} diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index c5a3d3018..0af407587 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -83,23 +83,22 @@ func (d sessionsDict) GetCompletedStages(sessionID string) []authtypes.LoginType return make([]authtypes.LoginType, 0) } -// AddCompletedStage records that a session has completed an auth stage. -func (d *sessionsDict) AddCompletedStage(sessionID string, stage authtypes.LoginType) { - // Return if the stage is already present - for _, completedStage := range d.GetCompletedStages(sessionID) { - if completedStage == stage { - return - } - } - d.sessions[sessionID] = append(d.GetCompletedStages(sessionID), stage) -} - func newSessionsDict() *sessionsDict { return &sessionsDict{ sessions: make(map[string][]authtypes.LoginType), } } +// AddCompletedSessionStage records that a session has completed an auth stage. +func AddCompletedSessionStage(sessionID string, stage authtypes.LoginType) { + for _, completedStage := range sessions.GetCompletedStages(sessionID) { + if completedStage == stage { + return + } + } + sessions.sessions[sessionID] = append(sessions.GetCompletedStages(sessionID), stage) +} + var ( // TODO: Remove old sessions. Need to do so on a session-specific timeout. // sessions stores the completed flow stages for all sessions. Referenced using their sessionID. @@ -530,7 +529,7 @@ func handleRegistrationFlow( } // Add Recaptcha to the list of completed registration stages - sessions.AddCompletedStage(sessionID, authtypes.LoginTypeRecaptcha) + AddCompletedSessionStage(sessionID, authtypes.LoginTypeRecaptcha) case authtypes.LoginTypeSharedSecret: // Check shared secret against config @@ -543,7 +542,7 @@ func handleRegistrationFlow( } // Add SharedSecret to the list of completed registration stages - sessions.AddCompletedStage(sessionID, authtypes.LoginTypeSharedSecret) + AddCompletedSessionStage(sessionID, authtypes.LoginTypeSharedSecret) case "": // Extract the access token from the request, if there's one to extract @@ -573,7 +572,7 @@ func handleRegistrationFlow( case authtypes.LoginTypeDummy: // there is nothing to do // Add Dummy to the list of completed registration stages - sessions.AddCompletedStage(sessionID, authtypes.LoginTypeDummy) + AddCompletedSessionStage(sessionID, authtypes.LoginTypeDummy) default: return util.JSONResponse{ diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index d36ed6957..d4b323a2d 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -245,6 +245,13 @@ func Setup( }), ).Methods(http.MethodGet, http.MethodPost, http.MethodOptions) + r0mux.Handle("/auth/{authType}/fallback/web", + common.MakeHTMLAPI("auth_fallback", func(w http.ResponseWriter, req *http.Request) *util.JSONResponse { + vars := mux.Vars(req) + return AuthFallback(w, req, vars["authType"], cfg) + }), + ).Methods(http.MethodGet, http.MethodPost, http.MethodOptions) + r0mux.Handle("/pushrules/", common.MakeExternalAPI("push_rules", func(req *http.Request) util.JSONResponse { // TODO: Implement push rules API diff --git a/common/httpapi.go b/common/httpapi.go index 99e15830a..bf634ff4a 100644 --- a/common/httpapi.go +++ b/common/httpapi.go @@ -10,6 +10,7 @@ import ( "github.com/matrix-org/util" opentracing "github.com/opentracing/opentracing-go" "github.com/opentracing/opentracing-go/ext" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" ) @@ -43,6 +44,24 @@ func MakeExternalAPI(metricsName string, f func(*http.Request) util.JSONResponse return http.HandlerFunc(withSpan) } +// MakeHTMLAPI adds Span metrics to the HTML Handler function +// This is used to serve HTML alongside JSON error messages +func MakeHTMLAPI(metricsName string, f func(http.ResponseWriter, *http.Request) *util.JSONResponse) http.Handler { + withSpan := func(w http.ResponseWriter, req *http.Request) { + span := opentracing.StartSpan(metricsName) + defer span.Finish() + req = req.WithContext(opentracing.ContextWithSpan(req.Context(), span)) + if err := f(w, req); err != nil { + h := util.MakeJSONAPI(util.NewJSONRequestHandler(func(req *http.Request) util.JSONResponse { + return *err + })) + h.ServeHTTP(w, req) + } + } + + return prometheus.InstrumentHandler(metricsName, http.HandlerFunc(withSpan)) +} + // MakeInternalAPI turns a util.JSONRequestHandler function into an http.Handler. // This is used for APIs that are internal to dendrite. // If we are passed a tracing context in the request headers then we use that From d63d2a50cdce891e6d4159919b304ce0c66ac0c5 Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Fri, 16 Aug 2019 01:45:11 +0800 Subject: [PATCH 45/49] Replace event content types with ones in gomatrixserverlib (#785) The types that are not in gomatrixserverlib are not replaced. This also updates the gomatrixserverlib dependency. --- clientapi/routing/createroom.go | 6 +- clientapi/routing/membership.go | 2 +- clientapi/routing/profile.go | 2 +- clientapi/threepid/invites.go | 10 ++-- common/eventcontent.go | 95 ++++--------------------------- federationapi/routing/threepid.go | 19 +++---- go.mod | 2 +- go.sum | 2 + 8 files changed, 33 insertions(+), 105 deletions(-) diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index 4a76e1b06..620246d28 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -106,7 +106,7 @@ func (r createRoomRequest) Validate() *util.JSONResponse { } } - var CreationContent common.CreateContent + var CreationContent gomatrixserverlib.CreateContent err = json.Unmarshal(creationContentBytes, &CreationContent) if err != nil { return &util.JSONResponse{ @@ -196,7 +196,7 @@ func createRoom( return httputil.LogThenError(req, err) } - membershipContent := common.MemberContent{ + membershipContent := gomatrixserverlib.MemberContent{ Membership: gomatrixserverlib.Join, DisplayName: profile.DisplayName, AvatarURL: profile.AvatarURL, @@ -246,7 +246,7 @@ func createRoom( {"m.room.member", userID, membershipContent}, {"m.room.power_levels", "", common.InitialPowerLevelsContent(userID)}, // TODO: m.room.canonical_alias - {"m.room.join_rules", "", common.JoinRulesContent{JoinRule: joinRules}}, + {"m.room.join_rules", "", gomatrixserverlib.JoinRuleContent{JoinRule: joinRules}}, {"m.room.history_visibility", "", common.HistoryVisibilityContent{HistoryVisibility: historyVisibility}}, } if r.GuestCanJoin { diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index 5e183fa0f..c71ac2de2 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -144,7 +144,7 @@ func buildMembershipEvent( membership = gomatrixserverlib.Leave } - content := common.MemberContent{ + content := gomatrixserverlib.MemberContent{ Membership: membership, DisplayName: profile.DisplayName, AvatarURL: profile.AvatarURL, diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index e8ea6cf13..a87c6f743 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -332,7 +332,7 @@ func buildMembershipEvents( StateKey: &userID, } - content := common.MemberContent{ + content := gomatrixserverlib.MemberContent{ Membership: gomatrixserverlib.Join, } diff --git a/clientapi/threepid/invites.go b/clientapi/threepid/invites.go index 251afb0d3..bfe5060a8 100644 --- a/clientapi/threepid/invites.go +++ b/clientapi/threepid/invites.go @@ -56,10 +56,10 @@ type idServerLookupResponse struct { // idServerLookupResponse represents the response described at https://matrix.org/docs/spec/client_server/r0.2.0.html#invitation-storage type idServerStoreInviteResponse struct { - PublicKey string `json:"public_key"` - Token string `json:"token"` - DisplayName string `json:"display_name"` - PublicKeys []common.PublicKey `json:"public_keys"` + PublicKey string `json:"public_key"` + Token string `json:"token"` + DisplayName string `json:"display_name"` + PublicKeys []gomatrixserverlib.PublicKey `json:"public_keys"` } var ( @@ -342,7 +342,7 @@ func emit3PIDInviteEvent( } validityURL := fmt.Sprintf("https://%s/_matrix/identity/api/v1/pubkey/isvalid", body.IDServer) - content := common.ThirdPartyInviteContent{ + content := gomatrixserverlib.ThirdPartyInviteContent{ DisplayName: res.DisplayName, KeyValidityURL: validityURL, PublicKey: res.PublicKey, diff --git a/common/eventcontent.go b/common/eventcontent.go index c45724fcd..c07c56276 100644 --- a/common/eventcontent.go +++ b/common/eventcontent.go @@ -14,55 +14,7 @@ package common -// CreateContent is the event content for http://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-create -type CreateContent struct { - Creator string `json:"creator"` - Federate *bool `json:"m.federate,omitempty"` - RoomVersion string `json:"room_version,omitempty"` - Predecessor PreviousRoom `json:"predecessor,omitempty"` -} - -// PreviousRoom is the "Previous Room" structure defined at https://matrix.org/docs/spec/client_server/r0.5.0#m-room-create -type PreviousRoom struct { - RoomID string `json:"room_id"` - EventID string `json:"event_id"` -} - -// MemberContent is the event content for http://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-member -type MemberContent struct { - Membership string `json:"membership"` - DisplayName string `json:"displayname,omitempty"` - AvatarURL string `json:"avatar_url,omitempty"` - Reason string `json:"reason,omitempty"` - ThirdPartyInvite *TPInvite `json:"third_party_invite,omitempty"` -} - -// TPInvite is the "Invite" structure defined at http://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-member -type TPInvite struct { - DisplayName string `json:"display_name"` - Signed TPInviteSigned `json:"signed"` -} - -// TPInviteSigned is the "signed" structure defined at http://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-member -type TPInviteSigned struct { - MXID string `json:"mxid"` - Signatures map[string]map[string]string `json:"signatures"` - Token string `json:"token"` -} - -// ThirdPartyInviteContent is the content event for https://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-third-party-invite -type ThirdPartyInviteContent struct { - DisplayName string `json:"display_name"` - KeyValidityURL string `json:"key_validity_url"` - PublicKey string `json:"public_key"` - PublicKeys []PublicKey `json:"public_keys"` -} - -// PublicKey is the PublicKeys structure in https://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-third-party-invite -type PublicKey struct { - KeyValidityURL string `json:"key_validity_url"` - PublicKey string `json:"public_key"` -} +import "github.com/matrix-org/gomatrixserverlib" // NameContent is the event content for https://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-name type NameContent struct { @@ -79,51 +31,26 @@ type GuestAccessContent struct { GuestAccess string `json:"guest_access"` } -// JoinRulesContent is the event content for http://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-join-rules -type JoinRulesContent struct { - JoinRule string `json:"join_rule"` -} - // HistoryVisibilityContent is the event content for http://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-history-visibility type HistoryVisibilityContent struct { HistoryVisibility string `json:"history_visibility"` } -// PowerLevelContent is the event content for http://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-power-levels -type PowerLevelContent struct { - EventsDefault int `json:"events_default"` - Invite int `json:"invite"` - StateDefault int `json:"state_default"` - Redact int `json:"redact"` - Ban int `json:"ban"` - UsersDefault int `json:"users_default"` - Events map[string]int `json:"events"` - Kick int `json:"kick"` - Users map[string]int `json:"users"` -} - // InitialPowerLevelsContent returns the initial values for m.room.power_levels on room creation // if they have not been specified. // http://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-power-levels // https://github.com/matrix-org/synapse/blob/v0.19.2/synapse/handlers/room.py#L294 -func InitialPowerLevelsContent(roomCreator string) PowerLevelContent { - return PowerLevelContent{ - EventsDefault: 0, - Invite: 0, - StateDefault: 50, - Redact: 50, - Ban: 50, - UsersDefault: 0, - Events: map[string]int{ - "m.room.name": 50, - "m.room.power_levels": 100, - "m.room.history_visibility": 100, - "m.room.canonical_alias": 50, - "m.room.avatar": 50, - }, - Kick: 50, - Users: map[string]int{roomCreator: 100}, +func InitialPowerLevelsContent(roomCreator string) (c gomatrixserverlib.PowerLevelContent) { + c.Defaults() + c.Events = map[string]int64{ + "m.room.name": 50, + "m.room.power_levels": 100, + "m.room.history_visibility": 100, + "m.room.canonical_alias": 50, + "m.room.avatar": 50, } + c.Users = map[string]int64{roomCreator: 100} + return c } // AliasesContent is the event content for http://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-aliases diff --git a/federationapi/routing/threepid.go b/federationapi/routing/threepid.go index cff311cc4..7fa02be91 100644 --- a/federationapi/routing/threepid.go +++ b/federationapi/routing/threepid.go @@ -27,7 +27,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" - "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/config" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" @@ -38,11 +37,11 @@ import ( ) type invite struct { - MXID string `json:"mxid"` - RoomID string `json:"room_id"` - Sender string `json:"sender"` - Token string `json:"token"` - Signed common.TPInviteSigned `json:"signed"` + MXID string `json:"mxid"` + RoomID string `json:"room_id"` + Sender string `json:"sender"` + Token string `json:"token"` + Signed gomatrixserverlib.MemberThirdPartyInviteSigned `json:"signed"` } type invites struct { @@ -199,11 +198,11 @@ func createInviteFrom3PIDInvite( return nil, err } - content := common.MemberContent{ + content := gomatrixserverlib.MemberContent{ AvatarURL: profile.AvatarURL, DisplayName: profile.DisplayName, Membership: gomatrixserverlib.Invite, - ThirdPartyInvite: &common.TPInvite{ + ThirdPartyInvite: &gomatrixserverlib.MemberThirdPartyInvite{ Signed: inv.Signed, }, } @@ -330,7 +329,7 @@ func sendToRemoteServer( func fillDisplayName( builder *gomatrixserverlib.EventBuilder, authEvents gomatrixserverlib.AuthEvents, ) error { - var content common.MemberContent + var content gomatrixserverlib.MemberContent if err := json.Unmarshal(builder.Content, &content); err != nil { return err } @@ -343,7 +342,7 @@ func fillDisplayName( return nil } - var thirdPartyInviteContent common.ThirdPartyInviteContent + var thirdPartyInviteContent gomatrixserverlib.ThirdPartyInviteContent if err := json.Unmarshal(thirdPartyInviteEvent.Content(), &thirdPartyInviteContent); err != nil { return err } diff --git a/go.mod b/go.mod index 8e14253ca..d51f0a33e 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/lib/pq v0.0.0-20170918175043-23da1db4f16d github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20190805173246-3a2199d5ecd6 + github.com/matrix-org/gomatrixserverlib v0.0.0-20190814163046-d6285a18401f github.com/matrix-org/naffka v0.0.0-20171115094957-662bfd0841d0 github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 github.com/matttproud/golang_protobuf_extensions v1.0.1 diff --git a/go.sum b/go.sum index 0d59d1dd6..56781c9a6 100644 --- a/go.sum +++ b/go.sum @@ -58,6 +58,8 @@ github.com/matrix-org/gomatrixserverlib v0.0.0-20190724145009-a6df10ef35d6 h1:B8 github.com/matrix-org/gomatrixserverlib v0.0.0-20190724145009-a6df10ef35d6/go.mod h1:sf0RcKOdiwJeTti7A313xsaejNUGYDq02MQZ4JD4w/E= github.com/matrix-org/gomatrixserverlib v0.0.0-20190805173246-3a2199d5ecd6 h1:xr69Hk6QM3RIN6JSvx3RpDowBGpHpDDqhqXCeySwYow= github.com/matrix-org/gomatrixserverlib v0.0.0-20190805173246-3a2199d5ecd6/go.mod h1:sf0RcKOdiwJeTti7A313xsaejNUGYDq02MQZ4JD4w/E= +github.com/matrix-org/gomatrixserverlib v0.0.0-20190814163046-d6285a18401f h1:20CZL7ApB7xgR7sZF9yD/qpsP51Sfx0TTgUJ3vKgnZQ= +github.com/matrix-org/gomatrixserverlib v0.0.0-20190814163046-d6285a18401f/go.mod h1:sf0RcKOdiwJeTti7A313xsaejNUGYDq02MQZ4JD4w/E= github.com/matrix-org/naffka v0.0.0-20171115094957-662bfd0841d0 h1:p7WTwG+aXM86+yVrYAiCMW3ZHSmotVvuRbjtt3jC+4A= github.com/matrix-org/naffka v0.0.0-20171115094957-662bfd0841d0/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/util v0.0.0-20171013132526-8b1c8ab81986 h1:TiWl4hLvezAhRPM8tPcPDFTysZ7k4T/1J4GPp/iqlZo= From bf5efbc31fa02ef438b450b78db5eef14b3ab3a7 Mon Sep 17 00:00:00 2001 From: Parminder Singh Date: Thu, 15 Aug 2019 23:29:17 +0530 Subject: [PATCH 46/49] Error when recaptcha enabled with empty configs (#786) --- common/config/config.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/common/config/config.go b/common/config/config.go index 9fcab8cf9..40232fb03 100644 --- a/common/config/config.go +++ b/common/config/config.go @@ -498,6 +498,11 @@ func (config *Dendrite) checkMatrix(configErrs *configErrors) { checkNotEmpty(configErrs, "matrix.server_name", string(config.Matrix.ServerName)) checkNotEmpty(configErrs, "matrix.private_key", string(config.Matrix.PrivateKeyPath)) checkNotZero(configErrs, "matrix.federation_certificates", int64(len(config.Matrix.FederationCertificatePaths))) + if config.Matrix.RecaptchaEnabled { + checkNotEmpty(configErrs, "matrix.recaptcha_public_key", string(config.Matrix.RecaptchaPublicKey)) + checkNotEmpty(configErrs, "matrix.recaptcha_private_key", string(config.Matrix.RecaptchaPrivateKey)) + checkNotEmpty(configErrs, "matrix.recaptcha_siteverify_api", string(config.Matrix.RecaptchaSiteVerifyAPI)) + } } // checkMedia verifies the parameters media.* are valid. From 0ed2dd0b154f147a5575fe60996005692946160a Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Fri, 16 Aug 2019 12:05:00 +0800 Subject: [PATCH 47/49] Fix data race in clientapi/routing/register.go (#787) --- clientapi/routing/register.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index 0af407587..d0f36a6fd 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -29,6 +29,7 @@ import ( "sort" "strconv" "strings" + "sync" "time" "github.com/matrix-org/dendrite/common/config" @@ -70,12 +71,17 @@ func init() { } // sessionsDict keeps track of completed auth stages for each session. +// It shouldn't be passed by value because it contains a mutex. type sessionsDict struct { + sync.Mutex sessions map[string][]authtypes.LoginType } // GetCompletedStages returns the completed stages for a session. -func (d sessionsDict) GetCompletedStages(sessionID string) []authtypes.LoginType { +func (d *sessionsDict) GetCompletedStages(sessionID string) []authtypes.LoginType { + d.Lock() + defer d.Unlock() + if completedStages, ok := d.sessions[sessionID]; ok { return completedStages } @@ -91,12 +97,15 @@ func newSessionsDict() *sessionsDict { // AddCompletedSessionStage records that a session has completed an auth stage. func AddCompletedSessionStage(sessionID string, stage authtypes.LoginType) { - for _, completedStage := range sessions.GetCompletedStages(sessionID) { + sessions.Lock() + defer sessions.Unlock() + + for _, completedStage := range sessions.sessions[sessionID] { if completedStage == stage { return } } - sessions.sessions[sessionID] = append(sessions.GetCompletedStages(sessionID), stage) + sessions.sessions[sessionID] = append(sessions.sessions[sessionID], stage) } var ( From a81917c3e72be13dd0a9680bde54f3ad35c846ab Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Tue, 20 Aug 2019 01:01:53 +0800 Subject: [PATCH 48/49] Make trailing slash on server key request optional (#788) Cherry-picked from 7e861b60fbd721b374ec929926b14e57dc60ec41 --- federationapi/routing/routing.go | 1 + testfile | 1 + 2 files changed, 2 insertions(+) diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index ed32c8904..9f576790b 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -64,6 +64,7 @@ func Setup( // {keyID} argument and always return a response containing all of the keys. v2keysmux.Handle("/server/{keyID}", localKeys).Methods(http.MethodGet) v2keysmux.Handle("/server/", localKeys).Methods(http.MethodGet) + v2keysmux.Handle("/server", localKeys).Methods(http.MethodGet) v1fedmux.Handle("/send/{txnID}", common.MakeFedAPI( "federation_send", cfg.Matrix.ServerName, keys, diff --git a/testfile b/testfile index cea6a4f46..17978913e 100644 --- a/testfile +++ b/testfile @@ -170,3 +170,4 @@ Deleted tags appear in an incremental v2 /sync Outbound federation can query profile data /event/ on joined room works /event/ does not allow access to events before the user joined +Federation key API allows unsigned requests for keys From 5eb63f1d1eafae0111d46def5cf5bf431d2e8169 Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Thu, 22 Aug 2019 19:47:52 +0800 Subject: [PATCH 49/49] Add joined hosts query APIs (#781) This adds two joined hosts query APIs to the federation sender for use of other components. --- federationsender/api/query.go | 98 +++++++++++++++++++++++++++++++++ federationsender/query/query.go | 55 ++++++++++++++++++ 2 files changed, 153 insertions(+) create mode 100644 federationsender/api/query.go create mode 100644 federationsender/query/query.go diff --git a/federationsender/api/query.go b/federationsender/api/query.go new file mode 100644 index 000000000..ebc6e833f --- /dev/null +++ b/federationsender/api/query.go @@ -0,0 +1,98 @@ +package api + +import ( + "context" + "net/http" + + commonHTTP "github.com/matrix-org/dendrite/common/http" + "github.com/matrix-org/gomatrixserverlib" + + "github.com/matrix-org/dendrite/federationsender/types" + "github.com/opentracing/opentracing-go" +) + +// QueryJoinedHostsInRoomRequest is a request to QueryJoinedHostsInRoom +type QueryJoinedHostsInRoomRequest struct { + RoomID string `json:"room_id"` +} + +// QueryJoinedHostsInRoomResponse is a response to QueryJoinedHostsInRoom +type QueryJoinedHostsInRoomResponse struct { + JoinedHosts []types.JoinedHost `json:"joined_hosts"` +} + +// QueryJoinedHostServerNamesRequest is a request to QueryJoinedHostServerNames +type QueryJoinedHostServerNamesInRoomRequest struct { + RoomID string `json:"room_id"` +} + +// QueryJoinedHostServerNamesResponse is a response to QueryJoinedHostServerNames +type QueryJoinedHostServerNamesInRoomResponse struct { + ServerNames []gomatrixserverlib.ServerName `json:"server_names"` +} + +// FederationSenderQueryAPI is used to query information from the federation sender. +type FederationSenderQueryAPI interface { + // Query the joined hosts and the membership events accounting for their participation in a room. + // Note that if a server has multiple users in the room, it will have multiple entries in the returned slice. + // See `QueryJoinedHostServerNamesInRoom` for a de-duplicated version. + QueryJoinedHostsInRoom( + ctx context.Context, + request *QueryJoinedHostsInRoomRequest, + response *QueryJoinedHostsInRoomResponse, + ) error + // Query the server names of the joined hosts in a room. + // Unlike QueryJoinedHostsInRoom, this function returns a de-duplicated slice + // containing only the server names (without information for membership events). + QueryJoinedHostServerNamesInRoom( + ctx context.Context, + request *QueryJoinedHostServerNamesInRoomRequest, + response *QueryJoinedHostServerNamesInRoomResponse, + ) error +} + +// FederationSenderQueryJoinedHostsInRoomPath is the HTTP path for the QueryJoinedHostsInRoom API. +const FederationSenderQueryJoinedHostsInRoomPath = "/api/federationsender/queryJoinedHostsInRoom" + +// FederationSenderQueryJoinedHostServerNamesInRoomPath is the HTTP path for the QueryJoinedHostServerNamesInRoom API. +const FederationSenderQueryJoinedHostServerNamesInRoomPath = "/api/federationsender/queryJoinedHostServerNamesInRoom" + +// NewFederationSenderQueryAPIHTTP creates a FederationSenderQueryAPI implemented by talking to a HTTP POST API. +// If httpClient is nil then it uses the http.DefaultClient +func NewFederationSenderQueryAPIHTTP(federationSenderURL string, httpClient *http.Client) FederationSenderQueryAPI { + if httpClient == nil { + httpClient = http.DefaultClient + } + return &httpFederationSenderQueryAPI{federationSenderURL, httpClient} +} + +type httpFederationSenderQueryAPI struct { + federationSenderURL string + httpClient *http.Client +} + +// QueryJoinedHostsInRoom implements FederationSenderQueryAPI +func (h *httpFederationSenderQueryAPI) QueryJoinedHostsInRoom( + ctx context.Context, + request *QueryJoinedHostsInRoomRequest, + response *QueryJoinedHostsInRoomResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "QueryJoinedHostsInRoom") + defer span.Finish() + + apiURL := h.federationSenderURL + FederationSenderQueryJoinedHostsInRoomPath + return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +// QueryJoinedHostServerNamesInRoom implements FederationSenderQueryAPI +func (h *httpFederationSenderQueryAPI) QueryJoinedHostServerNamesInRoom( + ctx context.Context, + request *QueryJoinedHostServerNamesInRoomRequest, + response *QueryJoinedHostServerNamesInRoomResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "QueryJoinedHostServerNamesInRoom") + defer span.Finish() + + apiURL := h.federationSenderURL + FederationSenderQueryJoinedHostServerNamesInRoomPath + return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} diff --git a/federationsender/query/query.go b/federationsender/query/query.go new file mode 100644 index 000000000..ec9242863 --- /dev/null +++ b/federationsender/query/query.go @@ -0,0 +1,55 @@ +package query + +import ( + "context" + + "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/federationsender/types" + "github.com/matrix-org/gomatrixserverlib" +) + +// FederationSenderQueryDatabase has the APIs needed to implement the query API. +type FederationSenderQueryDatabase interface { + GetJoinedHosts( + ctx context.Context, roomID string, + ) ([]types.JoinedHost, error) +} + +// FederationSenderQueryAPI is an implementation of api.FederationSenderQueryAPI +type FederationSenderQueryAPI struct { + DB FederationSenderQueryDatabase +} + +// QueryJoinedHostsInRoom implements api.FederationSenderQueryAPI +func (f *FederationSenderQueryAPI) QueryJoinedHostsInRoom( + ctx context.Context, + request *api.QueryJoinedHostsInRoomRequest, + response *api.QueryJoinedHostsInRoomResponse, +) (err error) { + response.JoinedHosts, err = f.DB.GetJoinedHosts(ctx, request.RoomID) + return +} + +// QueryJoinedHostServerNamesInRoom implements api.FederationSenderQueryAPI +func (f *FederationSenderQueryAPI) QueryJoinedHostServerNamesInRoom( + ctx context.Context, + request *api.QueryJoinedHostServerNamesInRoomRequest, + response *api.QueryJoinedHostServerNamesInRoomResponse, +) (err error) { + joinedHosts, err := f.DB.GetJoinedHosts(ctx, request.RoomID) + if err != nil { + return + } + + serverNamesSet := make(map[gomatrixserverlib.ServerName]bool, len(joinedHosts)) + for _, host := range joinedHosts { + serverNamesSet[host.ServerName] = true + } + + response.ServerNames = make([]gomatrixserverlib.ServerName, 0, len(serverNamesSet)) + for name := range serverNamesSet { + response.ServerNames = append(response.ServerNames, name) + } + + return +}