From 28f60bec4e8a7bc86c2a6564d28f3afa925b26d8 Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Wed, 14 Aug 2019 21:44:34 +0800 Subject: [PATCH 01/12] Update name of a passing test in testfile (#784) --- testfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testfile b/testfile index d04ab731c..cea6a4f46 100644 --- a/testfile +++ b/testfile @@ -143,7 +143,7 @@ Trying to get push rules with unknown rule_id fails with 404 Events come down the correct room local user can join room with version 5 User can invite local user to room with version 5 -Inbound federation can receive room-join requests +Inbound federation can receive v1 room-join requests Typing events appear in initial sync Typing events appear in incremental sync Typing events appear in gapped sync From d21a2fb152143b2ce7600213abe18013519ccedf Mon Sep 17 00:00:00 2001 From: Parminder Singh Date: Wed, 14 Aug 2019 23:04:49 +0530 Subject: [PATCH 02/12] Add auth fallback endpoint (#405) Also adds support for the recaptcha auth type. --- clientapi/routing/auth_fallback.go | 210 +++++++++++++++++++++++++++++ clientapi/routing/register.go | 27 ++-- clientapi/routing/routing.go | 7 + common/httpapi.go | 19 +++ 4 files changed, 249 insertions(+), 14 deletions(-) create mode 100644 clientapi/routing/auth_fallback.go diff --git a/clientapi/routing/auth_fallback.go b/clientapi/routing/auth_fallback.go new file mode 100644 index 000000000..cd4530d1b --- /dev/null +++ b/clientapi/routing/auth_fallback.go @@ -0,0 +1,210 @@ +// Copyright 2019 Parminder Singh +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routing + +import ( + "html/template" + "net/http" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/clientapi/httputil" + "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/util" +) + +// recaptchaTemplate is an HTML webpage template for recaptcha auth +const recaptchaTemplate = ` + + +Authentication + + + + + + +
+
+

+ Hello! We need to prevent computer programs and other automated + things from creating accounts on this server. +

+

+ Please verify that you're not a robot. +

+ +
+
+ +
+ +
+ + +` + +// successTemplate is an HTML template presented to the user after successful +// recaptcha completion +const successTemplate = ` + + +Success! + + + + +
+

Thank you!

+

You may now close this window and return to the application.

+
+ + +` + +// serveTemplate fills template data and serves it using http.ResponseWriter +func serveTemplate(w http.ResponseWriter, templateHTML string, data map[string]string) { + t := template.Must(template.New("response").Parse(templateHTML)) + if err := t.Execute(w, data); err != nil { + panic(err) + } +} + +// AuthFallback implements GET and POST /auth/{authType}/fallback/web?session={sessionID} +func AuthFallback( + w http.ResponseWriter, req *http.Request, authType string, + cfg config.Dendrite, +) *util.JSONResponse { + sessionID := req.URL.Query().Get("session") + + if sessionID == "" { + return writeHTTPMessage(w, req, + "Session ID not provided", + http.StatusBadRequest, + ) + } + + serveRecaptcha := func() { + data := map[string]string{ + "myUrl": req.URL.String(), + "session": sessionID, + "siteKey": cfg.Matrix.RecaptchaPublicKey, + } + serveTemplate(w, recaptchaTemplate, data) + } + + serveSuccess := func() { + data := map[string]string{} + serveTemplate(w, successTemplate, data) + } + + if req.Method == http.MethodGet { + // Handle Recaptcha + if authType == authtypes.LoginTypeRecaptcha { + if err := checkRecaptchaEnabled(&cfg, w, req); err != nil { + return err + } + + serveRecaptcha() + return nil + } + return &util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound("Unknown auth stage type"), + } + } else if req.Method == http.MethodPost { + // Handle Recaptcha + if authType == authtypes.LoginTypeRecaptcha { + if err := checkRecaptchaEnabled(&cfg, w, req); err != nil { + return err + } + + clientIP := req.RemoteAddr + err := req.ParseForm() + if err != nil { + res := httputil.LogThenError(req, err) + return &res + } + + response := req.Form.Get("g-recaptcha-response") + if err := validateRecaptcha(&cfg, response, clientIP); err != nil { + util.GetLogger(req.Context()).Error(err) + return err + } + + // Success. Add recaptcha as a completed login flow + AddCompletedSessionStage(sessionID, authtypes.LoginTypeRecaptcha) + + serveSuccess() + return nil + } + + return &util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound("Unknown auth stage type"), + } + } + return &util.JSONResponse{ + Code: http.StatusMethodNotAllowed, + JSON: jsonerror.NotFound("Bad method"), + } +} + +// checkRecaptchaEnabled creates an error response if recaptcha is not usable on homeserver. +func checkRecaptchaEnabled( + cfg *config.Dendrite, + w http.ResponseWriter, + req *http.Request, +) *util.JSONResponse { + if !cfg.Matrix.RecaptchaEnabled { + return writeHTTPMessage(w, req, + "Recaptcha login is disabled on this Homeserver", + http.StatusBadRequest, + ) + } + return nil +} + +// writeHTTPMessage writes the given header and message to the HTTP response writer. +// Returns an error JSONResponse obtained through httputil.LogThenError if the writing failed, otherwise nil. +func writeHTTPMessage( + w http.ResponseWriter, req *http.Request, + message string, header int, +) *util.JSONResponse { + w.WriteHeader(header) + _, err := w.Write([]byte(message)) + if err != nil { + res := httputil.LogThenError(req, err) + return &res + } + return nil +} diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index c5a3d3018..0af407587 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -83,23 +83,22 @@ func (d sessionsDict) GetCompletedStages(sessionID string) []authtypes.LoginType return make([]authtypes.LoginType, 0) } -// AddCompletedStage records that a session has completed an auth stage. -func (d *sessionsDict) AddCompletedStage(sessionID string, stage authtypes.LoginType) { - // Return if the stage is already present - for _, completedStage := range d.GetCompletedStages(sessionID) { - if completedStage == stage { - return - } - } - d.sessions[sessionID] = append(d.GetCompletedStages(sessionID), stage) -} - func newSessionsDict() *sessionsDict { return &sessionsDict{ sessions: make(map[string][]authtypes.LoginType), } } +// AddCompletedSessionStage records that a session has completed an auth stage. +func AddCompletedSessionStage(sessionID string, stage authtypes.LoginType) { + for _, completedStage := range sessions.GetCompletedStages(sessionID) { + if completedStage == stage { + return + } + } + sessions.sessions[sessionID] = append(sessions.GetCompletedStages(sessionID), stage) +} + var ( // TODO: Remove old sessions. Need to do so on a session-specific timeout. // sessions stores the completed flow stages for all sessions. Referenced using their sessionID. @@ -530,7 +529,7 @@ func handleRegistrationFlow( } // Add Recaptcha to the list of completed registration stages - sessions.AddCompletedStage(sessionID, authtypes.LoginTypeRecaptcha) + AddCompletedSessionStage(sessionID, authtypes.LoginTypeRecaptcha) case authtypes.LoginTypeSharedSecret: // Check shared secret against config @@ -543,7 +542,7 @@ func handleRegistrationFlow( } // Add SharedSecret to the list of completed registration stages - sessions.AddCompletedStage(sessionID, authtypes.LoginTypeSharedSecret) + AddCompletedSessionStage(sessionID, authtypes.LoginTypeSharedSecret) case "": // Extract the access token from the request, if there's one to extract @@ -573,7 +572,7 @@ func handleRegistrationFlow( case authtypes.LoginTypeDummy: // there is nothing to do // Add Dummy to the list of completed registration stages - sessions.AddCompletedStage(sessionID, authtypes.LoginTypeDummy) + AddCompletedSessionStage(sessionID, authtypes.LoginTypeDummy) default: return util.JSONResponse{ diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index d36ed6957..d4b323a2d 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -245,6 +245,13 @@ func Setup( }), ).Methods(http.MethodGet, http.MethodPost, http.MethodOptions) + r0mux.Handle("/auth/{authType}/fallback/web", + common.MakeHTMLAPI("auth_fallback", func(w http.ResponseWriter, req *http.Request) *util.JSONResponse { + vars := mux.Vars(req) + return AuthFallback(w, req, vars["authType"], cfg) + }), + ).Methods(http.MethodGet, http.MethodPost, http.MethodOptions) + r0mux.Handle("/pushrules/", common.MakeExternalAPI("push_rules", func(req *http.Request) util.JSONResponse { // TODO: Implement push rules API diff --git a/common/httpapi.go b/common/httpapi.go index 99e15830a..bf634ff4a 100644 --- a/common/httpapi.go +++ b/common/httpapi.go @@ -10,6 +10,7 @@ import ( "github.com/matrix-org/util" opentracing "github.com/opentracing/opentracing-go" "github.com/opentracing/opentracing-go/ext" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" ) @@ -43,6 +44,24 @@ func MakeExternalAPI(metricsName string, f func(*http.Request) util.JSONResponse return http.HandlerFunc(withSpan) } +// MakeHTMLAPI adds Span metrics to the HTML Handler function +// This is used to serve HTML alongside JSON error messages +func MakeHTMLAPI(metricsName string, f func(http.ResponseWriter, *http.Request) *util.JSONResponse) http.Handler { + withSpan := func(w http.ResponseWriter, req *http.Request) { + span := opentracing.StartSpan(metricsName) + defer span.Finish() + req = req.WithContext(opentracing.ContextWithSpan(req.Context(), span)) + if err := f(w, req); err != nil { + h := util.MakeJSONAPI(util.NewJSONRequestHandler(func(req *http.Request) util.JSONResponse { + return *err + })) + h.ServeHTTP(w, req) + } + } + + return prometheus.InstrumentHandler(metricsName, http.HandlerFunc(withSpan)) +} + // MakeInternalAPI turns a util.JSONRequestHandler function into an http.Handler. // This is used for APIs that are internal to dendrite. // If we are passed a tracing context in the request headers then we use that From d63d2a50cdce891e6d4159919b304ce0c66ac0c5 Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Fri, 16 Aug 2019 01:45:11 +0800 Subject: [PATCH 03/12] Replace event content types with ones in gomatrixserverlib (#785) The types that are not in gomatrixserverlib are not replaced. This also updates the gomatrixserverlib dependency. --- clientapi/routing/createroom.go | 6 +- clientapi/routing/membership.go | 2 +- clientapi/routing/profile.go | 2 +- clientapi/threepid/invites.go | 10 ++-- common/eventcontent.go | 95 ++++--------------------------- federationapi/routing/threepid.go | 19 +++---- go.mod | 2 +- go.sum | 2 + 8 files changed, 33 insertions(+), 105 deletions(-) diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index 4a76e1b06..620246d28 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -106,7 +106,7 @@ func (r createRoomRequest) Validate() *util.JSONResponse { } } - var CreationContent common.CreateContent + var CreationContent gomatrixserverlib.CreateContent err = json.Unmarshal(creationContentBytes, &CreationContent) if err != nil { return &util.JSONResponse{ @@ -196,7 +196,7 @@ func createRoom( return httputil.LogThenError(req, err) } - membershipContent := common.MemberContent{ + membershipContent := gomatrixserverlib.MemberContent{ Membership: gomatrixserverlib.Join, DisplayName: profile.DisplayName, AvatarURL: profile.AvatarURL, @@ -246,7 +246,7 @@ func createRoom( {"m.room.member", userID, membershipContent}, {"m.room.power_levels", "", common.InitialPowerLevelsContent(userID)}, // TODO: m.room.canonical_alias - {"m.room.join_rules", "", common.JoinRulesContent{JoinRule: joinRules}}, + {"m.room.join_rules", "", gomatrixserverlib.JoinRuleContent{JoinRule: joinRules}}, {"m.room.history_visibility", "", common.HistoryVisibilityContent{HistoryVisibility: historyVisibility}}, } if r.GuestCanJoin { diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index 5e183fa0f..c71ac2de2 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -144,7 +144,7 @@ func buildMembershipEvent( membership = gomatrixserverlib.Leave } - content := common.MemberContent{ + content := gomatrixserverlib.MemberContent{ Membership: membership, DisplayName: profile.DisplayName, AvatarURL: profile.AvatarURL, diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index e8ea6cf13..a87c6f743 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -332,7 +332,7 @@ func buildMembershipEvents( StateKey: &userID, } - content := common.MemberContent{ + content := gomatrixserverlib.MemberContent{ Membership: gomatrixserverlib.Join, } diff --git a/clientapi/threepid/invites.go b/clientapi/threepid/invites.go index 251afb0d3..bfe5060a8 100644 --- a/clientapi/threepid/invites.go +++ b/clientapi/threepid/invites.go @@ -56,10 +56,10 @@ type idServerLookupResponse struct { // idServerLookupResponse represents the response described at https://matrix.org/docs/spec/client_server/r0.2.0.html#invitation-storage type idServerStoreInviteResponse struct { - PublicKey string `json:"public_key"` - Token string `json:"token"` - DisplayName string `json:"display_name"` - PublicKeys []common.PublicKey `json:"public_keys"` + PublicKey string `json:"public_key"` + Token string `json:"token"` + DisplayName string `json:"display_name"` + PublicKeys []gomatrixserverlib.PublicKey `json:"public_keys"` } var ( @@ -342,7 +342,7 @@ func emit3PIDInviteEvent( } validityURL := fmt.Sprintf("https://%s/_matrix/identity/api/v1/pubkey/isvalid", body.IDServer) - content := common.ThirdPartyInviteContent{ + content := gomatrixserverlib.ThirdPartyInviteContent{ DisplayName: res.DisplayName, KeyValidityURL: validityURL, PublicKey: res.PublicKey, diff --git a/common/eventcontent.go b/common/eventcontent.go index c45724fcd..c07c56276 100644 --- a/common/eventcontent.go +++ b/common/eventcontent.go @@ -14,55 +14,7 @@ package common -// CreateContent is the event content for http://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-create -type CreateContent struct { - Creator string `json:"creator"` - Federate *bool `json:"m.federate,omitempty"` - RoomVersion string `json:"room_version,omitempty"` - Predecessor PreviousRoom `json:"predecessor,omitempty"` -} - -// PreviousRoom is the "Previous Room" structure defined at https://matrix.org/docs/spec/client_server/r0.5.0#m-room-create -type PreviousRoom struct { - RoomID string `json:"room_id"` - EventID string `json:"event_id"` -} - -// MemberContent is the event content for http://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-member -type MemberContent struct { - Membership string `json:"membership"` - DisplayName string `json:"displayname,omitempty"` - AvatarURL string `json:"avatar_url,omitempty"` - Reason string `json:"reason,omitempty"` - ThirdPartyInvite *TPInvite `json:"third_party_invite,omitempty"` -} - -// TPInvite is the "Invite" structure defined at http://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-member -type TPInvite struct { - DisplayName string `json:"display_name"` - Signed TPInviteSigned `json:"signed"` -} - -// TPInviteSigned is the "signed" structure defined at http://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-member -type TPInviteSigned struct { - MXID string `json:"mxid"` - Signatures map[string]map[string]string `json:"signatures"` - Token string `json:"token"` -} - -// ThirdPartyInviteContent is the content event for https://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-third-party-invite -type ThirdPartyInviteContent struct { - DisplayName string `json:"display_name"` - KeyValidityURL string `json:"key_validity_url"` - PublicKey string `json:"public_key"` - PublicKeys []PublicKey `json:"public_keys"` -} - -// PublicKey is the PublicKeys structure in https://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-third-party-invite -type PublicKey struct { - KeyValidityURL string `json:"key_validity_url"` - PublicKey string `json:"public_key"` -} +import "github.com/matrix-org/gomatrixserverlib" // NameContent is the event content for https://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-name type NameContent struct { @@ -79,51 +31,26 @@ type GuestAccessContent struct { GuestAccess string `json:"guest_access"` } -// JoinRulesContent is the event content for http://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-join-rules -type JoinRulesContent struct { - JoinRule string `json:"join_rule"` -} - // HistoryVisibilityContent is the event content for http://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-history-visibility type HistoryVisibilityContent struct { HistoryVisibility string `json:"history_visibility"` } -// PowerLevelContent is the event content for http://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-power-levels -type PowerLevelContent struct { - EventsDefault int `json:"events_default"` - Invite int `json:"invite"` - StateDefault int `json:"state_default"` - Redact int `json:"redact"` - Ban int `json:"ban"` - UsersDefault int `json:"users_default"` - Events map[string]int `json:"events"` - Kick int `json:"kick"` - Users map[string]int `json:"users"` -} - // InitialPowerLevelsContent returns the initial values for m.room.power_levels on room creation // if they have not been specified. // http://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-power-levels // https://github.com/matrix-org/synapse/blob/v0.19.2/synapse/handlers/room.py#L294 -func InitialPowerLevelsContent(roomCreator string) PowerLevelContent { - return PowerLevelContent{ - EventsDefault: 0, - Invite: 0, - StateDefault: 50, - Redact: 50, - Ban: 50, - UsersDefault: 0, - Events: map[string]int{ - "m.room.name": 50, - "m.room.power_levels": 100, - "m.room.history_visibility": 100, - "m.room.canonical_alias": 50, - "m.room.avatar": 50, - }, - Kick: 50, - Users: map[string]int{roomCreator: 100}, +func InitialPowerLevelsContent(roomCreator string) (c gomatrixserverlib.PowerLevelContent) { + c.Defaults() + c.Events = map[string]int64{ + "m.room.name": 50, + "m.room.power_levels": 100, + "m.room.history_visibility": 100, + "m.room.canonical_alias": 50, + "m.room.avatar": 50, } + c.Users = map[string]int64{roomCreator: 100} + return c } // AliasesContent is the event content for http://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-aliases diff --git a/federationapi/routing/threepid.go b/federationapi/routing/threepid.go index cff311cc4..7fa02be91 100644 --- a/federationapi/routing/threepid.go +++ b/federationapi/routing/threepid.go @@ -27,7 +27,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" - "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/config" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" @@ -38,11 +37,11 @@ import ( ) type invite struct { - MXID string `json:"mxid"` - RoomID string `json:"room_id"` - Sender string `json:"sender"` - Token string `json:"token"` - Signed common.TPInviteSigned `json:"signed"` + MXID string `json:"mxid"` + RoomID string `json:"room_id"` + Sender string `json:"sender"` + Token string `json:"token"` + Signed gomatrixserverlib.MemberThirdPartyInviteSigned `json:"signed"` } type invites struct { @@ -199,11 +198,11 @@ func createInviteFrom3PIDInvite( return nil, err } - content := common.MemberContent{ + content := gomatrixserverlib.MemberContent{ AvatarURL: profile.AvatarURL, DisplayName: profile.DisplayName, Membership: gomatrixserverlib.Invite, - ThirdPartyInvite: &common.TPInvite{ + ThirdPartyInvite: &gomatrixserverlib.MemberThirdPartyInvite{ Signed: inv.Signed, }, } @@ -330,7 +329,7 @@ func sendToRemoteServer( func fillDisplayName( builder *gomatrixserverlib.EventBuilder, authEvents gomatrixserverlib.AuthEvents, ) error { - var content common.MemberContent + var content gomatrixserverlib.MemberContent if err := json.Unmarshal(builder.Content, &content); err != nil { return err } @@ -343,7 +342,7 @@ func fillDisplayName( return nil } - var thirdPartyInviteContent common.ThirdPartyInviteContent + var thirdPartyInviteContent gomatrixserverlib.ThirdPartyInviteContent if err := json.Unmarshal(thirdPartyInviteEvent.Content(), &thirdPartyInviteContent); err != nil { return err } diff --git a/go.mod b/go.mod index 8e14253ca..d51f0a33e 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/lib/pq v0.0.0-20170918175043-23da1db4f16d github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20190805173246-3a2199d5ecd6 + github.com/matrix-org/gomatrixserverlib v0.0.0-20190814163046-d6285a18401f github.com/matrix-org/naffka v0.0.0-20171115094957-662bfd0841d0 github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 github.com/matttproud/golang_protobuf_extensions v1.0.1 diff --git a/go.sum b/go.sum index 0d59d1dd6..56781c9a6 100644 --- a/go.sum +++ b/go.sum @@ -58,6 +58,8 @@ github.com/matrix-org/gomatrixserverlib v0.0.0-20190724145009-a6df10ef35d6 h1:B8 github.com/matrix-org/gomatrixserverlib v0.0.0-20190724145009-a6df10ef35d6/go.mod h1:sf0RcKOdiwJeTti7A313xsaejNUGYDq02MQZ4JD4w/E= github.com/matrix-org/gomatrixserverlib v0.0.0-20190805173246-3a2199d5ecd6 h1:xr69Hk6QM3RIN6JSvx3RpDowBGpHpDDqhqXCeySwYow= github.com/matrix-org/gomatrixserverlib v0.0.0-20190805173246-3a2199d5ecd6/go.mod h1:sf0RcKOdiwJeTti7A313xsaejNUGYDq02MQZ4JD4w/E= +github.com/matrix-org/gomatrixserverlib v0.0.0-20190814163046-d6285a18401f h1:20CZL7ApB7xgR7sZF9yD/qpsP51Sfx0TTgUJ3vKgnZQ= +github.com/matrix-org/gomatrixserverlib v0.0.0-20190814163046-d6285a18401f/go.mod h1:sf0RcKOdiwJeTti7A313xsaejNUGYDq02MQZ4JD4w/E= github.com/matrix-org/naffka v0.0.0-20171115094957-662bfd0841d0 h1:p7WTwG+aXM86+yVrYAiCMW3ZHSmotVvuRbjtt3jC+4A= github.com/matrix-org/naffka v0.0.0-20171115094957-662bfd0841d0/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/util v0.0.0-20171013132526-8b1c8ab81986 h1:TiWl4hLvezAhRPM8tPcPDFTysZ7k4T/1J4GPp/iqlZo= From bf5efbc31fa02ef438b450b78db5eef14b3ab3a7 Mon Sep 17 00:00:00 2001 From: Parminder Singh Date: Thu, 15 Aug 2019 23:29:17 +0530 Subject: [PATCH 04/12] Error when recaptcha enabled with empty configs (#786) --- common/config/config.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/common/config/config.go b/common/config/config.go index 9fcab8cf9..40232fb03 100644 --- a/common/config/config.go +++ b/common/config/config.go @@ -498,6 +498,11 @@ func (config *Dendrite) checkMatrix(configErrs *configErrors) { checkNotEmpty(configErrs, "matrix.server_name", string(config.Matrix.ServerName)) checkNotEmpty(configErrs, "matrix.private_key", string(config.Matrix.PrivateKeyPath)) checkNotZero(configErrs, "matrix.federation_certificates", int64(len(config.Matrix.FederationCertificatePaths))) + if config.Matrix.RecaptchaEnabled { + checkNotEmpty(configErrs, "matrix.recaptcha_public_key", string(config.Matrix.RecaptchaPublicKey)) + checkNotEmpty(configErrs, "matrix.recaptcha_private_key", string(config.Matrix.RecaptchaPrivateKey)) + checkNotEmpty(configErrs, "matrix.recaptcha_siteverify_api", string(config.Matrix.RecaptchaSiteVerifyAPI)) + } } // checkMedia verifies the parameters media.* are valid. From 0ed2dd0b154f147a5575fe60996005692946160a Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Fri, 16 Aug 2019 12:05:00 +0800 Subject: [PATCH 05/12] Fix data race in clientapi/routing/register.go (#787) --- clientapi/routing/register.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index 0af407587..d0f36a6fd 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -29,6 +29,7 @@ import ( "sort" "strconv" "strings" + "sync" "time" "github.com/matrix-org/dendrite/common/config" @@ -70,12 +71,17 @@ func init() { } // sessionsDict keeps track of completed auth stages for each session. +// It shouldn't be passed by value because it contains a mutex. type sessionsDict struct { + sync.Mutex sessions map[string][]authtypes.LoginType } // GetCompletedStages returns the completed stages for a session. -func (d sessionsDict) GetCompletedStages(sessionID string) []authtypes.LoginType { +func (d *sessionsDict) GetCompletedStages(sessionID string) []authtypes.LoginType { + d.Lock() + defer d.Unlock() + if completedStages, ok := d.sessions[sessionID]; ok { return completedStages } @@ -91,12 +97,15 @@ func newSessionsDict() *sessionsDict { // AddCompletedSessionStage records that a session has completed an auth stage. func AddCompletedSessionStage(sessionID string, stage authtypes.LoginType) { - for _, completedStage := range sessions.GetCompletedStages(sessionID) { + sessions.Lock() + defer sessions.Unlock() + + for _, completedStage := range sessions.sessions[sessionID] { if completedStage == stage { return } } - sessions.sessions[sessionID] = append(sessions.GetCompletedStages(sessionID), stage) + sessions.sessions[sessionID] = append(sessions.sessions[sessionID], stage) } var ( From a81917c3e72be13dd0a9680bde54f3ad35c846ab Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Tue, 20 Aug 2019 01:01:53 +0800 Subject: [PATCH 06/12] Make trailing slash on server key request optional (#788) Cherry-picked from 7e861b60fbd721b374ec929926b14e57dc60ec41 --- federationapi/routing/routing.go | 1 + testfile | 1 + 2 files changed, 2 insertions(+) diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index ed32c8904..9f576790b 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -64,6 +64,7 @@ func Setup( // {keyID} argument and always return a response containing all of the keys. v2keysmux.Handle("/server/{keyID}", localKeys).Methods(http.MethodGet) v2keysmux.Handle("/server/", localKeys).Methods(http.MethodGet) + v2keysmux.Handle("/server", localKeys).Methods(http.MethodGet) v1fedmux.Handle("/send/{txnID}", common.MakeFedAPI( "federation_send", cfg.Matrix.ServerName, keys, diff --git a/testfile b/testfile index cea6a4f46..17978913e 100644 --- a/testfile +++ b/testfile @@ -170,3 +170,4 @@ Deleted tags appear in an incremental v2 /sync Outbound federation can query profile data /event/ on joined room works /event/ does not allow access to events before the user joined +Federation key API allows unsigned requests for keys From 5eb63f1d1eafae0111d46def5cf5bf431d2e8169 Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Thu, 22 Aug 2019 19:47:52 +0800 Subject: [PATCH 07/12] Add joined hosts query APIs (#781) This adds two joined hosts query APIs to the federation sender for use of other components. --- federationsender/api/query.go | 98 +++++++++++++++++++++++++++++++++ federationsender/query/query.go | 55 ++++++++++++++++++ 2 files changed, 153 insertions(+) create mode 100644 federationsender/api/query.go create mode 100644 federationsender/query/query.go diff --git a/federationsender/api/query.go b/federationsender/api/query.go new file mode 100644 index 000000000..ebc6e833f --- /dev/null +++ b/federationsender/api/query.go @@ -0,0 +1,98 @@ +package api + +import ( + "context" + "net/http" + + commonHTTP "github.com/matrix-org/dendrite/common/http" + "github.com/matrix-org/gomatrixserverlib" + + "github.com/matrix-org/dendrite/federationsender/types" + "github.com/opentracing/opentracing-go" +) + +// QueryJoinedHostsInRoomRequest is a request to QueryJoinedHostsInRoom +type QueryJoinedHostsInRoomRequest struct { + RoomID string `json:"room_id"` +} + +// QueryJoinedHostsInRoomResponse is a response to QueryJoinedHostsInRoom +type QueryJoinedHostsInRoomResponse struct { + JoinedHosts []types.JoinedHost `json:"joined_hosts"` +} + +// QueryJoinedHostServerNamesRequest is a request to QueryJoinedHostServerNames +type QueryJoinedHostServerNamesInRoomRequest struct { + RoomID string `json:"room_id"` +} + +// QueryJoinedHostServerNamesResponse is a response to QueryJoinedHostServerNames +type QueryJoinedHostServerNamesInRoomResponse struct { + ServerNames []gomatrixserverlib.ServerName `json:"server_names"` +} + +// FederationSenderQueryAPI is used to query information from the federation sender. +type FederationSenderQueryAPI interface { + // Query the joined hosts and the membership events accounting for their participation in a room. + // Note that if a server has multiple users in the room, it will have multiple entries in the returned slice. + // See `QueryJoinedHostServerNamesInRoom` for a de-duplicated version. + QueryJoinedHostsInRoom( + ctx context.Context, + request *QueryJoinedHostsInRoomRequest, + response *QueryJoinedHostsInRoomResponse, + ) error + // Query the server names of the joined hosts in a room. + // Unlike QueryJoinedHostsInRoom, this function returns a de-duplicated slice + // containing only the server names (without information for membership events). + QueryJoinedHostServerNamesInRoom( + ctx context.Context, + request *QueryJoinedHostServerNamesInRoomRequest, + response *QueryJoinedHostServerNamesInRoomResponse, + ) error +} + +// FederationSenderQueryJoinedHostsInRoomPath is the HTTP path for the QueryJoinedHostsInRoom API. +const FederationSenderQueryJoinedHostsInRoomPath = "/api/federationsender/queryJoinedHostsInRoom" + +// FederationSenderQueryJoinedHostServerNamesInRoomPath is the HTTP path for the QueryJoinedHostServerNamesInRoom API. +const FederationSenderQueryJoinedHostServerNamesInRoomPath = "/api/federationsender/queryJoinedHostServerNamesInRoom" + +// NewFederationSenderQueryAPIHTTP creates a FederationSenderQueryAPI implemented by talking to a HTTP POST API. +// If httpClient is nil then it uses the http.DefaultClient +func NewFederationSenderQueryAPIHTTP(federationSenderURL string, httpClient *http.Client) FederationSenderQueryAPI { + if httpClient == nil { + httpClient = http.DefaultClient + } + return &httpFederationSenderQueryAPI{federationSenderURL, httpClient} +} + +type httpFederationSenderQueryAPI struct { + federationSenderURL string + httpClient *http.Client +} + +// QueryJoinedHostsInRoom implements FederationSenderQueryAPI +func (h *httpFederationSenderQueryAPI) QueryJoinedHostsInRoom( + ctx context.Context, + request *QueryJoinedHostsInRoomRequest, + response *QueryJoinedHostsInRoomResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "QueryJoinedHostsInRoom") + defer span.Finish() + + apiURL := h.federationSenderURL + FederationSenderQueryJoinedHostsInRoomPath + return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +// QueryJoinedHostServerNamesInRoom implements FederationSenderQueryAPI +func (h *httpFederationSenderQueryAPI) QueryJoinedHostServerNamesInRoom( + ctx context.Context, + request *QueryJoinedHostServerNamesInRoomRequest, + response *QueryJoinedHostServerNamesInRoomResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "QueryJoinedHostServerNamesInRoom") + defer span.Finish() + + apiURL := h.federationSenderURL + FederationSenderQueryJoinedHostServerNamesInRoomPath + return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} diff --git a/federationsender/query/query.go b/federationsender/query/query.go new file mode 100644 index 000000000..ec9242863 --- /dev/null +++ b/federationsender/query/query.go @@ -0,0 +1,55 @@ +package query + +import ( + "context" + + "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/federationsender/types" + "github.com/matrix-org/gomatrixserverlib" +) + +// FederationSenderQueryDatabase has the APIs needed to implement the query API. +type FederationSenderQueryDatabase interface { + GetJoinedHosts( + ctx context.Context, roomID string, + ) ([]types.JoinedHost, error) +} + +// FederationSenderQueryAPI is an implementation of api.FederationSenderQueryAPI +type FederationSenderQueryAPI struct { + DB FederationSenderQueryDatabase +} + +// QueryJoinedHostsInRoom implements api.FederationSenderQueryAPI +func (f *FederationSenderQueryAPI) QueryJoinedHostsInRoom( + ctx context.Context, + request *api.QueryJoinedHostsInRoomRequest, + response *api.QueryJoinedHostsInRoomResponse, +) (err error) { + response.JoinedHosts, err = f.DB.GetJoinedHosts(ctx, request.RoomID) + return +} + +// QueryJoinedHostServerNamesInRoom implements api.FederationSenderQueryAPI +func (f *FederationSenderQueryAPI) QueryJoinedHostServerNamesInRoom( + ctx context.Context, + request *api.QueryJoinedHostServerNamesInRoomRequest, + response *api.QueryJoinedHostServerNamesInRoomResponse, +) (err error) { + joinedHosts, err := f.DB.GetJoinedHosts(ctx, request.RoomID) + if err != nil { + return + } + + serverNamesSet := make(map[gomatrixserverlib.ServerName]bool, len(joinedHosts)) + for _, host := range joinedHosts { + serverNamesSet[host.ServerName] = true + } + + response.ServerNames = make([]gomatrixserverlib.ServerName, 0, len(serverNamesSet)) + for name := range serverNamesSet { + response.ServerNames = append(response.ServerNames, name) + } + + return +} From 43308d2f3f8fcf9bdb3ec55d4e679b576cc19488 Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Sat, 24 Aug 2019 00:55:40 +0800 Subject: [PATCH 08/12] Associate transactions with session IDs instead of device IDs (#789) --- clientapi/auth/authtypes/device.go | 4 ++++ .../auth/storage/devices/devices_table.go | 19 +++++++++++++---- clientapi/routing/sendevent.go | 8 +++---- roomserver/api/input.go | 4 ++-- roomserver/input/events.go | 6 +++--- roomserver/storage/storage.go | 12 +++++------ roomserver/storage/transactions_table.go | 18 ++++++++-------- syncapi/storage/output_room_events_table.go | 21 ++++++++++--------- syncapi/storage/syncserver.go | 2 +- 9 files changed, 55 insertions(+), 39 deletions(-) diff --git a/clientapi/auth/authtypes/device.go b/clientapi/auth/authtypes/device.go index a6d3a7b08..930ab3956 100644 --- a/clientapi/auth/authtypes/device.go +++ b/clientapi/auth/authtypes/device.go @@ -21,5 +21,9 @@ type Device struct { // The access_token granted to this device. // This uniquely identifies the device from all other devices and clients. AccessToken string + // The unique ID of the session identified by the access token. + // Can be used as a secure substitution in places where data needs to be + // associated with access tokens. + SessionID int64 // TODO: display name, last used timestamp, keys, etc } diff --git a/clientapi/auth/storage/devices/devices_table.go b/clientapi/auth/storage/devices/devices_table.go index 60aa563a2..d011d25c9 100644 --- a/clientapi/auth/storage/devices/devices_table.go +++ b/clientapi/auth/storage/devices/devices_table.go @@ -27,11 +27,19 @@ import ( ) const devicesSchema = ` +-- This sequence is used for automatic allocation of session_id. +CREATE SEQUENCE IF NOT EXISTS device_session_id_seq START 1; + -- Stores data about devices. CREATE TABLE IF NOT EXISTS device_devices ( -- The access token granted to this device. This has to be the primary key -- so we can distinguish which device is making a given request. access_token TEXT NOT NULL PRIMARY KEY, + -- The auto-allocated unique ID of the session identified by the access token. + -- This can be used as a secure substitution of the access token in situations + -- where data is associated with access tokens (e.g. transaction storage), + -- so we don't have to store users' access tokens everywhere. + session_id BIGINT NOT NULL DEFAULT nextval('device_session_id_seq'), -- The device identifier. This only needs to uniquely identify a device for a given user, not globally. -- access_tokens will be clobbered based on the device ID for a user. device_id TEXT NOT NULL, @@ -51,10 +59,11 @@ CREATE UNIQUE INDEX IF NOT EXISTS device_localpart_id_idx ON device_devices(loca ` const insertDeviceSQL = "" + - "INSERT INTO device_devices(device_id, localpart, access_token, created_ts, display_name) VALUES ($1, $2, $3, $4, $5)" + "INSERT INTO device_devices(device_id, localpart, access_token, created_ts, display_name) VALUES ($1, $2, $3, $4, $5)" + + " RETURNING session_id" const selectDeviceByTokenSQL = "" + - "SELECT device_id, localpart FROM device_devices WHERE access_token = $1" + "SELECT session_id, device_id, localpart FROM device_devices WHERE access_token = $1" const selectDeviceByIDSQL = "" + "SELECT display_name FROM device_devices WHERE localpart = $1 and device_id = $2" @@ -120,14 +129,16 @@ func (s *devicesStatements) insertDevice( displayName *string, ) (*authtypes.Device, error) { createdTimeMS := time.Now().UnixNano() / 1000000 + var sessionID int64 stmt := common.TxStmt(txn, s.insertDeviceStmt) - if _, err := stmt.ExecContext(ctx, id, localpart, accessToken, createdTimeMS, displayName); err != nil { + if err := stmt.QueryRowContext(ctx, id, localpart, accessToken, createdTimeMS, displayName).Scan(&sessionID); err != nil { return nil, err } return &authtypes.Device{ ID: id, UserID: userutil.MakeUserID(localpart, s.serverName), AccessToken: accessToken, + SessionID: sessionID, }, nil } @@ -161,7 +172,7 @@ func (s *devicesStatements) selectDeviceByToken( var dev authtypes.Device var localpart string stmt := s.selectDeviceByTokenStmt - err := stmt.QueryRowContext(ctx, accessToken).Scan(&dev.ID, &localpart) + err := stmt.QueryRowContext(ctx, accessToken).Scan(&dev.SessionID, &dev.ID, &localpart) if err == nil { dev.UserID = userutil.MakeUserID(localpart, s.serverName) dev.AccessToken = accessToken diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go index 9696b360e..76e36cd46 100644 --- a/clientapi/routing/sendevent.go +++ b/clientapi/routing/sendevent.go @@ -60,18 +60,18 @@ func SendEvent( return *resErr } - var txnAndDeviceID *api.TransactionID + var txnAndSessionID *api.TransactionID if txnID != nil { - txnAndDeviceID = &api.TransactionID{ + txnAndSessionID = &api.TransactionID{ TransactionID: *txnID, - DeviceID: device.ID, + SessionID: device.SessionID, } } // pass the new event to the roomserver and receive the correct event ID // event ID in case of duplicate transaction is discarded eventID, err := producer.SendEvents( - req.Context(), []gomatrixserverlib.Event{*e}, cfg.Matrix.ServerName, txnAndDeviceID, + req.Context(), []gomatrixserverlib.Event{*e}, cfg.Matrix.ServerName, txnAndSessionID, ) if err != nil { return httputil.LogThenError(req, err) diff --git a/roomserver/api/input.go b/roomserver/api/input.go index 2c2e27c62..9643a927c 100644 --- a/roomserver/api/input.go +++ b/roomserver/api/input.go @@ -75,9 +75,9 @@ type InputRoomEvent struct { } // TransactionID contains the transaction ID sent by a client when sending an -// event, along with the ID of that device. +// event, along with the ID of the client session. type TransactionID struct { - DeviceID string `json:"device_id"` + SessionID int64 `json:"session_id"` TransactionID string `json:"id"` } diff --git a/roomserver/input/events.go b/roomserver/input/events.go index feb15b3e1..b30c39928 100644 --- a/roomserver/input/events.go +++ b/roomserver/input/events.go @@ -32,7 +32,7 @@ type RoomEventDatabase interface { StoreEvent( ctx context.Context, event gomatrixserverlib.Event, - txnAndDeviceID *api.TransactionID, + txnAndSessionID *api.TransactionID, authEventNIDs []types.EventNID, ) (types.RoomNID, types.StateAtEvent, error) // Look up the state entries for a list of string event IDs @@ -67,7 +67,7 @@ type RoomEventDatabase interface { // Returns an empty string if no such event exists. GetTransactionEventID( ctx context.Context, transactionID string, - deviceID string, userID string, + sessionID int64, userID string, ) (string, error) } @@ -100,7 +100,7 @@ func processRoomEvent( if input.TransactionID != nil { tdID := input.TransactionID eventID, err = db.GetTransactionEventID( - ctx, tdID.TransactionID, tdID.DeviceID, input.Event.Sender(), + ctx, tdID.TransactionID, tdID.SessionID, input.Event.Sender(), ) // On error OR event with the transaction already processed/processesing if err != nil || eventID != "" { diff --git a/roomserver/storage/storage.go b/roomserver/storage/storage.go index 71c13b7ca..7e8eb98c9 100644 --- a/roomserver/storage/storage.go +++ b/roomserver/storage/storage.go @@ -47,7 +47,7 @@ func Open(dataSourceName string) (*Database, error) { // StoreEvent implements input.EventDatabase func (d *Database) StoreEvent( ctx context.Context, event gomatrixserverlib.Event, - txnAndDeviceID *api.TransactionID, authEventNIDs []types.EventNID, + txnAndSessionID *api.TransactionID, authEventNIDs []types.EventNID, ) (types.RoomNID, types.StateAtEvent, error) { var ( roomNID types.RoomNID @@ -58,10 +58,10 @@ func (d *Database) StoreEvent( err error ) - if txnAndDeviceID != nil { + if txnAndSessionID != nil { if err = d.statements.insertTransaction( - ctx, txnAndDeviceID.TransactionID, - txnAndDeviceID.DeviceID, event.Sender(), event.EventID(), + ctx, txnAndSessionID.TransactionID, + txnAndSessionID.SessionID, event.Sender(), event.EventID(), ); err != nil { return 0, types.StateAtEvent{}, err } @@ -322,9 +322,9 @@ func (d *Database) GetLatestEventsForUpdate( // GetTransactionEventID implements input.EventDatabase func (d *Database) GetTransactionEventID( ctx context.Context, transactionID string, - deviceID string, userID string, + sessionID int64, userID string, ) (string, error) { - eventID, err := d.statements.selectTransactionEventID(ctx, transactionID, deviceID, userID) + eventID, err := d.statements.selectTransactionEventID(ctx, transactionID, sessionID, userID) if err == sql.ErrNoRows { return "", nil } diff --git a/roomserver/storage/transactions_table.go b/roomserver/storage/transactions_table.go index e9c904cc8..b98ea3f33 100644 --- a/roomserver/storage/transactions_table.go +++ b/roomserver/storage/transactions_table.go @@ -23,8 +23,8 @@ const transactionsSchema = ` CREATE TABLE IF NOT EXISTS roomserver_transactions ( -- The transaction ID of the event. transaction_id TEXT NOT NULL, - -- The device ID of the originating transaction. - device_id TEXT NOT NULL, + -- The session ID of the originating transaction. + session_id BIGINT NOT NULL, -- User ID of the sender who authored the event user_id TEXT NOT NULL, -- Event ID corresponding to the transaction @@ -32,16 +32,16 @@ CREATE TABLE IF NOT EXISTS roomserver_transactions ( event_id TEXT NOT NULL, -- A transaction ID is unique for a user and device -- This automatically creates an index. - PRIMARY KEY (transaction_id, device_id, user_id) + PRIMARY KEY (transaction_id, session_id, user_id) ); ` const insertTransactionSQL = "" + - "INSERT INTO roomserver_transactions (transaction_id, device_id, user_id, event_id)" + + "INSERT INTO roomserver_transactions (transaction_id, session_id, user_id, event_id)" + " VALUES ($1, $2, $3, $4)" const selectTransactionEventIDSQL = "" + "SELECT event_id FROM roomserver_transactions" + - " WHERE transaction_id = $1 AND device_id = $2 AND user_id = $3" + " WHERE transaction_id = $1 AND session_id = $2 AND user_id = $3" type transactionStatements struct { insertTransactionStmt *sql.Stmt @@ -63,12 +63,12 @@ func (s *transactionStatements) prepare(db *sql.DB) (err error) { func (s *transactionStatements) insertTransaction( ctx context.Context, transactionID string, - deviceID string, + sessionID int64, userID string, eventID string, ) (err error) { _, err = s.insertTransactionStmt.ExecContext( - ctx, transactionID, deviceID, userID, eventID, + ctx, transactionID, sessionID, userID, eventID, ) return } @@ -76,11 +76,11 @@ func (s *transactionStatements) insertTransaction( func (s *transactionStatements) selectTransactionEventID( ctx context.Context, transactionID string, - deviceID string, + sessionID int64, userID string, ) (eventID string, err error) { err = s.selectTransactionEventIDStmt.QueryRowContext( - ctx, transactionID, deviceID, userID, + ctx, transactionID, sessionID, userID, ).Scan(&eventID) return } diff --git a/syncapi/storage/output_room_events_table.go b/syncapi/storage/output_room_events_table.go index 8fbeb18c9..2df2a96a1 100644 --- a/syncapi/storage/output_room_events_table.go +++ b/syncapi/storage/output_room_events_table.go @@ -54,7 +54,7 @@ CREATE TABLE IF NOT EXISTS syncapi_output_room_events ( -- if there is no delta. add_state_ids TEXT[], remove_state_ids TEXT[], - device_id TEXT, -- The local device that sent the event, if any + session_id BIGINT, -- The client session that sent the event, if any transaction_id TEXT -- The transaction id used to send the event, if any ); -- for event selection @@ -63,14 +63,14 @@ CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_id_idx ON syncapi_output_room_ev const insertEventSQL = "" + "INSERT INTO syncapi_output_room_events (" + - "room_id, event_id, event_json, type, sender, contains_url, add_state_ids, remove_state_ids, device_id, transaction_id" + + "room_id, event_id, event_json, type, sender, contains_url, add_state_ids, remove_state_ids, session_id, transaction_id" + ") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) RETURNING id" const selectEventsSQL = "" + "SELECT id, event_json FROM syncapi_output_room_events WHERE event_id = ANY($1)" const selectRecentEventsSQL = "" + - "SELECT id, event_json, device_id, transaction_id FROM syncapi_output_room_events" + + "SELECT id, event_json, session_id, transaction_id FROM syncapi_output_room_events" + " WHERE room_id = $1 AND id > $2 AND id <= $3" + " ORDER BY id DESC LIMIT $4" @@ -221,9 +221,10 @@ func (s *outputRoomEventsStatements) insertEvent( event *gomatrixserverlib.Event, addState, removeState []string, transactionID *api.TransactionID, ) (streamPos int64, err error) { - var deviceID, txnID *string + var txnID *string + var sessionID *int64 if transactionID != nil { - deviceID = &transactionID.DeviceID + sessionID = &transactionID.SessionID txnID = &transactionID.TransactionID } @@ -246,7 +247,7 @@ func (s *outputRoomEventsStatements) insertEvent( containsURL, pq.StringArray(addState), pq.StringArray(removeState), - deviceID, + sessionID, txnID, ).Scan(&streamPos) return @@ -296,11 +297,11 @@ func rowsToStreamEvents(rows *sql.Rows) ([]streamEvent, error) { var ( streamPos int64 eventBytes []byte - deviceID *string + sessionID *int64 txnID *string transactionID *api.TransactionID ) - if err := rows.Scan(&streamPos, &eventBytes, &deviceID, &txnID); err != nil { + if err := rows.Scan(&streamPos, &eventBytes, &sessionID, &txnID); err != nil { return nil, err } // TODO: Handle redacted events @@ -309,9 +310,9 @@ func rowsToStreamEvents(rows *sql.Rows) ([]streamEvent, error) { return nil, err } - if deviceID != nil && txnID != nil { + if sessionID != nil && txnID != nil { transactionID = &api.TransactionID{ - DeviceID: *deviceID, + SessionID: *sessionID, TransactionID: *txnID, } } diff --git a/syncapi/storage/syncserver.go b/syncapi/storage/syncserver.go index fb883702c..cda44d2e3 100644 --- a/syncapi/storage/syncserver.go +++ b/syncapi/storage/syncserver.go @@ -893,7 +893,7 @@ func streamEventsToEvents(device *authtypes.Device, in []streamEvent) []gomatrix for i := 0; i < len(in); i++ { out[i] = in[i].Event if device != nil && in[i].transactionID != nil { - if device.UserID == in[i].Sender() && device.ID == in[i].transactionID.DeviceID { + if device.UserID == in[i].Sender() && device.SessionID == in[i].transactionID.SessionID { err := out[i].SetUnsignedField( "transaction_id", in[i].transactionID.TransactionID, ) From 5e25f6ba22d8bda6e03347c3f5e1df13164719b6 Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Sat, 24 Aug 2019 01:25:10 +0800 Subject: [PATCH 09/12] Set up queryAPI in federationsender (#791) --- federationsender/federationsender.go | 19 +++++++++++---- federationsender/query/query.go | 36 ++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/federationsender/federationsender.go b/federationsender/federationsender.go index 9b732b386..a318d2099 100644 --- a/federationsender/federationsender.go +++ b/federationsender/federationsender.go @@ -15,11 +15,15 @@ package federationsender import ( + "net/http" + "github.com/matrix-org/dendrite/common/basecomponent" + "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/federationsender/consumers" + "github.com/matrix-org/dendrite/federationsender/query" "github.com/matrix-org/dendrite/federationsender/queue" "github.com/matrix-org/dendrite/federationsender/storage" - "github.com/matrix-org/dendrite/roomserver/api" + roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" ) @@ -29,8 +33,8 @@ import ( func SetupFederationSenderComponent( base *basecomponent.BaseDendrite, federation *gomatrixserverlib.FederationClient, - queryAPI api.RoomserverQueryAPI, -) { + rsQueryAPI roomserverAPI.RoomserverQueryAPI, +) api.FederationSenderQueryAPI { federationSenderDB, err := storage.NewDatabase(string(base.Cfg.Database.FederationSender)) if err != nil { logrus.WithError(err).Panic("failed to connect to federation sender db") @@ -40,7 +44,7 @@ func SetupFederationSenderComponent( rsConsumer := consumers.NewOutputRoomEventConsumer( base.Cfg, base.KafkaConsumer, queues, - federationSenderDB, queryAPI, + federationSenderDB, rsQueryAPI, ) if err = rsConsumer.Start(); err != nil { logrus.WithError(err).Panic("failed to start room server consumer") @@ -52,4 +56,11 @@ func SetupFederationSenderComponent( if err := tsConsumer.Start(); err != nil { logrus.WithError(err).Panic("failed to start typing server consumer") } + + queryAPI := query.FederationSenderQueryAPI{ + DB: federationSenderDB, + } + queryAPI.SetupHTTP(http.DefaultServeMux) + + return &queryAPI } diff --git a/federationsender/query/query.go b/federationsender/query/query.go index ec9242863..088244826 100644 --- a/federationsender/query/query.go +++ b/federationsender/query/query.go @@ -2,10 +2,14 @@ package query import ( "context" + "encoding/json" + "net/http" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/federationsender/types" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" ) // FederationSenderQueryDatabase has the APIs needed to implement the query API. @@ -53,3 +57,35 @@ func (f *FederationSenderQueryAPI) QueryJoinedHostServerNamesInRoom( return } + +// SetupHTTP adds the FederationSenderQueryAPI handlers to the http.ServeMux. +func (f *FederationSenderQueryAPI) SetupHTTP(servMux *http.ServeMux) { + servMux.Handle( + api.FederationSenderQueryJoinedHostsInRoomPath, + common.MakeInternalAPI("QueryJoinedHostsInRoom", func(req *http.Request) util.JSONResponse { + var request api.QueryJoinedHostsInRoomRequest + var response api.QueryJoinedHostsInRoomResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := f.QueryJoinedHostsInRoom(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.FederationSenderQueryJoinedHostServerNamesInRoomPath, + common.MakeInternalAPI("QueryJoinedHostServerNamesInRoom", func(req *http.Request) util.JSONResponse { + var request api.QueryJoinedHostServerNamesInRoomRequest + var response api.QueryJoinedHostServerNamesInRoomResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := f.QueryJoinedHostServerNamesInRoom(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) +} From e91942c9e4ab46b4f3deefa6b456deca89fe8dc7 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Thu, 26 Sep 2019 14:20:34 +0100 Subject: [PATCH 10/12] Add 'Can paginate public room list' to testfile (#797) --- testfile | 1 + 1 file changed, 1 insertion(+) diff --git a/testfile b/testfile index 17978913e..38506d717 100644 --- a/testfile +++ b/testfile @@ -171,3 +171,4 @@ Outbound federation can query profile data /event/ on joined room works /event/ does not allow access to events before the user joined Federation key API allows unsigned requests for keys +Can paginate public room list From 7b454bdd27932d72dbe8edb0b40cb9d6e1cbf954 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Thu, 26 Sep 2019 14:27:14 +0100 Subject: [PATCH 11/12] Remove CircleCI (#793) Sytest has been switched to Buildkite. As such the `.circleci` folder has been removed. --- .circleci/config.yml | 32 ------------------------- .circleci/matrix_key.pem | 5 ---- .circleci/server.key | 52 ---------------------------------------- CONTRIBUTING.md | 4 +++- 4 files changed, 3 insertions(+), 90 deletions(-) delete mode 100644 .circleci/config.yml delete mode 100644 .circleci/matrix_key.pem delete mode 100644 .circleci/server.key diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 8b6f4db02..000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,32 +0,0 @@ -version: 2 -jobs: - dendrite: - docker: - - image: matrixdotorg/sytest-dendrite - working_directory: /src - steps: - - checkout - - # Set up dendrite - - run: - name: Build Dendrite - command: ./build.sh - - run: - name: Copy dummy keys to root - command: | - mv .circleci/matrix_key.pem . - mv .circleci/server.key . - - run: - name: Run sytest with whitelisted tests - command: /dendrite_sytest.sh - - - store_artifacts: - path: /logs - destination: logs - - store_test_results: - path: /logs -workflows: - version: 2 - build: - jobs: - - dendrite diff --git a/.circleci/matrix_key.pem b/.circleci/matrix_key.pem deleted file mode 100644 index 6691c74af..000000000 --- a/.circleci/matrix_key.pem +++ /dev/null @@ -1,5 +0,0 @@ ------BEGIN MATRIX PRIVATE KEY----- -Key-ID: ed25519:zXtB - -jDyHsx0EXbAfvM32yBEKQfIy1FHrmwtB1uMAbm5INBg= ------END MATRIX PRIVATE KEY----- diff --git a/.circleci/server.key b/.circleci/server.key deleted file mode 100644 index a5c485079..000000000 --- a/.circleci/server.key +++ /dev/null @@ -1,52 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQCanRCqP11MLIQh -nC26+A1oyBsFfH7auZ3pqE/WFDrCCIoc7ek7cF3fZU7q8OYI+Q9L5V8fobuLb6FB -iXD5zZ6pBAI0VNjAS8yi8VluXIv6pJKsVY3k2hGiU7xRoEhkzckZBaEiruspQbcX -ziNoWoueVBB1a4Eproqzy225cTcoprHsJIPXj0HpW/jKcmahmlM/OrqRAxTwxpb/ -moI6MWIeN4n7h55N6dU1ScVvBS7gZpZQ28d8akuvG3m8kE8q1OPFYGvrNeowD4sp -qDPFijhbygwpzDQlAWriPcqV9KhuGRnYRGTGvuluOttmpgNhNFVxVAlwZJuMVAMU -Jhek66ntKsxWkF5LsO8ls20hmHyyAsL7+rb2ZjuRtEwE8SwOstU2AIIXoSTtqXjX -zC8Ew0VB9MCInJoJC/+iKTLoDqXRZeDKGFx1A2F3Y+Er+Z41HcwgqKRsPqZ066yR -6iKAb5rzJutnEARtbSrNipy9nHE5hIgKJzgOnggcegypcAj3nqbfFFCZA2CFNXoG -XFkmBHEpz38pPLI5z6HpeZRRySoIyahk9IfSwM3aB1aUi//8CcpAodGvYGNQkQ3W -HkrZmM4MtC25I5RyMpYJQWKFpx1cOVPf2ASqaJ+IX1JJTv9dSdYHY/rxsxaiXiry -+uI7UITRvUKgAOrExfSAXco73bgUFwIDAQABAoICAQCP9QX7PhxEPH6aPKxnlWYG -1aozJYOHa6QYVlpfXV6IIyNVZD7w1OLSiaU9IydL23nelKZI8XGJllpyhuHl9Qlx -HQZga0+VW/4hCM7X7tt2d50JUG9ZUaFxnr2M0swU73X6Ej/B51OVilZLl+dn1kaB -GIxqh7ovcRA774EuVLei5fJriGQpZH1eJgAznujoNqSkDq5/Lntk48LcIqR2Qly0 -/ck/pTpEGSAnCZUGlbDbxyjWCIxozx/A3rguVb8ghi+9KtXQntZ6AT71fmMV3mgz -LqC8miFDA1rdY+MoVDAusrhZoPSkCEWYGL0HijNDYlLbvf874rDhq6diL0V8jOAd -PGOx5BY6VUWbSQAUtKpMuNSL6tidkOACGPwbuH7OIaG+yGZ0/Oiy3fureiAEg5VU -piyp6F7p1g0vgQEnj4CHiCQlX48bjC/mm8758DeaH8H5T++A8MOgRhgFVb9f01R+ -NMzszMziuVNDYe01cwdY1TXUx5b0o+opsbPm6sNp/7afL9Hou1epP9zQC0I8ulfP -fgrKTddMwlNjoBuDMQ8GqoK275YU4wtyhUMfjr3xQ0JwP46cZbhhc4nh6qcRSNTf -yVuKv/pT/bJcSmg5JOCS8qdK0BQhAvUin9HvgSAV9QmZVpxzT/xhqwuRlLDKW+VR -XyPt996f3L4CTXI9h88AQQKCAQEAycBChu3/ZKl8a90anOlv9PwmaaXfLBKH9Rkw -aeZrMilxTJAb+LEsmtj35rF5KPeBP6ARpX5gmvKJVzCDHT9YgNs+6C3E+l2f1/3a -TcjZKPTukT2gJdCgejhEgTzAwEse322GSptuyidtNpY7NgbAxP4VdDMOmPYbzufb -5BqxmfiGsfXgdvQkj8/MzHuGhhft4SU6ED/Ax+EPUWVV7kBr2995kGDF5z5CuJkb -SJjmVxAJZP/kC2Z/iPnP51G0hiCxHp7+gPY4mvvkHvhJGnGH/vutjRjoe28BENlP -MgB68S1/U3NGSUzWv86pT1OdHd+qynWj/NzF7Gp/T/ju8VZBXwKCAQEAxDAMSOfF -dizsU7cJbf6vxi6XJHjhwWUWD2vMznKz1D4mkByeY8aSOc8kQZsE5nd4ZgwkYTaZ -gItjGjM5y5dpKurfKdqQ+dA6PS03h3p+tp1lZp9/dI9X/DfkTO/LUdrfkVVcbQhE -zqc6C35qO98rhJdsRwhOF28mOc/4bbs0XjC5dEoBGyFt7Fbn2mYoCo4FSHl7WIq6 -TZR9pLAvxjqEZ6Dwrzpp9wtdLIQYPga+KVKcDT/DStThXDTCNt5PyDE9c8eImFww -u0T87Er5hSEQgodURxDOZh+9ktIfXzMtxiAJ3iDCEPc3NNnLCWfKMhwGsVTCCXj6 -WuHTOe79tOaQSQKCAQEAqBN52PsRl4TzWNEcyLhZQxmFzuIXKJpPlctkX/VMPL/1 -2bj89JR1+pLjA9e6fnyjuqPZz6uXQ77m2DJcKNOLId6Fa9wljAbPkZu0cLTw5YQX -8/wJHTfPWcLin2BDnG94yt5t0F3pUJTEEYPa1EmP8w1SRjn64Ue3JwpWUJREfWdk -n4GdfLwscXrGvVvzWGc7ECR5WOwj6OEAZ+kqS5BzyvtERRm6BcoCv9Mdvb9Tthhw -Gypri2vat/yWTbnt0QgPRtliYYG+6q8K/xoNnPAUQkLd9PxZQevaUXUY2yk3QxGK -T7VrSsmu5qB+wM2ByU9686xJ7/DlGu4mHjPerEQVtQKCAQBcM3iSitpyP4qRjWQR -HbDeIudFbMosaaWEedU28REynkLhV5HYsmnmYUNY0dHrvhoHW419YnuhveBFX+25 -kN8MHHXk5aNcxE+akLWYJimHCVGueScdUIC5OEtDHS8guQx48PUPCOPNeyn8XNzw -ZmG9Xqy0dWK+AK6mXOcUKvbhjWSbEmySo5NVj0JHkdsfmr9A4Fbntcr4yuCBlYve -TYIMccark3hZci3HzgzWmbSlFv3f/Cd787A19VWRE8nK+9k1oIDBmhIM8M8s/c9m -kbOApLkm7O8Tb7dYWQgFZbgNdOEuU5bhAk4fuHuDYBPWmPVMQdkvOnvuWlM61ubF -LdaBAoIBACDpbb5AQIYsWWOnoXuuGh+YY4kmnaBFpsbgEYkZSy92AaLr4Ibf49WN -oqNDX73YaJlURaGPYMC9J2Huq7TZcewH3SwkVA3N5UmDoijkM4juRfADAfVIMxB5 -+9paWeEfnYC/o377FTJIJ9hHJWIaWSoiJZLYDBmoYdxmk8DSHAJCeWsjYDzPybsH -7RyMPIa1u7lVdgOPEOBi1OIg7ASLxGKiHQtrYHq99GcaVvU/UxoNRMcSnPfY3G8R -pGah+EndSCb2F20ouDyvlKfOylAltH2BeNc3B4PeP7ZhlVr7bfyOAfC2Z7FNDm3J -+yaBExKfroZjsksctNAcAbgpuvhLLG8= ------END PRIVATE KEY----- diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dc962fee7..fa815f15b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -25,7 +25,9 @@ run](scripts/build-test-lint.sh). When a Pull Request is submitted, continuous integration jobs are run automatically to ensure the code builds and is relatively well-written. Checks are run on [Buildkite](https://buildkite.com/matrix-dot-org/dendrite/) and -[CircleCI](https://circleci.com/gh/matrix-org/dendrite/). +[CircleCI](https://circleci.com/gh/matrix-org/dendrite/). The Buildkite +pipeline can be found in Matrix.org's [pipelines +repository](https://github.com/matrix-org/pipelines). If a job fails, click the "details" button and you should be taken to the job's logs. From 49fd47c86313f036da3a9549e7d14def166f4ea7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Behouba=20Manass=C3=A9?= Date: Mon, 30 Sep 2019 19:25:04 +0300 Subject: [PATCH 12/12] selectAccountDataByType return ClientEvent pointer instead of slice of ClientEvent (#798) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This pull request is an attempt to fix #773. Signed-off-by: Kouame Behouba Manassé behouba@gmail.com --- .../storage/accounts/account_data_table.go | 25 ++++++------------- clientapi/auth/storage/accounts/storage.go | 4 +-- clientapi/routing/room_tagging.go | 14 +++++------ syncapi/sync/requestpool.go | 4 +-- 4 files changed, 18 insertions(+), 29 deletions(-) diff --git a/clientapi/auth/storage/accounts/account_data_table.go b/clientapi/auth/storage/accounts/account_data_table.go index 0d73cb312..0d6ad0933 100644 --- a/clientapi/auth/storage/accounts/account_data_table.go +++ b/clientapi/auth/storage/accounts/account_data_table.go @@ -120,28 +120,17 @@ func (s *accountDataStatements) selectAccountData( func (s *accountDataStatements) selectAccountDataByType( ctx context.Context, localpart, roomID, dataType string, -) (data []gomatrixserverlib.ClientEvent, err error) { - data = []gomatrixserverlib.ClientEvent{} - +) (data *gomatrixserverlib.ClientEvent, err error) { stmt := s.selectAccountDataByTypeStmt - rows, err := stmt.QueryContext(ctx, localpart, roomID, dataType) - if err != nil { + var content []byte + + if err = stmt.QueryRowContext(ctx, localpart, roomID, dataType).Scan(&content); err != nil { return } - for rows.Next() { - var content []byte - - if err = rows.Scan(&content); err != nil { - return - } - - ac := gomatrixserverlib.ClientEvent{ - Type: dataType, - Content: content, - } - - data = append(data, ac) + data = &gomatrixserverlib.ClientEvent{ + Type: dataType, + Content: content, } return diff --git a/clientapi/auth/storage/accounts/storage.go b/clientapi/auth/storage/accounts/storage.go index 41d75daad..020a38376 100644 --- a/clientapi/auth/storage/accounts/storage.go +++ b/clientapi/auth/storage/accounts/storage.go @@ -263,11 +263,11 @@ func (d *Database) GetAccountData(ctx context.Context, localpart string) ( // GetAccountDataByType returns account data matching a given // localpart, room ID and type. -// If no account data could be found, returns an empty array +// If no account data could be found, returns nil // Returns an error if there was an issue with the retrieval func (d *Database) GetAccountDataByType( ctx context.Context, localpart, roomID, dataType string, -) (data []gomatrixserverlib.ClientEvent, err error) { +) (data *gomatrixserverlib.ClientEvent, err error) { return d.accountDatas.selectAccountDataByType( ctx, localpart, roomID, dataType, ) diff --git a/clientapi/routing/room_tagging.go b/clientapi/routing/room_tagging.go index 6e7324cd8..487081c53 100644 --- a/clientapi/routing/room_tagging.go +++ b/clientapi/routing/room_tagging.go @@ -59,7 +59,7 @@ func GetTags( return httputil.LogThenError(req, err) } - if len(data) == 0 { + if data == nil { return util.JSONResponse{ Code: http.StatusOK, JSON: struct{}{}, @@ -68,7 +68,7 @@ func GetTags( return util.JSONResponse{ Code: http.StatusOK, - JSON: data[0].Content, + JSON: data.Content, } } @@ -103,8 +103,8 @@ func PutTag( } var tagContent gomatrix.TagContent - if len(data) > 0 { - if err = json.Unmarshal(data[0].Content, &tagContent); err != nil { + if data != nil { + if err = json.Unmarshal(data.Content, &tagContent); err != nil { return httputil.LogThenError(req, err) } } else { @@ -155,7 +155,7 @@ func DeleteTag( } // If there are no tags in the database, exit - if len(data) == 0 { + if data == nil { // Spec only defines 200 responses for this endpoint so we don't return anything else. return util.JSONResponse{ Code: http.StatusOK, @@ -164,7 +164,7 @@ func DeleteTag( } var tagContent gomatrix.TagContent - err = json.Unmarshal(data[0].Content, &tagContent) + err = json.Unmarshal(data.Content, &tagContent) if err != nil { return httputil.LogThenError(req, err) } @@ -204,7 +204,7 @@ func obtainSavedTags( userID string, roomID string, accountDB *accounts.Database, -) (string, []gomatrixserverlib.ClientEvent, error) { +) (string, *gomatrixserverlib.ClientEvent, error) { localpart, _, err := gomatrixserverlib.SplitID('@', userID) if err != nil { return "", nil, err diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index 6b95f4698..94a369001 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -196,13 +196,13 @@ func (rp *RequestPool) appendAccountData( events := []gomatrixserverlib.ClientEvent{} // Request the missing data from the database for _, dataType := range dataTypes { - evs, err := rp.accountDB.GetAccountDataByType( + event, err := rp.accountDB.GetAccountDataByType( req.ctx, localpart, roomID, dataType, ) if err != nil { return nil, err } - events = append(events, evs...) + events = append(events, *event) } // Append the data to the response