From ddf1c8adf1fd1441b76834df479d1ab5a132de88 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 17 Jun 2020 17:41:45 +0100 Subject: [PATCH 1/8] Hacks for supporting Riot iOS (#1148) * Join room body is optional * Support deprecated login by user/password * Implement dummy key upload endpoint * Make a very determinate end to /messages if we hit the create event in back-pagination * Linting --- clientapi/routing/joinroom.go | 4 +-- clientapi/routing/login.go | 67 +++++++++++++++++++++++------------ keyserver/routing/routing.go | 10 ++++++ syncapi/routing/messages.go | 45 ++++++++++++++--------- 4 files changed, 84 insertions(+), 42 deletions(-) diff --git a/clientapi/routing/joinroom.go b/clientapi/routing/joinroom.go index e190beefd..3871e4d4e 100644 --- a/clientapi/routing/joinroom.go +++ b/clientapi/routing/joinroom.go @@ -43,9 +43,7 @@ func JoinRoomByIDOrAlias( // If content was provided in the request then incude that // in the request. It'll get used as a part of the membership // event content. - if err := httputil.UnmarshalJSONRequest(req, &joinReq.Content); err != nil { - return *err - } + _ = httputil.UnmarshalJSONRequest(req, &joinReq.Content) // Work out our localpart for the client profile request. localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) diff --git a/clientapi/routing/login.go b/clientapi/routing/login.go index 1b894a566..dc0180da6 100644 --- a/clientapi/routing/login.go +++ b/clientapi/routing/login.go @@ -47,6 +47,7 @@ type loginIdentifier struct { type passwordRequest struct { Identifier loginIdentifier `json:"identifier"` + User string `json:"user"` // deprecated in favour of identifier Password string `json:"password"` // Both DeviceID and InitialDisplayName can be omitted, or empty strings ("") // Thus a pointer is needed to differentiate between the two @@ -81,6 +82,7 @@ func Login( } else if req.Method == http.MethodPost { var r passwordRequest var acc *api.Account + var errJSON *util.JSONResponse resErr := httputil.UnmarshalJSONRequest(req, &r) if resErr != nil { return *resErr @@ -93,30 +95,22 @@ func Login( JSON: jsonerror.BadJSON("'user' must be supplied."), } } - - util.GetLogger(req.Context()).WithField("user", r.Identifier.User).Info("Processing login request") - - localpart, err := userutil.ParseUsernameParam(r.Identifier.User, &cfg.Matrix.ServerName) - if err != nil { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.InvalidUsername(err.Error()), - } - } - - acc, err = accountDB.GetAccountByPassword(req.Context(), localpart, r.Password) - if err != nil { - // Technically we could tell them if the user does not exist by checking if err == sql.ErrNoRows - // but that would leak the existence of the user. - return util.JSONResponse{ - Code: http.StatusForbidden, - JSON: jsonerror.Forbidden("username or password was incorrect, or the account does not exist"), - } + acc, errJSON = r.processUsernamePasswordLoginRequest(req, accountDB, cfg, r.Identifier.User) + if errJSON != nil { + return *errJSON } default: - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.BadJSON("login identifier '" + r.Identifier.Type + "' not supported"), + // TODO: The below behaviour is deprecated but without it Riot iOS won't log in + if r.User != "" { + acc, errJSON = r.processUsernamePasswordLoginRequest(req, accountDB, cfg, r.User) + if errJSON != nil { + return *errJSON + } + } else { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.BadJSON("login identifier '" + r.Identifier.Type + "' not supported"), + } } } @@ -163,3 +157,32 @@ func getDevice( ) return } + +func (r *passwordRequest) processUsernamePasswordLoginRequest( + req *http.Request, accountDB accounts.Database, + cfg *config.Dendrite, username string, +) (acc *api.Account, errJSON *util.JSONResponse) { + util.GetLogger(req.Context()).WithField("user", username).Info("Processing login request") + + localpart, err := userutil.ParseUsernameParam(username, &cfg.Matrix.ServerName) + if err != nil { + errJSON = &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.InvalidUsername(err.Error()), + } + return + } + + acc, err = accountDB.GetAccountByPassword(req.Context(), localpart, r.Password) + if err != nil { + // Technically we could tell them if the user does not exist by checking if err == sql.ErrNoRows + // but that would leak the existence of the user. + errJSON = &util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("username or password was incorrect, or the account does not exist"), + } + return + } + + return +} diff --git a/keyserver/routing/routing.go b/keyserver/routing/routing.go index c09031d8a..dba43528f 100644 --- a/keyserver/routing/routing.go +++ b/keyserver/routing/routing.go @@ -36,9 +36,19 @@ func Setup( publicAPIMux *mux.Router, cfg *config.Dendrite, userAPI userapi.UserInternalAPI, ) { r0mux := publicAPIMux.PathPrefix(pathPrefixR0).Subrouter() + r0mux.Handle("/keys/query", httputil.MakeAuthAPI("queryKeys", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { return QueryKeys(req) }), ).Methods(http.MethodPost, http.MethodOptions) + + r0mux.Handle("/keys/upload/{keyID}", + httputil.MakeAuthAPI("keys_upload", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + return util.JSONResponse{ + Code: 200, + JSON: map[string]interface{}{}, + } + }), + ).Methods(http.MethodPost, http.MethodOptions) } diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index de5429db4..15add1b45 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -158,6 +158,7 @@ func OnIncomingMessagesRequest( util.GetLogger(req.Context()).WithError(err).Error("mreq.retrieveEvents failed") return jsonerror.InternalServerError() } + util.GetLogger(req.Context()).WithFields(logrus.Fields{ "from": from.String(), "to": to.String(), @@ -246,6 +247,12 @@ func (r *messagesReq) retrieveEvents() ( // change the way topological positions are defined (as depth isn't the most // reliable way to define it), it would be easier and less troublesome to // only have to change it in one place, i.e. the database. + start, end, err = r.getStartEnd(events) + + return clientEvents, start, end, err +} + +func (r *messagesReq) getStartEnd(events []gomatrixserverlib.HeaderedEvent) (start, end types.TopologyToken, err error) { start, err = r.db.EventPositionInTopology( r.ctx, events[0].EventID(), ) @@ -253,24 +260,28 @@ func (r *messagesReq) retrieveEvents() ( err = fmt.Errorf("EventPositionInTopology: for start event %s: %w", events[0].EventID(), err) return } - end, err = r.db.EventPositionInTopology( - r.ctx, events[len(events)-1].EventID(), - ) - if err != nil { - err = fmt.Errorf("EventPositionInTopology: for end event %s: %w", events[len(events)-1].EventID(), err) - return + if r.backwardOrdering && events[len(events)-1].Type() == gomatrixserverlib.MRoomCreate { + // We've hit the beginning of the room so there's really nowhere else + // to go. This seems to fix Riot iOS from looping on /messages endlessly. + end = types.NewTopologyToken(0, 0) + } else { + end, err = r.db.EventPositionInTopology( + r.ctx, events[len(events)-1].EventID(), + ) + if err != nil { + err = fmt.Errorf("EventPositionInTopology: for end event %s: %w", events[len(events)-1].EventID(), err) + return + } + if r.backwardOrdering { + // A stream/topological position is a cursor located between two events. + // While they are identified in the code by the event on their right (if + // we consider a left to right chronological order), tokens need to refer + // to them by the event on their left, therefore we need to decrement the + // end position we send in the response if we're going backward. + end.Decrement() + } } - - if r.backwardOrdering { - // A stream/topological position is a cursor located between two events. - // While they are identified in the code by the event on their right (if - // we consider a left to right chronological order), tokens need to refer - // to them by the event on their left, therefore we need to decrement the - // end position we send in the response if we're going backward. - end.Decrement() - } - - return clientEvents, start, end, err + return } // handleEmptyEventsSlice handles the case where the initial request to the From 3547a1768c36626c672e5c7834f297496f568b2f Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 18 Jun 2020 13:48:47 +0100 Subject: [PATCH 2/8] Fix embed Riot Web into Yggdrasil demo --- cmd/dendrite-demo-yggdrasil/embed/embed_other.go | 4 +++- cmd/dendrite-demo-yggdrasil/embed/embed_riotweb.go | 7 ++++--- cmd/dendrite-demo-yggdrasil/main.go | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/cmd/dendrite-demo-yggdrasil/embed/embed_other.go b/cmd/dendrite-demo-yggdrasil/embed/embed_other.go index a9108fad3..598881148 100644 --- a/cmd/dendrite-demo-yggdrasil/embed/embed_other.go +++ b/cmd/dendrite-demo-yggdrasil/embed/embed_other.go @@ -2,6 +2,8 @@ package embed -func Embed(_ int, _ string) { +import "github.com/gorilla/mux" + +func Embed(_ *mux.Router, _ int, _ string) { } diff --git a/cmd/dendrite-demo-yggdrasil/embed/embed_riotweb.go b/cmd/dendrite-demo-yggdrasil/embed/embed_riotweb.go index 360d0bc5b..a9e04a312 100644 --- a/cmd/dendrite-demo-yggdrasil/embed/embed_riotweb.go +++ b/cmd/dendrite-demo-yggdrasil/embed/embed_riotweb.go @@ -7,19 +7,20 @@ import ( "io" "net/http" + "github.com/gorilla/mux" "github.com/tidwall/sjson" ) // From within the Riot Web directory: // go run github.com/mjibson/esc -o /path/to/dendrite/internal/embed/fs_riotweb.go -private -pkg embed . -func Embed(listenPort int, serverName string) { +func Embed(rootMux *mux.Router, listenPort int, serverName string) { url := fmt.Sprintf("http://localhost:%d", listenPort) embeddedFS := _escFS(false) embeddedServ := http.FileServer(embeddedFS) - http.DefaultServeMux.Handle("/", embeddedServ) - http.DefaultServeMux.HandleFunc("/config.json", func(w http.ResponseWriter, _ *http.Request) { + rootMux.Handle("/", embeddedServ) + rootMux.HandleFunc("/config.json", func(w http.ResponseWriter, _ *http.Request) { configFile, err := embeddedFS.Open("/config.sample.json") if err != nil { w.WriteHeader(500) diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index be15da472..328aa0629 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -154,7 +154,7 @@ func main() { logrus.WithError(err).Panicf("failed to connect to public rooms db") } - embed.Embed(*instancePort, "Yggdrasil Demo") + embed.Embed(base.BaseMux, *instancePort, "Yggdrasil Demo") monolith := setup.Monolith{ Config: base.Cfg, From dc0bac85d5bad933d32ee63f8bc1aef6348ca6e9 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 18 Jun 2020 18:36:03 +0100 Subject: [PATCH 3/8] Refactor account data (#1150) * Refactor account data * Tweak database fetching * Tweaks * Restore syncProducer notification * Various tweaks, update tag behaviour * Fix initial sync --- clientapi/routing/account_data.go | 55 ++++---- clientapi/routing/room_tagging.go | 127 +++++++----------- clientapi/routing/routing.go | 14 +- syncapi/sync/requestpool.go | 75 +++++++---- userapi/api/api.go | 27 ++-- userapi/internal/api.go | 31 ++++- userapi/inthttp/client.go | 10 ++ userapi/storage/accounts/interface.go | 7 +- .../accounts/postgres/account_data_table.go | 48 +++---- userapi/storage/accounts/postgres/storage.go | 13 +- .../accounts/sqlite3/account_data_table.go | 50 +++---- userapi/storage/accounts/sqlite3/storage.go | 13 +- 12 files changed, 248 insertions(+), 222 deletions(-) diff --git a/clientapi/routing/account_data.go b/clientapi/routing/account_data.go index 68e0dc5da..d5fafedb1 100644 --- a/clientapi/routing/account_data.go +++ b/clientapi/routing/account_data.go @@ -16,21 +16,20 @@ package routing import ( "encoding/json" + "fmt" "io/ioutil" "net/http" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/userapi/api" - "github.com/matrix-org/dendrite/userapi/storage/accounts" - "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) // GetAccountData implements GET /user/{userId}/[rooms/{roomid}/]account_data/{type} func GetAccountData( - req *http.Request, accountDB accounts.Database, device *api.Device, + req *http.Request, userAPI api.UserInternalAPI, device *api.Device, userID string, roomID string, dataType string, ) util.JSONResponse { if userID != device.UserID { @@ -40,18 +39,28 @@ func GetAccountData( } } - localpart, _, err := gomatrixserverlib.SplitID('@', userID) - if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") - return jsonerror.InternalServerError() + dataReq := api.QueryAccountDataRequest{ + UserID: userID, + DataType: dataType, + RoomID: roomID, + } + dataRes := api.QueryAccountDataResponse{} + if err := userAPI.QueryAccountData(req.Context(), &dataReq, &dataRes); err != nil { + util.GetLogger(req.Context()).WithError(err).Error("userAPI.QueryAccountData failed") + return util.ErrorResponse(fmt.Errorf("userAPI.QueryAccountData: %w", err)) } - if data, err := accountDB.GetAccountDataByType( - req.Context(), localpart, roomID, dataType, - ); err == nil { + var data json.RawMessage + var ok bool + if roomID != "" { + data, ok = dataRes.RoomAccountData[roomID][dataType] + } else { + data, ok = dataRes.GlobalAccountData[dataType] + } + if ok { return util.JSONResponse{ Code: http.StatusOK, - JSON: data.Content, + JSON: data, } } @@ -63,7 +72,7 @@ func GetAccountData( // SaveAccountData implements PUT /user/{userId}/[rooms/{roomId}/]account_data/{type} func SaveAccountData( - req *http.Request, accountDB accounts.Database, device *api.Device, + req *http.Request, userAPI api.UserInternalAPI, device *api.Device, userID string, roomID string, dataType string, syncProducer *producers.SyncAPIProducer, ) util.JSONResponse { if userID != device.UserID { @@ -73,12 +82,6 @@ func SaveAccountData( } } - localpart, _, err := gomatrixserverlib.SplitID('@', userID) - if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") - return jsonerror.InternalServerError() - } - defer req.Body.Close() // nolint: errcheck if req.Body == http.NoBody { @@ -101,13 +104,19 @@ func SaveAccountData( } } - if err := accountDB.SaveAccountData( - req.Context(), localpart, roomID, dataType, string(body), - ); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("accountDB.SaveAccountData failed") - return jsonerror.InternalServerError() + dataReq := api.InputAccountDataRequest{ + UserID: userID, + DataType: dataType, + RoomID: roomID, + AccountData: json.RawMessage(body), + } + dataRes := api.InputAccountDataResponse{} + if err := userAPI.InputAccountData(req.Context(), &dataReq, &dataRes); err != nil { + util.GetLogger(req.Context()).WithError(err).Error("userAPI.QueryAccountData failed") + return util.ErrorResponse(err) } + // TODO: user API should do this since it's account data if err := syncProducer.SendData(userID, roomID, dataType); err != nil { util.GetLogger(req.Context()).WithError(err).Error("syncProducer.SendData failed") return jsonerror.InternalServerError() diff --git a/clientapi/routing/room_tagging.go b/clientapi/routing/room_tagging.go index b1cfcca86..c683cc949 100644 --- a/clientapi/routing/room_tagging.go +++ b/clientapi/routing/room_tagging.go @@ -24,23 +24,14 @@ import ( "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/userapi/api" - "github.com/matrix-org/dendrite/userapi/storage/accounts" "github.com/matrix-org/gomatrix" - "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) -// newTag creates and returns a new gomatrix.TagContent -func newTag() gomatrix.TagContent { - return gomatrix.TagContent{ - Tags: make(map[string]gomatrix.TagProperties), - } -} - // GetTags implements GET /_matrix/client/r0/user/{userID}/rooms/{roomID}/tags func GetTags( req *http.Request, - accountDB accounts.Database, + userAPI api.UserInternalAPI, device *api.Device, userID string, roomID string, @@ -54,22 +45,15 @@ func GetTags( } } - _, data, err := obtainSavedTags(req, userID, roomID, accountDB) + tagContent, err := obtainSavedTags(req, userID, roomID, userAPI) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("obtainSavedTags failed") return jsonerror.InternalServerError() } - if data == nil { - return util.JSONResponse{ - Code: http.StatusOK, - JSON: struct{}{}, - } - } - return util.JSONResponse{ Code: http.StatusOK, - JSON: data.Content, + JSON: tagContent, } } @@ -78,7 +62,7 @@ func GetTags( // the tag to the "map" and saving the new "map" to the DB func PutTag( req *http.Request, - accountDB accounts.Database, + userAPI api.UserInternalAPI, device *api.Device, userID string, roomID string, @@ -98,34 +82,25 @@ func PutTag( return *reqErr } - localpart, data, err := obtainSavedTags(req, userID, roomID, accountDB) + tagContent, err := obtainSavedTags(req, userID, roomID, userAPI) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("obtainSavedTags failed") return jsonerror.InternalServerError() } - var tagContent gomatrix.TagContent - if data != nil { - if err = json.Unmarshal(data.Content, &tagContent); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("json.Unmarshal failed") - return jsonerror.InternalServerError() - } - } else { - tagContent = newTag() + if tagContent.Tags == nil { + tagContent.Tags = make(map[string]gomatrix.TagProperties) } tagContent.Tags[tag] = properties - if err = saveTagData(req, localpart, roomID, accountDB, tagContent); err != nil { + + if err = saveTagData(req, userID, roomID, userAPI, tagContent); err != nil { util.GetLogger(req.Context()).WithError(err).Error("saveTagData failed") return jsonerror.InternalServerError() } - // Send data to syncProducer in order to inform clients of changes - // Run in a goroutine in order to prevent blocking the tag request response - go func() { - if err := syncProducer.SendData(userID, roomID, "m.tag"); err != nil { - logrus.WithError(err).Error("Failed to send m.tag account data update to syncapi") - } - }() + if err = syncProducer.SendData(userID, roomID, "m.tag"); err != nil { + logrus.WithError(err).Error("Failed to send m.tag account data update to syncapi") + } return util.JSONResponse{ Code: http.StatusOK, @@ -138,7 +113,7 @@ func PutTag( // the "map" and then saving the new "map" in the DB func DeleteTag( req *http.Request, - accountDB accounts.Database, + userAPI api.UserInternalAPI, device *api.Device, userID string, roomID string, @@ -153,28 +128,12 @@ func DeleteTag( } } - localpart, data, err := obtainSavedTags(req, userID, roomID, accountDB) + tagContent, err := obtainSavedTags(req, userID, roomID, userAPI) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("obtainSavedTags failed") return jsonerror.InternalServerError() } - // If there are no tags in the database, exit - if data == nil { - // Spec only defines 200 responses for this endpoint so we don't return anything else. - return util.JSONResponse{ - Code: http.StatusOK, - JSON: struct{}{}, - } - } - - var tagContent gomatrix.TagContent - err = json.Unmarshal(data.Content, &tagContent) - if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("json.Unmarshal failed") - return jsonerror.InternalServerError() - } - // Check whether the tag to be deleted exists if _, ok := tagContent.Tags[tag]; ok { delete(tagContent.Tags, tag) @@ -185,18 +144,16 @@ func DeleteTag( JSON: struct{}{}, } } - if err = saveTagData(req, localpart, roomID, accountDB, tagContent); err != nil { + + if err = saveTagData(req, userID, roomID, userAPI, tagContent); err != nil { util.GetLogger(req.Context()).WithError(err).Error("saveTagData failed") return jsonerror.InternalServerError() } - // Send data to syncProducer in order to inform clients of changes - // Run in a goroutine in order to prevent blocking the tag request response - go func() { - if err := syncProducer.SendData(userID, roomID, "m.tag"); err != nil { - logrus.WithError(err).Error("Failed to send m.tag account data update to syncapi") - } - }() + // TODO: user API should do this since it's account data + if err := syncProducer.SendData(userID, roomID, "m.tag"); err != nil { + logrus.WithError(err).Error("Failed to send m.tag account data update to syncapi") + } return util.JSONResponse{ Code: http.StatusOK, @@ -210,32 +167,46 @@ func obtainSavedTags( req *http.Request, userID string, roomID string, - accountDB accounts.Database, -) (string, *gomatrixserverlib.ClientEvent, error) { - localpart, _, err := gomatrixserverlib.SplitID('@', userID) - if err != nil { - return "", nil, err + userAPI api.UserInternalAPI, +) (tags gomatrix.TagContent, err error) { + dataReq := api.QueryAccountDataRequest{ + UserID: userID, + RoomID: roomID, + DataType: "m.tag", } - - data, err := accountDB.GetAccountDataByType( - req.Context(), localpart, roomID, "m.tag", - ) - - return localpart, data, err + dataRes := api.QueryAccountDataResponse{} + err = userAPI.QueryAccountData(req.Context(), &dataReq, &dataRes) + if err != nil { + return + } + data, ok := dataRes.RoomAccountData[roomID]["m.tag"] + if !ok { + return + } + if err = json.Unmarshal(data, &tags); err != nil { + return + } + return tags, nil } // saveTagData saves the provided tag data into the database func saveTagData( req *http.Request, - localpart string, + userID string, roomID string, - accountDB accounts.Database, + userAPI api.UserInternalAPI, Tag gomatrix.TagContent, ) error { newTagData, err := json.Marshal(Tag) if err != nil { return err } - - return accountDB.SaveAccountData(req.Context(), localpart, roomID, "m.tag", string(newTagData)) + dataReq := api.InputAccountDataRequest{ + UserID: userID, + RoomID: roomID, + DataType: "m.tag", + AccountData: json.RawMessage(newTagData), + } + dataRes := api.InputAccountDataResponse{} + return userAPI.InputAccountData(req.Context(), &dataReq, &dataRes) } diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 41c7fb18e..e91b07ac7 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -476,7 +476,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SaveAccountData(req, accountDB, device, vars["userID"], "", vars["type"], syncProducer) + return SaveAccountData(req, userAPI, device, vars["userID"], "", vars["type"], syncProducer) }), ).Methods(http.MethodPut, http.MethodOptions) @@ -486,7 +486,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SaveAccountData(req, accountDB, device, vars["userID"], vars["roomID"], vars["type"], syncProducer) + return SaveAccountData(req, userAPI, device, vars["userID"], vars["roomID"], vars["type"], syncProducer) }), ).Methods(http.MethodPut, http.MethodOptions) @@ -496,7 +496,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return GetAccountData(req, accountDB, device, vars["userID"], "", vars["type"]) + return GetAccountData(req, userAPI, device, vars["userID"], "", vars["type"]) }), ).Methods(http.MethodGet) @@ -506,7 +506,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return GetAccountData(req, accountDB, device, vars["userID"], vars["roomID"], vars["type"]) + return GetAccountData(req, userAPI, device, vars["userID"], vars["roomID"], vars["type"]) }), ).Methods(http.MethodGet) @@ -604,7 +604,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return GetTags(req, accountDB, device, vars["userId"], vars["roomId"], syncProducer) + return GetTags(req, userAPI, device, vars["userId"], vars["roomId"], syncProducer) }), ).Methods(http.MethodGet, http.MethodOptions) @@ -614,7 +614,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return PutTag(req, accountDB, device, vars["userId"], vars["roomId"], vars["tag"], syncProducer) + return PutTag(req, userAPI, device, vars["userId"], vars["roomId"], vars["tag"], syncProducer) }), ).Methods(http.MethodPut, http.MethodOptions) @@ -624,7 +624,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return DeleteTag(req, accountDB, device, vars["userId"], vars["roomId"], vars["tag"], syncProducer) + return DeleteTag(req, userAPI, device, vars["userId"], vars["roomId"], vars["tag"], syncProducer) }), ).Methods(http.MethodDelete, http.MethodOptions) diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index 26b925eac..8d51689e3 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -205,22 +205,34 @@ func (rp *RequestPool) appendAccountData( if req.since == nil { // If this is the initial sync, we don't need to check if a data has // already been sent. Instead, we send the whole batch. - var res userapi.QueryAccountDataResponse - err := rp.userAPI.QueryAccountData(req.ctx, &userapi.QueryAccountDataRequest{ + dataReq := &userapi.QueryAccountDataRequest{ UserID: userID, - }, &res) - if err != nil { + } + dataRes := &userapi.QueryAccountDataResponse{} + if err := rp.userAPI.QueryAccountData(req.ctx, dataReq, dataRes); err != nil { return nil, err } - data.AccountData.Events = res.GlobalAccountData - + for datatype, databody := range dataRes.GlobalAccountData { + data.AccountData.Events = append( + data.AccountData.Events, + gomatrixserverlib.ClientEvent{ + Type: datatype, + Content: gomatrixserverlib.RawJSON(databody), + }, + ) + } for r, j := range data.Rooms.Join { - if len(res.RoomAccountData[r]) > 0 { - j.AccountData.Events = res.RoomAccountData[r] + for datatype, databody := range dataRes.RoomAccountData[r] { + j.AccountData.Events = append( + j.AccountData.Events, + gomatrixserverlib.ClientEvent{ + Type: datatype, + Content: gomatrixserverlib.RawJSON(databody), + }, + ) data.Rooms.Join[r] = j } } - return data, nil } @@ -249,33 +261,42 @@ func (rp *RequestPool) appendAccountData( // Iterate over the rooms for roomID, dataTypes := range dataTypes { - events := []gomatrixserverlib.ClientEvent{} // Request the missing data from the database for _, dataType := range dataTypes { - var res userapi.QueryAccountDataResponse - err = rp.userAPI.QueryAccountData(req.ctx, &userapi.QueryAccountDataRequest{ + dataReq := userapi.QueryAccountDataRequest{ UserID: userID, RoomID: roomID, DataType: dataType, - }, &res) + } + dataRes := userapi.QueryAccountDataResponse{} + err = rp.userAPI.QueryAccountData(req.ctx, &dataReq, &dataRes) if err != nil { - return nil, err + continue } - if len(res.RoomAccountData[roomID]) > 0 { - events = append(events, res.RoomAccountData[roomID]...) - } else if len(res.GlobalAccountData) > 0 { - events = append(events, res.GlobalAccountData...) + if roomID == "" { + if globalData, ok := dataRes.GlobalAccountData[dataType]; ok { + data.AccountData.Events = append( + data.AccountData.Events, + gomatrixserverlib.ClientEvent{ + Type: dataType, + Content: gomatrixserverlib.RawJSON(globalData), + }, + ) + } + } else { + if roomData, ok := dataRes.RoomAccountData[roomID][dataType]; ok { + joinData := data.Rooms.Join[roomID] + joinData.AccountData.Events = append( + joinData.AccountData.Events, + gomatrixserverlib.ClientEvent{ + Type: dataType, + Content: gomatrixserverlib.RawJSON(roomData), + }, + ) + data.Rooms.Join[roomID] = joinData + } } } - - // Append the data to the response - if len(roomID) > 0 { - jr := data.Rooms.Join[roomID] - jr.AccountData.Events = events - data.Rooms.Join[roomID] = jr - } else { - data.AccountData.Events = events - } } return data, nil diff --git a/userapi/api/api.go b/userapi/api/api.go index c953a5bac..a80adf2d8 100644 --- a/userapi/api/api.go +++ b/userapi/api/api.go @@ -16,12 +16,14 @@ package api import ( "context" + "encoding/json" "github.com/matrix-org/gomatrixserverlib" ) // UserInternalAPI is the internal API for information about users and devices. type UserInternalAPI interface { + InputAccountData(ctx context.Context, req *InputAccountDataRequest, res *InputAccountDataResponse) error PerformAccountCreation(ctx context.Context, req *PerformAccountCreationRequest, res *PerformAccountCreationResponse) error PerformDeviceCreation(ctx context.Context, req *PerformDeviceCreationRequest, res *PerformDeviceCreationResponse) error QueryProfile(ctx context.Context, req *QueryProfileRequest, res *QueryProfileResponse) error @@ -30,6 +32,18 @@ type UserInternalAPI interface { QueryAccountData(ctx context.Context, req *QueryAccountDataRequest, res *QueryAccountDataResponse) error } +// InputAccountDataRequest is the request for InputAccountData +type InputAccountDataRequest struct { + UserID string // required: the user to set account data for + RoomID string // optional: the room to associate the account data with + DataType string // optional: the data type of the data + AccountData json.RawMessage // required: the message content +} + +// InputAccountDataResponse is the response for InputAccountData +type InputAccountDataResponse struct { +} + // QueryAccessTokenRequest is the request for QueryAccessToken type QueryAccessTokenRequest struct { AccessToken string @@ -46,18 +60,15 @@ type QueryAccessTokenResponse struct { // QueryAccountDataRequest is the request for QueryAccountData type QueryAccountDataRequest struct { - UserID string // required: the user to get account data for. - // TODO: This is a terribly confusing API shape :/ - DataType string // optional: if specified returns only a single event matching this data type. - // optional: Only used if DataType is set. If blank returns global account data matching the data type. - // If set, returns only room account data matching this data type. - RoomID string + UserID string // required: the user to get account data for. + RoomID string // optional: the room ID, or global account data if not specified. + DataType string // optional: the data type, or all types if not specified. } // QueryAccountDataResponse is the response for QueryAccountData type QueryAccountDataResponse struct { - GlobalAccountData []gomatrixserverlib.ClientEvent - RoomAccountData map[string][]gomatrixserverlib.ClientEvent + GlobalAccountData map[string]json.RawMessage // type -> data + RoomAccountData map[string]map[string]json.RawMessage // room -> type -> data } // QueryDevicesRequest is the request for QueryDevices diff --git a/userapi/internal/api.go b/userapi/internal/api.go index ae021f575..b081eca49 100644 --- a/userapi/internal/api.go +++ b/userapi/internal/api.go @@ -17,6 +17,7 @@ package internal import ( "context" "database/sql" + "encoding/json" "errors" "fmt" @@ -38,6 +39,20 @@ type UserInternalAPI struct { AppServices []config.ApplicationService } +func (a *UserInternalAPI) InputAccountData(ctx context.Context, req *api.InputAccountDataRequest, res *api.InputAccountDataResponse) error { + local, domain, err := gomatrixserverlib.SplitID('@', req.UserID) + if err != nil { + return err + } + if domain != a.ServerName { + return fmt.Errorf("cannot query profile of remote users: got %s want %s", domain, a.ServerName) + } + if req.DataType == "" { + return fmt.Errorf("data type must not be empty") + } + return a.AccountDB.SaveAccountData(ctx, local, req.RoomID, req.DataType, req.AccountData) +} + func (a *UserInternalAPI) PerformAccountCreation(ctx context.Context, req *api.PerformAccountCreationRequest, res *api.PerformAccountCreationResponse) error { if req.AccountType == api.AccountTypeGuest { acc, err := a.AccountDB.CreateGuestAccount(ctx) @@ -130,17 +145,21 @@ func (a *UserInternalAPI) QueryAccountData(ctx context.Context, req *api.QueryAc return fmt.Errorf("cannot query account data of remote users: got %s want %s", domain, a.ServerName) } if req.DataType != "" { - var event *gomatrixserverlib.ClientEvent - event, err = a.AccountDB.GetAccountDataByType(ctx, local, req.RoomID, req.DataType) + var data json.RawMessage + data, err = a.AccountDB.GetAccountDataByType(ctx, local, req.RoomID, req.DataType) if err != nil { return err } - if event != nil { + res.RoomAccountData = make(map[string]map[string]json.RawMessage) + res.GlobalAccountData = make(map[string]json.RawMessage) + if data != nil { if req.RoomID != "" { - res.RoomAccountData = make(map[string][]gomatrixserverlib.ClientEvent) - res.RoomAccountData[req.RoomID] = []gomatrixserverlib.ClientEvent{*event} + if _, ok := res.RoomAccountData[req.RoomID]; !ok { + res.RoomAccountData[req.RoomID] = make(map[string]json.RawMessage) + } + res.RoomAccountData[req.RoomID][req.DataType] = data } else { - res.GlobalAccountData = append(res.GlobalAccountData, *event) + res.GlobalAccountData[req.DataType] = data } } return nil diff --git a/userapi/inthttp/client.go b/userapi/inthttp/client.go index 0e9628c58..4ab0d690e 100644 --- a/userapi/inthttp/client.go +++ b/userapi/inthttp/client.go @@ -26,6 +26,8 @@ import ( // HTTP paths for the internal HTTP APIs const ( + InputAccountDataPath = "/userapi/inputAccountData" + PerformDeviceCreationPath = "/userapi/performDeviceCreation" PerformAccountCreationPath = "/userapi/performAccountCreation" @@ -55,6 +57,14 @@ type httpUserInternalAPI struct { httpClient *http.Client } +func (h *httpUserInternalAPI) InputAccountData(ctx context.Context, req *api.InputAccountDataRequest, res *api.InputAccountDataResponse) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "InputAccountData") + defer span.Finish() + + apiURL := h.apiURL + InputAccountDataPath + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res) +} + func (h *httpUserInternalAPI) PerformAccountCreation( ctx context.Context, request *api.PerformAccountCreationRequest, diff --git a/userapi/storage/accounts/interface.go b/userapi/storage/accounts/interface.go index 13e3e2895..c6692879b 100644 --- a/userapi/storage/accounts/interface.go +++ b/userapi/storage/accounts/interface.go @@ -16,6 +16,7 @@ package accounts import ( "context" + "encoding/json" "errors" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" @@ -39,13 +40,13 @@ type Database interface { GetMembershipInRoomByLocalpart(ctx context.Context, localpart, roomID string) (authtypes.Membership, error) GetRoomIDsByLocalPart(ctx context.Context, localpart string) ([]string, error) GetMembershipsByLocalpart(ctx context.Context, localpart string) (memberships []authtypes.Membership, err error) - SaveAccountData(ctx context.Context, localpart, roomID, dataType, content string) error - GetAccountData(ctx context.Context, localpart string) (global []gomatrixserverlib.ClientEvent, rooms map[string][]gomatrixserverlib.ClientEvent, err error) + SaveAccountData(ctx context.Context, localpart, roomID, dataType string, content json.RawMessage) error + GetAccountData(ctx context.Context, localpart string) (global map[string]json.RawMessage, rooms map[string]map[string]json.RawMessage, err error) // GetAccountDataByType returns account data matching a given // localpart, room ID and type. // If no account data could be found, returns nil // Returns an error if there was an issue with the retrieval - GetAccountDataByType(ctx context.Context, localpart, roomID, dataType string) (data *gomatrixserverlib.ClientEvent, err error) + GetAccountDataByType(ctx context.Context, localpart, roomID, dataType string) (data json.RawMessage, err error) GetNewNumericLocalpart(ctx context.Context) (int64, error) SaveThreePIDAssociation(ctx context.Context, threepid, localpart, medium string) (err error) RemoveThreePIDAssociation(ctx context.Context, threepid string, medium string) (err error) diff --git a/userapi/storage/accounts/postgres/account_data_table.go b/userapi/storage/accounts/postgres/account_data_table.go index 2f16c5c02..90c79e878 100644 --- a/userapi/storage/accounts/postgres/account_data_table.go +++ b/userapi/storage/accounts/postgres/account_data_table.go @@ -17,9 +17,9 @@ package postgres import ( "context" "database/sql" + "encoding/json" "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/gomatrixserverlib" ) const accountDataSchema = ` @@ -73,7 +73,7 @@ func (s *accountDataStatements) prepare(db *sql.DB) (err error) { } func (s *accountDataStatements) insertAccountData( - ctx context.Context, txn *sql.Tx, localpart, roomID, dataType, content string, + ctx context.Context, txn *sql.Tx, localpart, roomID, dataType string, content json.RawMessage, ) (err error) { stmt := txn.Stmt(s.insertAccountDataStmt) _, err = stmt.ExecContext(ctx, localpart, roomID, dataType, content) @@ -83,18 +83,18 @@ func (s *accountDataStatements) insertAccountData( func (s *accountDataStatements) selectAccountData( ctx context.Context, localpart string, ) ( - global []gomatrixserverlib.ClientEvent, - rooms map[string][]gomatrixserverlib.ClientEvent, - err error, + /* global */ map[string]json.RawMessage, + /* rooms */ map[string]map[string]json.RawMessage, + error, ) { rows, err := s.selectAccountDataStmt.QueryContext(ctx, localpart) if err != nil { - return + return nil, nil, err } defer internal.CloseAndLogIfError(ctx, rows, "selectAccountData: rows.close() failed") - global = []gomatrixserverlib.ClientEvent{} - rooms = make(map[string][]gomatrixserverlib.ClientEvent) + global := map[string]json.RawMessage{} + rooms := map[string]map[string]json.RawMessage{} for rows.Next() { var roomID string @@ -102,41 +102,33 @@ func (s *accountDataStatements) selectAccountData( var content []byte if err = rows.Scan(&roomID, &dataType, &content); err != nil { - return + return nil, nil, err } - ac := gomatrixserverlib.ClientEvent{ - Type: dataType, - Content: content, - } - - if len(roomID) > 0 { - rooms[roomID] = append(rooms[roomID], ac) + if roomID != "" { + if _, ok := rooms[roomID]; !ok { + rooms[roomID] = map[string]json.RawMessage{} + } + rooms[roomID][dataType] = content } else { - global = append(global, ac) + global[dataType] = content } } + return global, rooms, rows.Err() } func (s *accountDataStatements) selectAccountDataByType( ctx context.Context, localpart, roomID, dataType string, -) (data *gomatrixserverlib.ClientEvent, err error) { +) (data json.RawMessage, err error) { + var bytes []byte stmt := s.selectAccountDataByTypeStmt - var content []byte - - if err = stmt.QueryRowContext(ctx, localpart, roomID, dataType).Scan(&content); err != nil { + if err = stmt.QueryRowContext(ctx, localpart, roomID, dataType).Scan(&bytes); err != nil { if err == sql.ErrNoRows { return nil, nil } - return } - - data = &gomatrixserverlib.ClientEvent{ - Type: dataType, - Content: content, - } - + data = json.RawMessage(bytes) return } diff --git a/userapi/storage/accounts/postgres/storage.go b/userapi/storage/accounts/postgres/storage.go index 2b88cb70a..e55099800 100644 --- a/userapi/storage/accounts/postgres/storage.go +++ b/userapi/storage/accounts/postgres/storage.go @@ -17,6 +17,7 @@ package postgres import ( "context" "database/sql" + "encoding/json" "errors" "strconv" @@ -169,7 +170,7 @@ func (d *Database) createAccount( return nil, err } - if err := d.accountDatas.insertAccountData(ctx, txn, localpart, "", "m.push_rules", `{ + if err := d.accountDatas.insertAccountData(ctx, txn, localpart, "", "m.push_rules", json.RawMessage(`{ "global": { "content": [], "override": [], @@ -177,7 +178,7 @@ func (d *Database) createAccount( "sender": [], "underride": [] } - }`); err != nil { + }`)); err != nil { return nil, err } return d.accounts.insertAccount(ctx, txn, localpart, hash, appserviceID) @@ -295,7 +296,7 @@ func (d *Database) newMembership( // update the corresponding row with the new content // Returns a SQL error if there was an issue with the insertion/update func (d *Database) SaveAccountData( - ctx context.Context, localpart, roomID, dataType, content string, + ctx context.Context, localpart, roomID, dataType string, content json.RawMessage, ) error { return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { return d.accountDatas.insertAccountData(ctx, txn, localpart, roomID, dataType, content) @@ -306,8 +307,8 @@ func (d *Database) SaveAccountData( // If no account data could be found, returns an empty arrays // Returns an error if there was an issue with the retrieval func (d *Database) GetAccountData(ctx context.Context, localpart string) ( - global []gomatrixserverlib.ClientEvent, - rooms map[string][]gomatrixserverlib.ClientEvent, + global map[string]json.RawMessage, + rooms map[string]map[string]json.RawMessage, err error, ) { return d.accountDatas.selectAccountData(ctx, localpart) @@ -319,7 +320,7 @@ func (d *Database) GetAccountData(ctx context.Context, localpart string) ( // Returns an error if there was an issue with the retrieval func (d *Database) GetAccountDataByType( ctx context.Context, localpart, roomID, dataType string, -) (data *gomatrixserverlib.ClientEvent, err error) { +) (data json.RawMessage, err error) { return d.accountDatas.selectAccountDataByType( ctx, localpart, roomID, dataType, ) diff --git a/userapi/storage/accounts/sqlite3/account_data_table.go b/userapi/storage/accounts/sqlite3/account_data_table.go index b6bb63617..d048dbd19 100644 --- a/userapi/storage/accounts/sqlite3/account_data_table.go +++ b/userapi/storage/accounts/sqlite3/account_data_table.go @@ -17,8 +17,7 @@ package sqlite3 import ( "context" "database/sql" - - "github.com/matrix-org/gomatrixserverlib" + "encoding/json" ) const accountDataSchema = ` @@ -72,7 +71,7 @@ func (s *accountDataStatements) prepare(db *sql.DB) (err error) { } func (s *accountDataStatements) insertAccountData( - ctx context.Context, txn *sql.Tx, localpart, roomID, dataType, content string, + ctx context.Context, txn *sql.Tx, localpart, roomID, dataType string, content json.RawMessage, ) (err error) { _, err = txn.Stmt(s.insertAccountDataStmt).ExecContext(ctx, localpart, roomID, dataType, content) return @@ -81,17 +80,17 @@ func (s *accountDataStatements) insertAccountData( func (s *accountDataStatements) selectAccountData( ctx context.Context, localpart string, ) ( - global []gomatrixserverlib.ClientEvent, - rooms map[string][]gomatrixserverlib.ClientEvent, - err error, + /* global */ map[string]json.RawMessage, + /* rooms */ map[string]map[string]json.RawMessage, + error, ) { rows, err := s.selectAccountDataStmt.QueryContext(ctx, localpart) if err != nil { - return + return nil, nil, err } - global = []gomatrixserverlib.ClientEvent{} - rooms = make(map[string][]gomatrixserverlib.ClientEvent) + global := map[string]json.RawMessage{} + rooms := map[string]map[string]json.RawMessage{} for rows.Next() { var roomID string @@ -99,42 +98,33 @@ func (s *accountDataStatements) selectAccountData( var content []byte if err = rows.Scan(&roomID, &dataType, &content); err != nil { - return + return nil, nil, err } - ac := gomatrixserverlib.ClientEvent{ - Type: dataType, - Content: content, - } - - if len(roomID) > 0 { - rooms[roomID] = append(rooms[roomID], ac) + if roomID != "" { + if _, ok := rooms[roomID]; !ok { + rooms[roomID] = map[string]json.RawMessage{} + } + rooms[roomID][dataType] = content } else { - global = append(global, ac) + global[dataType] = content } } - return + return global, rooms, nil } func (s *accountDataStatements) selectAccountDataByType( ctx context.Context, localpart, roomID, dataType string, -) (data *gomatrixserverlib.ClientEvent, err error) { +) (data json.RawMessage, err error) { + var bytes []byte stmt := s.selectAccountDataByTypeStmt - var content []byte - - if err = stmt.QueryRowContext(ctx, localpart, roomID, dataType).Scan(&content); err != nil { + if err = stmt.QueryRowContext(ctx, localpart, roomID, dataType).Scan(&bytes); err != nil { if err == sql.ErrNoRows { return nil, nil } - return } - - data = &gomatrixserverlib.ClientEvent{ - Type: dataType, - Content: content, - } - + data = json.RawMessage(bytes) return } diff --git a/userapi/storage/accounts/sqlite3/storage.go b/userapi/storage/accounts/sqlite3/storage.go index 4dd755a70..dbf6606c3 100644 --- a/userapi/storage/accounts/sqlite3/storage.go +++ b/userapi/storage/accounts/sqlite3/storage.go @@ -17,6 +17,7 @@ package sqlite3 import ( "context" "database/sql" + "encoding/json" "errors" "strconv" "sync" @@ -180,7 +181,7 @@ func (d *Database) createAccount( return nil, err } - if err := d.accountDatas.insertAccountData(ctx, txn, localpart, "", "m.push_rules", `{ + if err := d.accountDatas.insertAccountData(ctx, txn, localpart, "", "m.push_rules", json.RawMessage(`{ "global": { "content": [], "override": [], @@ -188,7 +189,7 @@ func (d *Database) createAccount( "sender": [], "underride": [] } - }`); err != nil { + }`)); err != nil { return nil, err } return d.accounts.insertAccount(ctx, txn, localpart, hash, appserviceID) @@ -306,7 +307,7 @@ func (d *Database) newMembership( // update the corresponding row with the new content // Returns a SQL error if there was an issue with the insertion/update func (d *Database) SaveAccountData( - ctx context.Context, localpart, roomID, dataType, content string, + ctx context.Context, localpart, roomID, dataType string, content json.RawMessage, ) error { return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { return d.accountDatas.insertAccountData(ctx, txn, localpart, roomID, dataType, content) @@ -317,8 +318,8 @@ func (d *Database) SaveAccountData( // If no account data could be found, returns an empty arrays // Returns an error if there was an issue with the retrieval func (d *Database) GetAccountData(ctx context.Context, localpart string) ( - global []gomatrixserverlib.ClientEvent, - rooms map[string][]gomatrixserverlib.ClientEvent, + global map[string]json.RawMessage, + rooms map[string]map[string]json.RawMessage, err error, ) { return d.accountDatas.selectAccountData(ctx, localpart) @@ -330,7 +331,7 @@ func (d *Database) GetAccountData(ctx context.Context, localpart string) ( // Returns an error if there was an issue with the retrieval func (d *Database) GetAccountDataByType( ctx context.Context, localpart, roomID, dataType string, -) (data *gomatrixserverlib.ClientEvent, err error) { +) (data json.RawMessage, err error) { return d.accountDatas.selectAccountDataByType( ctx, localpart, roomID, dataType, ) From 9e3d771a32059b96c5595b08bc1f13a481ca800b Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 19 Jun 2020 09:18:09 +0100 Subject: [PATCH 4/8] Fix comment in InputAccountDataRequest --- userapi/api/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/userapi/api/api.go b/userapi/api/api.go index a80adf2d8..cf0f05633 100644 --- a/userapi/api/api.go +++ b/userapi/api/api.go @@ -36,7 +36,7 @@ type UserInternalAPI interface { type InputAccountDataRequest struct { UserID string // required: the user to set account data for RoomID string // optional: the room to associate the account data with - DataType string // optional: the data type of the data + DataType string // required: the data type of the data AccountData json.RawMessage // required: the message content } From 72444e4a4f31b70dfb7ddd875ef874f637699dce Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 19 Jun 2020 09:37:19 +0100 Subject: [PATCH 5/8] User API polylith component (#1151) * User API polylith component * Add docker-pull.sh --- build/docker/config/dendrite-config.yaml | 1 + build/docker/docker-compose.polylith.yml | 11 +++++++++++ build/docker/images-build.sh | 1 + build/docker/images-pull.sh | 16 ++++++++++++++++ build/docker/images-push.sh | 1 + dendrite-config.yaml | 6 ++++-- docs/INSTALL.md | 10 ++++++++++ internal/config/config.go | 1 + 8 files changed, 45 insertions(+), 2 deletions(-) create mode 100755 build/docker/images-pull.sh diff --git a/build/docker/config/dendrite-config.yaml b/build/docker/config/dendrite-config.yaml index 26dc272ab..f421b2e71 100644 --- a/build/docker/config/dendrite-config.yaml +++ b/build/docker/config/dendrite-config.yaml @@ -117,6 +117,7 @@ listen: federation_sender: "federation_sender:7776" edu_server: "edu_server:7777" key_server: "key_server:7779" + user_api: "user_api:7780" # The configuration for tracing the dendrite components. tracing: diff --git a/build/docker/docker-compose.polylith.yml b/build/docker/docker-compose.polylith.yml index 178604093..af2ae4e81 100644 --- a/build/docker/docker-compose.polylith.yml +++ b/build/docker/docker-compose.polylith.yml @@ -152,6 +152,17 @@ services: networks: - internal + user_api: + hostname: user_api + image: matrixdotorg/dendrite:userapi + command: [ + "--config=dendrite.yaml" + ] + volumes: + - ./config:/etc/dendrite + networks: + - internal + networks: internal: attachable: true diff --git a/build/docker/images-build.sh b/build/docker/images-build.sh index 6bc058962..e43d2d040 100755 --- a/build/docker/images-build.sh +++ b/build/docker/images-build.sh @@ -18,3 +18,4 @@ docker build -t matrixdotorg/dendrite:publicroomsapi --build-arg component=de docker build -t matrixdotorg/dendrite:roomserver --build-arg component=dendrite-room-server -f build/docker/Dockerfile.component . docker build -t matrixdotorg/dendrite:syncapi --build-arg component=dendrite-sync-api-server -f build/docker/Dockerfile.component . docker build -t matrixdotorg/dendrite:serverkeyapi --build-arg component=dendrite-server-key-api-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite:userapi --build-arg component=dendrite-user-api-server -f build/docker/Dockerfile.component . diff --git a/build/docker/images-pull.sh b/build/docker/images-pull.sh new file mode 100755 index 000000000..edccf4a33 --- /dev/null +++ b/build/docker/images-pull.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +docker pull matrixdotorg/dendrite:monolith + +docker pull matrixdotorg/dendrite:clientapi +docker pull matrixdotorg/dendrite:clientproxy +docker pull matrixdotorg/dendrite:eduserver +docker pull matrixdotorg/dendrite:federationapi +docker pull matrixdotorg/dendrite:federationsender +docker pull matrixdotorg/dendrite:federationproxy +docker pull matrixdotorg/dendrite:keyserver +docker pull matrixdotorg/dendrite:mediaapi +docker pull matrixdotorg/dendrite:publicroomsapi +docker pull matrixdotorg/dendrite:roomserver +docker pull matrixdotorg/dendrite:syncapi +docker pull matrixdotorg/dendrite:userapi diff --git a/build/docker/images-push.sh b/build/docker/images-push.sh index b39d98d65..2b4303872 100755 --- a/build/docker/images-push.sh +++ b/build/docker/images-push.sh @@ -13,3 +13,4 @@ docker push matrixdotorg/dendrite:mediaapi docker push matrixdotorg/dendrite:publicroomsapi docker push matrixdotorg/dendrite:roomserver docker push matrixdotorg/dendrite:syncapi +docker push matrixdotorg/dendrite:userapi diff --git a/dendrite-config.yaml b/dendrite-config.yaml index 52793cda4..73bfec247 100644 --- a/dendrite-config.yaml +++ b/dendrite-config.yaml @@ -108,7 +108,9 @@ kafka: output_send_to_device_event: eduServerSendToDeviceOutput user_updates: userUpdates -# The postgres connection configs for connecting to the databases e.g a postgres:// URI +# The postgres connection configs for connecting to the databases, e.g. +# for Postgres: postgres://username:password@hostname/database +# for SQLite: file:filename.db or file:///path/to/filename.db database: account: "postgres://dendrite:itsasecret@localhost/dendrite_account?sslmode=disable" device: "postgres://dendrite:itsasecret@localhost/dendrite_device?sslmode=disable" @@ -122,7 +124,7 @@ database: max_open_conns: 100 max_idle_conns: 2 conn_max_lifetime: -1 - # If using naffka you need to specify a naffka database + # If 'use_naffka: true' set above then you need to specify a naffka database # naffka: "postgres://dendrite:itsasecret@localhost/dendrite_naffka?sslmode=disable" # The TCP host:port pairs to bind the internal HTTP APIs to. diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 184b777d8..b4c81a42b 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -329,3 +329,13 @@ finished). ```bash ./bin/dendrite-key-server --config dendrite.yaml ``` + +### User server + +This manages user accounts, device access tokens and user account data, +amongst other things. + +```bash +./bin/dendrite-user-api-server --config dendrite.yaml +``` + diff --git a/internal/config/config.go b/internal/config/config.go index 2bd56ad9e..baa82be23 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -226,6 +226,7 @@ type Dendrite struct { ServerKeyAPI Address `yaml:"server_key_api"` AppServiceAPI Address `yaml:"appservice_api"` SyncAPI Address `yaml:"sync_api"` + UserAPI Address `yaml:"user_api"` RoomServer Address `yaml:"room_server"` FederationSender Address `yaml:"federation_sender"` PublicRoomsAPI Address `yaml:"public_rooms_api"` From 7f26b0cd13534ba235c2bf0a7220d3462ca8e9da Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 19 Jun 2020 13:29:27 +0100 Subject: [PATCH 6/8] Bind build support, further Yggdrasil demo updates (#1152) * Add gobind builds for Yggdrasil demo * Massage client API a bit * Fix build * Fix gobind build * Fix gobind client API setup * Tweaks * Tweaks * Update sytest-whitelist, add comment * Default to sending push rules on initial sync --- build/gobind/build.sh | 6 + build/gobind/monolith.go | 161 ++++++++++++++++++ build/gobind/platform_ios.go | 25 +++ build/gobind/platform_other.go | 12 ++ clientapi/routing/device.go | 9 +- clientapi/routing/joinroom.go | 1 + cmd/dendrite-demo-yggdrasil/main.go | 49 +----- cmd/dendrite-demo-yggdrasil/yggconn/client.go | 74 ++++++++ cmd/dendrite-demo-yggdrasil/yggconn/node.go | 27 +-- go.mod | 1 + go.sum | 15 ++ syncapi/sync/request.go | 4 + syncapi/sync/requestpool.go | 3 +- syncapi/types/types.go | 11 +- sytest-whitelist | 3 + 15 files changed, 336 insertions(+), 65 deletions(-) create mode 100644 build/gobind/build.sh create mode 100644 build/gobind/monolith.go create mode 100644 build/gobind/platform_ios.go create mode 100644 build/gobind/platform_other.go create mode 100644 cmd/dendrite-demo-yggdrasil/yggconn/client.go diff --git a/build/gobind/build.sh b/build/gobind/build.sh new file mode 100644 index 000000000..3a80d374a --- /dev/null +++ b/build/gobind/build.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +gomobile bind -v \ + -ldflags "-X $github.com/yggdrasil-network/yggdrasil-go/src/version.buildName=riot-ios-p2p" \ + -target ios \ + github.com/matrix-org/dendrite/build/gobind \ No newline at end of file diff --git a/build/gobind/monolith.go b/build/gobind/monolith.go new file mode 100644 index 000000000..750babad8 --- /dev/null +++ b/build/gobind/monolith.go @@ -0,0 +1,161 @@ +package gobind + +import ( + "context" + "crypto/tls" + "fmt" + "net" + "net/http" + "time" + + "github.com/matrix-org/dendrite/appservice" + "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" + "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/yggconn" + "github.com/matrix-org/dendrite/eduserver" + "github.com/matrix-org/dendrite/eduserver/cache" + "github.com/matrix-org/dendrite/federationsender" + "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/internal/setup" + "github.com/matrix-org/dendrite/publicroomsapi/storage" + "github.com/matrix-org/dendrite/roomserver" + "github.com/matrix-org/dendrite/userapi" + "github.com/matrix-org/gomatrixserverlib" + "github.com/sirupsen/logrus" +) + +type DendriteMonolith struct { + StorageDirectory string + listener net.Listener +} + +func (m *DendriteMonolith) BaseURL() string { + return fmt.Sprintf("http://%s", m.listener.Addr().String()) +} + +func (m *DendriteMonolith) Start() { + logger := logrus.Logger{ + Out: BindLogger{}, + } + logrus.SetOutput(BindLogger{}) + + var err error + m.listener, err = net.Listen("tcp", "localhost:65432") + if err != nil { + panic(err) + } + + ygg, err := yggconn.Setup("dendrite", "", m.StorageDirectory) + if err != nil { + panic(err) + } + + cfg := &config.Dendrite{} + cfg.SetDefaults() + cfg.Matrix.ServerName = gomatrixserverlib.ServerName(ygg.DerivedServerName()) + cfg.Matrix.PrivateKey = ygg.SigningPrivateKey() + cfg.Matrix.KeyID = gomatrixserverlib.KeyID(signing.KeyID) + cfg.Kafka.UseNaffka = true + cfg.Kafka.Topics.OutputRoomEvent = "roomserverOutput" + cfg.Kafka.Topics.OutputClientData = "clientapiOutput" + cfg.Kafka.Topics.OutputTypingEvent = "typingServerOutput" + cfg.Kafka.Topics.OutputSendToDeviceEvent = "sendToDeviceOutput" + cfg.Database.Account = config.DataSource(fmt.Sprintf("file:%s/dendrite-account.db", m.StorageDirectory)) + cfg.Database.Device = config.DataSource(fmt.Sprintf("file:%s/dendrite-device.db", m.StorageDirectory)) + cfg.Database.MediaAPI = config.DataSource(fmt.Sprintf("file:%s/dendrite-mediaapi.db", m.StorageDirectory)) + cfg.Database.SyncAPI = config.DataSource(fmt.Sprintf("file:%s/dendrite-syncapi.db", m.StorageDirectory)) + cfg.Database.RoomServer = config.DataSource(fmt.Sprintf("file:%s/dendrite-roomserver.db", m.StorageDirectory)) + cfg.Database.ServerKey = config.DataSource(fmt.Sprintf("file:%s/dendrite-serverkey.db", m.StorageDirectory)) + cfg.Database.FederationSender = config.DataSource(fmt.Sprintf("file:%s/dendrite-federationsender.db", m.StorageDirectory)) + cfg.Database.AppService = config.DataSource(fmt.Sprintf("file:%s/dendrite-appservice.db", m.StorageDirectory)) + cfg.Database.PublicRoomsAPI = config.DataSource(fmt.Sprintf("file:%s/dendrite-publicroomsa.db", m.StorageDirectory)) + cfg.Database.Naffka = config.DataSource(fmt.Sprintf("file:%s/dendrite-naffka.db", m.StorageDirectory)) + if err = cfg.Derive(); err != nil { + panic(err) + } + + base := setup.NewBaseDendrite(cfg, "Monolith", false) + defer base.Close() // nolint: errcheck + + accountDB := base.CreateAccountsDB() + deviceDB := base.CreateDeviceDB() + federation := ygg.CreateFederationClient(base) + + serverKeyAPI := &signing.YggdrasilKeys{} + keyRing := serverKeyAPI.KeyRing() + userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, cfg.Derived.ApplicationServices) + + rsAPI := roomserver.NewInternalAPI( + base, keyRing, federation, + ) + + eduInputAPI := eduserver.NewInternalAPI( + base, cache.New(), userAPI, + ) + + asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI) + + fsAPI := federationsender.NewInternalAPI( + base, federation, rsAPI, keyRing, + ) + + // The underlying roomserver implementation needs to be able to call the fedsender. + // This is different to rsAPI which can be the http client which doesn't need this dependency + rsAPI.SetFederationSenderAPI(fsAPI) + + publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties(), cfg.Matrix.ServerName) + if err != nil { + logrus.WithError(err).Panicf("failed to connect to public rooms db") + } + + monolith := setup.Monolith{ + Config: base.Cfg, + AccountDB: accountDB, + DeviceDB: deviceDB, + Client: ygg.CreateClient(base), + FedClient: federation, + KeyRing: keyRing, + KafkaConsumer: base.KafkaConsumer, + KafkaProducer: base.KafkaProducer, + + AppserviceAPI: asAPI, + EDUInternalAPI: eduInputAPI, + FederationSenderAPI: fsAPI, + RoomserverAPI: rsAPI, + UserAPI: userAPI, + //ServerKeyAPI: serverKeyAPI, + + PublicRoomsDB: publicRoomsDB, + } + monolith.AddAllPublicRoutes(base.PublicAPIMux) + + httputil.SetupHTTPAPI( + base.BaseMux, + base.PublicAPIMux, + base.InternalAPIMux, + cfg, + base.UseHTTPAPIs, + ) + + // Build both ends of a HTTP multiplex. + httpServer := &http.Server{ + Addr: ":0", + TLSNextProto: map[string]func(*http.Server, *tls.Conn, http.Handler){}, + ReadTimeout: 15 * time.Second, + WriteTimeout: 45 * time.Second, + IdleTimeout: 60 * time.Second, + BaseContext: func(_ net.Listener) context.Context { + return context.Background() + }, + Handler: base.BaseMux, + } + + go func() { + logger.Info("Listening on ", ygg.DerivedServerName()) + logger.Fatal(httpServer.Serve(ygg)) + }() + go func() { + logger.Info("Listening on ", m.BaseURL()) + logger.Fatal(httpServer.Serve(m.listener)) + }() +} diff --git a/build/gobind/platform_ios.go b/build/gobind/platform_ios.go new file mode 100644 index 000000000..01f8a6a04 --- /dev/null +++ b/build/gobind/platform_ios.go @@ -0,0 +1,25 @@ +// +build ios + +package gobind + +/* +#cgo CFLAGS: -x objective-c +#cgo LDFLAGS: -framework Foundation +#import +void Log(const char *text) { + NSString *nss = [NSString stringWithUTF8String:text]; + NSLog(@"%@", nss); +} +*/ +import "C" +import "unsafe" + +type BindLogger struct { +} + +func (nsl BindLogger) Write(p []byte) (n int, err error) { + p = append(p, 0) + cstr := (*C.char)(unsafe.Pointer(&p[0])) + C.Log(cstr) + return len(p), nil +} diff --git a/build/gobind/platform_other.go b/build/gobind/platform_other.go new file mode 100644 index 000000000..fdfb13bc0 --- /dev/null +++ b/build/gobind/platform_other.go @@ -0,0 +1,12 @@ +// +build !ios + +package gobind + +import "log" + +type BindLogger struct{} + +func (nsl BindLogger) Write(p []byte) (n int, err error) { + log.Println(string(p)) + return len(p), nil +} diff --git a/clientapi/routing/device.go b/clientapi/routing/device.go index 403937c9c..51a15a882 100644 --- a/clientapi/routing/device.go +++ b/clientapi/routing/device.go @@ -26,9 +26,12 @@ import ( "github.com/matrix-org/util" ) +// https://matrix.org/docs/spec/client_server/r0.6.1#get-matrix-client-r0-devices type deviceJSON struct { - DeviceID string `json:"device_id"` - UserID string `json:"user_id"` + DeviceID string `json:"device_id"` + DisplayName string `json:"display_name"` + LastSeenIP string `json:"last_seen_ip"` + LastSeenTS uint64 `json:"last_seen_ts"` } type devicesJSON struct { @@ -70,7 +73,6 @@ func GetDeviceByID( Code: http.StatusOK, JSON: deviceJSON{ DeviceID: dev.ID, - UserID: dev.UserID, }, } } @@ -98,7 +100,6 @@ func GetDevicesByLocalpart( for _, dev := range deviceList { res.Devices = append(res.Devices, deviceJSON{ DeviceID: dev.ID, - UserID: dev.UserID, }) } diff --git a/clientapi/routing/joinroom.go b/clientapi/routing/joinroom.go index 3871e4d4e..db890d03f 100644 --- a/clientapi/routing/joinroom.go +++ b/clientapi/routing/joinroom.go @@ -37,6 +37,7 @@ func JoinRoomByIDOrAlias( joinReq := roomserverAPI.PerformJoinRequest{ RoomIDOrAlias: roomIDOrAlias, UserID: device.UserID, + Content: map[string]interface{}{}, } joinRes := roomserverAPI.PerformJoinResponse{} diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index 328aa0629..db05ecb76 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -47,52 +47,11 @@ var ( instancePeer = flag.String("peer", "", "an internet Yggdrasil peer to connect to") ) -type yggroundtripper struct { - inner *http.Transport -} - -func (y *yggroundtripper) RoundTrip(req *http.Request) (*http.Response, error) { - req.URL.Scheme = "http" - return y.inner.RoundTrip(req) -} - -func createFederationClient( - base *setup.BaseDendrite, n *yggconn.Node, -) *gomatrixserverlib.FederationClient { - tr := &http.Transport{} - tr.RegisterProtocol( - "matrix", &yggroundtripper{ - inner: &http.Transport{ - ResponseHeaderTimeout: 15 * time.Second, - IdleConnTimeout: 60 * time.Second, - DialContext: n.DialerContext, - }, - }, - ) - return gomatrixserverlib.NewFederationClientWithTransport( - base.Cfg.Matrix.ServerName, base.Cfg.Matrix.KeyID, base.Cfg.Matrix.PrivateKey, tr, - ) -} - -func createClient(n *yggconn.Node) *gomatrixserverlib.Client { - tr := &http.Transport{} - tr.RegisterProtocol( - "matrix", &yggroundtripper{ - inner: &http.Transport{ - ResponseHeaderTimeout: 15 * time.Second, - IdleConnTimeout: 60 * time.Second, - DialContext: n.DialerContext, - }, - }, - ) - return gomatrixserverlib.NewClientWithTransport(tr) -} - // nolint:gocyclo func main() { flag.Parse() - ygg, err := yggconn.Setup(*instanceName, *instancePeer) + ygg, err := yggconn.Setup(*instanceName, *instancePeer, ".") if err != nil { panic(err) } @@ -125,7 +84,7 @@ func main() { accountDB := base.CreateAccountsDB() deviceDB := base.CreateDeviceDB() - federation := createFederationClient(base, ygg) + federation := ygg.CreateFederationClient(base) serverKeyAPI := &signing.YggdrasilKeys{} keyRing := serverKeyAPI.KeyRing() @@ -160,7 +119,7 @@ func main() { Config: base.Cfg, AccountDB: accountDB, DeviceDB: deviceDB, - Client: createClient(ygg), + Client: ygg.CreateClient(base), FedClient: federation, KeyRing: keyRing, KafkaConsumer: base.KafkaConsumer, @@ -203,7 +162,7 @@ func main() { logrus.Fatal(httpServer.Serve(ygg)) }() go func() { - httpBindAddr := fmt.Sprintf("localhost:%d", *instancePort) + httpBindAddr := fmt.Sprintf(":%d", *instancePort) logrus.Info("Listening on ", httpBindAddr) logrus.Fatal(http.ListenAndServe(httpBindAddr, base.BaseMux)) }() diff --git a/cmd/dendrite-demo-yggdrasil/yggconn/client.go b/cmd/dendrite-demo-yggdrasil/yggconn/client.go new file mode 100644 index 000000000..399993e3e --- /dev/null +++ b/cmd/dendrite-demo-yggdrasil/yggconn/client.go @@ -0,0 +1,74 @@ +package yggconn + +import ( + "context" + "crypto/ed25519" + "encoding/hex" + "fmt" + "net" + "net/http" + "strings" + "time" + + "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/convert" + "github.com/matrix-org/dendrite/internal/setup" + "github.com/matrix-org/gomatrixserverlib" +) + +func (n *Node) yggdialer(_, address string) (net.Conn, error) { + tokens := strings.Split(address, ":") + raw, err := hex.DecodeString(tokens[0]) + if err != nil { + return nil, fmt.Errorf("hex.DecodeString: %w", err) + } + converted := convert.Ed25519PublicKeyToCurve25519(ed25519.PublicKey(raw)) + convhex := hex.EncodeToString(converted) + return n.Dial("curve25519", convhex) +} + +func (n *Node) yggdialerctx(ctx context.Context, network, address string) (net.Conn, error) { + return n.yggdialer(network, address) +} + +type yggroundtripper struct { + inner *http.Transport +} + +func (y *yggroundtripper) RoundTrip(req *http.Request) (*http.Response, error) { + req.URL.Scheme = "http" + return y.inner.RoundTrip(req) +} + +func (n *Node) CreateClient( + base *setup.BaseDendrite, +) *gomatrixserverlib.Client { + tr := &http.Transport{} + tr.RegisterProtocol( + "matrix", &yggroundtripper{ + inner: &http.Transport{ + ResponseHeaderTimeout: 15 * time.Second, + IdleConnTimeout: 60 * time.Second, + DialContext: n.yggdialerctx, + }, + }, + ) + return gomatrixserverlib.NewClientWithTransport(tr) +} + +func (n *Node) CreateFederationClient( + base *setup.BaseDendrite, +) *gomatrixserverlib.FederationClient { + tr := &http.Transport{} + tr.RegisterProtocol( + "matrix", &yggroundtripper{ + inner: &http.Transport{ + ResponseHeaderTimeout: 15 * time.Second, + IdleConnTimeout: 60 * time.Second, + DialContext: n.yggdialerctx, + }, + }, + ) + return gomatrixserverlib.NewFederationClientWithTransport( + base.Cfg.Matrix.ServerName, base.Cfg.Matrix.KeyID, base.Cfg.Matrix.PrivateKey, tr, + ) +} diff --git a/cmd/dendrite-demo-yggdrasil/yggconn/node.go b/cmd/dendrite-demo-yggdrasil/yggconn/node.go index b225e1cf9..c335f2eac 100644 --- a/cmd/dendrite-demo-yggdrasil/yggconn/node.go +++ b/cmd/dendrite-demo-yggdrasil/yggconn/node.go @@ -67,7 +67,7 @@ func (n *Node) DialerContext(ctx context.Context, network, address string) (net. } // nolint:gocyclo -func Setup(instanceName, instancePeer string) (*Node, error) { +func Setup(instanceName, instancePeer, storageDirectory string) (*Node, error) { n := &Node{ core: &yggdrasil.Core{}, config: yggdrasilconfig.GenerateConfig(), @@ -77,7 +77,7 @@ func Setup(instanceName, instancePeer string) (*Node, error) { incoming: make(chan *yamux.Stream), } - yggfile := fmt.Sprintf("%s-yggdrasil.conf", instanceName) + yggfile := fmt.Sprintf("%s/%s-yggdrasil.conf", storageDirectory, instanceName) if _, err := os.Stat(yggfile); !os.IsNotExist(err) { yggconf, e := ioutil.ReadFile(yggfile) if e != nil { @@ -87,7 +87,7 @@ func Setup(instanceName, instancePeer string) (*Node, error) { panic(err) } } else { - n.config.AdminListen = fmt.Sprintf("unix://./%s-yggdrasil.sock", instanceName) + n.config.AdminListen = "none" // fmt.Sprintf("unix://%s/%s-yggdrasil.sock", storageDirectory, instanceName) n.config.MulticastInterfaces = []string{".*"} n.config.EncryptionPrivateKey = hex.EncodeToString(n.EncryptionPrivateKey()) n.config.EncryptionPublicKey = hex.EncodeToString(n.EncryptionPublicKey()) @@ -114,20 +114,22 @@ func Setup(instanceName, instancePeer string) (*Node, error) { panic(err) } } - if err = n.admin.Init(n.core, n.state, n.log, nil); err != nil { - panic(err) - } - if err = n.admin.Start(); err != nil { - panic(err) - } + /* + if err = n.admin.Init(n.core, n.state, n.log, nil); err != nil { + panic(err) + } + if err = n.admin.Start(); err != nil { + panic(err) + } + */ if err = n.multicast.Init(n.core, n.state, n.log, nil); err != nil { panic(err) } if err = n.multicast.Start(); err != nil { panic(err) } - n.admin.SetupAdminHandlers(n.admin) - n.multicast.SetupAdminHandlers(n.admin) + //n.admin.SetupAdminHandlers(n.admin) + //n.multicast.SetupAdminHandlers(n.admin) n.listener, err = n.core.ConnListen() if err != nil { panic(err) @@ -137,6 +139,9 @@ func Setup(instanceName, instancePeer string) (*Node, error) { panic(err) } + n.log.Println("Public curve25519:", n.core.EncryptionPublicKey()) + n.log.Println("Public ed25519:", n.core.SigningPublicKey()) + go n.listenFromYgg() return n, nil diff --git a/go.mod b/go.mod index ff73ad078..4473dc97f 100644 --- a/go.mod +++ b/go.mod @@ -38,6 +38,7 @@ require ( github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20200530233943-aec82d7a391b go.uber.org/atomic v1.4.0 golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d + golang.org/x/mobile v0.0.0-20200329125638-4c31acba0007 // indirect gopkg.in/h2non/bimg.v1 v1.0.18 gopkg.in/yaml.v2 v2.2.8 ) diff --git a/go.sum b/go.sum index f62b14b62..97abb7b98 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,7 @@ github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOv github.com/Arceliar/phony v0.0.0-20191006174943-d0c68492aca0 h1:p3puK8Sl2xK+2FnnIvY/C0N1aqJo2kbEsdAzU+Tnv48= github.com/Arceliar/phony v0.0.0-20191006174943-d0c68492aca0/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= github.com/Shopify/sarama v1.26.1 h1:3jnfWKD7gVwbB1KSy/lE0szA9duPuSFLViK0o/d3DgA= github.com/Shopify/sarama v1.26.1/go.mod h1:NbSGBSSndYaIhRcBtY9V0U7AyH+x71bG668AuWys/yU= @@ -589,6 +590,7 @@ golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -599,9 +601,18 @@ golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d h1:1ZiEyfaQIg3Qh0EoqpwAakHVhecoE5wlSg5GjnafJGw= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20200329125638-4c31acba0007 h1:JxsyO7zPDWn1rBZW8FV5RFwCKqYeXnyaS/VQPLpXu6I= +golang.org/x/mobile v0.0.0-20200329125638-4c31acba0007/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -671,6 +682,10 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd h1:/e+gpKk9r3dJobndpTytxS2gOy6m5uvpg+ISQoEcusQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69 h1:yBHHx+XZqXJBm6Exke3N7V9gnlsyXxoCPEb1yVenjfk= +golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= diff --git a/syncapi/sync/request.go b/syncapi/sync/request.go index beeaa40f7..5dd92c853 100644 --- a/syncapi/sync/request.go +++ b/syncapi/sync/request.go @@ -62,6 +62,10 @@ func newSyncRequest(req *http.Request, device userapi.Device) (*syncRequest, err } since = &tok } + if since == nil { + tok := types.NewStreamToken(0, 0) + since = &tok + } timelineLimit := defaultTimelineLimit // TODO: read from stored filters too filterQuery := req.URL.Query().Get("filter") diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index 8d51689e3..743c63a62 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -256,7 +256,8 @@ func (rp *RequestPool) appendAccountData( } if len(dataTypes) == 0 { - return data, nil + // TODO: this fixes the sytest but is it the right thing to do? + dataTypes[""] = []string{"m.push_rules"} } // Iterate over the rooms diff --git a/syncapi/types/types.go b/syncapi/types/types.go index c1f09fba5..1094416a1 100644 --- a/syncapi/types/types.go +++ b/syncapi/types/types.go @@ -98,6 +98,9 @@ func (t *StreamingToken) PDUPosition() StreamPosition { func (t *StreamingToken) EDUPosition() StreamPosition { return t.Positions[1] } +func (t *StreamingToken) String() string { + return t.syncToken.String() +} // IsAfter returns true if ANY position in this token is greater than `other`. func (t *StreamingToken) IsAfter(other StreamingToken) bool { @@ -220,8 +223,8 @@ func NewTopologyTokenFromString(tok string) (token TopologyToken, err error) { err = fmt.Errorf("token %s is not a topology token", tok) return } - if len(t.Positions) != 2 { - err = fmt.Errorf("token %s wrong number of values, got %d want 2", tok, len(t.Positions)) + if len(t.Positions) < 2 { + err = fmt.Errorf("token %s wrong number of values, got %d want at least 2", tok, len(t.Positions)) return } return TopologyToken{ @@ -247,8 +250,8 @@ func NewStreamTokenFromString(tok string) (token StreamingToken, err error) { err = fmt.Errorf("token %s is not a streaming token", tok) return } - if len(t.Positions) != 2 { - err = fmt.Errorf("token %s wrong number of values, got %d want 2", tok, len(t.Positions)) + if len(t.Positions) < 2 { + err = fmt.Errorf("token %s wrong number of values, got %d want at least 2", tok, len(t.Positions)) return } return StreamingToken{ diff --git a/sytest-whitelist b/sytest-whitelist index 971f8cf0d..7edc79e64 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -347,3 +347,6 @@ GET /rooms/:room_id/state/m.room.topic gets topic GET /rooms/:room_id/state fetches entire room state Setting room topic reports m.room.topic to myself setting 'm.room.name' respects room powerlevel +Syncing a new room with a large timeline limit isn't limited +Left rooms appear in the leave section of sync +Banned rooms appear in the leave section of sync From 61e0482fef7aa34c0261e41fd90aa5c5e4890bfc Mon Sep 17 00:00:00 2001 From: Ashley Nelson Date: Sat, 20 Jun 2020 15:28:30 -0500 Subject: [PATCH 7/8] Add appservices component to docker scripts (#1153) Signed-off-by: Ashley Nelson --- build/docker/config/dendrite-config.yaml | 1 + build/docker/docker-compose.polylith.yml | 14 ++++++++++ build/docker/images-build.sh | 1 + build/docker/images-pull.sh | 1 + build/docker/images-push.sh | 1 + build/scripts/build-test-lint.sh | 2 +- cmd/dendrite-user-api-server/main.go | 35 ++++++++++++++++++++++++ 7 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 cmd/dendrite-user-api-server/main.go diff --git a/build/docker/config/dendrite-config.yaml b/build/docker/config/dendrite-config.yaml index f421b2e71..53d9f7b02 100644 --- a/build/docker/config/dendrite-config.yaml +++ b/build/docker/config/dendrite-config.yaml @@ -118,6 +118,7 @@ listen: edu_server: "edu_server:7777" key_server: "key_server:7779" user_api: "user_api:7780" + appservice_api: "appservice_api:7781" # The configuration for tracing the dendrite components. tracing: diff --git a/build/docker/docker-compose.polylith.yml b/build/docker/docker-compose.polylith.yml index af2ae4e81..d424d43b1 100644 --- a/build/docker/docker-compose.polylith.yml +++ b/build/docker/docker-compose.polylith.yml @@ -163,6 +163,20 @@ services: networks: - internal + appservice_api: + hostname: appservice_api + image: matrixdotorg/dendrite:appservice + command: [ + "--config=dendrite.yaml" + ] + volumes: + - ./config:/etc/dendrite + networks: + - internal + depends_on: + - room_server + - user_api + networks: internal: attachable: true diff --git a/build/docker/images-build.sh b/build/docker/images-build.sh index e43d2d040..9ee5a09de 100755 --- a/build/docker/images-build.sh +++ b/build/docker/images-build.sh @@ -6,6 +6,7 @@ docker build -f build/docker/Dockerfile -t matrixdotorg/dendrite:latest . docker build -t matrixdotorg/dendrite:monolith --build-arg component=dendrite-monolith-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite:appservice --build-arg component=dendrite-appservice-server -f build/docker/Dockerfile.component . docker build -t matrixdotorg/dendrite:clientapi --build-arg component=dendrite-client-api-server -f build/docker/Dockerfile.component . docker build -t matrixdotorg/dendrite:clientproxy --build-arg component=client-api-proxy -f build/docker/Dockerfile.component . docker build -t matrixdotorg/dendrite:eduserver --build-arg component=dendrite-edu-server -f build/docker/Dockerfile.component . diff --git a/build/docker/images-pull.sh b/build/docker/images-pull.sh index edccf4a33..da08a7325 100755 --- a/build/docker/images-pull.sh +++ b/build/docker/images-pull.sh @@ -2,6 +2,7 @@ docker pull matrixdotorg/dendrite:monolith +docker pull matrixdotorg/dendrite:appservice docker pull matrixdotorg/dendrite:clientapi docker pull matrixdotorg/dendrite:clientproxy docker pull matrixdotorg/dendrite:eduserver diff --git a/build/docker/images-push.sh b/build/docker/images-push.sh index 2b4303872..d8f8758a0 100755 --- a/build/docker/images-push.sh +++ b/build/docker/images-push.sh @@ -2,6 +2,7 @@ docker push matrixdotorg/dendrite:monolith +docker push matrixdotorg/dendrite:appservice docker push matrixdotorg/dendrite:clientapi docker push matrixdotorg/dendrite:clientproxy docker push matrixdotorg/dendrite:eduserver diff --git a/build/scripts/build-test-lint.sh b/build/scripts/build-test-lint.sh index 4b18ca2f8..8f0b775b1 100755 --- a/build/scripts/build-test-lint.sh +++ b/build/scripts/build-test-lint.sh @@ -10,7 +10,7 @@ set -eu echo "Checking that it builds..." go build ./cmd/... -./scripts/find-lint.sh +./build/scripts/find-lint.sh echo "Testing..." go test -v ./... diff --git a/cmd/dendrite-user-api-server/main.go b/cmd/dendrite-user-api-server/main.go new file mode 100644 index 000000000..4257da3f3 --- /dev/null +++ b/cmd/dendrite-user-api-server/main.go @@ -0,0 +1,35 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "github.com/matrix-org/dendrite/internal/setup" + "github.com/matrix-org/dendrite/userapi" +) + +func main() { + cfg := setup.ParseFlags(false) + base := setup.NewBaseDendrite(cfg, "UserAPI", true) + defer base.Close() // nolint: errcheck + + accountDB := base.CreateAccountsDB() + deviceDB := base.CreateDeviceDB() + + userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, cfg.Derived.ApplicationServices) + + userapi.AddInternalRoutes(base.InternalAPIMux, userAPI) + + base.SetupAndServeHTTP(string(base.Cfg.Bind.UserAPI), string(base.Cfg.Listen.UserAPI)) +} From a1352cbd9ed156da546ac67ae5194b30f607fed6 Mon Sep 17 00:00:00 2001 From: Martin Honermeyer Date: Tue, 23 Jun 2020 10:51:38 +0200 Subject: [PATCH 8/8] Push serverkeyapi docker image to registry as well (#1154) Signed-off-by: Martin Honermeyer --- build/docker/images-push.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/build/docker/images-push.sh b/build/docker/images-push.sh index d8f8758a0..1ac60b921 100755 --- a/build/docker/images-push.sh +++ b/build/docker/images-push.sh @@ -14,4 +14,5 @@ docker push matrixdotorg/dendrite:mediaapi docker push matrixdotorg/dendrite:publicroomsapi docker push matrixdotorg/dendrite:roomserver docker push matrixdotorg/dendrite:syncapi +docker push matrixdotorg/dendrite:serverkeyapi docker push matrixdotorg/dendrite:userapi