From cdd1fdcd3d791cfb965506ac28d96cf111043085 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 25 Jun 2017 01:06:07 +0100 Subject: [PATCH 1/4] trivial typo in usage output --- src/github.com/matrix-org/dendrite/cmd/client-api-proxy/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/github.com/matrix-org/dendrite/cmd/client-api-proxy/main.go b/src/github.com/matrix-org/dendrite/cmd/client-api-proxy/main.go index 8aab6575d..f2b64d44a 100644 --- a/src/github.com/matrix-org/dendrite/cmd/client-api-proxy/main.go +++ b/src/github.com/matrix-org/dendrite/cmd/client-api-proxy/main.go @@ -106,7 +106,7 @@ func main() { if *syncServerURL == "" { flag.Usage() - fmt.Fprintln(os.Stderr, "no --sync-server-url specified.") + fmt.Fprintln(os.Stderr, "no --sync-api-server-url specified.") os.Exit(1) } From eb029116b0a0bf15523b43a3eacf99d339546962 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 25 Jun 2017 01:20:04 +0100 Subject: [PATCH 2/4] trivial fix to actually parse fed-api-server's CLI args --- .../dendrite/cmd/dendrite-federation-api-server/main.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/github.com/matrix-org/dendrite/cmd/dendrite-federation-api-server/main.go b/src/github.com/matrix-org/dendrite/cmd/dendrite-federation-api-server/main.go index f9576ce90..f4f19cdcd 100644 --- a/src/github.com/matrix-org/dendrite/cmd/dendrite-federation-api-server/main.go +++ b/src/github.com/matrix-org/dendrite/cmd/dendrite-federation-api-server/main.go @@ -38,6 +38,8 @@ var ( func main() { common.SetupLogging(logDir) + flag.Parse() + if *configPath == "" { log.Fatal("--config must be supplied") } From 524475f8a3a3efa3cca53d2be7f3a71f7d6bb2eb Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 27 Jun 2017 12:37:25 +0100 Subject: [PATCH 3/4] Stub APIs needed to let riot join a room (#147) * Stub APIs needed to let riot join a room * Fix comments --- .../dendrite/clientapi/jsonerror/jsonerror.go | 6 ++ .../dendrite/clientapi/readers/directory.go | 87 ++++++++++++++++ .../dendrite/clientapi/routing/routing.go | 99 +++++++++++++++++++ 3 files changed, 192 insertions(+) create mode 100644 src/github.com/matrix-org/dendrite/clientapi/readers/directory.go diff --git a/src/github.com/matrix-org/dendrite/clientapi/jsonerror/jsonerror.go b/src/github.com/matrix-org/dendrite/clientapi/jsonerror/jsonerror.go index b271eeb65..3a424ddeb 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/jsonerror/jsonerror.go +++ b/src/github.com/matrix-org/dendrite/clientapi/jsonerror/jsonerror.go @@ -85,6 +85,12 @@ func WeakPassword(msg string) *MatrixError { return &MatrixError{"M_WEAK_PASSWORD", msg} } +// GuestAccessForbidden is an error which is returned when the client is +// forbidden from accessing a resource as a guest. +func GuestAccessForbidden(msg string) *MatrixError { + return &MatrixError{"M_GUEST_ACCESS_FORBIDDEN", msg} +} + // LimitExceededError is a rate-limiting error. type LimitExceededError struct { MatrixError diff --git a/src/github.com/matrix-org/dendrite/clientapi/readers/directory.go b/src/github.com/matrix-org/dendrite/clientapi/readers/directory.go new file mode 100644 index 000000000..db8e75247 --- /dev/null +++ b/src/github.com/matrix-org/dendrite/clientapi/readers/directory.go @@ -0,0 +1,87 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package readers + +import ( + "fmt" + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/clientapi/httputil" + "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/gomatrix" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" + "net/http" + "strings" +) + +// DirectoryRoom looks up a room alias +func DirectoryRoom( + req *http.Request, + device *authtypes.Device, + roomAlias string, + federation *gomatrixserverlib.FederationClient, + cfg *config.Dendrite, +) util.JSONResponse { + domain, err := domainFromID(roomAlias) + if err != nil { + return util.JSONResponse{ + Code: 400, + JSON: jsonerror.BadJSON("Room alias must be in the form '#localpart:domain'"), + } + } + + if domain == cfg.Matrix.ServerName { + // TODO: Implement lookup up local room aliases. + panic(fmt.Errorf("Looking up local room aliases is not implemented")) + } else { + resp, err := federation.LookupRoomAlias(domain, roomAlias) + if err != nil { + switch x := err.(type) { + case gomatrix.HTTPError: + if x.Code == 404 { + return util.JSONResponse{ + Code: 404, + JSON: jsonerror.NotFound("Room alias not found"), + } + } + } + // TODO: Return 502 if the remote server errored. + // TODO: Return 504 if the remote server timed out. + return httputil.LogThenError(req, err) + } + + return util.JSONResponse{ + Code: 200, + JSON: resp, + } + } +} + +// domainFromID returns everything after the first ":" character to extract +// the domain part of a matrix ID. +// TODO: duplicated from gomatrixserverlib. +func domainFromID(id string) (gomatrixserverlib.ServerName, error) { + // IDs have the format: SIGIL LOCALPART ":" DOMAIN + // Split on the first ":" character since the domain can contain ":" + // characters. + parts := strings.SplitN(id, ":", 2) + if len(parts) != 2 { + // The ID must have a ":" character. + return "", fmt.Errorf("invalid ID: %q", id) + } + // Return everything after the first ":" character. + return gomatrixserverlib.ServerName(parts[1]), nil +} diff --git a/src/github.com/matrix-org/dendrite/clientapi/routing/routing.go b/src/github.com/matrix-org/dendrite/clientapi/routing/routing.go index f6c0989fb..ce895c4bc 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/routing/routing.go +++ b/src/github.com/matrix-org/dendrite/clientapi/routing/routing.go @@ -22,6 +22,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" + "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/clientapi/readers" "github.com/matrix-org/dendrite/clientapi/writers" @@ -34,6 +35,7 @@ import ( ) const pathPrefixR0 = "/_matrix/client/r0" +const pathPrefixUnstable = "/_matrix/client/unstable" // Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client // to clients which need to make outbound HTTP requests. @@ -46,7 +48,25 @@ func Setup( keyRing gomatrixserverlib.KeyRing, ) { apiMux := mux.NewRouter() + + apiMux.Handle("/_matrix/client/versions", + common.MakeAPI("versions", func(req *http.Request) util.JSONResponse { + return util.JSONResponse{ + Code: 200, + JSON: struct { + Versions []string `json:"versions"` + }{[]string{ + "r0.0.1", + "r0.1.0", + "r0.2.0", + }}, + } + }), + ) + r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter() + unstableMux := apiMux.PathPrefix(pathPrefixUnstable).Subrouter() + r0mux.Handle("/createRoom", common.MakeAuthAPI("createRoom", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return writers.CreateRoom(req, device, cfg, producer) @@ -85,6 +105,13 @@ func Setup( return writers.Register(req, accountDB, deviceDB) })) + r0mux.Handle("/directory/room/{roomAlias}", + common.MakeAuthAPI("directory_room", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars := mux.Vars(req) + return readers.DirectoryRoom(req, device, vars["roomAlias"], federation, &cfg) + }), + ) + // Stub endpoints required by Riot r0mux.Handle("/login", @@ -166,6 +193,78 @@ func Setup( }), ) + r0mux.Handle("/voip/turnServer", + common.MakeAPI("turn_server", func(req *http.Request) util.JSONResponse { + // TODO: Return credentials for a turn server if one is configured. + return util.JSONResponse{ + Code: 200, + JSON: struct{}{}, + } + }), + ) + + r0mux.Handle("/publicRooms", + common.MakeAPI("public_rooms", func(req *http.Request) util.JSONResponse { + // TODO: Return a list of public rooms + return util.JSONResponse{ + Code: 200, + JSON: struct { + Chunk []struct{} `json:"chunk"` + Start string `json:"start"` + End string `json:"end"` + }{[]struct{}{}, "", ""}, + } + }), + ) + + unstableMux.Handle("/thirdparty/protocols", + common.MakeAPI("thirdparty_protocols", func(req *http.Request) util.JSONResponse { + // TODO: Return the third party protcols + return util.JSONResponse{ + Code: 200, + JSON: struct{}{}, + } + }), + ) + + r0mux.Handle("/rooms/{roomID}/initialSync", + common.MakeAPI("rooms_initial_sync", func(req *http.Request) util.JSONResponse { + // TODO: Allow people to peek into rooms. + return util.JSONResponse{ + Code: 403, + JSON: jsonerror.GuestAccessForbidden("Guest access not implemented"), + } + }), + ) + + r0mux.Handle("/profile/{userID}/displayname", + common.MakeAPI("profile_displayname", func(req *http.Request) util.JSONResponse { + // TODO: Set and get the displayname + return util.JSONResponse{Code: 200, JSON: struct{}{}} + }), + ) + + r0mux.Handle("/user/{userID}/account_data/{type}", + common.MakeAPI("user_account_data", func(req *http.Request) util.JSONResponse { + // TODO: Set and get the account_data + return util.JSONResponse{Code: 200, JSON: struct{}{}} + }), + ) + + r0mux.Handle("/rooms/{roomID}/read_markers", + common.MakeAPI("rooms_read_markers", func(req *http.Request) util.JSONResponse { + // TODO: return the read_markers. + return util.JSONResponse{Code: 200, JSON: struct{}{}} + }), + ) + + r0mux.Handle("/rooms/{roomID}/typing/{userID}", + common.MakeAPI("rooms_typing", func(req *http.Request) util.JSONResponse { + // TODO: handling typing + return util.JSONResponse{Code: 200, JSON: struct{}{}} + }), + ) + servMux.Handle("/metrics", prometheus.Handler()) servMux.Handle("/api/", http.StripPrefix("/api", apiMux)) } From 572f6c399d9fe4b4d513ad65a276b32847ebd6ba Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 27 Jun 2017 13:20:04 +0100 Subject: [PATCH 4/4] Include the state before the event in roomserver output (#143) * Include the state before the event in roomserver output * Fix roomserver integration test * Remove VisibilityEventIDs from the JSON * More comments * Remove spurious TODO --- .../cmd/roomserver-integration-tests/main.go | 3 +- .../dendrite/roomserver/api/output.go | 71 +++++++++++++------ .../roomserver/input/latest_events.go | 35 ++++++++- 3 files changed, 83 insertions(+), 26 deletions(-) diff --git a/src/github.com/matrix-org/dendrite/cmd/roomserver-integration-tests/main.go b/src/github.com/matrix-org/dendrite/cmd/roomserver-integration-tests/main.go index 940293f8e..b303a06ba 100644 --- a/src/github.com/matrix-org/dendrite/cmd/roomserver-integration-tests/main.go +++ b/src/github.com/matrix-org/dendrite/cmd/roomserver-integration-tests/main.go @@ -340,7 +340,8 @@ func main() { "state_key":"@richvdh:matrix.org", "type":"m.room.member" }, - "VisibilityEventIDs":null, + "StateBeforeRemovesEventIDs":["$1463671339126270PnVwC:matrix.org"], + "StateBeforeAddsEventIDs":null, "LatestEventIDs":["$1463671339126270PnVwC:matrix.org"], "AddsStateEventIDs":["$1463671337126266wrSBX:matrix.org", "$1463671339126270PnVwC:matrix.org"], "RemovesStateEventIDs":null, diff --git a/src/github.com/matrix-org/dendrite/roomserver/api/output.go b/src/github.com/matrix-org/dendrite/roomserver/api/output.go index 29e76ed86..0b2aee64e 100644 --- a/src/github.com/matrix-org/dendrite/roomserver/api/output.go +++ b/src/github.com/matrix-org/dendrite/roomserver/api/output.go @@ -19,12 +19,17 @@ import ( ) // An OutputRoomEvent is written when the roomserver receives a new event. +// It contains the full matrix room event and enough information for a +// consumer to construct the current state of the room and the state before the +// event. +// +// When we talk about state in a matrix room we are talking about the state +// after a list of events. The current state is the state after the latest +// event IDs in the room. The state before an event is the state after its +// prev_events. type OutputRoomEvent struct { // The JSON bytes of the event. Event []byte - // The state event IDs needed to determine who can see this event. - // This can be used to tell which users to send the event to. - VisibilityEventIDs []string // The latest events in the room after this event. // This can be used to set the prev events for new events in the room. // This also can be used to get the full current state after this event. @@ -40,9 +45,27 @@ type OutputRoomEvent struct { // This is used by consumers to check if they can safely update their // current state using the delta supplied in AddsStateEventIDs and // RemovesStateEventIDs. + // // If the LastSentEventID doesn't match what they were expecting it to be // they can use the LatestEventIDs to request the full current state. LastSentEventID string + // The state event IDs that are part of the state at the event, but not + // part of the current state. Together with the StateBeforeRemovesEventIDs + // this can be used to construct the state before the event from the + // current state. The StateBeforeAddsEventIDs and StateBeforeRemovesEventIDs + // delta is applied after the AddsStateEventIDs and RemovesStateEventIDs. + // + // Consumers need to know the state at each event in order to determine + // which users and servers are allowed to see the event. This information + // is needed to apply the history visibility rules and to tell which + // servers we need to push events to over federation. + // + // The state is given as a delta against the current state because they are + // usually either the same state, or differ by just a couple of events. + StateBeforeAddsEventIDs []string + // The state event IDs that are part of the current state, but not part + // of the state at the event. + StateBeforeRemovesEventIDs []string } // UnmarshalJSON implements json.Unmarshaller @@ -52,12 +75,13 @@ func (ore *OutputRoomEvent) UnmarshalJSON(data []byte) error { // We use json.RawMessage so that the event JSON is sent as JSON rather than // being base64 encoded which is the default for []byte. var content struct { - Event *json.RawMessage - VisibilityEventIDs []string - LatestEventIDs []string - AddsStateEventIDs []string - RemovesStateEventIDs []string - LastSentEventID string + Event *json.RawMessage + LatestEventIDs []string + AddsStateEventIDs []string + RemovesStateEventIDs []string + LastSentEventID string + StateBeforeAddsEventIDs []string + StateBeforeRemovesEventIDs []string } if err := json.Unmarshal(data, &content); err != nil { return err @@ -65,11 +89,12 @@ func (ore *OutputRoomEvent) UnmarshalJSON(data []byte) error { if content.Event != nil { ore.Event = []byte(*content.Event) } - ore.VisibilityEventIDs = content.VisibilityEventIDs ore.LatestEventIDs = content.LatestEventIDs ore.AddsStateEventIDs = content.AddsStateEventIDs ore.RemovesStateEventIDs = content.RemovesStateEventIDs ore.LastSentEventID = content.LastSentEventID + ore.StateBeforeAddsEventIDs = content.StateBeforeAddsEventIDs + ore.StateBeforeRemovesEventIDs = content.StateBeforeRemovesEventIDs return nil } @@ -81,19 +106,21 @@ func (ore OutputRoomEvent) MarshalJSON() ([]byte, error) { // being base64 encoded which is the default for []byte. event := json.RawMessage(ore.Event) content := struct { - Event *json.RawMessage - VisibilityEventIDs []string - LatestEventIDs []string - AddsStateEventIDs []string - RemovesStateEventIDs []string - LastSentEventID string + Event *json.RawMessage + LatestEventIDs []string + AddsStateEventIDs []string + RemovesStateEventIDs []string + LastSentEventID string + StateBeforeAddsEventIDs []string + StateBeforeRemovesEventIDs []string }{ - Event: &event, - VisibilityEventIDs: ore.VisibilityEventIDs, - LatestEventIDs: ore.LatestEventIDs, - AddsStateEventIDs: ore.AddsStateEventIDs, - RemovesStateEventIDs: ore.RemovesStateEventIDs, - LastSentEventID: ore.LastSentEventID, + Event: &event, + LatestEventIDs: ore.LatestEventIDs, + AddsStateEventIDs: ore.AddsStateEventIDs, + RemovesStateEventIDs: ore.RemovesStateEventIDs, + LastSentEventID: ore.LastSentEventID, + StateBeforeAddsEventIDs: ore.StateBeforeAddsEventIDs, + StateBeforeRemovesEventIDs: ore.StateBeforeRemovesEventIDs, } return json.Marshal(&content) } diff --git a/src/github.com/matrix-org/dendrite/roomserver/input/latest_events.go b/src/github.com/matrix-org/dendrite/roomserver/input/latest_events.go index ad5c15a7e..ce7ac0bf6 100644 --- a/src/github.com/matrix-org/dendrite/roomserver/input/latest_events.go +++ b/src/github.com/matrix-org/dendrite/roomserver/input/latest_events.go @@ -20,6 +20,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/state" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" ) // updateLatestEvents updates the list of latest events for this room in the database and writes the @@ -110,6 +111,13 @@ func doUpdateLatestEvents( return err } + stateBeforeEventRemoves, stateBeforeEventAdds, err := state.DifferenceBetweeenStateSnapshots( + db, newStateNID, stateAtEvent.BeforeStateSnapshotNID, + ) + if err != nil { + return err + } + // Send the event to the output logs. // We do this inside the database transaction to ensure that we only mark an event as sent if we sent it. // (n.b. this means that it's possible that the same event will be sent twice if the transaction fails but @@ -118,7 +126,10 @@ func doUpdateLatestEvents( // send the event asynchronously but we would need to ensure that 1) the events are written to the log in // the correct order, 2) that pending writes are resent across restarts. In order to avoid writing all the // necessary bookkeeping we'll keep the event sending synchronous for now. - if err = writeEvent(db, ow, lastEventIDSent, event, newLatest, removed, added); err != nil { + if err = writeEvent( + db, ow, lastEventIDSent, event, newLatest, removed, added, + stateBeforeEventRemoves, stateBeforeEventAdds, + ); err != nil { return err } @@ -170,6 +181,7 @@ func writeEvent( db RoomEventDatabase, ow OutputRoomEventWriter, lastEventIDSent string, event gomatrixserverlib.Event, latest []types.StateAtEventAndReference, removed, added []types.StateEntry, + stateBeforeEventRemoves, stateBeforeEventAdds []types.StateEntry, ) error { latestEventIDs := make([]string, len(latest)) @@ -190,6 +202,13 @@ func writeEvent( for _, entry := range removed { stateEventNIDs = append(stateEventNIDs, entry.EventNID) } + for _, entry := range stateBeforeEventRemoves { + stateEventNIDs = append(stateEventNIDs, entry.EventNID) + } + for _, entry := range stateBeforeEventAdds { + stateEventNIDs = append(stateEventNIDs, entry.EventNID) + } + stateEventNIDs = stateEventNIDs[:util.SortAndUnique(eventNIDSorter(stateEventNIDs))] eventIDMap, err := db.EventIDs(stateEventNIDs) if err != nil { return err @@ -200,7 +219,17 @@ func writeEvent( for _, entry := range removed { ore.RemovesStateEventIDs = append(ore.RemovesStateEventIDs, eventIDMap[entry.EventNID]) } - - // TODO: Fill out VisibilityStateIDs + for _, entry := range stateBeforeEventRemoves { + ore.StateBeforeRemovesEventIDs = append(ore.StateBeforeRemovesEventIDs, eventIDMap[entry.EventNID]) + } + for _, entry := range stateBeforeEventAdds { + ore.StateBeforeAddsEventIDs = append(ore.StateBeforeAddsEventIDs, eventIDMap[entry.EventNID]) + } return ow.WriteOutputRoomEvent(ore) } + +type eventNIDSorter []types.EventNID + +func (s eventNIDSorter) Len() int { return len(s) } +func (s eventNIDSorter) Less(i, j int) bool { return s[i] < s[j] } +func (s eventNIDSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] }