From ec7a0e42aea91d881eea8be7ca9fdf2cd69eeaef Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 3 Dec 2020 10:55:17 +0000 Subject: [PATCH 01/10] Simplify create-account (#1608) --- cmd/create-account/main.go | 39 +++++++++++++++----------------------- 1 file changed, 15 insertions(+), 24 deletions(-) diff --git a/cmd/create-account/main.go b/cmd/create-account/main.go index 6aa1e7281..bba2d55d6 100644 --- a/cmd/create-account/main.go +++ b/cmd/create-account/main.go @@ -20,24 +20,27 @@ import ( "fmt" "os" + "github.com/matrix-org/dendrite/setup" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/userapi/storage/accounts" - "github.com/matrix-org/gomatrixserverlib" + "github.com/sirupsen/logrus" ) const usage = `Usage: %s -Generate a new Matrix account for testing purposes. +Creates a new user account on the homeserver. + +Example: + + ./create-account --config dendrite.yaml --username alice --password foobarbaz Arguments: ` var ( - database = flag.String("database", "", "The location of the account database.") - username = flag.String("username", "", "The user ID localpart to register e.g 'alice' in '@alice:localhost'.") - password = flag.String("password", "", "Optional. The password to register with. If not specified, this account will be password-less.") - serverNameStr = flag.String("servername", "localhost", "The Matrix server domain which will form the domain part of the user ID.") + username = flag.String("username", "", "The username of the account to register (specify the localpart only, e.g. 'alice' for '@alice:domain.com')") + password = flag.String("password", "", "The password to associate with the account (optional, account will be password-less if not specified)") ) func main() { @@ -45,36 +48,24 @@ func main() { fmt.Fprintf(os.Stderr, usage, os.Args[0]) flag.PrintDefaults() } - - flag.Parse() + cfg := setup.ParseFlags(true) if *username == "" { flag.Usage() - fmt.Println("Missing --username") os.Exit(1) } - if *database == "" { - flag.Usage() - fmt.Println("Missing --database") - os.Exit(1) - } - - serverName := gomatrixserverlib.ServerName(*serverNameStr) - accountDB, err := accounts.NewDatabase(&config.DatabaseOptions{ - ConnectionString: config.DataSource(*database), - }, serverName) + ConnectionString: cfg.UserAPI.AccountDatabase.ConnectionString, + }, cfg.Global.ServerName) if err != nil { - fmt.Println(err.Error()) - os.Exit(1) + logrus.Fatalln("Failed to connect to the database:", err.Error()) } _, err = accountDB.CreateAccount(context.Background(), *username, *password, "") if err != nil { - fmt.Println(err.Error()) - os.Exit(1) + logrus.Fatalln("Failed to create the account:", err.Error()) } - fmt.Println("Created account") + logrus.Infoln("Created account", *username) } From 2b03d24358aeac14ba7c8c63e35012d6e91c1509 Mon Sep 17 00:00:00 2001 From: alexkursell Date: Thu, 3 Dec 2020 06:01:49 -0500 Subject: [PATCH 02/10] Fix /joined_members API response (#1606) * Fix /joined_members API response * Fix golint issue --- clientapi/routing/memberships.go | 11 +++++++++-- sytest-whitelist | 1 + 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/clientapi/routing/memberships.go b/clientapi/routing/memberships.go index 457f81cbd..513fcefd7 100644 --- a/clientapi/routing/memberships.go +++ b/clientapi/routing/memberships.go @@ -44,6 +44,13 @@ type joinedMember struct { AvatarURL string `json:"avatar_url"` } +// The database stores 'displayname' without an underscore. +// Deserialize into this and then change to the actual API response +type databaseJoinedMember struct { + DisplayName string `json:"displayname"` + AvatarURL string `json:"avatar_url"` +} + // GetMemberships implements GET /rooms/{roomId}/members func GetMemberships( req *http.Request, device *userapi.Device, roomID string, joinedOnly bool, @@ -72,12 +79,12 @@ func GetMemberships( var res getJoinedMembersResponse res.Joined = make(map[string]joinedMember) for _, ev := range queryRes.JoinEvents { - var content joinedMember + var content databaseJoinedMember if err := json.Unmarshal(ev.Content, &content); err != nil { util.GetLogger(req.Context()).WithError(err).Error("failed to unmarshal event content") return jsonerror.InternalServerError() } - res.Joined[ev.Sender] = content + res.Joined[ev.Sender] = joinedMember(content) } return util.JSONResponse{ Code: http.StatusOK, diff --git a/sytest-whitelist b/sytest-whitelist index ffcb1785a..17027041d 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -503,3 +503,4 @@ Forgetting room does not show up in v2 /sync Can forget room you've been kicked from Can re-join room if re-invited /whois +/joined_members return joined members From be7d8595be0533207f8942b129c16f3844550712 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 3 Dec 2020 11:11:46 +0000 Subject: [PATCH 03/10] Peeking updates (#1607) * Add unpeek * Don't allow peeks into encrypted rooms * Fix send tests * Update consumers --- clientapi/routing/peekroom.go | 25 ++++ clientapi/routing/routing.go | 14 +++ federationapi/routing/send_test.go | 7 ++ roomserver/api/api.go | 6 + roomserver/api/api_trace.go | 9 ++ roomserver/api/output.go | 11 ++ roomserver/api/perform.go | 11 ++ roomserver/internal/api.go | 8 ++ roomserver/internal/perform/perform_peek.go | 10 +- roomserver/internal/perform/perform_unpeek.go | 118 ++++++++++++++++++ roomserver/inthttp/client.go | 18 +++ roomserver/inthttp/server.go | 11 ++ syncapi/consumers/roomserver.go | 22 ++++ syncapi/storage/interface.go | 3 + syncapi/storage/shared/syncserver.go | 17 +++ syncapi/sync/notifier.go | 12 ++ 16 files changed, 300 insertions(+), 2 deletions(-) create mode 100644 roomserver/internal/perform/perform_unpeek.go diff --git a/clientapi/routing/peekroom.go b/clientapi/routing/peekroom.go index d96f91d0f..26aa64ce1 100644 --- a/clientapi/routing/peekroom.go +++ b/clientapi/routing/peekroom.go @@ -77,3 +77,28 @@ func PeekRoomByIDOrAlias( }{peekRes.RoomID}, } } + +func UnpeekRoomByID( + req *http.Request, + device *api.Device, + rsAPI roomserverAPI.RoomserverInternalAPI, + accountDB accounts.Database, + roomID string, +) util.JSONResponse { + unpeekReq := roomserverAPI.PerformUnpeekRequest{ + RoomID: roomID, + UserID: device.UserID, + DeviceID: device.ID, + } + unpeekRes := roomserverAPI.PerformUnpeekResponse{} + + rsAPI.PerformUnpeek(req.Context(), &unpeekReq, &unpeekRes) + if unpeekRes.Error != nil { + return unpeekRes.Error.JSONResponse() + } + + return util.JSONResponse{ + Code: http.StatusOK, + JSON: struct{}{}, + } +} diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index e6315b68f..8dbfc551d 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -106,6 +106,9 @@ func Setup( ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/peek/{roomIDOrAlias}", httputil.MakeAuthAPI(gomatrixserverlib.Peek, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + if r := rateLimits.rateLimit(req); r != nil { + return *r + } vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -148,6 +151,17 @@ func Setup( ) }), ).Methods(http.MethodPost, http.MethodOptions) + r0mux.Handle("/rooms/{roomID}/unpeek", + httputil.MakeAuthAPI("unpeek", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return UnpeekRoomByID( + req, device, rsAPI, accountDB, vars["roomID"], + ) + }), + ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/ban", httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) diff --git a/federationapi/routing/send_test.go b/federationapi/routing/send_test.go index 9b9db873b..a9ee78830 100644 --- a/federationapi/routing/send_test.go +++ b/federationapi/routing/send_test.go @@ -131,6 +131,13 @@ func (t *testRoomserverAPI) PerformPeek( ) { } +func (t *testRoomserverAPI) PerformUnpeek( + ctx context.Context, + req *api.PerformUnpeekRequest, + res *api.PerformUnpeekResponse, +) { +} + func (t *testRoomserverAPI) PerformPublish( ctx context.Context, req *api.PerformPublishRequest, diff --git a/roomserver/api/api.go b/roomserver/api/api.go index 2683918a8..bef2bb3fa 100644 --- a/roomserver/api/api.go +++ b/roomserver/api/api.go @@ -42,6 +42,12 @@ type RoomserverInternalAPI interface { res *PerformPeekResponse, ) + PerformUnpeek( + ctx context.Context, + req *PerformUnpeekRequest, + res *PerformUnpeekResponse, + ) + PerformPublish( ctx context.Context, req *PerformPublishRequest, diff --git a/roomserver/api/api_trace.go b/roomserver/api/api_trace.go index e625fb04a..eb2b2e1d4 100644 --- a/roomserver/api/api_trace.go +++ b/roomserver/api/api_trace.go @@ -46,6 +46,15 @@ func (t *RoomserverInternalAPITrace) PerformPeek( util.GetLogger(ctx).Infof("PerformPeek req=%+v res=%+v", js(req), js(res)) } +func (t *RoomserverInternalAPITrace) PerformUnpeek( + ctx context.Context, + req *PerformUnpeekRequest, + res *PerformUnpeekResponse, +) { + t.Impl.PerformUnpeek(ctx, req, res) + util.GetLogger(ctx).Infof("PerformUnpeek req=%+v res=%+v", js(req), js(res)) +} + func (t *RoomserverInternalAPITrace) PerformJoin( ctx context.Context, req *PerformJoinRequest, diff --git a/roomserver/api/output.go b/roomserver/api/output.go index fb512a933..2993813cb 100644 --- a/roomserver/api/output.go +++ b/roomserver/api/output.go @@ -51,6 +51,8 @@ const ( // OutputTypeNewPeek indicates that the kafka event is an OutputNewPeek OutputTypeNewPeek OutputType = "new_peek" + // OutputTypeRetirePeek indicates that the kafka event is an OutputRetirePeek + OutputTypeRetirePeek OutputType = "retire_peek" ) // An OutputEvent is an entry in the roomserver output kafka log. @@ -70,6 +72,8 @@ type OutputEvent struct { RedactedEvent *OutputRedactedEvent `json:"redacted_event,omitempty"` // The content of event with type OutputTypeNewPeek NewPeek *OutputNewPeek `json:"new_peek,omitempty"` + // The content of event with type OutputTypeRetirePeek + RetirePeek *OutputRetirePeek `json:"retire_peek,omitempty"` } // Type of the OutputNewRoomEvent. @@ -240,3 +244,10 @@ type OutputNewPeek struct { UserID string DeviceID string } + +// An OutputRetirePeek is written whenever a user stops peeking into a room. +type OutputRetirePeek struct { + RoomID string + UserID string + DeviceID string +} diff --git a/roomserver/api/perform.go b/roomserver/api/perform.go index ec561f11e..ae2d6d975 100644 --- a/roomserver/api/perform.go +++ b/roomserver/api/perform.go @@ -123,6 +123,17 @@ type PerformPeekResponse struct { Error *PerformError } +type PerformUnpeekRequest struct { + RoomID string `json:"room_id"` + UserID string `json:"user_id"` + DeviceID string `json:"device_id"` +} + +type PerformUnpeekResponse struct { + // If non-nil, the join request failed. Contains more information why it failed. + Error *PerformError +} + // PerformBackfillRequest is a request to PerformBackfill. type PerformBackfillRequest struct { // The room to backfill diff --git a/roomserver/internal/api.go b/roomserver/internal/api.go index c825c13d4..1ad971ecb 100644 --- a/roomserver/internal/api.go +++ b/roomserver/internal/api.go @@ -23,6 +23,7 @@ type RoomserverInternalAPI struct { *perform.Inviter *perform.Joiner *perform.Peeker + *perform.Unpeeker *perform.Leaver *perform.Publisher *perform.Backfiller @@ -94,6 +95,13 @@ func (r *RoomserverInternalAPI) SetFederationSenderAPI(fsAPI fsAPI.FederationSen FSAPI: r.fsAPI, Inputer: r.Inputer, } + r.Unpeeker = &perform.Unpeeker{ + ServerName: r.Cfg.Matrix.ServerName, + Cfg: r.Cfg, + DB: r.DB, + FSAPI: r.fsAPI, + Inputer: r.Inputer, + } r.Leaver = &perform.Leaver{ Cfg: r.Cfg, DB: r.DB, diff --git a/roomserver/internal/perform/perform_peek.go b/roomserver/internal/perform/perform_peek.go index 66d1bdb21..2f4694c86 100644 --- a/roomserver/internal/perform/perform_peek.go +++ b/roomserver/internal/perform/perform_peek.go @@ -163,8 +163,7 @@ func (r *Peeker) performPeekRoomByID( // XXX: we should probably factor out history_visibility checks into a common utility method somewhere // which handles the default value etc. var worldReadable = false - ev, _ := r.DB.GetStateEvent(ctx, roomID, "m.room.history_visibility", "") - if ev != nil { + if ev, _ := r.DB.GetStateEvent(ctx, roomID, "m.room.history_visibility", ""); ev != nil { content := map[string]string{} if err = json.Unmarshal(ev.Content(), &content); err != nil { util.GetLogger(ctx).WithError(err).Error("json.Unmarshal for history visibility failed") @@ -182,6 +181,13 @@ func (r *Peeker) performPeekRoomByID( } } + if ev, _ := r.DB.GetStateEvent(ctx, roomID, "m.room.encryption", ""); ev != nil { + return "", &api.PerformError{ + Code: api.PerformErrorNotAllowed, + Msg: "Cannot peek into an encrypted room", + } + } + // TODO: handle federated peeks err = r.Inputer.WriteOutputEvents(roomID, []api.OutputEvent{ diff --git a/roomserver/internal/perform/perform_unpeek.go b/roomserver/internal/perform/perform_unpeek.go new file mode 100644 index 000000000..f71e0007c --- /dev/null +++ b/roomserver/internal/perform/perform_unpeek.go @@ -0,0 +1,118 @@ +// Copyright 2020 New Vector Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package perform + +import ( + "context" + "fmt" + "strings" + + fsAPI "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/internal/input" + "github.com/matrix-org/dendrite/roomserver/storage" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/gomatrixserverlib" +) + +type Unpeeker struct { + ServerName gomatrixserverlib.ServerName + Cfg *config.RoomServer + FSAPI fsAPI.FederationSenderInternalAPI + DB storage.Database + + Inputer *input.Inputer +} + +// PerformPeek handles peeking into matrix rooms, including over federation by talking to the federationsender. +func (r *Unpeeker) PerformUnpeek( + ctx context.Context, + req *api.PerformUnpeekRequest, + res *api.PerformUnpeekResponse, +) { + if err := r.performUnpeek(ctx, req); err != nil { + perr, ok := err.(*api.PerformError) + if ok { + res.Error = perr + } else { + res.Error = &api.PerformError{ + Msg: err.Error(), + } + } + } +} + +func (r *Unpeeker) performUnpeek( + ctx context.Context, + req *api.PerformUnpeekRequest, +) error { + // FIXME: there's way too much duplication with performJoin + _, domain, err := gomatrixserverlib.SplitID('@', req.UserID) + if err != nil { + return &api.PerformError{ + Code: api.PerformErrorBadRequest, + Msg: fmt.Sprintf("Supplied user ID %q in incorrect format", req.UserID), + } + } + if domain != r.Cfg.Matrix.ServerName { + return &api.PerformError{ + Code: api.PerformErrorBadRequest, + Msg: fmt.Sprintf("User %q does not belong to this homeserver", req.UserID), + } + } + if strings.HasPrefix(req.RoomID, "!") { + return r.performUnpeekRoomByID(ctx, req) + } + return &api.PerformError{ + Code: api.PerformErrorBadRequest, + Msg: fmt.Sprintf("Room ID %q is invalid", req.RoomID), + } +} + +func (r *Unpeeker) performUnpeekRoomByID( + _ context.Context, + req *api.PerformUnpeekRequest, +) (err error) { + // Get the domain part of the room ID. + _, _, err = gomatrixserverlib.SplitID('!', req.RoomID) + if err != nil { + return &api.PerformError{ + Code: api.PerformErrorBadRequest, + Msg: fmt.Sprintf("Room ID %q is invalid: %s", req.RoomID, err), + } + } + + // TODO: handle federated peeks + + err = r.Inputer.WriteOutputEvents(req.RoomID, []api.OutputEvent{ + { + Type: api.OutputTypeRetirePeek, + RetirePeek: &api.OutputRetirePeek{ + RoomID: req.RoomID, + UserID: req.UserID, + DeviceID: req.DeviceID, + }, + }, + }) + if err != nil { + return + } + + // By this point, if req.RoomIDOrAlias contained an alias, then + // it will have been overwritten with a room ID by performPeekRoomByAlias. + // We should now include this in the response so that the CS API can + // return the right room ID. + return nil +} diff --git a/roomserver/inthttp/client.go b/roomserver/inthttp/client.go index f5b66ca6a..e496b81e0 100644 --- a/roomserver/inthttp/client.go +++ b/roomserver/inthttp/client.go @@ -27,6 +27,7 @@ const ( // Perform operations RoomserverPerformInvitePath = "/roomserver/performInvite" RoomserverPerformPeekPath = "/roomserver/performPeek" + RoomserverPerformUnpeekPath = "/roomserver/performUnpeek" RoomserverPerformJoinPath = "/roomserver/performJoin" RoomserverPerformLeavePath = "/roomserver/performLeave" RoomserverPerformBackfillPath = "/roomserver/performBackfill" @@ -209,6 +210,23 @@ func (h *httpRoomserverInternalAPI) PerformPeek( } } +func (h *httpRoomserverInternalAPI) PerformUnpeek( + ctx context.Context, + request *api.PerformUnpeekRequest, + response *api.PerformUnpeekResponse, +) { + span, ctx := opentracing.StartSpanFromContext(ctx, "PerformUnpeek") + defer span.Finish() + + apiURL := h.roomserverURL + RoomserverPerformUnpeekPath + err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + if err != nil { + response.Error = &api.PerformError{ + Msg: fmt.Sprintf("failed to communicate with roomserver: %s", err), + } + } +} + func (h *httpRoomserverInternalAPI) PerformLeave( ctx context.Context, request *api.PerformLeaveRequest, diff --git a/roomserver/inthttp/server.go b/roomserver/inthttp/server.go index 2bc8f82df..ac1fc25b6 100644 --- a/roomserver/inthttp/server.go +++ b/roomserver/inthttp/server.go @@ -72,6 +72,17 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) + internalAPIMux.Handle(RoomserverPerformPeekPath, + httputil.MakeInternalAPI("performUnpeek", func(req *http.Request) util.JSONResponse { + var request api.PerformUnpeekRequest + var response api.PerformUnpeekResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + r.PerformUnpeek(req.Context(), &request, &response) + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) internalAPIMux.Handle(RoomserverPerformPublishPath, httputil.MakeInternalAPI("performPublish", func(req *http.Request) util.JSONResponse { var request api.PerformPublishRequest diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index 4d453f133..11d75a683 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -105,6 +105,8 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { return s.onRetireInviteEvent(context.TODO(), *output.RetireInviteEvent) case api.OutputTypeNewPeek: return s.onNewPeek(context.TODO(), *output.NewPeek) + case api.OutputTypeRetirePeek: + return s.onRetirePeek(context.TODO(), *output.RetirePeek) case api.OutputTypeRedactedEvent: return s.onRedactEvent(context.TODO(), *output.RedactedEvent) default: @@ -309,6 +311,26 @@ func (s *OutputRoomEventConsumer) onNewPeek( return nil } +func (s *OutputRoomEventConsumer) onRetirePeek( + ctx context.Context, msg api.OutputRetirePeek, +) error { + sp, err := s.db.DeletePeek(ctx, msg.RoomID, msg.UserID, msg.DeviceID) + if err != nil { + // panic rather than continue with an inconsistent database + log.WithFields(log.Fields{ + log.ErrorKey: err, + }).Panicf("roomserver output log: write peek failure") + return nil + } + // tell the notifier about the new peek so it knows to wake up new devices + s.notifier.OnRetirePeek(msg.RoomID, msg.UserID, msg.DeviceID) + + // we need to wake up the users who might need to now be peeking into this room, + // so we send in a dummy event to trigger a wakeup + s.notifier.OnNewEvent(nil, msg.RoomID, nil, types.NewStreamToken(sp, 0, nil)) + return nil +} + func (s *OutputRoomEventConsumer) updateStateEvent(event *gomatrixserverlib.HeaderedEvent) (*gomatrixserverlib.HeaderedEvent, error) { if event.StateKey() == nil { return event, nil diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index eaa0f64f3..456ca1b1d 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -91,6 +91,9 @@ type Database interface { // AddPeek adds a new peek to our DB for a given room by a given user's device. // Returns an error if there was a problem communicating with the database. AddPeek(ctx context.Context, RoomID, UserID, DeviceID string) (types.StreamPosition, error) + // DeletePeek removes an existing peek from the database for a given room by a user's device. + // Returns an error if there was a problem communicating with the database. + DeletePeek(ctx context.Context, roomID, userID, deviceID string) (sp types.StreamPosition, err error) // DeletePeek deletes all peeks for a given room by a given user // Returns an error if there was a problem communicating with the database. DeletePeeks(ctx context.Context, RoomID, UserID string) (types.StreamPosition, error) diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go index fd8ca0412..6c35a7653 100644 --- a/syncapi/storage/shared/syncserver.go +++ b/syncapi/storage/shared/syncserver.go @@ -178,6 +178,23 @@ func (d *Database) AddPeek( return } +// DeletePeeks tracks the fact that a user has stopped peeking from the specified +// device. If the peeks was successfully deleted this returns the stream ID it was +// stored at. Returns an error if there was a problem communicating with the database. +func (d *Database) DeletePeek( + ctx context.Context, roomID, userID, deviceID string, +) (sp types.StreamPosition, err error) { + err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + sp, err = d.Peeks.DeletePeek(ctx, txn, roomID, userID, deviceID) + return err + }) + if err == sql.ErrNoRows { + sp = 0 + err = nil + } + return +} + // DeletePeeks tracks the fact that a user has stopped peeking from all devices // If the peeks was successfully deleted this returns the stream ID it was stored at. // Returns an error if there was a problem communicating with the database. diff --git a/syncapi/sync/notifier.go b/syncapi/sync/notifier.go index daa3a1d8c..1d8cd624c 100644 --- a/syncapi/sync/notifier.go +++ b/syncapi/sync/notifier.go @@ -137,6 +137,18 @@ func (n *Notifier) OnNewPeek( // by calling OnNewEvent. } +func (n *Notifier) OnRetirePeek( + roomID, userID, deviceID string, +) { + n.streamLock.Lock() + defer n.streamLock.Unlock() + + n.removePeekingDevice(roomID, userID, deviceID) + + // we don't wake up devices here given the roomserver consumer will do this shortly afterwards + // by calling OnRetireEvent. +} + func (n *Notifier) OnNewSendToDevice( userID string, deviceIDs []string, posUpdate types.StreamingToken, From 71327b8efac8d5f35a0fdcf06deb35b6eb1dd474 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 3 Dec 2020 13:22:32 +0000 Subject: [PATCH 04/10] Add FAQ.md --- docs/FAQ.md | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 docs/FAQ.md diff --git a/docs/FAQ.md b/docs/FAQ.md new file mode 100644 index 000000000..5b1cec981 --- /dev/null +++ b/docs/FAQ.md @@ -0,0 +1,52 @@ +# Frequently Asked Questions + +## Is Dendrite stable? + +Mostly, although there are still bugs and missing features. If you are a confident power user and you are happy to spend some time debugging things when they go wrong, then please try out Dendrite. If you are a community, organisation or business that demands stability and uptime, then Dendrite is not for you yet - please install Synapse instead. + +## Is Dendrite feature-complete? + +No, although a good portion of the Matrix specification has been implemented. Mostly missing are client features - see the readme at the root of the repository for more information. + +## Is there a migration path from Synapse to Dendrite? + +No, not at present. There will be in the future when Dendrite reaches version 1.0. + +## I've installed Dendrite but federation isn't working + +Check the [Federation Tester](https://federationtester.matrix.org). You need at least: + +* A valid DNS name +* A valid TLS certificate for that DNS name +* Either DNS SRV records or well-known files + +## Does Dendrite work with my favourite client? + +It should do, although we are aware of some minor issues: + +* **Element Android**: registration does not work, but logging in with an existing account does +* **Hydrogen**: occasionally sync can fail due to gaps in the `since` parameter, but clearing the cache fixes this + +## Does Dendrite support push notifications? + +No, not yet. This is a planned feature. + +## Does Dendrite support application services/bridges? + +Possibly - Dendrite does have some application service support but it is not well tested. Please let us know by raising a GitHub issue if you try it and run into problems. + +## Dendrite is using a lot of CPU + +Generally speaking, you should expect to see some CPU spikes, particularly if you are joining or participating in large rooms. However, constant/sustained high CPU usage is not expected - if you are experiencing that, please join `#dendrite-dev:matrix.org` and let us know, or file a GitHub issue. + +## Dendrite is using a lot of RAM + +A lot of users report that Dendrite is using a lot of RAM, sometimes even gigabytes of it. This is usually due to Go's allocator behaviour, which tries to hold onto allocated memory until the operating system wants to reclaim it for something else. This can make the memory usage look significantly inflated in tools like `top`/`htop` when actually most of that memory is not really in use at all. + +If you want to prevent this behaviour so that the Go runtime releases memory normally, start Dendrite using the `GODEBUG=madvdontneed=1` environment variable. It is also expected that the allocator behaviour will be changed again in Go 1.16 so that it does not hold onto memory unnecessarily in this way. + +If you are running with `GODEBUG=madvdontneed=1` and still see hugely inflated memory usage then that's quite possibly a bug - please join `#dendrite-dev:matrix.org` and let us know, or file a GitHub issue. + +## Dendrite is running out of PostgreSQL database connections + +You may need to revisit the connection limit of your PostgreSQL server and/or make changes to the `max_connections` lines in your Dendrite configuration. Be aware that each Dendrite component opens its own database connections and has its own connection limit, even in monolith mode! \ No newline at end of file From a4bf9921add5bc33d283e277c31868a6e79f1e89 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 3 Dec 2020 13:27:48 +0000 Subject: [PATCH 05/10] Update FAQ.md --- docs/FAQ.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/FAQ.md b/docs/FAQ.md index 5b1cec981..f69b20266 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -35,6 +35,14 @@ No, not yet. This is a planned feature. Possibly - Dendrite does have some application service support but it is not well tested. Please let us know by raising a GitHub issue if you try it and run into problems. +## Is it possible to prevent communication with the outside world? + +Yes, you can do this by disabling federation - set `disable_federation` to `true` in the `global` section of the Dendrite configuration file. + +## Should I use PostgreSQL or SQLite for my databases? + +Please use PostgreSQL wherever possible, especially if you are planning to run a homeserver that caters to more than a couple of users. + ## Dendrite is using a lot of CPU Generally speaking, you should expect to see some CPU spikes, particularly if you are joining or participating in large rooms. However, constant/sustained high CPU usage is not expected - if you are experiencing that, please join `#dendrite-dev:matrix.org` and let us know, or file a GitHub issue. @@ -49,4 +57,4 @@ If you are running with `GODEBUG=madvdontneed=1` and still see hugely inflated m ## Dendrite is running out of PostgreSQL database connections -You may need to revisit the connection limit of your PostgreSQL server and/or make changes to the `max_connections` lines in your Dendrite configuration. Be aware that each Dendrite component opens its own database connections and has its own connection limit, even in monolith mode! \ No newline at end of file +You may need to revisit the connection limit of your PostgreSQL server and/or make changes to the `max_connections` lines in your Dendrite configuration. Be aware that each Dendrite component opens its own database connections and has its own connection limit, even in monolith mode! From eef8f88092ffbe894dc0bbeee487315a05416577 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 3 Dec 2020 13:28:41 +0000 Subject: [PATCH 06/10] Update FAQ.md --- docs/FAQ.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/FAQ.md b/docs/FAQ.md index f69b20266..9267aed32 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -1,18 +1,18 @@ # Frequently Asked Questions -## Is Dendrite stable? +### Is Dendrite stable? Mostly, although there are still bugs and missing features. If you are a confident power user and you are happy to spend some time debugging things when they go wrong, then please try out Dendrite. If you are a community, organisation or business that demands stability and uptime, then Dendrite is not for you yet - please install Synapse instead. -## Is Dendrite feature-complete? +### Is Dendrite feature-complete? No, although a good portion of the Matrix specification has been implemented. Mostly missing are client features - see the readme at the root of the repository for more information. -## Is there a migration path from Synapse to Dendrite? +### Is there a migration path from Synapse to Dendrite? No, not at present. There will be in the future when Dendrite reaches version 1.0. -## I've installed Dendrite but federation isn't working +### I've installed Dendrite but federation isn't working Check the [Federation Tester](https://federationtester.matrix.org). You need at least: @@ -20,34 +20,34 @@ Check the [Federation Tester](https://federationtester.matrix.org). You need at * A valid TLS certificate for that DNS name * Either DNS SRV records or well-known files -## Does Dendrite work with my favourite client? +### Does Dendrite work with my favourite client? It should do, although we are aware of some minor issues: * **Element Android**: registration does not work, but logging in with an existing account does * **Hydrogen**: occasionally sync can fail due to gaps in the `since` parameter, but clearing the cache fixes this -## Does Dendrite support push notifications? +### Does Dendrite support push notifications? No, not yet. This is a planned feature. -## Does Dendrite support application services/bridges? +### Does Dendrite support application services/bridges? Possibly - Dendrite does have some application service support but it is not well tested. Please let us know by raising a GitHub issue if you try it and run into problems. -## Is it possible to prevent communication with the outside world? +### Is it possible to prevent communication with the outside world? Yes, you can do this by disabling federation - set `disable_federation` to `true` in the `global` section of the Dendrite configuration file. -## Should I use PostgreSQL or SQLite for my databases? +### Should I use PostgreSQL or SQLite for my databases? Please use PostgreSQL wherever possible, especially if you are planning to run a homeserver that caters to more than a couple of users. -## Dendrite is using a lot of CPU +### Dendrite is using a lot of CPU Generally speaking, you should expect to see some CPU spikes, particularly if you are joining or participating in large rooms. However, constant/sustained high CPU usage is not expected - if you are experiencing that, please join `#dendrite-dev:matrix.org` and let us know, or file a GitHub issue. -## Dendrite is using a lot of RAM +### Dendrite is using a lot of RAM A lot of users report that Dendrite is using a lot of RAM, sometimes even gigabytes of it. This is usually due to Go's allocator behaviour, which tries to hold onto allocated memory until the operating system wants to reclaim it for something else. This can make the memory usage look significantly inflated in tools like `top`/`htop` when actually most of that memory is not really in use at all. @@ -55,6 +55,6 @@ If you want to prevent this behaviour so that the Go runtime releases memory nor If you are running with `GODEBUG=madvdontneed=1` and still see hugely inflated memory usage then that's quite possibly a bug - please join `#dendrite-dev:matrix.org` and let us know, or file a GitHub issue. -## Dendrite is running out of PostgreSQL database connections +### Dendrite is running out of PostgreSQL database connections You may need to revisit the connection limit of your PostgreSQL server and/or make changes to the `max_connections` lines in your Dendrite configuration. Be aware that each Dendrite component opens its own database connections and has its own connection limit, even in monolith mode! From 246866a13184bd382eb79e3944511fbe717802d8 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 3 Dec 2020 13:46:56 +0000 Subject: [PATCH 07/10] Add PROFILING.md --- docs/PROFILING.md | 79 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 docs/PROFILING.md diff --git a/docs/PROFILING.md b/docs/PROFILING.md new file mode 100644 index 000000000..9184d1b13 --- /dev/null +++ b/docs/PROFILING.md @@ -0,0 +1,79 @@ +# Profiling Dendrite + +If you are running into problems with Dendrite using excessive resources (e.g. CPU or RAM) then you can use the profiler to work out what is happening. + +Dendrite contains an embedded profiler called `pprof`, which is a part of the standard Go toolchain. + +### Enable the profiler + +To enable the profiler, start Dendrite with the `PPROFLISTEN` environment variable. This variable specifies which address and port to listen on, e.g. + +``` +PPROFLISTEN=localhost:65432 ./bin/dendrite-monolith-server ... +``` + +If pprof has been enabled successfully, a log line at startup will show that pprof is listening: + +``` +WARN[2020-12-03T13:32:33.669405000Z] [/Users/neilalexander/Desktop/dendrite/internal/log.go:87] SetupPprof + Starting pprof on localhost:65432 +``` + +### Profiling CPU usage + +To examine where CPU time is going, you can call the `profile` endpoint: + +``` +http://localhost:65432/debug/pprof/profile?seconds=30 +``` + +The profile will run for the specified number of `seconds` and then will produce a result. + +If you have Go installed and want to explore the profile, you can invoke `go tool pprof` to start the profile directly. The `-http=` parameter will instruct `go tool pprof` to start a web server providing a view of the captured profile: + +``` +go tool pprof -http=localhost:23456 http://localhost:65432/debug/pprof/profile?seconds=30 +``` + +You can then visit `http://localhost:23456` in your web browser to see a visual representation of the profile. Particularly usefully, in the "View" menu, you can select "Flame Graph" to see a proportional interactive graph of CPU usage. + +If you don't have the Go tools installed but just want to capture the profile to send to someone else, you can instead use `curl` to download the profiler results: + +``` +curl -O http://localhost:65432/debug/pprof/profile?seconds=30 +``` + +This will block for the specified number of seconds, capturing information about what Dendrite is doing, and then produces a `profile` file, which you can send onward. + +### Profiling memory usage + +To examine where memory usage is going, you can call the `heap` endpoint: + +``` +http://localhost:65432/debug/pprof/heap +``` + +The profile will return almost instantly. + +If you have Go installed and want to explore the profile, you can invoke `go tool pprof` to start the profile directly. The `-http=` parameter will instruct `go tool pprof` to start a web server providing a view of the captured profile: + +``` +go tool pprof -http=localhost:23456 http://localhost:65432/debug/pprof/heap +``` + +You can then visit `http://localhost:23456` in your web browser to see a visual representation of the profile. The "Sample" menu lets you select between four different memory profiles: + +* `inuse_space`: Shows how much actual heap memory is allocated per function (this is generally the most useful profile when diagnosing high memory usage) +* `inuse_objects`: Shows how many heap objects are allocated per function +* `alloc_space`: Shows how much memory has been allocated per function (although that memory may have since been deallocated) +* `alloc_objects`: Shows how many allocations have been made per function (although that memory may have since been deallocated) + +Also in the "View" menu, you can select "Flame Graph" to see a proportional interactive graph of the memory usage. + +If you don't have the Go tools installed but just want to capture the profile to send to someone else, you can instead use `curl` to download the profiler results: + +``` +curl -O http://localhost:65432/debug/pprof/heap +`` + +This will almost instantly produce a `heap` file, which you can send onward. From 253b05ccde45730a9149626c3af4ee97e8517d9f Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 3 Dec 2020 13:48:02 +0000 Subject: [PATCH 08/10] Update PROFILING.md --- docs/PROFILING.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/PROFILING.md b/docs/PROFILING.md index 9184d1b13..136a757b9 100644 --- a/docs/PROFILING.md +++ b/docs/PROFILING.md @@ -4,7 +4,7 @@ If you are running into problems with Dendrite using excessive resources (e.g. C Dendrite contains an embedded profiler called `pprof`, which is a part of the standard Go toolchain. -### Enable the profiler +## Enable the profiler To enable the profiler, start Dendrite with the `PPROFLISTEN` environment variable. This variable specifies which address and port to listen on, e.g. @@ -19,7 +19,9 @@ WARN[2020-12-03T13:32:33.669405000Z] [/Users/neilalexander/Desktop/dendrite/inte Starting pprof on localhost:65432 ``` -### Profiling CPU usage +All examples from this point forward assume `PPROFLISTEN=localhost:65432` but you may need to adjust as necessary for your setup. + +## Profiling CPU usage To examine where CPU time is going, you can call the `profile` endpoint: @@ -45,7 +47,7 @@ curl -O http://localhost:65432/debug/pprof/profile?seconds=30 This will block for the specified number of seconds, capturing information about what Dendrite is doing, and then produces a `profile` file, which you can send onward. -### Profiling memory usage +## Profiling memory usage To examine where memory usage is going, you can call the `heap` endpoint: @@ -74,6 +76,6 @@ If you don't have the Go tools installed but just want to capture the profile to ``` curl -O http://localhost:65432/debug/pprof/heap -`` +``` This will almost instantly produce a `heap` file, which you can send onward. From 52905ffb82409fca9665ada0c8da40e05b58b8be Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 3 Dec 2020 13:49:34 +0000 Subject: [PATCH 09/10] Update PROFILING.md --- docs/PROFILING.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/PROFILING.md b/docs/PROFILING.md index 136a757b9..b026a8aed 100644 --- a/docs/PROFILING.md +++ b/docs/PROFILING.md @@ -31,6 +31,8 @@ http://localhost:65432/debug/pprof/profile?seconds=30 The profile will run for the specified number of `seconds` and then will produce a result. +### Examine a profile using the Go toolchain + If you have Go installed and want to explore the profile, you can invoke `go tool pprof` to start the profile directly. The `-http=` parameter will instruct `go tool pprof` to start a web server providing a view of the captured profile: ``` @@ -39,6 +41,8 @@ go tool pprof -http=localhost:23456 http://localhost:65432/debug/pprof/profile?s You can then visit `http://localhost:23456` in your web browser to see a visual representation of the profile. Particularly usefully, in the "View" menu, you can select "Flame Graph" to see a proportional interactive graph of CPU usage. +### Download a profile to send to someone else + If you don't have the Go tools installed but just want to capture the profile to send to someone else, you can instead use `curl` to download the profiler results: ``` @@ -57,6 +61,8 @@ http://localhost:65432/debug/pprof/heap The profile will return almost instantly. +### Examine a profile using the Go toolchain + If you have Go installed and want to explore the profile, you can invoke `go tool pprof` to start the profile directly. The `-http=` parameter will instruct `go tool pprof` to start a web server providing a view of the captured profile: ``` @@ -72,6 +78,8 @@ You can then visit `http://localhost:23456` in your web browser to see a visual Also in the "View" menu, you can select "Flame Graph" to see a proportional interactive graph of the memory usage. +### Download a profile to send to someone else + If you don't have the Go tools installed but just want to capture the profile to send to someone else, you can instead use `curl` to download the profiler results: ``` From 19b1d40d6479ac73cf4073c23f9ee291e3d5d112 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petter=20Jakub=20=C3=98kland?= Date: Thu, 3 Dec 2020 13:52:00 +0000 Subject: [PATCH 10/10] Hiawatha reverse proxy sample for monolith and fixes to polylith sample (#1609) * Created polylith reverse proxy sample for Hiawatha * Create monolith-sample.conf * Added timeout to proxies and binding explanation. * Fixed typo. * Consistency with regards to polylith-sample.conf. --- docs/hiawatha/monolith-sample.conf | 17 +++++++++++++++++ docs/hiawatha/polylith-sample.conf | 22 +++++++++++++++++----- 2 files changed, 34 insertions(+), 5 deletions(-) create mode 100644 docs/hiawatha/monolith-sample.conf diff --git a/docs/hiawatha/monolith-sample.conf b/docs/hiawatha/monolith-sample.conf new file mode 100644 index 000000000..8285c0bd6 --- /dev/null +++ b/docs/hiawatha/monolith-sample.conf @@ -0,0 +1,17 @@ +# Depending on which port is used for federation (.well-known/matrix/server or SRV record), +# ensure there's a binding for that port in the configuration. Replace "FEDPORT" with port +# number, (e.g. "8448"), and "IPV4" with your server's ipv4 address (separate binding for +# each ip address, e.g. if you use both ipv4 and ipv6 addresses). + +Binding { + Port = FEDPORT + Interface = IPV4 + TLScertFile = /path/to/fullchainandprivkey.pem +} + +VirtualHost { + ... + ReverseProxy = /_matrix http://localhost:8008 600 + ... + +} diff --git a/docs/hiawatha/polylith-sample.conf b/docs/hiawatha/polylith-sample.conf index 99730efdb..5ed0cb5ae 100644 --- a/docs/hiawatha/polylith-sample.conf +++ b/docs/hiawatha/polylith-sample.conf @@ -1,3 +1,15 @@ +# Depending on which port is used for federation (.well-known/matrix/server or SRV record), +# ensure there's a binding for that port in the configuration. Replace "FEDPORT" with port +# number, (e.g. "8448"), and "IPV4" with your server's ipv4 address (separate binding for +# each ip address, e.g. if you use both ipv4 and ipv6 addresses). + +Binding { + Port = FEDPORT + Interface = IPV4 + TLScertFile = /path/to/fullchainandprivkey.pem +} + + VirtualHost { ... # route requests to: @@ -7,10 +19,10 @@ VirtualHost { # /_matrix/client/.*/keys/changes # /_matrix/client/.*/rooms/{roomId}/messages # to sync_api - ReverseProxy = /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/messages) http://localhost:8073 - ReverseProxy = /_matrix/client http://localhost:8071 - ReverseProxy = /_matrix/federation http://localhost:8072 - ReverseProxy = /_matrix/key http://localhost:8072 - ReverseProxy = /_matrix/media http://localhost:8074 + ReverseProxy = /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/messages) http://localhost:8073 600 + ReverseProxy = /_matrix/client http://localhost:8071 600 + ReverseProxy = /_matrix/federation http://localhost:8072 600 + ReverseProxy = /_matrix/key http://localhost:8072 600 + ReverseProxy = /_matrix/media http://localhost:8074 600 ... }