From bb8dcb09a1b0bb38ea1e834e739fe5415ff3ec6b Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@googlemail.com> Date: Fri, 17 Nov 2017 10:27:28 +0000 Subject: [PATCH 1/8] use voip turnServers struct from gomatrix for dedup and consistency (#344) Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- .../matrix-org/dendrite/clientapi/routing/voip.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/github.com/matrix-org/dendrite/clientapi/routing/voip.go b/src/github.com/matrix-org/dendrite/clientapi/routing/voip.go index c4c48bd73..1a2a87527 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/routing/voip.go +++ b/src/github.com/matrix-org/dendrite/clientapi/routing/voip.go @@ -26,16 +26,10 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/gomatrix" "github.com/matrix-org/util" ) -type turnServerResponse struct { - Username string `json:"username"` - Password string `json:"password"` - URIs []string `json:"uris"` - TTL int `json:"ttl"` -} - // RequestTurnServer implements: // GET /voip/turnServer func RequestTurnServer(req *http.Request, device *authtypes.Device, cfg config.Dendrite) util.JSONResponse { @@ -52,7 +46,7 @@ func RequestTurnServer(req *http.Request, device *authtypes.Device, cfg config.D // Duration checked at startup, err not possible duration, _ := time.ParseDuration(turnConfig.UserLifetime) - resp := turnServerResponse{ + resp := gomatrix.RespTurnServer{ URIs: turnConfig.URIs, TTL: int(duration.Seconds()), } From 19a716e7da3ac8ec5867cc85b7e93d439f77d7e0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Sat, 18 Nov 2017 15:06:51 +0000 Subject: [PATCH 2/8] Fix create filter API (#342) --- .../dendrite/clientapi/auth/storage/accounts/filter_table.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/filter_table.go b/src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/filter_table.go index 9e3b7d6e6..81bae4545 100644 --- a/src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/filter_table.go +++ b/src/github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/filter_table.go @@ -92,11 +92,11 @@ func (s *filterStatements) insertFilter( // Check if filter already exists in the database err = s.selectFilterIDByContentStmt.QueryRowContext(ctx, localpart, filterJSON).Scan(&existingFilterID) - if err != nil { + if err != nil && err != sql.ErrNoRows { return "", err } // If it does, return the existing ID - if len(existingFilterID) != 0 { + if existingFilterID != "" { return existingFilterID, err } From ea53558cca358f0b72141922ab37a8b41ee775f6 Mon Sep 17 00:00:00 2001 From: Ross Schulman Date: Mon, 20 Nov 2017 09:33:49 -0500 Subject: [PATCH 3/8] Implement room_alias federation end point (#338) * Add room alias query endpoint * Try to fix indentation problems * Fix linting errors and use of httpReq.FormValue Signed-off-by: Ross Schulman * Run gofmt * Check for empty alias parameter and fix route URL Signed-off-by: Ross Schulman * Fix some linting errors Signed-off-by: Ross Schulman * Delete extra copy of directory route --- .../dendrite-federation-api-server/main.go | 3 +- .../cmd/dendrite-monolith-server/main.go | 2 +- .../dendrite/federationapi/routing/query.go | 96 +++++++++++++++++++ .../dendrite/federationapi/routing/routing.go | 10 ++ 4 files changed, 109 insertions(+), 2 deletions(-) create mode 100644 src/github.com/matrix-org/dendrite/federationapi/routing/query.go diff --git a/src/github.com/matrix-org/dendrite/cmd/dendrite-federation-api-server/main.go b/src/github.com/matrix-org/dendrite/cmd/dendrite-federation-api-server/main.go index ba981d8cf..53587ee20 100644 --- a/src/github.com/matrix-org/dendrite/cmd/dendrite-federation-api-server/main.go +++ b/src/github.com/matrix-org/dendrite/cmd/dendrite-federation-api-server/main.go @@ -80,6 +80,7 @@ func main() { queryAPI := api.NewRoomserverQueryAPIHTTP(cfg.RoomServerURL(), nil) inputAPI := api.NewRoomserverInputAPIHTTP(cfg.RoomServerURL(), nil) + aliasAPI := api.NewRoomserverAliasAPIHTTP(cfg.RoomServerURL(), nil) roomserverProducer := producers.NewRoomserverProducer(inputAPI) @@ -90,7 +91,7 @@ func main() { log.Info("Starting federation API server on ", cfg.Listen.FederationAPI) api := mux.NewRouter() - routing.Setup(api, *cfg, queryAPI, roomserverProducer, keyRing, federation, accountDB) + routing.Setup(api, *cfg, queryAPI, aliasAPI, roomserverProducer, keyRing, federation, accountDB) common.SetupHTTPAPI(http.DefaultServeMux, api) log.Fatal(http.ListenAndServe(string(cfg.Listen.FederationAPI), nil)) diff --git a/src/github.com/matrix-org/dendrite/cmd/dendrite-monolith-server/main.go b/src/github.com/matrix-org/dendrite/cmd/dendrite-monolith-server/main.go index fafd4cb82..05fc4252b 100644 --- a/src/github.com/matrix-org/dendrite/cmd/dendrite-monolith-server/main.go +++ b/src/github.com/matrix-org/dendrite/cmd/dendrite-monolith-server/main.go @@ -348,7 +348,7 @@ func (m *monolith) setupAPIs() { ), m.syncAPIDB, m.deviceDB) federationapi_routing.Setup( - m.api, *m.cfg, m.queryAPI, m.roomServerProducer, m.keyRing, m.federation, + m.api, *m.cfg, m.queryAPI, m.aliasAPI, m.roomServerProducer, m.keyRing, m.federation, m.accountDB, ) diff --git a/src/github.com/matrix-org/dendrite/federationapi/routing/query.go b/src/github.com/matrix-org/dendrite/federationapi/routing/query.go new file mode 100644 index 000000000..ef4b8961a --- /dev/null +++ b/src/github.com/matrix-org/dendrite/federationapi/routing/query.go @@ -0,0 +1,96 @@ +// Copyright 2017 New Vector Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routing + +import ( + "fmt" + "net/http" + + "github.com/matrix-org/dendrite/clientapi/httputil" + "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/gomatrix" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" +) + +// RoomAliasToID converts the queried alias into a room ID and returns it +func RoomAliasToID( + httpReq *http.Request, + federation *gomatrixserverlib.FederationClient, + cfg config.Dendrite, + aliasAPI api.RoomserverAliasAPI, +) util.JSONResponse { + roomAlias := httpReq.FormValue("alias") + if roomAlias == "" { + return util.JSONResponse{ + Code: 400, + JSON: jsonerror.BadJSON("Must supply room alias parameter."), + } + } + _, domain, err := gomatrixserverlib.SplitID('#', roomAlias) + if err != nil { + return util.JSONResponse{ + Code: 400, + JSON: jsonerror.BadJSON("Room alias must be in the form '#localpart:domain'"), + } + } + + var resp gomatrixserverlib.RespDirectory + + if domain == cfg.Matrix.ServerName { + queryReq := api.GetAliasRoomIDRequest{Alias: roomAlias} + var queryRes api.GetAliasRoomIDResponse + if err = aliasAPI.GetAliasRoomID(httpReq.Context(), &queryReq, &queryRes); err != nil { + return httputil.LogThenError(httpReq, err) + } + + if queryRes.RoomID == "" { + // TODO: List servers that are aware of this room alias + resp = gomatrixserverlib.RespDirectory{ + RoomID: queryRes.RoomID, + Servers: []gomatrixserverlib.ServerName{}, + } + } else { + // If the response doesn't contain a non-empty string, return an error + return util.JSONResponse{ + Code: 404, + JSON: jsonerror.NotFound(fmt.Sprintf("Room alias %s not found", roomAlias)), + } + } + } else { + resp, err = federation.LookupRoomAlias(httpReq.Context(), domain, roomAlias) + if err != nil { + switch x := err.(type) { + case gomatrix.HTTPError: + if x.Code == 404 { + return util.JSONResponse{ + Code: 404, + JSON: jsonerror.NotFound("Room alias not found"), + } + } + } + // TODO: Return 502 if the remote server errored. + // TODO: Return 504 if the remote server timed out. + return httputil.LogThenError(httpReq, err) + } + } + + return util.JSONResponse{ + Code: 200, + JSON: resp, + } +} diff --git a/src/github.com/matrix-org/dendrite/federationapi/routing/routing.go b/src/github.com/matrix-org/dendrite/federationapi/routing/routing.go index c50afd6e0..b23d738f9 100644 --- a/src/github.com/matrix-org/dendrite/federationapi/routing/routing.go +++ b/src/github.com/matrix-org/dendrite/federationapi/routing/routing.go @@ -38,6 +38,7 @@ func Setup( apiMux *mux.Router, cfg config.Dendrite, query api.RoomserverQueryAPI, + aliasAPI api.RoomserverAliasAPI, producer *producers.RoomserverProducer, keys gomatrixserverlib.KeyRing, federation *gomatrixserverlib.FederationClient, @@ -105,6 +106,15 @@ func Setup( }, )).Methods("GET") + v1fedmux.Handle("/query/directory/", common.MakeFedAPI( + "federation_query_room_alias", cfg.Matrix.ServerName, keys, + func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { + return RoomAliasToID( + httpReq, federation, cfg, aliasAPI, + ) + }, + )).Methods("GET") + v1fedmux.Handle("/query/profile", common.MakeFedAPI( "federation_query_profile", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { From e5f5bb5961bbca704a6b037b56603f9af6233afb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 21 Nov 2017 12:13:01 +0000 Subject: [PATCH 4/8] Update version of kafka (#347) * Update version of kafka * Update path * Fix setting config option --- scripts/install-local-kafka.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/install-local-kafka.sh b/scripts/install-local-kafka.sh index 869d28914..41bc7bd20 100755 --- a/scripts/install-local-kafka.sh +++ b/scripts/install-local-kafka.sh @@ -5,9 +5,9 @@ set -eu # The mirror to download kafka from is picked from the list of mirrors at -# https://www.apache.org/dyn/closer.cgi?path=/kafka/0.10.2.0/kafka_2.11-0.10.2.0.tgz +# https://www.apache.org/dyn/closer.cgi?path=/kafka/0.10.2.0/kafka_2.11-0.11.0.2.tgz # TODO: Check the signature since we are downloading over HTTP. -MIRROR=http://apache.mirror.anlx.net/kafka/0.10.2.0/kafka_2.11-0.10.2.0.tgz +MIRROR=http://apache.mirror.anlx.net/kafka/0.11.0.2/kafka_2.11-0.11.0.2.tgz # Only download the kafka if it isn't already downloaded. test -f kafka.tgz || wget $MIRROR -O kafka.tgz @@ -18,7 +18,7 @@ mkdir -p kafka && tar xzf kafka.tgz -C kafka --strip-components 1 kafka/bin/zookeeper-server-start.sh -daemon kafka/config/zookeeper.properties # Enable topic deletion so that the integration tests can create a fresh topic # for each test run. -echo "delete.topic.enable=true" >> kafka/config/server.properties +echo -e "\n\ndelete.topic.enable=true" >> kafka/config/server.properties # Start the kafka server running in the background. # By default the kafka listens on localhost:9092 kafka/bin/kafka-server-start.sh -daemon kafka/config/server.properties From d44dc2d5e6b7b1fde02fd6bef526394bcf05f571 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 22 Nov 2017 09:35:25 +0000 Subject: [PATCH 5/8] Code Style: Prefer WithFields over Infof (#349) * Code Style: Prefer WithFields over Infof * Update CODE_STYLE.md --- CODE_STYLE.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/CODE_STYLE.md b/CODE_STYLE.md index 65d0daf4c..e8208172c 100644 --- a/CODE_STYLE.md +++ b/CODE_STYLE.md @@ -52,6 +52,29 @@ github. These can be added just before merging of the PR to master, and the issue number should be added to the comment, e.g. `// TODO(#324): ...` +## Logging + +We generally prefer to log with static log messages and include any dynamic +information in fields. + +```golang +logger := util.GetLogger(ctx) + +// Not recommended +logger.Infof("Finished processing keys for %s, number of keys %d", name, numKeys) + +// Recommended +logger.WithFields(logrus.Fields{ + "numberOfKeys": numKeys, + "entityName": name, +}).Info("Finished processing keys") +``` + +This is useful when logging to systems that natively understand log fields, as +it allows people to search and process the fields without having to parse the +log message. + + ## Visual Studio Code If you use VSCode then the following is an example of a workspace setting that From f42f44391f7240da3990b465cd999b8734e923bd Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 22 Nov 2017 09:51:12 +0000 Subject: [PATCH 6/8] Fix /sync when we have no events (#341) * Fix /sync when we have no events We used a since token of 0 to mean that no token was given. However, naffka streams start at 0. This causes clients to get stuck spinning forever until an event is sent. This changes it so that we pass around pointers instead, with nil meaning a since token wasn't given. * Comment * Fix unit tests * Comments * Fix typo --- .../dendrite/syncapi/sync/notifier_test.go | 4 ++-- .../matrix-org/dendrite/syncapi/sync/request.go | 13 ++++++++----- .../matrix-org/dendrite/syncapi/sync/requestpool.go | 12 ++++++------ 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/github.com/matrix-org/dendrite/syncapi/sync/notifier_test.go b/src/github.com/matrix-org/dendrite/syncapi/sync/notifier_test.go index 6ee259681..79c5a2872 100644 --- a/src/github.com/matrix-org/dendrite/syncapi/sync/notifier_test.go +++ b/src/github.com/matrix-org/dendrite/syncapi/sync/notifier_test.go @@ -264,7 +264,7 @@ func waitForEvents(n *Notifier, req syncRequest) (types.StreamPosition, error) { return types.StreamPosition(0), fmt.Errorf( "waitForEvents timed out waiting for %s (pos=%d)", req.userID, req.since, ) - case <-listener.GetNotifyChannel(req.since): + case <-listener.GetNotifyChannel(*req.since): p := listener.GetStreamPosition() return p, nil } @@ -282,7 +282,7 @@ func newTestSyncRequest(userID string, since types.StreamPosition) syncRequest { return syncRequest{ userID: userID, timeout: 1 * time.Minute, - since: since, + since: &since, wantFullState: false, limit: defaultTimelineLimit, log: util.GetLogger(context.TODO()), diff --git a/src/github.com/matrix-org/dendrite/syncapi/sync/request.go b/src/github.com/matrix-org/dendrite/syncapi/sync/request.go index 7dec55fe1..7f5259814 100644 --- a/src/github.com/matrix-org/dendrite/syncapi/sync/request.go +++ b/src/github.com/matrix-org/dendrite/syncapi/sync/request.go @@ -34,7 +34,7 @@ type syncRequest struct { userID string limit int timeout time.Duration - since types.StreamPosition + since *types.StreamPosition // nil means that no since token was supplied wantFullState bool log *log.Entry } @@ -70,13 +70,16 @@ func getTimeout(timeoutMS string) time.Duration { return time.Duration(i) * time.Millisecond } -func getSyncStreamPosition(since string) (types.StreamPosition, error) { +// getSyncStreamPosition tries to parse a 'since' token taken from the API to a +// stream position. If the string is empty then (nil, nil) is returned. +func getSyncStreamPosition(since string) (*types.StreamPosition, error) { if since == "" { - return types.StreamPosition(0), nil + return nil, nil } i, err := strconv.Atoi(since) if err != nil { - return types.StreamPosition(0), err + return nil, err } - return types.StreamPosition(i), nil + token := types.StreamPosition(i) + return &token, nil } diff --git a/src/github.com/matrix-org/dendrite/syncapi/sync/requestpool.go b/src/github.com/matrix-org/dendrite/syncapi/sync/requestpool.go index 3b6775618..15993b774 100644 --- a/src/github.com/matrix-org/dendrite/syncapi/sync/requestpool.go +++ b/src/github.com/matrix-org/dendrite/syncapi/sync/requestpool.go @@ -64,7 +64,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *authtype currPos := rp.notifier.CurrentPosition() // If this is an initial sync or timeout=0 we return immediately - if syncReq.since == types.StreamPosition(0) || syncReq.timeout == 0 { + if syncReq.since == nil || syncReq.timeout == 0 { syncData, err := rp.currentSyncForUser(*syncReq, currPos) if err != nil { return httputil.LogThenError(req, err) @@ -93,7 +93,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *authtype case <-timer.C: return util.JSONResponse{ Code: 200, - JSON: types.NewResponse(syncReq.since), + JSON: types.NewResponse(currPos), } // Or for the request to be cancelled case <-req.Context().Done(): @@ -121,10 +121,10 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *authtype func (rp *RequestPool) currentSyncForUser(req syncRequest, currentPos types.StreamPosition) (res *types.Response, err error) { // TODO: handle ignored users - if req.since == types.StreamPosition(0) { + if req.since == nil { res, err = rp.db.CompleteSync(req.ctx, req.userID, req.limit) } else { - res, err = rp.db.IncrementalSync(req.ctx, req.userID, req.since, currentPos, req.limit) + res, err = rp.db.IncrementalSync(req.ctx, req.userID, *req.since, currentPos, req.limit) } if err != nil { @@ -148,7 +148,7 @@ func (rp *RequestPool) appendAccountData( return nil, err } - if req.since == types.StreamPosition(0) { + if req.since == nil { // If this is the initial sync, we don't need to check if a data has // already been sent. Instead, we send the whole batch. var global []gomatrixserverlib.ClientEvent @@ -170,7 +170,7 @@ func (rp *RequestPool) appendAccountData( } // Sync is not initial, get all account data since the latest sync - dataTypes, err := rp.db.GetAccountDataInRange(req.ctx, userID, req.since, currentPos) + dataTypes, err := rp.db.GetAccountDataInRange(req.ctx, userID, *req.since, currentPos) if err != nil { return nil, err } From 0d72e34acf1b30727bf368c6201c1f20d1440bcd Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 22 Nov 2017 15:45:03 +0000 Subject: [PATCH 7/8] Fix panic if upload name wasn't supplied (#351) --- src/github.com/matrix-org/dendrite/mediaapi/routing/upload.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/github.com/matrix-org/dendrite/mediaapi/routing/upload.go b/src/github.com/matrix-org/dendrite/mediaapi/routing/upload.go index aafc0230a..46eb242d8 100644 --- a/src/github.com/matrix-org/dendrite/mediaapi/routing/upload.go +++ b/src/github.com/matrix-org/dendrite/mediaapi/routing/upload.go @@ -21,6 +21,7 @@ import ( "net/http" "net/url" "path" + "strings" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/common/config" @@ -188,7 +189,7 @@ func (r *uploadRequest) Validate(maxFileSizeBytes config.FileSizeBytes) *util.JS JSON: jsonerror.Unknown("HTTP Content-Type request header must be set."), } } - if r.MediaMetadata.UploadName[0] == '~' { + if strings.HasPrefix(string(r.MediaMetadata.UploadName), "~") { return &util.JSONResponse{ Code: 400, JSON: jsonerror.Unknown("File name must not begin with '~'."), From 9476a266bd4f71b19dd1123a4e2593d8af5f63e5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 22 Nov 2017 15:46:49 +0000 Subject: [PATCH 8/8] Add database.naffka to example config (#350) * Add database.naffka to example config * Update dendrite-config.yaml * Add naffka creation to INSTALL --- INSTALL.md | 2 +- dendrite-config.yaml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/INSTALL.md b/INSTALL.md index 54addd285..f2345bc8f 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -64,7 +64,7 @@ Dendrite requires a postgres database engine, version 9.5 or later. ``` * Create databases: ```bash - for i in account device mediaapi syncapi roomserver serverkey federationsender publicroomsapi; do + for i in account device mediaapi syncapi roomserver serverkey federationsender publicroomsapi naffka; do sudo -u postgres createdb -O dendrite dendrite_$i done ``` diff --git a/dendrite-config.yaml b/dendrite-config.yaml index 9baba8f94..6e2326595 100644 --- a/dendrite-config.yaml +++ b/dendrite-config.yaml @@ -80,6 +80,7 @@ kafka: # Naffka can only be used when running dendrite as a single monolithic server. # Kafka can be used both with a monolithic server and when running the # components as separate servers. + # If enabled database.naffka must also be specified. use_naffka: false # The names of the kafka topics to use. topics: @@ -97,6 +98,8 @@ database: server_key: "postgres://dendrite:itsasecret@localhost/dendrite_serverkey?sslmode=disable" federation_sender: "postgres://dendrite:itsasecret@localhost/dendrite_federationsender?sslmode=disable" public_rooms_api: "postgres://dendrite:itsasecret@localhost/dendrite_publicroomsapi?sslmode=disable" + # If using naffka you need to specify a naffka database + # naffka: "postgres://dendrite:itsasecret@localhost/dendrite_naffka?sslmode=disable" # The TCP host:port pairs to bind the internal HTTP APIs to. # These shouldn't be exposed to the public internet.