diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml index 9f44877fe..9d755a244 100644 --- a/.buildkite/pipeline.yaml +++ b/.buildkite/pipeline.yaml @@ -1,50 +1,49 @@ steps: - command: - - "go build ./cmd/..." - label: ":hammer-and-wrench: Build / :go: 1.11" - env: - GOGC: "400" - DENDRITE_LINT_DISABLE_GC: "1" + # https://github.com/golangci/golangci-lint#memory-usage-of-golangci-lint + - "GOGC=20 ./scripts/find-lint.sh" + label: "\U0001F9F9 Lint / :go: 1.12" + agents: + # Use a larger instance as linting takes a looot of memory + queue: "medium" plugins: - docker#v3.0.1: - image: "golang:1.11-alpine" + image: "golang:1.12" - - command: - - "go test ./..." - label: ":female-scientist: Unit tests / :go: 1.11" - env: - GOGC: "400" - DENDRITE_LINT_DISABLE_GC: "1" - plugins: - - docker#v3.0.1: - image: "golang:1.11-alpine" + - wait - command: - "go build ./cmd/..." - label: ":hammer-and-wrench: Build / :go: 1.12" - env: - GOGC: "400" - DENDRITE_LINT_DISABLE_GC: "1" + label: "\U0001F528 Build / :go: 1.11" plugins: - docker#v3.0.1: - image: "golang:1.12-alpine" + image: "golang:1.11" + retry: + automatic: + - exit_status: 128 + limit: 3 + + - command: + - "go build ./cmd/..." + label: "\U0001F528 Build / :go: 1.12" + plugins: + - docker#v3.0.1: + image: "golang:1.12" + retry: + automatic: + - exit_status: 128 + limit: 3 - command: - "go test ./..." - label: ":female-scientist: Unit tests / :go: 1.12" - env: - GOGC: "400" - DENDRITE_LINT_DISABLE_GC: "1" + label: "\U0001F9EA Unit tests / :go: 1.11" plugins: - docker#v3.0.1: - image: "golang:1.12-alpine" + image: "golang:1.11" - command: - - "./scripts/find-lint.sh" - label: ":lower_left_crayon: Lint / :go: 1.12" - env: - GOGC: "400" - DENDRITE_LINT_DISABLE_GC: "1" + - "go test ./..." + label: "\U0001F9EA Unit tests / :go: 1.12" plugins: - docker#v3.0.1: - image: "golang:1.12-alpine" + image: "golang:1.12" diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 000000000..0d0f51bd2 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,280 @@ +# Config file for golangci-lint + +# options for analysis running +run: + # default concurrency is a available CPU number + concurrency: 4 + + # timeout for analysis, e.g. 30s, 5m, default is 1m + deadline: 30m + + # exit code when at least one issue was found, default is 1 + issues-exit-code: 1 + + # include test files or not, default is true + tests: true + + # list of build tags, all linters use it. Default is empty list. + #build-tags: + # - mytag + + # which dirs to skip: they won't be analyzed; + # can use regexp here: generated.*, regexp is applied on full path; + # default value is empty list, but next dirs are always skipped independently + # from this option's value: + # vendor$, third_party$, testdata$, examples$, Godeps$, builtin$ + skip-dirs: + - bin + - docs + + # which files to skip: they will be analyzed, but issues from them + # won't be reported. Default value is empty list, but there is + # no need to include all autogenerated files, we confidently recognize + # autogenerated files. If it's not please let us know. + skip-files: + - ".*\\.md$" + - ".*\\.sh$" + - "^cmd/syncserver-integration-tests/testdata.go$" + + # by default isn't set. If set we pass it to "go list -mod={option}". From "go help modules": + # If invoked with -mod=readonly, the go command is disallowed from the implicit + # automatic updating of go.mod described above. Instead, it fails when any changes + # to go.mod are needed. This setting is most useful to check that go.mod does + # not need updates, such as in a continuous integration and testing system. + # If invoked with -mod=vendor, the go command assumes that the vendor + # directory holds the correct copies of dependencies and ignores + # the dependency descriptions in go.mod. + #modules-download-mode: (release|readonly|vendor) + + +# output configuration options +output: + # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number" + format: colored-line-number + + # print lines of code with issue, default is true + print-issued-lines: true + + # print linter name in the end of issue text, default is true + print-linter-name: true + + +# all available settings of specific linters +linters-settings: + errcheck: + # report about not checking of errors in type assertions: `a := b.(MyStruct)`; + # default is false: such cases aren't reported by default. + check-type-assertions: false + + # report about assignment of errors to blank identifier: `num, _ := strconv.Atoi(numStr)`; + # default is false: such cases aren't reported by default. + check-blank: false + + # [deprecated] comma-separated list of pairs of the form pkg:regex + # the regex is used to ignore names within pkg. (default "fmt:.*"). + # see https://github.com/kisielk/errcheck#the-deprecated-method for details + #ignore: fmt:.*,io/ioutil:^Read.* + + # path to a file containing a list of functions to exclude from checking + # see https://github.com/kisielk/errcheck#excluding-functions for details + #exclude: /path/to/file.txt + govet: + # report about shadowed variables + check-shadowing: true + + # settings per analyzer + settings: + printf: # analyzer name, run `go tool vet help` to see all analyzers + funcs: # run `go tool vet help printf` to see available settings for `printf` analyzer + - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof + - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf + - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf + - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf + golint: + # minimal confidence for issues, default is 0.8 + min-confidence: 0.8 + gofmt: + # simplify code: gofmt with `-s` option, true by default + simplify: true + goimports: + # put imports beginning with prefix after 3rd-party packages; + # it's a comma-separated list of prefixes + #local-prefixes: github.com/org/project + gocyclo: + # minimal code complexity to report, 30 by default (but we recommend 10-20) + min-complexity: 12 + maligned: + # print struct with more effective memory layout or not, false by default + suggest-new: true + dupl: + # tokens count to trigger issue, 150 by default + threshold: 100 + goconst: + # minimal length of string constant, 3 by default + min-len: 3 + # minimal occurrences count to trigger, 3 by default + min-occurrences: 3 + depguard: + list-type: blacklist + include-go-root: false + packages: + # - github.com/davecgh/go-spew/spew + misspell: + # Correct spellings using locale preferences for US or UK. + # Default is to use a neutral variety of English. + # Setting locale to US will correct the British spelling of 'colour' to 'color'. + locale: UK + ignore-words: + # - someword + lll: + # max line length, lines longer will be reported. Default is 120. + # '\t' is counted as 1 character by default, and can be changed with the tab-width option + line-length: 96 + # tab width in spaces. Default to 1. + tab-width: 1 + unused: + # treat code as a program (not a library) and report unused exported identifiers; default is false. + # XXX: if you enable this setting, unused will report a lot of false-positives in text editors: + # if it's called for subdir of a project it can't find funcs usages. All text editor integrations + # with golangci-lint call it on a directory with the changed file. + check-exported: false + unparam: + # Inspect exported functions, default is false. Set to true if no external program/library imports your code. + # XXX: if you enable this setting, unparam will report a lot of false-positives in text editors: + # if it's called for subdir of a project it can't find external interfaces. All text editor integrations + # with golangci-lint call it on a directory with the changed file. + check-exported: false + nakedret: + # make an issue if func has more lines of code than this setting and it has naked returns; default is 30 + max-func-lines: 60 + prealloc: + # XXX: we don't recommend using this linter before doing performance profiling. + # For most programs usage of prealloc will be a premature optimization. + + # Report preallocation suggestions only on simple loops that have no returns/breaks/continues/gotos in them. + # True by default. + simple: true + range-loops: true # Report preallocation suggestions on range loops, true by default + for-loops: false # Report preallocation suggestions on for loops, false by default + gocritic: + # Which checks should be enabled; can't be combined with 'disabled-checks'; + # See https://go-critic.github.io/overview#checks-overview + # To check which checks are enabled run `GL_DEBUG=gocritic golangci-lint run` + # By default list of stable checks is used. + #enabled-checks: + + # Which checks should be disabled; can't be combined with 'enabled-checks'; default is empty + #disabled-checks: + + # Enable multiple checks by tags, run `GL_DEBUG=gocritic golangci-lint` run to see all tags and checks. + # Empty list by default. See https://github.com/go-critic/go-critic#usage -> section "Tags". + #enabled-tags: + # - performance + + settings: # settings passed to gocritic + captLocal: # must be valid enabled check name + paramsOnly: true + #rangeValCopy: + # sizeThreshold: 32 + +linters: + enable: + - deadcode + - errcheck + - goconst + - gocyclo + - goimports # Does everything gofmt does + - gosimple + - ineffassign + - megacheck + - misspell # Check code comments, whereas misspell in CI checks *.md files + - nakedret + - staticcheck + - structcheck + - unparam + - unused + - varcheck + enable-all: false + disable: + - bodyclose + - depguard + - dupl + - gochecknoglobals + - gochecknoinits + - gocritic + - gofmt + - golint + - gosec # Should turn back on soon + - interfacer + - lll + - maligned + - prealloc # Should turn back on soon + - scopelint + - stylecheck + - typecheck # Should turn back on soon + - unconvert # Should turn back on soon + disable-all: false + presets: + fast: false + + +issues: + # List of regexps of issue texts to exclude, empty list by default. + # But independently from this option we use default exclude patterns, + # it can be disabled by `exclude-use-default: false`. To list all + # excluded by default patterns execute `golangci-lint run --help` + exclude: + # - abcdef + + # Excluding configuration per-path, per-linter, per-text and per-source + exclude-rules: + # Exclude some linters from running on tests files. + - path: _test\.go + linters: + - gocyclo + - errcheck + - dupl + - gosec + + # Exclude known linters from partially hard-vendored code, + # which is impossible to exclude via "nolint" comments. + - path: internal/hmac/ + text: "weak cryptographic primitive" + linters: + - gosec + + # Exclude some staticcheck messages + - linters: + - staticcheck + text: "SA9003:" + + # Exclude lll issues for long lines with go:generate + - linters: + - lll + source: "^//go:generate " + + # Independently from option `exclude` we use default exclude patterns, + # it can be disabled by this option. To list all + # excluded by default patterns execute `golangci-lint run --help`. + # Default value for this option is true. + exclude-use-default: false + + # Maximum issues count per one linter. Set to 0 to disable. Default is 50. + max-issues-per-linter: 0 + + # Maximum count of issues with the same text. Set to 0 to disable. Default is 3. + max-same-issues: 0 + + # Show only new issues: if there are unstaged changes or untracked files, + # only those changes are analyzed, else only changes in HEAD~ are analyzed. + # It's a super-useful option for integration of golangci-lint into existing + # large codebase. It's not practical to fix all existing issues at the moment + # of integration: much better don't allow issues in new code. + # Default is false. + new: false + + # Show only new issues created after git revision `REV` + #new-from-rev: REV + + # Show only new issues created in git patch with set file path. + #new-from-patch: path/to/patch/file diff --git a/README.md b/README.md index 7ba64490f..8eadaf431 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Dendrite [![Build Status](https://travis-ci.org/matrix-org/dendrite.svg?branch=master)](https://travis-ci.org/matrix-org/dendrite) [![CircleCI](https://circleci.com/gh/matrix-org/dendrite.svg?style=svg)](https://circleci.com/gh/matrix-org/dendrite) [![Dendrite Dev on Matrix](https://img.shields.io/matrix/dendrite-dev:matrix.org.svg?label=%23dendrite-dev%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite-dev:matrix.org) [![Dendrite on Matrix](https://img.shields.io/matrix/dendrite:matrix.org.svg?label=%23dendrite%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite:matrix.org) +# Dendrite [![Build Status](https://badge.buildkite.com/4be40938ab19f2bbc4a6c6724517353ee3ec1422e279faf374.svg)](https://buildkite.com/matrix-dot-org/dendrite) [![CircleCI](https://circleci.com/gh/matrix-org/dendrite.svg?style=svg)](https://circleci.com/gh/matrix-org/dendrite) [![Dendrite Dev on Matrix](https://img.shields.io/matrix/dendrite-dev:matrix.org.svg?label=%23dendrite-dev%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite-dev:matrix.org) [![Dendrite on Matrix](https://img.shields.io/matrix/dendrite:matrix.org.svg?label=%23dendrite%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite:matrix.org) Dendrite will be a matrix homeserver written in go. diff --git a/appservice/api/query.go b/appservice/api/query.go index 9ec214486..8ce3b4e04 100644 --- a/appservice/api/query.go +++ b/appservice/api/query.go @@ -134,9 +134,9 @@ func (h *httpAppServiceQueryAPI) UserIDExists( return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } -// RetreiveUserProfile is a wrapper that queries both the local database and +// RetrieveUserProfile is a wrapper that queries both the local database and // application services for a given user's profile -func RetreiveUserProfile( +func RetrieveUserProfile( ctx context.Context, userID string, asAPI AppServiceQueryAPI, diff --git a/appservice/routing/routing.go b/appservice/routing/routing.go index f0b8461dc..3c19c8401 100644 --- a/appservice/routing/routing.go +++ b/appservice/routing/routing.go @@ -31,6 +31,10 @@ const pathPrefixApp = "/_matrix/app/r0" // Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client // to clients which need to make outbound HTTP requests. +// +// Due to Setup being used to call many other functions, a gocyclo nolint is +// applied: +// nolint: gocyclo func Setup( apiMux *mux.Router, cfg config.Dendrite, // nolint: unparam queryAPI api.RoomserverQueryAPI, aliasAPI api.RoomserverAliasAPI, // nolint: unparam diff --git a/build.sh b/build.sh index eed4d52f5..9a8050f3c 100755 --- a/build.sh +++ b/build.sh @@ -1,3 +1,3 @@ -#!/bin/bash +#!/bin/sh GOBIN=$PWD/`dirname $0`/bin go install -v ./cmd/... diff --git a/clientapi/auth/auth.go b/clientapi/auth/auth.go index 00943fb83..f51cfea26 100644 --- a/clientapi/auth/auth.go +++ b/clientapi/auth/auth.go @@ -130,7 +130,7 @@ func VerifyUserFromRequest( return nil, &util.JSONResponse{ Code: http.StatusUnauthorized, - JSON: jsonerror.UnknownToken("Unrecognized access token"), + JSON: jsonerror.UnknownToken("Unrecognized access token"), // nolint: misspell } } diff --git a/clientapi/auth/storage/accounts/storage.go b/clientapi/auth/storage/accounts/storage.go index 2650470b8..27c0a176a 100644 --- a/clientapi/auth/storage/accounts/storage.go +++ b/clientapi/auth/storage/accounts/storage.go @@ -23,6 +23,7 @@ import ( "github.com/matrix-org/dendrite/common" "github.com/matrix-org/gomatrixserverlib" "golang.org/x/crypto/bcrypt" + // Import the postgres database driver. _ "github.com/lib/pq" ) diff --git a/clientapi/jsonerror/jsonerror.go b/clientapi/jsonerror/jsonerror.go index fa15d9d8e..8df1fead2 100644 --- a/clientapi/jsonerror/jsonerror.go +++ b/clientapi/jsonerror/jsonerror.go @@ -87,7 +87,7 @@ func MissingToken(msg string) *MatrixError { } // UnknownToken is an error when the client tries to access a resource which -// requires authentication and supplies an unrecognized token +// requires authentication and supplies an unrecognised token func UnknownToken(msg string) *MatrixError { return &MatrixError{"M_UNKNOWN_TOKEN", msg} } diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index a7187c495..220ba6ae8 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -163,7 +163,7 @@ func createRoom( "roomID": roomID, }).Info("Creating new room") - profile, err := appserviceAPI.RetreiveUserProfile(req.Context(), userID, asAPI, accountDB) + profile, err := appserviceAPI.RetrieveUserProfile(req.Context(), userID, asAPI, accountDB) if err != nil { return httputil.LogThenError(req, err) } diff --git a/clientapi/routing/directory.go b/clientapi/routing/directory.go index b23dfbfb6..ab85e86a9 100644 --- a/clientapi/routing/directory.go +++ b/clientapi/routing/directory.go @@ -117,12 +117,16 @@ func SetLocalAlias( // 1. The new method for checking for things matching an AS's namespace // 2. Using an overall Regex object for all AS's just like we did for usernames for _, appservice := range cfg.Derived.ApplicationServices { - if aliasNamespaces, ok := appservice.NamespaceMap["aliases"]; ok { - for _, namespace := range aliasNamespaces { - if namespace.Exclusive && namespace.RegexpObject.MatchString(alias) { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.ASExclusive("Alias is reserved by an application service"), + // Don't prevent AS from creating aliases in its own namespace + // Note that Dendrite uses SenderLocalpart as UserID for AS users + if device.UserID != appservice.SenderLocalpart { + if aliasNamespaces, ok := appservice.NamespaceMap["aliases"]; ok { + for _, namespace := range aliasNamespaces { + if namespace.Exclusive && namespace.RegexpObject.MatchString(alias) { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.ASExclusive("Alias is reserved by an application service"), + } } } } diff --git a/clientapi/routing/filter.go b/clientapi/routing/filter.go index 109c55da1..1ed91cd2f 100644 --- a/clientapi/routing/filter.go +++ b/clientapi/routing/filter.go @@ -62,7 +62,7 @@ func GetFilter( filter := gomatrix.Filter{} err = json.Unmarshal(res, &filter) if err != nil { - httputil.LogThenError(req, err) + return httputil.LogThenError(req, err) } return util.JSONResponse{ diff --git a/clientapi/routing/joinroom.go b/clientapi/routing/joinroom.go index c98688de0..9c02a93ca 100644 --- a/clientapi/routing/joinroom.go +++ b/clientapi/routing/joinroom.go @@ -86,7 +86,10 @@ func JoinRoomByIDOrAlias( } return util.JSONResponse{ Code: http.StatusBadRequest, - JSON: jsonerror.BadJSON("Invalid first character for room ID or alias"), + JSON: jsonerror.BadJSON( + fmt.Sprintf("Invalid first character '%s' for room ID or alias", + string([]rune(roomIDOrAlias)[0])), // Wrapping with []rune makes this call UTF-8 safe + ), } } diff --git a/clientapi/routing/login.go b/clientapi/routing/login.go index cb2218805..2e2d409f6 100644 --- a/clientapi/routing/login.go +++ b/clientapi/routing/login.go @@ -19,6 +19,7 @@ import ( "context" "database/sql" + "github.com/matrix-org/dendrite/clientapi/auth" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" @@ -106,7 +107,7 @@ func Login( token, err := auth.GenerateAccessToken() if err != nil { - httputil.LogThenError(req, err) + return httputil.LogThenError(req, err) } dev, err := getDevice(req.Context(), r, deviceDB, acc, localpart, token) diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index b308de79a..61898fecd 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -58,27 +58,12 @@ func SendMembership( } } - inviteStored, err := threepid.CheckAndProcessInvite( - req.Context(), device, &body, cfg, queryAPI, accountDB, producer, + inviteStored, jsonErrResp := checkAndProcessThreepid( + req, device, &body, cfg, queryAPI, accountDB, producer, membership, roomID, evTime, ) - if err == threepid.ErrMissingParameter { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.BadJSON(err.Error()), - } - } else if err == threepid.ErrNotTrusted { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.NotTrusted(body.IDServer), - } - } else if err == common.ErrRoomNoExists { - return util.JSONResponse{ - Code: http.StatusNotFound, - JSON: jsonerror.NotFound(err.Error()), - } - } else if err != nil { - return httputil.LogThenError(req, err) + if jsonErrResp != nil { + return *jsonErrResp } // If an invite has been stored on an identity server, it means that a @@ -114,9 +99,18 @@ func SendMembership( return httputil.LogThenError(req, err) } + var returnData interface{} = struct{}{} + + // The join membership requires the room id to be sent in the response + if membership == "join" { + returnData = struct { + RoomID string `json:"room_id"` + }{roomID} + } + return util.JSONResponse{ Code: http.StatusOK, - JSON: struct{}{}, + JSON: returnData, } } @@ -182,7 +176,7 @@ func loadProfile( var profile *authtypes.Profile if serverName == cfg.Matrix.ServerName { - profile, err = appserviceAPI.RetreiveUserProfile(ctx, userID, asAPI, accountDB) + profile, err = appserviceAPI.RetrieveUserProfile(ctx, userID, asAPI, accountDB) } else { profile = &authtypes.Profile{} } @@ -215,3 +209,41 @@ func getMembershipStateKey( return } + +func checkAndProcessThreepid( + req *http.Request, + device *authtypes.Device, + body *threepid.MembershipRequest, + cfg config.Dendrite, + queryAPI roomserverAPI.RoomserverQueryAPI, + accountDB *accounts.Database, + producer *producers.RoomserverProducer, + membership, roomID string, + evTime time.Time, +) (inviteStored bool, errRes *util.JSONResponse) { + + inviteStored, err := threepid.CheckAndProcessInvite( + req.Context(), device, body, cfg, queryAPI, accountDB, producer, + membership, roomID, evTime, + ) + if err == threepid.ErrMissingParameter { + return inviteStored, &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.BadJSON(err.Error()), + } + } else if err == threepid.ErrNotTrusted { + return inviteStored, &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.NotTrusted(body.IDServer), + } + } else if err == common.ErrRoomNoExists { + return inviteStored, &util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound(err.Error()), + } + } else if err != nil { + er := httputil.LogThenError(req, err) + return inviteStored, &er + } + return +} diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index e57d16fbf..eb1acab74 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -43,7 +43,7 @@ func GetProfile( JSON: jsonerror.NotFound("Bad method"), } } - profile, err := appserviceAPI.RetreiveUserProfile(req.Context(), userID, asAPI, accountDB) + profile, err := appserviceAPI.RetrieveUserProfile(req.Context(), userID, asAPI, accountDB) if err != nil { return httputil.LogThenError(req, err) } @@ -62,7 +62,7 @@ func GetProfile( func GetAvatarURL( req *http.Request, accountDB *accounts.Database, userID string, asAPI appserviceAPI.AppServiceQueryAPI, ) util.JSONResponse { - profile, err := appserviceAPI.RetreiveUserProfile(req.Context(), userID, asAPI, accountDB) + profile, err := appserviceAPI.RetrieveUserProfile(req.Context(), userID, asAPI, accountDB) if err != nil { return httputil.LogThenError(req, err) } @@ -160,7 +160,7 @@ func SetAvatarURL( func GetDisplayName( req *http.Request, accountDB *accounts.Database, userID string, asAPI appserviceAPI.AppServiceQueryAPI, ) util.JSONResponse { - profile, err := appserviceAPI.RetreiveUserProfile(req.Context(), userID, asAPI, accountDB) + profile, err := appserviceAPI.RetrieveUserProfile(req.Context(), userID, asAPI, accountDB) if err != nil { return httputil.LogThenError(req, err) } diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index b1522e82b..243f9dd23 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -243,8 +243,8 @@ func validateRecaptcha( ) *util.JSONResponse { if !cfg.Matrix.RecaptchaEnabled { return &util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.BadJSON("Captcha registration is disabled"), + Code: http.StatusConflict, + JSON: jsonerror.Unknown("Captcha registration is disabled"), } } @@ -279,8 +279,8 @@ func validateRecaptcha( body, err := ioutil.ReadAll(resp.Body) if err != nil { return &util.JSONResponse{ - Code: http.StatusInternalServerError, - JSON: jsonerror.BadJSON("Error in contacting captcha server" + err.Error()), + Code: http.StatusGatewayTimeout, + JSON: jsonerror.Unknown("Error in contacting captcha server" + err.Error()), } } err = json.Unmarshal(body, &r) diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 2787fccd7..63ac27471 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -41,6 +41,10 @@ const pathPrefixUnstable = "/_matrix/client/unstable" // Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client // to clients which need to make outbound HTTP requests. +// +// Due to Setup being used to call many other functions, a gocyclo nolint is +// applied: +// nolint: gocyclo func Setup( apiMux *mux.Router, cfg config.Dendrite, producer *producers.RoomserverProducer, @@ -90,7 +94,10 @@ func Setup( ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/join/{roomIDOrAlias}", common.MakeAuthAPI("join", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return JoinRoomByIDOrAlias( req, device, vars["roomIDOrAlias"], cfg, federation, producer, queryAPI, aliasAPI, keyRing, accountDB, ) @@ -98,19 +105,28 @@ func Setup( ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/{membership:(?:join|kick|ban|unban|leave|invite)}", common.MakeAuthAPI("membership", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return SendMembership(req, accountDB, device, vars["roomID"], vars["membership"], cfg, queryAPI, asAPI, producer) }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/send/{eventType}", common.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, nil, cfg, queryAPI, producer, nil) }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/send/{eventType}/{txnID}", common.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } txnID := vars["txnID"] return SendEvent(req, device, vars["roomID"], vars["eventType"], &txnID, nil, cfg, queryAPI, producer, transactionsCache) @@ -118,7 +134,10 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/state/{eventType:[^/]+/?}", common.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } emptyString := "" eventType := vars["eventType"] // If there's a trailing slash, remove it @@ -130,7 +149,10 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/state/{eventType}/{stateKey}", common.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } stateKey := vars["stateKey"] return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, &stateKey, cfg, queryAPI, producer, nil) }), @@ -150,21 +172,30 @@ func Setup( r0mux.Handle("/directory/room/{roomAlias}", common.MakeExternalAPI("directory_room", func(req *http.Request) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return DirectoryRoom(req, vars["roomAlias"], federation, &cfg, aliasAPI) }), ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/directory/room/{roomAlias}", common.MakeAuthAPI("directory_room", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return SetLocalAlias(req, device, vars["roomAlias"], &cfg, aliasAPI) }), ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/directory/room/{roomAlias}", common.MakeAuthAPI("directory_room", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return RemoveLocalAlias(req, device, vars["roomAlias"], aliasAPI) }), ).Methods(http.MethodDelete, http.MethodOptions) @@ -183,7 +214,10 @@ func Setup( r0mux.Handle("/rooms/{roomID}/typing/{userID}", common.MakeAuthAPI("rooms_typing", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return SendTyping(req, device, vars["roomID"], vars["userID"], accountDB, typingProducer) }), ).Methods(http.MethodPut, http.MethodOptions) @@ -223,14 +257,20 @@ func Setup( r0mux.Handle("/user/{userId}/filter", common.MakeAuthAPI("put_filter", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return PutFilter(req, device, accountDB, vars["userId"]) }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/user/{userId}/filter/{filterId}", common.MakeAuthAPI("get_filter", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return GetFilter(req, device, accountDB, vars["userId"], vars["filterId"]) }), ).Methods(http.MethodGet, http.MethodOptions) @@ -239,21 +279,30 @@ func Setup( r0mux.Handle("/profile/{userID}", common.MakeExternalAPI("profile", func(req *http.Request) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return GetProfile(req, accountDB, vars["userID"], asAPI) }), ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/profile/{userID}/avatar_url", common.MakeExternalAPI("profile_avatar_url", func(req *http.Request) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return GetAvatarURL(req, accountDB, vars["userID"], asAPI) }), ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/profile/{userID}/avatar_url", common.MakeAuthAPI("profile_avatar_url", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return SetAvatarURL(req, accountDB, device, vars["userID"], userUpdateProducer, &cfg, producer, queryAPI) }), ).Methods(http.MethodPut, http.MethodOptions) @@ -262,14 +311,20 @@ func Setup( r0mux.Handle("/profile/{userID}/displayname", common.MakeExternalAPI("profile_displayname", func(req *http.Request) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return GetDisplayName(req, accountDB, vars["userID"], asAPI) }), ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/profile/{userID}/displayname", common.MakeAuthAPI("profile_displayname", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return SetDisplayName(req, accountDB, device, vars["userID"], userUpdateProducer, &cfg, producer, queryAPI) }), ).Methods(http.MethodPut, http.MethodOptions) @@ -339,28 +394,40 @@ func Setup( r0mux.Handle("/user/{userID}/account_data/{type}", common.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return SaveAccountData(req, accountDB, device, vars["userID"], "", vars["type"], syncProducer) }), ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/user/{userID}/rooms/{roomID}/account_data/{type}", common.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return SaveAccountData(req, accountDB, device, vars["userID"], vars["roomID"], vars["type"], syncProducer) }), ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/members", common.MakeAuthAPI("rooms_members", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return GetMemberships(req, device, vars["roomID"], false, cfg, queryAPI) }), ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/joined_members", common.MakeAuthAPI("rooms_members", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return GetMemberships(req, device, vars["roomID"], true, cfg, queryAPI) }), ).Methods(http.MethodGet, http.MethodOptions) @@ -380,14 +447,20 @@ func Setup( r0mux.Handle("/devices/{deviceID}", common.MakeAuthAPI("get_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return GetDeviceByID(req, deviceDB, device, vars["deviceID"]) }), ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/devices/{deviceID}", common.MakeAuthAPI("device_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return UpdateDeviceByID(req, deviceDB, device, vars["deviceID"]) }), ).Methods(http.MethodPut, http.MethodOptions) diff --git a/cmd/client-api-proxy/main.go b/cmd/client-api-proxy/main.go index b602016d5..27991c109 100644 --- a/cmd/client-api-proxy/main.go +++ b/cmd/client-api-proxy/main.go @@ -17,13 +17,14 @@ package main import ( "flag" "fmt" - log "github.com/sirupsen/logrus" "net/http" "net/http/httputil" "net/url" "os" "strings" "time" + + log "github.com/sirupsen/logrus" ) const usage = `Usage: %s diff --git a/cmd/federation-api-proxy/main.go b/cmd/federation-api-proxy/main.go index fc7a9e57a..fa90482d5 100644 --- a/cmd/federation-api-proxy/main.go +++ b/cmd/federation-api-proxy/main.go @@ -17,13 +17,14 @@ package main import ( "flag" "fmt" - log "github.com/sirupsen/logrus" "net/http" "net/http/httputil" "net/url" "os" "strings" "time" + + log "github.com/sirupsen/logrus" ) const usage = `Usage: %s diff --git a/cmd/kafka-producer/main.go b/cmd/kafka-producer/main.go index c35c587d8..8a4340f21 100644 --- a/cmd/kafka-producer/main.go +++ b/cmd/kafka-producer/main.go @@ -18,9 +18,10 @@ import ( "bufio" "flag" "fmt" - "github.com/Shopify/sarama" "os" "strings" + + "github.com/Shopify/sarama" ) const usage = `Usage: %s diff --git a/common/basecomponent/base.go b/common/basecomponent/base.go index d1f507544..6a20aca3b 100644 --- a/common/basecomponent/base.go +++ b/common/basecomponent/base.go @@ -71,7 +71,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string) *BaseDendrite { componentName: componentName, tracerCloser: closer, Cfg: cfg, - APIMux: mux.NewRouter(), + APIMux: mux.NewRouter().UseEncodedPath(), KafkaConsumer: kafkaConsumer, KafkaProducer: kafkaProducer, } diff --git a/common/config/config.go b/common/config/config.go index 16e50aead..9fcab8cf9 100644 --- a/common/config/config.go +++ b/common/config/config.go @@ -585,7 +585,7 @@ func (config *Dendrite) check(monolithic bool) error { } // Due to how Golang manages its interface types, this condition is not redundant. - // In order to get the proper behavior, it is necessary to return an explicit nil + // In order to get the proper behaviour, it is necessary to return an explicit nil // and not a nil configErrors. // This is because the following equalities hold: // error(nil) == nil diff --git a/common/log.go b/common/log.go index 7daa069c4..89a705822 100644 --- a/common/log.go +++ b/common/log.go @@ -76,7 +76,7 @@ func SetupHookLogging(hooks []config.LogrusHook, componentName string) { // Check we received a proper logging level level, err := logrus.ParseLevel(hook.Level) if err != nil { - logrus.Fatalf("Unrecognized logging level %s: %q", hook.Level, err) + logrus.Fatalf("Unrecognised logging level %s: %q", hook.Level, err) } // Perform a first filter on the logs according to the lowest level of all @@ -90,7 +90,7 @@ func SetupHookLogging(hooks []config.LogrusHook, componentName string) { checkFileHookParams(hook.Params) setupFileHook(hook, level, componentName) default: - logrus.Fatalf("Unrecognized logging hook type: %s", hook.Type) + logrus.Fatalf("Unrecognised logging hook type: %s", hook.Type) } } } diff --git a/common/partition_offset_table.go b/common/partition_offset_table.go index bb23755c7..bf37e2ed5 100644 --- a/common/partition_offset_table.go +++ b/common/partition_offset_table.go @@ -50,7 +50,6 @@ type PartitionOffsetStatements struct { // Prepare converts the raw SQL statements into prepared statements. // Takes a prefix to prepend to the table name used to store the partition offsets. // This allows multiple components to share the same database schema. -// nolint: safesql func (s *PartitionOffsetStatements) Prepare(db *sql.DB, prefix string) (err error) { _, err = db.Exec(strings.Replace(partitionOffsetsSchema, "${prefix}", prefix, -1)) if err != nil { diff --git a/common/routing.go b/common/routing.go new file mode 100644 index 000000000..330912cde --- /dev/null +++ b/common/routing.go @@ -0,0 +1,35 @@ +// Copyright 2019 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + "net/url" +) + +// URLDecodeMapValues is a function that iterates through each of the items in a +// map, URL decodes the value, and returns a new map with the decoded values +// under the same key names +func URLDecodeMapValues(vmap map[string]string) (map[string]string, error) { + decoded := make(map[string]string, len(vmap)) + for key, value := range vmap { + decodedVal, err := url.QueryUnescape(value) + if err != nil { + return make(map[string]string), err + } + decoded[key] = decodedVal + } + + return decoded, nil +} diff --git a/common/test/client.go b/common/test/client.go index d1c40e53f..a38540ac9 100644 --- a/common/test/client.go +++ b/common/test/client.go @@ -34,7 +34,7 @@ type Request struct { LastErr *LastRequestErr } -// LastRequestErr is a synchronized error wrapper +// LastRequestErr is a synchronised error wrapper // Useful for obtaining the last error from a set of requests type LastRequestErr struct { sync.Mutex diff --git a/common/types.go b/common/types.go index e539774e2..6888d3806 100644 --- a/common/types.go +++ b/common/types.go @@ -43,7 +43,7 @@ type DisplayName struct { // WeakBoolean is a type that will Unmarshal to true or false even if the encoded // representation is "true"/1 or "false"/0, as well as whatever other forms are -// recognized by strconv.ParseBool +// recognised by strconv.ParseBool type WeakBoolean bool // UnmarshalJSON is overridden here to allow strings vaguely representing a true diff --git a/docs/sytest.md b/docs/sytest.md index a0f7d85c1..e936dc493 100644 --- a/docs/sytest.md +++ b/docs/sytest.md @@ -1,8 +1,78 @@ # SyTest Dendrite uses [SyTest](https://github.com/matrix-org/sytest) for its -integration testing. When creating a new PR, add the test IDs that your PR -should allow to pass to `testfile` in dendrite's root directory. Not all PRs -need to make new tests pass. If we find your PR should be making a test pass we -may ask you to add to that file, as generally Dendrite's progress can be +integration testing. When creating a new PR, add the test IDs (see below) that +your PR should allow to pass to `testfile` in dendrite's root directory. Not all +PRs need to make new tests pass. If we find your PR should be making a test pass +we may ask you to add to that file, as generally Dendrite's progress can be tracked through the amount of SyTest tests it passes. + +## Finding out which tests to add + +We recommend you run the tests locally by manually setting up SyTest or using a +SyTest docker image. After running the tests, a script will print the tests you +need to add to `testfile` for you. + +You should proceed after you see no build problems for dendrite after running: + +```sh +./build.sh +``` + +### Manually Setting up SyTest + +Make sure you have Perl v5+ installed, and get SyTest with: + +(Note that this guide assumes your SyTest checkout is next to your +`dendrite` checkout.) + +```sh +git clone -b develop https://github.com/matrix-org/sytest +cd sytest +./install-deps.pl +``` + +Set up the database: + +```sh +sudo -u postgres psql -c "CREATE USER dendrite PASSWORD 'itsasecret'" +sudo -u postgres psql -c "CREATE DATABASE sytest_template OWNER dendrite" +mkdir -p "server-0" +cat > "server-0/database.yaml" << EOF +args: + user: dendrite + database: dendrite + host: 127.0.0.1 +type: pg +EOF +``` + +Run the tests: + +```sh +./run-tests.pl -I Dendrite::Monolith -d ../dendrite/bin -W ../dendrite/testfile -O tap --all | tee results.tap +``` + +where `tee` lets you see the results while they're being piped to the file. + +Once the tests are complete, run the helper script to see if you need to add +any newly passing test names to `testfile` in the project's root directory: + +```sh +../dendrite/show-expected-fail-tests.sh results.tap ../dendrite/testfile +``` + +If the script prints nothing/exits with 0, then you're good to go. + +### Using a SyTest Docker image + +Ensure you have the latest image for SyTest, then run the tests: + +```sh +docker pull matrixdotorg/sytest-dendrite +docker run --rm -v /path/to/dendrite/:/src/ matrixdotorg/sytest-dendrite +``` + +where `/path/to/dendrite/` should be replaced with the actual path to your +dendrite source code. The output should tell you if you need to add any tests to +`testfile`. diff --git a/federationapi/routing/profile.go b/federationapi/routing/profile.go index aa4fcdc42..2b478cfbf 100644 --- a/federationapi/routing/profile.go +++ b/federationapi/routing/profile.go @@ -53,7 +53,7 @@ func GetProfile( return httputil.LogThenError(httpReq, err) } - profile, err := appserviceAPI.RetreiveUserProfile(httpReq.Context(), userID, asAPI, accountDB) + profile, err := appserviceAPI.RetrieveUserProfile(httpReq.Context(), userID, asAPI, accountDB) if err != nil { return httputil.LogThenError(httpReq, err) } diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index 035d54aa1..16704e0b2 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -35,6 +35,10 @@ const ( ) // Setup registers HTTP handlers with the given ServeMux. +// +// Due to Setup being used to call many other functions, a gocyclo nolint is +// applied: +// nolint: gocyclo func Setup( apiMux *mux.Router, cfg config.Dendrite, @@ -64,7 +68,10 @@ func Setup( v1fedmux.Handle("/send/{txnID}/", common.MakeFedAPI( "federation_send", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars := mux.Vars(httpReq) + vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) + if err != nil { + return util.ErrorResponse(err) + } return Send( httpReq, request, gomatrixserverlib.TransactionID(vars["txnID"]), cfg, query, producer, keys, federation, @@ -75,7 +82,10 @@ func Setup( v1fedmux.Handle("/invite/{roomID}/{eventID}", common.MakeFedAPI( "federation_invite", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars := mux.Vars(httpReq) + vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) + if err != nil { + return util.ErrorResponse(err) + } return Invite( httpReq, request, vars["roomID"], vars["eventID"], cfg, producer, keys, @@ -92,7 +102,10 @@ func Setup( v1fedmux.Handle("/exchange_third_party_invite/{roomID}", common.MakeFedAPI( "exchange_third_party_invite", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars := mux.Vars(httpReq) + vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) + if err != nil { + return util.ErrorResponse(err) + } return ExchangeThirdPartyInvite( httpReq, request, vars["roomID"], query, cfg, federation, producer, ) @@ -102,7 +115,10 @@ func Setup( v1fedmux.Handle("/event/{eventID}", common.MakeFedAPI( "federation_get_event", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars := mux.Vars(httpReq) + vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) + if err != nil { + return util.ErrorResponse(err) + } return GetEvent( httpReq.Context(), request, query, vars["eventID"], ) @@ -112,7 +128,10 @@ func Setup( v1fedmux.Handle("/state/{roomID}", common.MakeFedAPI( "federation_get_event_auth", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars := mux.Vars(httpReq) + vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) + if err != nil { + return util.ErrorResponse(err) + } return GetState( httpReq.Context(), request, query, vars["roomID"], ) @@ -122,7 +141,10 @@ func Setup( v1fedmux.Handle("/state_ids/{roomID}", common.MakeFedAPI( "federation_get_event_auth", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars := mux.Vars(httpReq) + vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) + if err != nil { + return util.ErrorResponse(err) + } return GetStateIDs( httpReq.Context(), request, query, vars["roomID"], ) @@ -150,7 +172,10 @@ func Setup( v1fedmux.Handle("/user/devices/{userID}", common.MakeFedAPI( "federation_user_devices", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars := mux.Vars(httpReq) + vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) + if err != nil { + return util.ErrorResponse(err) + } return GetUserDevices( httpReq, deviceDB, vars["userID"], ) @@ -160,7 +185,10 @@ func Setup( v1fedmux.Handle("/make_join/{roomID}/{userID}", common.MakeFedAPI( "federation_make_join", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars := mux.Vars(httpReq) + vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) + if err != nil { + return util.ErrorResponse(err) + } roomID := vars["roomID"] userID := vars["userID"] return MakeJoin( @@ -172,7 +200,10 @@ func Setup( v1fedmux.Handle("/send_join/{roomID}/{userID}", common.MakeFedAPI( "federation_send_join", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars := mux.Vars(httpReq) + vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) + if err != nil { + return util.ErrorResponse(err) + } roomID := vars["roomID"] userID := vars["userID"] return SendJoin( @@ -184,7 +215,10 @@ func Setup( v1fedmux.Handle("/make_leave/{roomID}/{userID}", common.MakeFedAPI( "federation_make_leave", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars := mux.Vars(httpReq) + vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) + if err != nil { + return util.ErrorResponse(err) + } roomID := vars["roomID"] userID := vars["userID"] return MakeLeave( @@ -196,7 +230,10 @@ func Setup( v1fedmux.Handle("/send_leave/{roomID}/{userID}", common.MakeFedAPI( "federation_send_leave", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars := mux.Vars(httpReq) + vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) + if err != nil { + return util.ErrorResponse(err) + } roomID := vars["roomID"] userID := vars["userID"] return SendLeave( @@ -215,7 +252,10 @@ func Setup( v1fedmux.Handle("/get_missing_events/{roomID}", common.MakeFedAPI( "federation_get_missing_events", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars := mux.Vars(httpReq) + vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) + if err != nil { + return util.ErrorResponse(err) + } return GetMissingEvents(httpReq, request, query, vars["roomID"]) }, )).Methods(http.MethodPost) @@ -223,7 +263,10 @@ func Setup( v1fedmux.Handle("/backfill/{roomID}/", common.MakeFedAPI( "federation_backfill", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars := mux.Vars(httpReq) + vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) + if err != nil { + return util.ErrorResponse(err) + } return Backfill(httpReq, request, query, vars["roomID"], cfg) }, )).Methods(http.MethodGet) diff --git a/federationapi/routing/state.go b/federationapi/routing/state.go index 130f8a4f7..86cf1cf54 100644 --- a/federationapi/routing/state.go +++ b/federationapi/routing/state.go @@ -75,7 +75,8 @@ func parseEventIDParam( ) (eventID string, resErr *util.JSONResponse) { URL, err := url.Parse(request.RequestURI()) if err != nil { - *resErr = util.ErrorResponse(err) + response := util.ErrorResponse(err) + resErr = &response return } @@ -102,6 +103,10 @@ func getState( return nil, resErr } + if event.RoomID() != roomID { + return nil, &util.JSONResponse{Code: http.StatusNotFound, JSON: nil} + } + prevEventIDs := getIDsFromEventRef(event.PrevEvents()) authEventIDs := getIDsFromEventRef(event.AuthEvents()) diff --git a/federationapi/routing/threepid.go b/federationapi/routing/threepid.go index 27796067b..05ca8892e 100644 --- a/federationapi/routing/threepid.go +++ b/federationapi/routing/threepid.go @@ -194,7 +194,7 @@ func createInviteFrom3PIDInvite( StateKey: &inv.MXID, } - profile, err := appserviceAPI.RetreiveUserProfile(ctx, inv.MXID, asAPI, accountDB) + profile, err := appserviceAPI.RetrieveUserProfile(ctx, inv.MXID, asAPI, accountDB) if err != nil { return nil, err } diff --git a/federationsender/queue/queue.go b/federationsender/queue/queue.go index 4a38dc086..6a05c5f07 100644 --- a/federationsender/queue/queue.go +++ b/federationsender/queue/queue.go @@ -96,9 +96,11 @@ func (oqs *OutgoingQueues) SendEDU( // Remove our own server from the list of destinations. destinations = filterDestinations(oqs.origin, destinations) - log.WithFields(log.Fields{ - "destinations": destinations, "edu_type": e.Type, - }).Info("Sending EDU event") + if len(destinations) > 0 { + log.WithFields(log.Fields{ + "destinations": destinations, "edu_type": e.Type, + }).Info("Sending EDU event") + } oqs.queuesMutex.Lock() defer oqs.queuesMutex.Unlock() diff --git a/go.mod b/go.mod index 117248a7e..eecb6af6d 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd github.com/crossdock/crossdock-go v0.0.0-20160816171116-049aabb0122b - github.com/davecgh/go-spew v1.1.0 + github.com/davecgh/go-spew v1.1.1 github.com/eapache/go-resiliency v0.0.0-20160104191539-b86b1ec0dd42 github.com/eapache/go-xerial-snappy v0.0.0-20160609142408-bb955e01b934 github.com/eapache/queue v1.1.0 @@ -16,16 +16,21 @@ require ( github.com/golang/snappy v0.0.0-20170119014723-7db9049039a0 github.com/google/shlex v0.0.0-20150127133951-6f45313302b9 github.com/gorilla/context v1.1.1 - github.com/gorilla/mux v1.3.0 + github.com/gorilla/mux v1.7.3 github.com/jaegertracing/jaeger-client-go v0.0.0-20170921145708-3ad49a1d839b github.com/jaegertracing/jaeger-lib v0.0.0-20170920222118-21a3da6d66fe github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6 github.com/lib/pq v0.0.0-20170918175043-23da1db4f16d github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 +<<<<<<< HEAD github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 github.com/matrix-org/gomatrixserverlib v0.0.0-20181109104322-1c2cbc0872f0 +======= + github.com/matrix-org/gomatrix v0.0.0-20190130130140-385f072fe9af + github.com/matrix-org/gomatrixserverlib v0.0.0-20190619132215-178ed5e3b8e2 +>>>>>>> e2251199a49ab0bb846c02ba37e1cd437a7f725b github.com/matrix-org/naffka v0.0.0-20171115094957-662bfd0841d0 - github.com/matrix-org/util v0.0.0-20171013132526-8b1c8ab81986 + github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 github.com/matttproud/golang_protobuf_extensions v1.0.1 github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 github.com/nicksnyder/go-i18n v1.8.1 @@ -40,11 +45,11 @@ require ( github.com/prometheus/common v0.0.0-20170108231212-dd2f054febf4 github.com/prometheus/procfs v0.0.0-20170128160123-1878d9fbb537 github.com/rcrowley/go-metrics v0.0.0-20161128210544-1f30fe9094a5 - github.com/sirupsen/logrus v0.0.0-20170822132746-89742aefa4b2 - github.com/stretchr/testify v0.0.0-20170809224252-890a5c3458b4 - github.com/tidwall/gjson v1.0.2 - github.com/tidwall/match v0.0.0-20171002075945-1731857f09b1 - github.com/tidwall/sjson v1.0.0 + github.com/sirupsen/logrus v1.3.0 + github.com/stretchr/testify v1.2.2 + github.com/tidwall/gjson v1.1.5 + github.com/tidwall/match v1.0.1 + github.com/tidwall/sjson v1.0.3 github.com/uber-go/atomic v1.3.0 github.com/uber/jaeger-client-go v2.15.0+incompatible github.com/uber/jaeger-lib v1.5.0 @@ -52,14 +57,14 @@ require ( go.uber.org/atomic v1.3.0 go.uber.org/multierr v0.0.0-20170829224307-fb7d312c2c04 go.uber.org/zap v1.7.1 - golang.org/x/crypto v0.0.0-20181106171534-e4dc69e5b2fd - golang.org/x/net v0.0.0-20170927055102-0a9397675ba3 - golang.org/x/sys v0.0.0-20171012164349-43eea11bc926 + golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613 + golang.org/x/net v0.0.0-20190301231341-16b79f2e4e95 + golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 gopkg.in/Shopify/sarama.v1 v1.11.0 gopkg.in/airbrake/gobrake.v2 v2.0.9 gopkg.in/alecthomas/kingpin.v3-unstable v3.0.0-20170727041045-23bcc3c4eae3 gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 gopkg.in/h2non/bimg.v1 v1.0.18 - gopkg.in/macaroon.v2 v2.0.0 - gopkg.in/yaml.v2 v2.0.0-20171116090243-287cf08546ab + gopkg.in/macaroon.v2 v2.1.0 + gopkg.in/yaml.v2 v2.2.2 ) diff --git a/go.sum b/go.sum index b225d4bf0..b69606fd1 100644 --- a/go.sum +++ b/go.sum @@ -10,40 +10,62 @@ github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE github.com/crossdock/crossdock-go v0.0.0-20160816171116-049aabb0122b/go.mod h1:v9FBN7gdVTpiD/+LZ7Po0UKvROyT87uLVxTHVky/dlQ= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/eapache/go-resiliency v0.0.0-20160104191539-b86b1ec0dd42 h1:f8ERmXYuaC+kCSv2w+y3rBK/oVu6If4DEm3jywJJ0hc= github.com/eapache/go-resiliency v0.0.0-20160104191539-b86b1ec0dd42/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20160609142408-bb955e01b934 h1:oGLoaVIefp3tiOgi7+KInR/nNPvEpPM6GFo+El7fd14= github.com/eapache/go-xerial-snappy v0.0.0-20160609142408-bb955e01b934/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/frankban/quicktest v1.0.0/go.mod h1:R98jIehRai+d1/3Hv2//jOVCTJhW1VBavT6B6CuGq2k= github.com/golang/protobuf v0.0.0-20161117033126-8ee79997227b h1:fE/yi9pibxGEc0gSJuEShcsBXE2d5FW3OudsjE9tKzQ= github.com/golang/protobuf v0.0.0-20161117033126-8ee79997227b/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.0-20170119014723-7db9049039a0 h1:FMElzTwkd/2jQ2QzLEzt97JRgvFhYhnYiaQSwZ7tuyU= github.com/golang/snappy v0.0.0-20170119014723-7db9049039a0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/shlex v0.0.0-20150127133951-6f45313302b9/go.mod h1:RpwtwJQFrIEPstU94h88MWPXP2ektJZ8cZ0YntAmXiE= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.3.0 h1:HwSEKGN6U5T2aAQTfu5pW8fiwjSp3IgwdRbkICydk/c= github.com/gorilla/mux v1.3.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= +github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= github.com/jaegertracing/jaeger-client-go v0.0.0-20170921145708-3ad49a1d839b/go.mod h1:HWG7INeOG1ZE17I/S8eeb+svquXmBS/hf1Obi6hJUyQ= github.com/jaegertracing/jaeger-lib v0.0.0-20170920222118-21a3da6d66fe/go.mod h1:VqeqQrZmZr9G4WdLw4ei9tAHU54iJRkfoFHvTTQn4jQ= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6 h1:KAZ1BW2TCmT6PRihDPpocIy1QTtsAsrx6TneU/4+CMg= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v0.0.0-20170918175043-23da1db4f16d h1:Hdtccv31GWxWoCzWsIhZXy5NxEktzAkA8lywhTKu8O4= github.com/lib/pq v0.0.0-20170918175043-23da1db4f16d/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 h1:nMX2t7hbGF0NYDYySx0pCqEKGKAeZIiSqlWSspetlhY= github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg= github.com/matrix-org/gomatrix v0.0.0-20171003113848-a7fc80c8060c h1:aZap604NyBGhAUE0CyNHz6+Pryye5A5mHnYyO4KPPW8= github.com/matrix-org/gomatrix v0.0.0-20171003113848-a7fc80c8060c/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= +<<<<<<< HEAD github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bhrnp3Ky1qgx/fzCtCALOoGYylh2tpS9K4= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= +======= +github.com/matrix-org/gomatrix v0.0.0-20190130130140-385f072fe9af h1:piaIBNQGIHnni27xRB7VKkEwoWCgAmeuYf8pxAyG0bI= +github.com/matrix-org/gomatrix v0.0.0-20190130130140-385f072fe9af/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= +>>>>>>> e2251199a49ab0bb846c02ba37e1cd437a7f725b github.com/matrix-org/gomatrixserverlib v0.0.0-20181109104322-1c2cbc0872f0 h1:3UzhmERBbis4ZaB3imEbZwtDjGz/oVRC2cLLEajCzJA= github.com/matrix-org/gomatrixserverlib v0.0.0-20181109104322-1c2cbc0872f0/go.mod h1:YHyhIQUmuXyKtoVfDUMk/DyU93Taamlu6nPZkij/JtA= +github.com/matrix-org/gomatrixserverlib v0.0.0-20190619132215-178ed5e3b8e2 h1:pYajAEdi3sowj4iSunqctchhcMNW3rDjeeH0T4uDkMY= +github.com/matrix-org/gomatrixserverlib v0.0.0-20190619132215-178ed5e3b8e2/go.mod h1:sf0RcKOdiwJeTti7A313xsaejNUGYDq02MQZ4JD4w/E= github.com/matrix-org/naffka v0.0.0-20171115094957-662bfd0841d0 h1:p7WTwG+aXM86+yVrYAiCMW3ZHSmotVvuRbjtt3jC+4A= github.com/matrix-org/naffka v0.0.0-20171115094957-662bfd0841d0/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/util v0.0.0-20171013132526-8b1c8ab81986 h1:TiWl4hLvezAhRPM8tPcPDFTysZ7k4T/1J4GPp/iqlZo= github.com/matrix-org/util v0.0.0-20171013132526-8b1c8ab81986/go.mod h1:lePuOiXLNDott7NZfnQvJk0lAZ5HgvIuWGhel6J+RLA= +github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 h1:W7l5CP4V7wPyPb4tYE11dbmeAOwtFQBTW0rf4OonOS8= +github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5/go.mod h1:lePuOiXLNDott7NZfnQvJk0lAZ5HgvIuWGhel6J+RLA= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.1.4/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 h1:BvoENQQU+fZ9uukda/RzCAL/191HHwJA5b13R6diVlY= github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= github.com/nicksnyder/go-i18n v1.8.1/go.mod h1:HrK7VCrbOvQoUAQ7Vpy7i87N7JZZZ7R2xBGjv0j365Q= @@ -69,13 +91,23 @@ github.com/rcrowley/go-metrics v0.0.0-20161128210544-1f30fe9094a5 h1:gwcdIpH6NU2 github.com/rcrowley/go-metrics v0.0.0-20161128210544-1f30fe9094a5/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/sirupsen/logrus v0.0.0-20170822132746-89742aefa4b2 h1:+8J/sCAVv2Y9Ct1BKszDFJEVWv6Aynr2O4FYGUg6+Mc= github.com/sirupsen/logrus v0.0.0-20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= +github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME= +github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v0.0.0-20170809224252-890a5c3458b4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/tidwall/gjson v1.0.2 h1:5BsM7kyEAHAUGEGDkEKO9Mdyiuw6QQ6TSDdarP0Nnmk= github.com/tidwall/gjson v1.0.2/go.mod h1:c/nTNbUr0E0OrXEhq1pwa8iEgc2DOt4ZZqAt1HtCkPA= +github.com/tidwall/gjson v1.1.5 h1:QysILxBeUEY3GTLA0fQVgkQG1zme8NxGvhh2SSqWNwI= +github.com/tidwall/gjson v1.1.5/go.mod h1:c/nTNbUr0E0OrXEhq1pwa8iEgc2DOt4ZZqAt1HtCkPA= github.com/tidwall/match v0.0.0-20171002075945-1731857f09b1 h1:pWIN9LOlFRCJFqWIOEbHLvY0WWJddsjH2FQ6N0HKZdU= github.com/tidwall/match v0.0.0-20171002075945-1731857f09b1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= +github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= +github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= github.com/tidwall/sjson v1.0.0 h1:hOrzQPtGKlKAudQVmU43GkxEgG8TOgKyiKUyb7sE0rs= github.com/tidwall/sjson v1.0.0/go.mod h1:bURseu1nuBkFpIES5cz6zBtjmYeOQmEESshn7VpF15Y= +github.com/tidwall/sjson v1.0.3 h1:DeF+0LZqvIt4fKYw41aPB29ZGlvwVkHKktoXJ1YW9Y8= +github.com/tidwall/sjson v1.0.3/go.mod h1:bURseu1nuBkFpIES5cz6zBtjmYeOQmEESshn7VpF15Y= github.com/uber-go/atomic v1.3.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= github.com/uber/jaeger-client-go v2.15.0+incompatible h1:NP3qsSqNxh8VYr956ur1N/1C1PjvOJnJykCzcD5QHbk= github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= @@ -85,18 +117,31 @@ github.com/uber/tchannel-go v0.0.0-20170927010734-b3e26487e291/go.mod h1:Rrgz1eL go.uber.org/atomic v1.3.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v0.0.0-20170829224307-fb7d312c2c04/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.7.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181106171534-e4dc69e5b2fd h1:VtIkGDhk0ph3t+THbvXHfMZ8QHgsBO39Nh52+74pq7w= golang.org/x/crypto v0.0.0-20181106171534-e4dc69e5b2fd/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613 h1:MQ/ZZiDsUapFFiMS+vzwXkCTeEKaum+Do5rINYJDmxc= +golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/net v0.0.0-20170927055102-0a9397675ba3 h1:tTDpczhDVjW6WN3DinzKcw5juwkDTVn22I7MNlfxSXM= golang.org/x/net v0.0.0-20170927055102-0a9397675ba3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190301231341-16b79f2e4e95 h1:fY7Dsw114eJN4boqzVSbpVHO6rTdhq6/GnXeu+PKnzU= +golang.org/x/net v0.0.0-20190301231341-16b79f2e4e95/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/sys v0.0.0-20171012164349-43eea11bc926 h1:PY6OU86NqbyZiOzaPnDw6oOjAGtYQqIua16z6y9QkwE= golang.org/x/sys v0.0.0-20171012164349-43eea11bc926/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 h1:I6FyU15t786LL7oL/hn43zqTuEGr4PN7F4XJ1p4E3Y8= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= gopkg.in/Shopify/sarama.v1 v1.11.0 h1:/3kaCyeYaPbr59IBjeqhIcUOB1vXlIVqXAYa5g5C5F0= gopkg.in/Shopify/sarama.v1 v1.11.0/go.mod h1:AxnvoaevB2nBjNK17cG61A3LleFcWFwVBHBt+cot4Oc= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v3-unstable v3.0.0-20170727041045-23bcc3c4eae3/go.mod h1:3HH7i1SgMqlzxCcBmUHW657sD4Kvv9sC3HpL3YukzwA= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/h2non/bimg.v1 v1.0.18/go.mod h1:PgsZL7dLwUbsGm1NYps320GxGgvQNTnecMCZqxV11So= +gopkg.in/h2non/gock.v1 v1.0.14/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE= gopkg.in/macaroon.v2 v2.0.0/go.mod h1:+I6LnTMkm/uV5ew/0nsulNjL16SK4+C8yDmRUzHR17I= +gopkg.in/macaroon.v2 v2.1.0/go.mod h1:OUb+TQP/OP0WOerC2Jp/3CwhIKyIa9kQjuc7H24e6/o= gopkg.in/yaml.v2 v2.0.0-20171116090243-287cf08546ab h1:yZ6iByf7GKeJ3gsd1Dr/xaj1DyJ//wxKX1Cdh8LhoAw= gopkg.in/yaml.v2 v2.0.0-20171116090243-287cf08546ab/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/linter-fast.json b/linter-fast.json deleted file mode 100644 index 68d518444..000000000 --- a/linter-fast.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "Vendor": true, - "Cyclo": 12, - "Deadline": "5m", - "Enable": [ - "vetshadow", - "deadcode", - "gocyclo", - "ineffassign", - "misspell", - "errcheck", - "vet", - "gofmt", - "goconst" - ] -} diff --git a/linter.json b/linter.json deleted file mode 100644 index 1f0550aaf..000000000 --- a/linter.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "Vendor": true, - "Cyclo": 12, - "Deadline": "5m", - "Enable": [ - "vetshadow", - "deadcode", - "gocyclo", - "golint", - "varcheck", - "structcheck", - "ineffassign", - "misspell", - "unparam", - "errcheck", - "vet", - "gofmt", - "goconst" - ] -} diff --git a/mediaapi/routing/download.go b/mediaapi/routing/download.go index 9c8f43c44..38c436367 100644 --- a/mediaapi/routing/download.go +++ b/mediaapi/routing/download.go @@ -305,6 +305,10 @@ func (r *downloadRequest) respondFromLocalFile( }).Info("Responding with file") responseFile = file responseMetadata = r.MediaMetadata + + if len(responseMetadata.UploadName) > 0 { + w.Header().Set("Content-Disposition", fmt.Sprintf(`inline; filename*=utf-8"%s"`, responseMetadata.UploadName)) + } } w.Header().Set("Content-Type", string(responseMetadata.ContentType)) diff --git a/mediaapi/routing/routing.go b/mediaapi/routing/routing.go index fb983ccc2..5bcce1772 100644 --- a/mediaapi/routing/routing.go +++ b/mediaapi/routing/routing.go @@ -34,6 +34,10 @@ import ( const pathPrefixR0 = "/_matrix/media/r0" // Setup registers the media API HTTP handlers +// +// Due to Setup being used to call many other functions, a gocyclo nolint is +// applied: +// nolint: gocyclo func Setup( apiMux *mux.Router, cfg *config.Dendrite, @@ -87,7 +91,7 @@ func makeDownloadAPI( // Content-Type will be overridden in case of returning file data, else we respond with JSON-formatted errors w.Header().Set("Content-Type", "application/json") - vars := mux.Vars(req) + vars, _ := common.URLDecodeMapValues(mux.Vars(req)) Download( w, req, diff --git a/mediaapi/thumbnailer/thumbnailer_nfnt.go b/mediaapi/thumbnailer/thumbnailer_nfnt.go index 43bf8efba..5df6ce4be 100644 --- a/mediaapi/thumbnailer/thumbnailer_nfnt.go +++ b/mediaapi/thumbnailer/thumbnailer_nfnt.go @@ -20,9 +20,11 @@ import ( "context" "image" "image/draw" + // Imported for gif codec _ "image/gif" "image/jpeg" + // Imported for png codec _ "image/png" "os" @@ -258,9 +260,6 @@ func adjustSize(dst types.Path, img image.Image, w, h int, crop bool, logger *lo out = target } else { out = resize.Thumbnail(uint(w), uint(h), img, resize.Lanczos3) - if err != nil { - return -1, -1, err - } } if err = writeFile(out, string(dst)); err != nil { diff --git a/publicroomsapi/routing/routing.go b/publicroomsapi/routing/routing.go index 6a4b79b7e..3a1c9eb58 100644 --- a/publicroomsapi/routing/routing.go +++ b/publicroomsapi/routing/routing.go @@ -30,6 +30,10 @@ import ( const pathPrefixR0 = "/_matrix/client/r0" // Setup configures the given mux with publicroomsapi server listeners +// +// Due to Setup being used to call many other functions, a gocyclo nolint is +// applied: +// nolint: gocyclo func Setup(apiMux *mux.Router, deviceDB *devices.Database, publicRoomsDB *storage.PublicRoomsServerDatabase) { r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter() @@ -41,14 +45,20 @@ func Setup(apiMux *mux.Router, deviceDB *devices.Database, publicRoomsDB *storag r0mux.Handle("/directory/list/room/{roomID}", common.MakeExternalAPI("directory_list", func(req *http.Request) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return directory.GetVisibility(req, publicRoomsDB, vars["roomID"]) }), ).Methods(http.MethodGet, http.MethodOptions) // TODO: Add AS support r0mux.Handle("/directory/list/room/{roomID}", common.MakeAuthAPI("directory_list", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return directory.SetVisibility(req, publicRoomsDB, vars["roomID"]) }), ).Methods(http.MethodPut, http.MethodOptions) diff --git a/publicroomsapi/storage/public_rooms_table.go b/publicroomsapi/storage/public_rooms_table.go index 85d65c2cc..5e1eb3e12 100644 --- a/publicroomsapi/storage/public_rooms_table.go +++ b/publicroomsapi/storage/public_rooms_table.go @@ -134,7 +134,6 @@ type publicRoomsStatements struct { updateRoomAttributeStmts map[string]*sql.Stmt } -// nolint: safesql func (s *publicRoomsStatements) prepare(db *sql.DB) (err error) { _, err = db.Exec(publicRoomsSchema) if err != nil { diff --git a/roomserver/alias/alias.go b/roomserver/alias/alias.go index 27279aad8..f699e3362 100644 --- a/roomserver/alias/alias.go +++ b/roomserver/alias/alias.go @@ -96,12 +96,21 @@ func (r *RoomserverAliasAPI) GetRoomIDForAlias( return err } - // No rooms found locally, try our application services by making a call to - // the appservice component - aliasReq := appserviceAPI.RoomAliasExistsRequest{Alias: request.Alias} - var aliasResp appserviceAPI.RoomAliasExistsResponse - if err = r.AppserviceAPI.RoomAliasExists(ctx, &aliasReq, &aliasResp); err != nil { - return err + if roomID == "" { + // No room found locally, try our application services by making a call to + // the appservice component + aliasReq := appserviceAPI.RoomAliasExistsRequest{Alias: request.Alias} + var aliasResp appserviceAPI.RoomAliasExistsResponse + if err = r.AppserviceAPI.RoomAliasExists(ctx, &aliasReq, &aliasResp); err != nil { + return err + } + + if aliasResp.AliasExists { + roomID, err = r.DB.GetRoomIDForAlias(ctx, request.Alias) + if err != nil { + return err + } + } } response.RoomID = roomID @@ -268,6 +277,20 @@ func (r *RoomserverAliasAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) + servMux.Handle( + roomserverAPI.RoomserverGetAliasesForRoomIDPath, + common.MakeInternalAPI("getAliasesForRoomID", func(req *http.Request) util.JSONResponse { + var request roomserverAPI.GetAliasesForRoomIDRequest + var response roomserverAPI.GetAliasesForRoomIDResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.GetAliasesForRoomID(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) servMux.Handle( roomserverAPI.RoomserverRemoveRoomAliasPath, common.MakeInternalAPI("removeRoomAlias", func(req *http.Request) util.JSONResponse { diff --git a/roomserver/alias/alias_test.go b/roomserver/alias/alias_test.go new file mode 100644 index 000000000..4b9ca022d --- /dev/null +++ b/roomserver/alias/alias_test.go @@ -0,0 +1,196 @@ +// Copyright 2019 Serra Allgood +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package alias + +import ( + "context" + "fmt" + "strings" + "testing" + + appserviceAPI "github.com/matrix-org/dendrite/appservice/api" + roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" +) + +type MockRoomserverAliasAPIDatabase struct { + mode string + attempts int +} + +// These methods can be essentially noop +func (db MockRoomserverAliasAPIDatabase) SetRoomAlias(ctx context.Context, alias string, roomID string) error { + return nil +} + +func (db MockRoomserverAliasAPIDatabase) GetAliasesForRoomID(ctx context.Context, roomID string) ([]string, error) { + aliases := make([]string, 0) + return aliases, nil +} + +func (db MockRoomserverAliasAPIDatabase) RemoveRoomAlias(ctx context.Context, alias string) error { + return nil +} + +// This method needs to change depending on test case +func (db *MockRoomserverAliasAPIDatabase) GetRoomIDForAlias( + ctx context.Context, + alias string, +) (string, error) { + switch db.mode { + case "empty": + return "", nil + case "error": + return "", fmt.Errorf("found an error from GetRoomIDForAlias") + case "found": + return "123", nil + case "emptyFound": + switch db.attempts { + case 0: + db.attempts = 1 + return "", nil + case 1: + db.attempts = 0 + return "123", nil + default: + return "", nil + } + default: + return "", fmt.Errorf("unknown option used") + } +} + +type MockAppServiceQueryAPI struct { + mode string +} + +// This method can be noop +func (q MockAppServiceQueryAPI) UserIDExists( + ctx context.Context, + req *appserviceAPI.UserIDExistsRequest, + resp *appserviceAPI.UserIDExistsResponse, +) error { + return nil +} + +func (q MockAppServiceQueryAPI) RoomAliasExists( + ctx context.Context, + req *appserviceAPI.RoomAliasExistsRequest, + resp *appserviceAPI.RoomAliasExistsResponse, +) error { + switch q.mode { + case "error": + return fmt.Errorf("found an error from RoomAliasExists") + case "found": + resp.AliasExists = true + return nil + case "empty": + resp.AliasExists = false + return nil + default: + return fmt.Errorf("Unknown option used") + } +} + +func TestGetRoomIDForAlias(t *testing.T) { + type arguments struct { + ctx context.Context + request *roomserverAPI.GetRoomIDForAliasRequest + response *roomserverAPI.GetRoomIDForAliasResponse + } + args := arguments{ + context.Background(), + &roomserverAPI.GetRoomIDForAliasRequest{}, + &roomserverAPI.GetRoomIDForAliasResponse{}, + } + type testCase struct { + name string + dbMode string + queryMode string + wantError bool + errorMsg string + want string + } + tt := []testCase{ + { + "found local alias", + "found", + "error", + false, + "", + "123", + }, + { + "found appservice alias", + "emptyFound", + "found", + false, + "", + "123", + }, + { + "error returned from DB", + "error", + "", + true, + "GetRoomIDForAlias", + "", + }, + { + "error returned from appserviceAPI", + "empty", + "error", + true, + "RoomAliasExists", + "", + }, + { + "no errors but no alias", + "empty", + "empty", + false, + "", + "", + }, + } + + setup := func(dbMode, queryMode string) *RoomserverAliasAPI { + mockAliasAPIDB := &MockRoomserverAliasAPIDatabase{dbMode, 0} + mockAppServiceQueryAPI := MockAppServiceQueryAPI{queryMode} + + return &RoomserverAliasAPI{ + DB: mockAliasAPIDB, + AppserviceAPI: mockAppServiceQueryAPI, + } + } + + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + aliasAPI := setup(tc.dbMode, tc.queryMode) + + err := aliasAPI.GetRoomIDForAlias(args.ctx, args.request, args.response) + if tc.wantError { + if err == nil { + t.Fatalf("Got no error; wanted error from %s", tc.errorMsg) + } else if !strings.Contains(err.Error(), tc.errorMsg) { + t.Fatalf("Got %s; wanted error from %s", err, tc.errorMsg) + } + } else if err != nil { + t.Fatalf("Got %s; wanted no error", err) + } else if args.response.RoomID != tc.want { + t.Errorf("Got '%s'; wanted '%s'", args.response.RoomID, tc.want) + } + }) + } +} diff --git a/roomserver/storage/prepare.go b/roomserver/storage/prepare.go index 61c49a3c1..b19765992 100644 --- a/roomserver/storage/prepare.go +++ b/roomserver/storage/prepare.go @@ -25,7 +25,6 @@ type statementList []struct { } // prepare the SQL for each statement in the list and assign the result to the prepared statement. -// nolint: safesql func (s statementList) prepare(db *sql.DB) (err error) { for _, statement := range s { if *statement.statement, err = db.Prepare(statement.sql); err != nil { diff --git a/scripts/find-lint.sh b/scripts/find-lint.sh index a8201072b..25b311f94 100755 --- a/scripts/find-lint.sh +++ b/scripts/find-lint.sh @@ -3,44 +3,34 @@ # Runs the linters against dendrite # The linters can take a lot of resources and are slow, so they can be -# configured using two environment variables: +# configured using the following environment variables: # # - `DENDRITE_LINT_CONCURRENCY` - number of concurrent linters to run, -# gometalinter defaults this to 8 -# - `DENDRITE_LINT_DISABLE_GC` - if set then the the go gc will be disabled -# when running the linters, speeding them up but using much more memory. +# golangci-lint defaults this to NumCPU +# - `GOGC` - how often to perform garbage collection during golangci-lint runs. +# Essentially a ratio of memory/speed. See https://github.com/golangci/golangci-lint#memory-usage-of-golangci-lint +# for more info. set -eux cd `dirname $0`/.. -# gometalinter doesn't seem to work without this. -# We should move from gometalinter asap as per https://github.com/matrix-org/dendrite/issues/697 so this is a temporary -# fix. -export GO111MODULE=off - args="" if [ ${1:-""} = "fast" ] -then args="--config=linter-fast.json" -else args="--config=linter.json" +then args="--fast" fi -if [ -n "${DENDRITE_LINT_CONCURRENCY:-}" ] -then args="$args --concurrency=$DENDRITE_LINT_CONCURRENCY" -fi +echo "Installing golangci-lint..." -if [ -z "${DENDRITE_LINT_DISABLE_GC:-}" ] -then args="$args --enable-gc" -fi - -echo "Installing lint search engine..." -go get github.com/alecthomas/gometalinter/ - -gometalinter --config=linter.json ./... --install +# Make a backup of go.{mod,sum} first +# TODO: Once go 1.13 is out, use go get's -mod=readonly option +# https://github.com/golang/go/issues/30667 +cp go.mod go.mod.bak && cp go.sum go.sum.bak +go get github.com/golangci/golangci-lint/cmd/golangci-lint echo "Looking for lint..." -gometalinter ./... $args +golangci-lint run $args -echo "Double checking spelling..." -misspell -error src *.md +# Restore go.{mod,sum} +mv go.mod.bak go.mod && mv go.sum.bak go.sum diff --git a/show-expected-fail-tests.sh b/show-expected-fail-tests.sh new file mode 100755 index 000000000..80b842ab1 --- /dev/null +++ b/show-expected-fail-tests.sh @@ -0,0 +1,45 @@ +#! /bin/bash + +results_file=$1 +testfile=$2 + +fail_build=0 + +if [ ! -f "$results_file" ]; then + echo "ERROR: Specified results file ${results_file} doesn't exist." + fail_build=1 +fi + +if [ ! -f "$testfile" ]; then + echo "ERROR: Specified testfile ${testfile} doesn't exist." + fail_build=1 +fi + +[ "$fail_build" = 0 ] || exit 1 + +passed_but_expected_fail=$(grep ' # TODO passed but expected fail' ${results_file} | sed -E 's/^ok [0-9]+ (\(expected fail\) )?//' | sed -E 's/( \([0-9]+ subtests\))? # TODO passed but expected fail$//') +tests_to_add="" +already_in_testfile="" + +while read -r test_id; do + [ "${test_id}" = "" ] && continue + grep "${test_id}" "${testfile}" > /dev/null 2>&1 + if [ "$?" != "0" ]; then + tests_to_add="${tests_to_add}${test_id}\n" + fail_build=1 + else + already_in_testfile="${already_in_testfile}${test_id}\n" + fi +done <<< "${passed_but_expected_fail}" + +if [ -n "${tests_to_add}" ]; then + echo "ERROR: The following passed tests are not present in testfile. Please append them to the file:" + echo -e "${tests_to_add}" +fi + +if [ -n "${already_in_testfile}" ]; then + echo "WARN: Tests in testfile still marked as expected fail:" + echo -e "${already_in_testfile}" +fi + +exit ${fail_build} diff --git a/syncapi/consumers/clientapi.go b/syncapi/consumers/clientapi.go index d05a76920..f0db56427 100644 --- a/syncapi/consumers/clientapi.go +++ b/syncapi/consumers/clientapi.go @@ -22,6 +22,7 @@ import ( "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/sync" + "github.com/matrix-org/dendrite/syncapi/types" log "github.com/sirupsen/logrus" sarama "gopkg.in/Shopify/sarama.v1" ) @@ -29,7 +30,7 @@ import ( // OutputClientDataConsumer consumes events that originated in the client API server. type OutputClientDataConsumer struct { clientAPIConsumer *common.ContinualConsumer - db *storage.SyncServerDatabase + db *storage.SyncServerDatasource notifier *sync.Notifier } @@ -38,7 +39,7 @@ func NewOutputClientDataConsumer( cfg *config.Dendrite, kafkaConsumer sarama.Consumer, n *sync.Notifier, - store *storage.SyncServerDatabase, + store *storage.SyncServerDatasource, ) *OutputClientDataConsumer { consumer := common.ContinualConsumer{ @@ -78,7 +79,7 @@ func (s *OutputClientDataConsumer) onMessage(msg *sarama.ConsumerMessage) error "room_id": output.RoomID, }).Info("received data from client API server") - syncStreamPos, err := s.db.UpsertAccountData( + pduPos, err := s.db.UpsertAccountData( context.TODO(), string(msg.Key), output.RoomID, output.Type, ) if err != nil { @@ -89,7 +90,7 @@ func (s *OutputClientDataConsumer) onMessage(msg *sarama.ConsumerMessage) error }).Panicf("could not save account data") } - s.notifier.OnNewEvent(nil, string(msg.Key), syncStreamPos) + s.notifier.OnNewEvent(nil, "", []string{string(msg.Key)}, types.SyncPosition{PDUPosition: pduPos}) return nil } diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index 273b6aea1..e4f1ab460 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -33,7 +33,7 @@ import ( // OutputRoomEventConsumer consumes events that originated in the room server. type OutputRoomEventConsumer struct { roomServerConsumer *common.ContinualConsumer - db *storage.SyncServerDatabase + db *storage.SyncServerDatasource notifier *sync.Notifier query api.RoomserverQueryAPI } @@ -43,7 +43,7 @@ func NewOutputRoomEventConsumer( cfg *config.Dendrite, kafkaConsumer sarama.Consumer, n *sync.Notifier, - store *storage.SyncServerDatabase, + store *storage.SyncServerDatasource, queryAPI api.RoomserverQueryAPI, ) *OutputRoomEventConsumer { @@ -126,7 +126,7 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent( } } - syncStreamPos, err := s.db.WriteEvent( + pduPos, err := s.db.WriteEvent( ctx, &ev, addsStateEvents, @@ -134,10 +134,6 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent( msg.RemovesStateEventIDs, msg.TransactionID, ) - if err != nil { - return err - } - if err != nil { // panic rather than continue with an inconsistent database log.WithFields(log.Fields{ @@ -148,7 +144,7 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent( }).Panicf("roomserver output log: write event failure") return nil } - s.notifier.OnNewEvent(&ev, "", types.StreamPosition(syncStreamPos)) + s.notifier.OnNewEvent(&ev, "", nil, types.SyncPosition{PDUPosition: pduPos}) return nil } @@ -156,7 +152,7 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent( func (s *OutputRoomEventConsumer) onNewInviteEvent( ctx context.Context, msg api.OutputNewInviteEvent, ) error { - syncStreamPos, err := s.db.AddInviteEvent(ctx, msg.Event) + pduPos, err := s.db.AddInviteEvent(ctx, msg.Event) if err != nil { // panic rather than continue with an inconsistent database log.WithFields(log.Fields{ @@ -165,7 +161,7 @@ func (s *OutputRoomEventConsumer) onNewInviteEvent( }).Panicf("roomserver output log: write invite failure") return nil } - s.notifier.OnNewEvent(&msg.Event, "", syncStreamPos) + s.notifier.OnNewEvent(&msg.Event, "", nil, types.SyncPosition{PDUPosition: pduPos}) return nil } diff --git a/syncapi/consumers/typingserver.go b/syncapi/consumers/typingserver.go new file mode 100644 index 000000000..5d998a18a --- /dev/null +++ b/syncapi/consumers/typingserver.go @@ -0,0 +1,96 @@ +// Copyright 2019 Alex Chen +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package consumers + +import ( + "encoding/json" + + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/dendrite/syncapi/sync" + "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/dendrite/typingserver/api" + log "github.com/sirupsen/logrus" + sarama "gopkg.in/Shopify/sarama.v1" +) + +// OutputTypingEventConsumer consumes events that originated in the typing server. +type OutputTypingEventConsumer struct { + typingConsumer *common.ContinualConsumer + db *storage.SyncServerDatasource + notifier *sync.Notifier +} + +// NewOutputTypingEventConsumer creates a new OutputTypingEventConsumer. +// Call Start() to begin consuming from the typing server. +func NewOutputTypingEventConsumer( + cfg *config.Dendrite, + kafkaConsumer sarama.Consumer, + n *sync.Notifier, + store *storage.SyncServerDatasource, +) *OutputTypingEventConsumer { + + consumer := common.ContinualConsumer{ + Topic: string(cfg.Kafka.Topics.OutputTypingEvent), + Consumer: kafkaConsumer, + PartitionStore: store, + } + + s := &OutputTypingEventConsumer{ + typingConsumer: &consumer, + db: store, + notifier: n, + } + + consumer.ProcessMessage = s.onMessage + + return s +} + +// Start consuming from typing api +func (s *OutputTypingEventConsumer) Start() error { + s.db.SetTypingTimeoutCallback(func(userID, roomID string, latestSyncPosition int64) { + s.notifier.OnNewEvent(nil, roomID, nil, types.SyncPosition{TypingPosition: latestSyncPosition}) + }) + + return s.typingConsumer.Start() +} + +func (s *OutputTypingEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { + var output api.OutputTypingEvent + if err := json.Unmarshal(msg.Value, &output); err != nil { + // If the message was invalid, log it and move on to the next message in the stream + log.WithError(err).Errorf("typing server output log: message parse failure") + return nil + } + + log.WithFields(log.Fields{ + "room_id": output.Event.RoomID, + "user_id": output.Event.UserID, + "typing": output.Event.Typing, + }).Debug("received data from typing server") + + var typingPos int64 + typingEvent := output.Event + if typingEvent.Typing { + typingPos = s.db.AddTypingUser(typingEvent.UserID, typingEvent.RoomID, output.ExpireTime) + } else { + typingPos = s.db.RemoveTypingUser(typingEvent.UserID, typingEvent.RoomID) + } + + s.notifier.OnNewEvent(nil, output.Event.RoomID, nil, types.SyncPosition{TypingPosition: typingPos}) + return nil +} diff --git a/syncapi/routing/routing.go b/syncapi/routing/routing.go index 93d939c30..0f5019fc3 100644 --- a/syncapi/routing/routing.go +++ b/syncapi/routing/routing.go @@ -30,7 +30,11 @@ import ( const pathPrefixR0 = "/_matrix/client/r0" // Setup configures the given mux with sync-server listeners -func Setup(apiMux *mux.Router, srp *sync.RequestPool, syncDB *storage.SyncServerDatabase, deviceDB *devices.Database) { +// +// Due to Setup being used to call many other functions, a gocyclo nolint is +// applied: +// nolint: gocyclo +func Setup(apiMux *mux.Router, srp *sync.RequestPool, syncDB *storage.SyncServerDatasource, deviceDB *devices.Database) { r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter() authData := auth.Data{ @@ -45,17 +49,26 @@ func Setup(apiMux *mux.Router, srp *sync.RequestPool, syncDB *storage.SyncServer })).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/state", common.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return OnIncomingStateRequest(req, syncDB, vars["roomID"]) })).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/state/{type}", common.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return OnIncomingStateTypeRequest(req, syncDB, vars["roomID"], vars["type"], "") })).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", common.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return OnIncomingStateTypeRequest(req, syncDB, vars["roomID"], vars["type"], vars["stateKey"]) })).Methods(http.MethodGet, http.MethodOptions) } diff --git a/syncapi/routing/state.go b/syncapi/routing/state.go index 6b98a0b7b..5571a0525 100644 --- a/syncapi/routing/state.go +++ b/syncapi/routing/state.go @@ -40,7 +40,7 @@ type stateEventInStateResp struct { // TODO: Check if the user is in the room. If not, check if the room's history // is publicly visible. Current behaviour is returning an empty array if the // user cannot see the room's history. -func OnIncomingStateRequest(req *http.Request, db *storage.SyncServerDatabase, roomID string) util.JSONResponse { +func OnIncomingStateRequest(req *http.Request, db *storage.SyncServerDatasource, roomID string) util.JSONResponse { // TODO(#287): Auth request and handle the case where the user has left (where // we should return the state at the poin they left) @@ -84,7 +84,7 @@ func OnIncomingStateRequest(req *http.Request, db *storage.SyncServerDatabase, r // /rooms/{roomID}/state/{type}/{statekey} request. It will look in current // state to see if there is an event with that type and state key, if there // is then (by default) we return the content, otherwise a 404. -func OnIncomingStateTypeRequest(req *http.Request, db *storage.SyncServerDatabase, roomID string, evType, stateKey string) util.JSONResponse { +func OnIncomingStateTypeRequest(req *http.Request, db *storage.SyncServerDatasource, roomID string, evType, stateKey string) util.JSONResponse { // TODO(#287): Auth request and handle the case where the user has left (where // we should return the state at the poin they left) diff --git a/syncapi/storage/account_data_table.go b/syncapi/storage/account_data_table.go index d4d74d158..9b73ce7d6 100644 --- a/syncapi/storage/account_data_table.go +++ b/syncapi/storage/account_data_table.go @@ -19,8 +19,6 @@ import ( "database/sql" "github.com/matrix-org/dendrite/common" - - "github.com/matrix-org/dendrite/syncapi/types" ) const accountDataSchema = ` @@ -94,7 +92,7 @@ func (s *accountDataStatements) insertAccountData( func (s *accountDataStatements) selectAccountDataInRange( ctx context.Context, userID string, - oldPos, newPos types.StreamPosition, + oldPos, newPos int64, ) (data map[string][]string, err error) { data = make(map[string][]string) diff --git a/syncapi/storage/output_room_events_table.go b/syncapi/storage/output_room_events_table.go index 035db9882..34632aedf 100644 --- a/syncapi/storage/output_room_events_table.go +++ b/syncapi/storage/output_room_events_table.go @@ -23,7 +23,6 @@ import ( "github.com/lib/pq" "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" log "github.com/sirupsen/logrus" ) @@ -109,11 +108,11 @@ func (s *outputRoomEventsStatements) prepare(db *sql.DB) (err error) { return } -// selectStateInRange returns the state events between the two given stream positions, exclusive of oldPos, inclusive of newPos. +// selectStateInRange returns the state events between the two given PDU stream positions, exclusive of oldPos, inclusive of newPos. // Results are bucketed based on the room ID. If the same state is overwritten multiple times between the // two positions, only the most recent state is returned. func (s *outputRoomEventsStatements) selectStateInRange( - ctx context.Context, txn *sql.Tx, oldPos, newPos types.StreamPosition, + ctx context.Context, txn *sql.Tx, oldPos, newPos int64, ) (map[string]map[string]bool, map[string]streamEvent, error) { stmt := common.TxStmt(txn, s.selectStateInRangeStmt) @@ -171,7 +170,7 @@ func (s *outputRoomEventsStatements) selectStateInRange( eventIDToEvent[ev.EventID()] = streamEvent{ Event: ev, - streamPosition: types.StreamPosition(streamPos), + streamPosition: streamPos, } } @@ -223,7 +222,7 @@ func (s *outputRoomEventsStatements) insertEvent( // RecentEventsInRoom returns the most recent events in the given room, up to a maximum of 'limit'. func (s *outputRoomEventsStatements) selectRecentEvents( ctx context.Context, txn *sql.Tx, - roomID string, fromPos, toPos types.StreamPosition, limit int, + roomID string, fromPos, toPos int64, limit int, ) ([]streamEvent, error) { stmt := common.TxStmt(txn, s.selectRecentEventsStmt) rows, err := stmt.QueryContext(ctx, roomID, fromPos, toPos, limit) @@ -236,7 +235,7 @@ func (s *outputRoomEventsStatements) selectRecentEvents( return nil, err } // The events need to be returned from oldest to latest, which isn't - // necessary the way the SQL query returns them, so a sort is necessary to + // necessarily the way the SQL query returns them, so a sort is necessary to // ensure the events are in the right order in the slice. sort.SliceStable(events, func(i int, j int) bool { return events[i].streamPosition < events[j].streamPosition @@ -286,7 +285,7 @@ func rowsToStreamEvents(rows *sql.Rows) ([]streamEvent, error) { result = append(result, streamEvent{ Event: ev, - streamPosition: types.StreamPosition(streamPos), + streamPosition: streamPos, transactionID: transactionID, }) } diff --git a/syncapi/storage/syncserver.go b/syncapi/storage/syncserver.go index ec973e2c1..b4d7ccbd2 100644 --- a/syncapi/storage/syncserver.go +++ b/syncapi/storage/syncserver.go @@ -17,16 +17,21 @@ package storage import ( "context" "database/sql" + "encoding/json" "fmt" + "strconv" + "time" "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/roomserver/api" + // Import the postgres database driver. _ "github.com/lib/pq" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/dendrite/typingserver/cache" "github.com/matrix-org/gomatrixserverlib" ) @@ -34,33 +39,35 @@ type stateDelta struct { roomID string stateEvents []gomatrixserverlib.Event membership string - // The stream position of the latest membership event for this user, if applicable. + // The PDU stream position of the latest membership event for this user, if applicable. // Can be 0 if there is no membership event in this delta. - membershipPos types.StreamPosition + membershipPos int64 } -// Same as gomatrixserverlib.Event but also has the stream position for this event. +// Same as gomatrixserverlib.Event but also has the PDU stream position for this event. type streamEvent struct { gomatrixserverlib.Event - streamPosition types.StreamPosition + streamPosition int64 transactionID *api.TransactionID } -// SyncServerDatabase represents a sync server database -type SyncServerDatabase struct { +// SyncServerDatabase represents a sync server datasource which manages +// both the database for PDUs and caches for EDUs. +type SyncServerDatasource struct { db *sql.DB common.PartitionOffsetStatements accountData accountDataStatements events outputRoomEventsStatements roomstate currentRoomStateStatements invites inviteEventsStatements + typingCache *cache.TypingCache } // NewSyncServerDatabase creates a new sync server database -func NewSyncServerDatabase(dataSourceName string) (*SyncServerDatabase, error) { - var d SyncServerDatabase +func NewSyncServerDatasource(dbDataSourceName string) (*SyncServerDatasource, error) { + var d SyncServerDatasource var err error - if d.db, err = sql.Open("postgres", dataSourceName); err != nil { + if d.db, err = sql.Open("postgres", dbDataSourceName); err != nil { return nil, err } if err = d.PartitionOffsetStatements.Prepare(d.db, "syncapi"); err != nil { @@ -78,11 +85,12 @@ func NewSyncServerDatabase(dataSourceName string) (*SyncServerDatabase, error) { if err := d.invites.prepare(d.db); err != nil { return nil, err } + d.typingCache = cache.NewTypingCache() return &d, nil } // AllJoinedUsersInRooms returns a map of room ID to a list of all joined user IDs. -func (d *SyncServerDatabase) AllJoinedUsersInRooms(ctx context.Context) (map[string][]string, error) { +func (d *SyncServerDatasource) AllJoinedUsersInRooms(ctx context.Context) (map[string][]string, error) { return d.roomstate.selectJoinedUsers(ctx) } @@ -91,7 +99,7 @@ func (d *SyncServerDatabase) AllJoinedUsersInRooms(ctx context.Context) (map[str // If an event is not found in the database then it will be omitted from the list. // Returns an error if there was a problem talking with the database. // Does not include any transaction IDs in the returned events. -func (d *SyncServerDatabase) Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.Event, error) { +func (d *SyncServerDatasource) Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.Event, error) { streamEvents, err := d.events.selectEvents(ctx, nil, eventIDs) if err != nil { return nil, err @@ -103,38 +111,38 @@ func (d *SyncServerDatabase) Events(ctx context.Context, eventIDs []string) ([]g } // WriteEvent into the database. It is not safe to call this function from multiple goroutines, as it would create races -// when generating the stream position for this event. Returns the sync stream position for the inserted event. +// when generating the sync stream position for this event. Returns the sync stream position for the inserted event. // Returns an error if there was a problem inserting this event. -func (d *SyncServerDatabase) WriteEvent( +func (d *SyncServerDatasource) WriteEvent( ctx context.Context, ev *gomatrixserverlib.Event, addStateEvents []gomatrixserverlib.Event, addStateEventIDs, removeStateEventIDs []string, transactionID *api.TransactionID, -) (streamPos types.StreamPosition, returnErr error) { +) (pduPosition int64, returnErr error) { returnErr = common.WithTransaction(d.db, func(txn *sql.Tx) error { var err error pos, err := d.events.insertEvent(ctx, txn, ev, addStateEventIDs, removeStateEventIDs, transactionID) if err != nil { return err } - streamPos = types.StreamPosition(pos) + pduPosition = pos if len(addStateEvents) == 0 && len(removeStateEventIDs) == 0 { // Nothing to do, the event may have just been a message event. return nil } - return d.updateRoomState(ctx, txn, removeStateEventIDs, addStateEvents, streamPos) + return d.updateRoomState(ctx, txn, removeStateEventIDs, addStateEvents, pduPosition) }) return } -func (d *SyncServerDatabase) updateRoomState( +func (d *SyncServerDatasource) updateRoomState( ctx context.Context, txn *sql.Tx, removedEventIDs []string, addedEvents []gomatrixserverlib.Event, - streamPos types.StreamPosition, + pduPosition int64, ) error { // remove first, then add, as we do not ever delete state, but do replace state which is a remove followed by an add. for _, eventID := range removedEventIDs { @@ -156,7 +164,7 @@ func (d *SyncServerDatabase) updateRoomState( } membership = &value } - if err := d.roomstate.upsertRoomState(ctx, txn, event, membership, int64(streamPos)); err != nil { + if err := d.roomstate.upsertRoomState(ctx, txn, event, membership, pduPosition); err != nil { return err } } @@ -167,7 +175,7 @@ func (d *SyncServerDatabase) updateRoomState( // GetStateEvent returns the Matrix state event of a given type for a given room with a given state key // If no event could be found, returns nil // If there was an issue during the retrieval, returns an error -func (d *SyncServerDatabase) GetStateEvent( +func (d *SyncServerDatasource) GetStateEvent( ctx context.Context, roomID, evType, stateKey string, ) (*gomatrixserverlib.Event, error) { return d.roomstate.selectStateEvent(ctx, roomID, evType, stateKey) @@ -176,7 +184,7 @@ func (d *SyncServerDatabase) GetStateEvent( // GetStateEventsForRoom fetches the state events for a given room. // Returns an empty slice if no state events could be found for this room. // Returns an error if there was an issue with the retrieval. -func (d *SyncServerDatabase) GetStateEventsForRoom( +func (d *SyncServerDatasource) GetStateEventsForRoom( ctx context.Context, roomID string, ) (stateEvents []gomatrixserverlib.Event, err error) { err = common.WithTransaction(d.db, func(txn *sql.Tx) error { @@ -186,46 +194,49 @@ func (d *SyncServerDatabase) GetStateEventsForRoom( return } -// SyncStreamPosition returns the latest position in the sync stream. Returns 0 if there are no events yet. -func (d *SyncServerDatabase) SyncStreamPosition(ctx context.Context) (types.StreamPosition, error) { - return d.syncStreamPositionTx(ctx, nil) +// SyncPosition returns the latest positions for syncing. +func (d *SyncServerDatasource) SyncPosition(ctx context.Context) (types.SyncPosition, error) { + return d.syncPositionTx(ctx, nil) } -func (d *SyncServerDatabase) syncStreamPositionTx( +func (d *SyncServerDatasource) syncPositionTx( ctx context.Context, txn *sql.Tx, -) (types.StreamPosition, error) { - maxID, err := d.events.selectMaxEventID(ctx, txn) +) (sp types.SyncPosition, err error) { + + maxEventID, err := d.events.selectMaxEventID(ctx, txn) if err != nil { - return 0, err + return sp, err } maxAccountDataID, err := d.accountData.selectMaxAccountDataID(ctx, txn) if err != nil { - return 0, err + return sp, err } - if maxAccountDataID > maxID { - maxID = maxAccountDataID + if maxAccountDataID > maxEventID { + maxEventID = maxAccountDataID } maxInviteID, err := d.invites.selectMaxInviteID(ctx, txn) if err != nil { - return 0, err + return sp, err } - if maxInviteID > maxID { - maxID = maxInviteID + if maxInviteID > maxEventID { + maxEventID = maxInviteID } - return types.StreamPosition(maxID), nil + sp.PDUPosition = maxEventID + + sp.TypingPosition = d.typingCache.GetLatestSyncPosition() + + return } -// IncrementalSync returns all the data needed in order to create an incremental -// sync response for the given user. Events returned will include any client -// transaction IDs associated with the given device. These transaction IDs come -// from when the device sent the event via an API that included a transaction -// ID. -func (d *SyncServerDatabase) IncrementalSync( +// addPDUDeltaToResponse adds all PDU deltas to a sync response. +// IDs of all rooms the user joined are returned so EDU deltas can be added for them. +func (d *SyncServerDatasource) addPDUDeltaToResponse( ctx context.Context, device authtypes.Device, - fromPos, toPos types.StreamPosition, + fromPos, toPos int64, numRecentEventsPerRoom int, -) (*types.Response, error) { + res *types.Response, +) ([]string, error) { txn, err := d.db.BeginTx(ctx, &txReadOnlySnapshot) if err != nil { return nil, err @@ -234,7 +245,7 @@ func (d *SyncServerDatabase) IncrementalSync( defer common.EndTransaction(txn, &succeeded) // Work out which rooms to return in the response. This is done by getting not only the currently - // joined rooms, but also which rooms have membership transitions for this user between the 2 stream positions. + // joined rooms, but also which rooms have membership transitions for this user between the 2 PDU stream positions. // This works out what the 'state' key should be for each room as well as which membership block // to put the room into. deltas, err := d.getStateDeltas(ctx, &device, txn, fromPos, toPos, device.UserID) @@ -242,8 +253,9 @@ func (d *SyncServerDatabase) IncrementalSync( return nil, err } - res := types.NewResponse(toPos) + joinedRoomIDs := make([]string, 0, len(deltas)) for _, delta := range deltas { + joinedRoomIDs = append(joinedRoomIDs, delta.roomID) err = d.addRoomDeltaToResponse(ctx, &device, txn, fromPos, toPos, delta, numRecentEventsPerRoom, res) if err != nil { return nil, err @@ -256,52 +268,151 @@ func (d *SyncServerDatabase) IncrementalSync( } succeeded = true + return joinedRoomIDs, nil +} + +// addTypingDeltaToResponse adds all typing notifications to a sync response +// since the specified position. +func (d *SyncServerDatasource) addTypingDeltaToResponse( + since int64, + joinedRoomIDs []string, + res *types.Response, +) error { + var jr types.JoinResponse + var ok bool + var err error + for _, roomID := range joinedRoomIDs { + if typingUsers, updated := d.typingCache.GetTypingUsersIfUpdatedAfter( + roomID, since, + ); updated { + ev := gomatrixserverlib.ClientEvent{ + Type: gomatrixserverlib.MTyping, + } + ev.Content, err = json.Marshal(map[string]interface{}{ + "user_ids": typingUsers, + }) + if err != nil { + return err + } + + if jr, ok = res.Rooms.Join[roomID]; !ok { + jr = *types.NewJoinResponse() + } + jr.Ephemeral.Events = append(jr.Ephemeral.Events, ev) + res.Rooms.Join[roomID] = jr + } + } + return nil +} + +// addEDUDeltaToResponse adds updates for EDUs of each type since fromPos if +// the positions of that type are not equal in fromPos and toPos. +func (d *SyncServerDatasource) addEDUDeltaToResponse( + fromPos, toPos types.SyncPosition, + joinedRoomIDs []string, + res *types.Response, +) (err error) { + + if fromPos.TypingPosition != toPos.TypingPosition { + err = d.addTypingDeltaToResponse( + fromPos.TypingPosition, joinedRoomIDs, res, + ) + } + + return +} + +// IncrementalSync returns all the data needed in order to create an incremental +// sync response for the given user. Events returned will include any client +// transaction IDs associated with the given device. These transaction IDs come +// from when the device sent the event via an API that included a transaction +// ID. +func (d *SyncServerDatasource) IncrementalSync( + ctx context.Context, + device authtypes.Device, + fromPos, toPos types.SyncPosition, + numRecentEventsPerRoom int, +) (*types.Response, error) { + nextBatchPos := fromPos.WithUpdates(toPos) + res := types.NewResponse(nextBatchPos) + + var joinedRoomIDs []string + var err error + if fromPos.PDUPosition != toPos.PDUPosition { + joinedRoomIDs, err = d.addPDUDeltaToResponse( + ctx, device, fromPos.PDUPosition, toPos.PDUPosition, numRecentEventsPerRoom, res, + ) + } else { + joinedRoomIDs, err = d.roomstate.selectRoomIDsWithMembership( + ctx, nil, device.UserID, "join", + ) + } + if err != nil { + return nil, err + } + + err = d.addEDUDeltaToResponse( + fromPos, toPos, joinedRoomIDs, res, + ) + if err != nil { + return nil, err + } + return res, nil } -// CompleteSync a complete /sync API response for the given user. -func (d *SyncServerDatabase) CompleteSync( - ctx context.Context, userID string, numRecentEventsPerRoom int, -) (*types.Response, error) { +// getResponseWithPDUsForCompleteSync creates a response and adds all PDUs needed +// to it. It returns toPos and joinedRoomIDs for use of adding EDUs. +func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( + ctx context.Context, + userID string, + numRecentEventsPerRoom int, +) ( + res *types.Response, + toPos types.SyncPosition, + joinedRoomIDs []string, + err error, +) { // This needs to be all done in a transaction as we need to do multiple SELECTs, and we need to have - // a consistent view of the database throughout. This includes extracting the sync stream position. + // a consistent view of the database throughout. This includes extracting the sync position. // This does have the unfortunate side-effect that all the matrixy logic resides in this function, // but it's better to not hide the fact that this is being done in a transaction. txn, err := d.db.BeginTx(ctx, &txReadOnlySnapshot) if err != nil { - return nil, err + return } var succeeded bool defer common.EndTransaction(txn, &succeeded) - // Get the current stream position which we will base the sync response on. - pos, err := d.syncStreamPositionTx(ctx, txn) + // Get the current sync position which we will base the sync response on. + toPos, err = d.syncPositionTx(ctx, txn) if err != nil { - return nil, err + return } + res = types.NewResponse(toPos) + // Extract room state and recent events for all rooms the user is joined to. - roomIDs, err := d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, "join") + joinedRoomIDs, err = d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, "join") if err != nil { - return nil, err + return } // Build up a /sync response. Add joined rooms. - res := types.NewResponse(pos) - for _, roomID := range roomIDs { + for _, roomID := range joinedRoomIDs { var stateEvents []gomatrixserverlib.Event stateEvents, err = d.roomstate.selectCurrentState(ctx, txn, roomID) if err != nil { - return nil, err + return } // TODO: When filters are added, we may need to call this multiple times to get enough events. // See: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L316 var recentStreamEvents []streamEvent recentStreamEvents, err = d.events.selectRecentEvents( - ctx, txn, roomID, types.StreamPosition(0), pos, numRecentEventsPerRoom, + ctx, txn, roomID, 0, toPos.PDUPosition, numRecentEventsPerRoom, ) if err != nil { - return nil, err + return } // We don't include a device here as we don't need to send down @@ -310,10 +421,12 @@ func (d *SyncServerDatabase) CompleteSync( stateEvents = removeDuplicates(stateEvents, recentEvents) jr := types.NewJoinResponse() - if prevBatch := recentStreamEvents[0].streamPosition - 1; prevBatch > 0 { - jr.Timeline.PrevBatch = types.StreamPosition(prevBatch).String() + if prevPDUPos := recentStreamEvents[0].streamPosition - 1; prevPDUPos > 0 { + // Use the short form of batch token for prev_batch + jr.Timeline.PrevBatch = strconv.FormatInt(prevPDUPos, 10) } else { - jr.Timeline.PrevBatch = types.StreamPosition(1).String() + // Use the short form of batch token for prev_batch + jr.Timeline.PrevBatch = "1" } jr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = true @@ -321,12 +434,34 @@ func (d *SyncServerDatabase) CompleteSync( res.Rooms.Join[roomID] = *jr } - if err = d.addInvitesToResponse(ctx, txn, userID, 0, pos, res); err != nil { - return nil, err + if err = d.addInvitesToResponse(ctx, txn, userID, 0, toPos.PDUPosition, res); err != nil { + return } succeeded = true - return res, err + return res, toPos, joinedRoomIDs, err +} + +// CompleteSync returns a complete /sync API response for the given user. +func (d *SyncServerDatasource) CompleteSync( + ctx context.Context, userID string, numRecentEventsPerRoom int, +) (*types.Response, error) { + res, toPos, joinedRoomIDs, err := d.getResponseWithPDUsForCompleteSync( + ctx, userID, numRecentEventsPerRoom, + ) + if err != nil { + return nil, err + } + + // Use a zero value SyncPosition for fromPos so all EDU states are added. + err = d.addEDUDeltaToResponse( + types.SyncPosition{}, toPos, joinedRoomIDs, res, + ) + if err != nil { + return nil, err + } + + return res, nil } var txReadOnlySnapshot = sql.TxOptions{ @@ -344,8 +479,8 @@ var txReadOnlySnapshot = sql.TxOptions{ // Returns a map following the format data[roomID] = []dataTypes // If no data is retrieved, returns an empty map // If there was an issue with the retrieval, returns an error -func (d *SyncServerDatabase) GetAccountDataInRange( - ctx context.Context, userID string, oldPos, newPos types.StreamPosition, +func (d *SyncServerDatasource) GetAccountDataInRange( + ctx context.Context, userID string, oldPos, newPos int64, ) (map[string][]string, error) { return d.accountData.selectAccountDataInRange(ctx, userID, oldPos, newPos) } @@ -356,26 +491,24 @@ func (d *SyncServerDatabase) GetAccountDataInRange( // If no data with the given type, user ID and room ID exists in the database, // creates a new row, else update the existing one // Returns an error if there was an issue with the upsert -func (d *SyncServerDatabase) UpsertAccountData( +func (d *SyncServerDatasource) UpsertAccountData( ctx context.Context, userID, roomID, dataType string, -) (types.StreamPosition, error) { - pos, err := d.accountData.insertAccountData(ctx, userID, roomID, dataType) - return types.StreamPosition(pos), err +) (int64, error) { + return d.accountData.insertAccountData(ctx, userID, roomID, dataType) } // AddInviteEvent stores a new invite event for a user. // If the invite was successfully stored this returns the stream ID it was stored at. // Returns an error if there was a problem communicating with the database. -func (d *SyncServerDatabase) AddInviteEvent( +func (d *SyncServerDatasource) AddInviteEvent( ctx context.Context, inviteEvent gomatrixserverlib.Event, -) (types.StreamPosition, error) { - pos, err := d.invites.insertInviteEvent(ctx, inviteEvent) - return types.StreamPosition(pos), err +) (int64, error) { + return d.invites.insertInviteEvent(ctx, inviteEvent) } // RetireInviteEvent removes an old invite event from the database. // Returns an error if there was a problem communicating with the database. -func (d *SyncServerDatabase) RetireInviteEvent( +func (d *SyncServerDatasource) RetireInviteEvent( ctx context.Context, inviteEventID string, ) error { // TODO: Record that invite has been retired in a stream so that we can @@ -384,10 +517,30 @@ func (d *SyncServerDatabase) RetireInviteEvent( return err } -func (d *SyncServerDatabase) addInvitesToResponse( +func (d *SyncServerDatasource) SetTypingTimeoutCallback(fn cache.TimeoutCallbackFn) { + d.typingCache.SetTimeoutCallback(fn) +} + +// AddTypingUser adds a typing user to the typing cache. +// Returns the newly calculated sync position for typing notifications. +func (d *SyncServerDatasource) AddTypingUser( + userID, roomID string, expireTime *time.Time, +) int64 { + return d.typingCache.AddTypingUser(userID, roomID, expireTime) +} + +// RemoveTypingUser removes a typing user from the typing cache. +// Returns the newly calculated sync position for typing notifications. +func (d *SyncServerDatasource) RemoveTypingUser( + userID, roomID string, +) int64 { + return d.typingCache.RemoveUser(userID, roomID) +} + +func (d *SyncServerDatasource) addInvitesToResponse( ctx context.Context, txn *sql.Tx, userID string, - fromPos, toPos types.StreamPosition, + fromPos, toPos int64, res *types.Response, ) error { invites, err := d.invites.selectInviteEventsInRange( @@ -408,11 +561,11 @@ func (d *SyncServerDatabase) addInvitesToResponse( } // addRoomDeltaToResponse adds a room state delta to a sync response -func (d *SyncServerDatabase) addRoomDeltaToResponse( +func (d *SyncServerDatasource) addRoomDeltaToResponse( ctx context.Context, device *authtypes.Device, txn *sql.Tx, - fromPos, toPos types.StreamPosition, + fromPos, toPos int64, delta stateDelta, numRecentEventsPerRoom int, res *types.Response, @@ -444,10 +597,12 @@ func (d *SyncServerDatabase) addRoomDeltaToResponse( switch delta.membership { case "join": jr := types.NewJoinResponse() - if prevBatch := recentStreamEvents[0].streamPosition - 1; prevBatch > 0 { - jr.Timeline.PrevBatch = types.StreamPosition(prevBatch).String() + if prevPDUPos := recentStreamEvents[0].streamPosition - 1; prevPDUPos > 0 { + // Use the short form of batch token for prev_batch + jr.Timeline.PrevBatch = strconv.FormatInt(prevPDUPos, 10) } else { - jr.Timeline.PrevBatch = types.StreamPosition(1).String() + // Use the short form of batch token for prev_batch + jr.Timeline.PrevBatch = "1" } jr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true @@ -459,10 +614,12 @@ func (d *SyncServerDatabase) addRoomDeltaToResponse( // TODO: recentEvents may contain events that this user is not allowed to see because they are // no longer in the room. lr := types.NewLeaveResponse() - if prevBatch := recentStreamEvents[0].streamPosition - 1; prevBatch > 0 { - lr.Timeline.PrevBatch = types.StreamPosition(prevBatch).String() + if prevPDUPos := recentStreamEvents[0].streamPosition - 1; prevPDUPos > 0 { + // Use the short form of batch token for prev_batch + lr.Timeline.PrevBatch = strconv.FormatInt(prevPDUPos, 10) } else { - lr.Timeline.PrevBatch = types.StreamPosition(1).String() + // Use the short form of batch token for prev_batch + lr.Timeline.PrevBatch = "1" } lr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync) lr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true @@ -475,7 +632,7 @@ func (d *SyncServerDatabase) addRoomDeltaToResponse( // fetchStateEvents converts the set of event IDs into a set of events. It will fetch any which are missing from the database. // Returns a map of room ID to list of events. -func (d *SyncServerDatabase) fetchStateEvents( +func (d *SyncServerDatasource) fetchStateEvents( ctx context.Context, txn *sql.Tx, roomIDToEventIDSet map[string]map[string]bool, eventIDToEvent map[string]streamEvent, @@ -520,7 +677,7 @@ func (d *SyncServerDatabase) fetchStateEvents( return stateBetween, nil } -func (d *SyncServerDatabase) fetchMissingStateEvents( +func (d *SyncServerDatasource) fetchMissingStateEvents( ctx context.Context, txn *sql.Tx, eventIDs []string, ) ([]streamEvent, error) { // Fetch from the events table first so we pick up the stream ID for the @@ -559,9 +716,9 @@ func (d *SyncServerDatabase) fetchMissingStateEvents( return events, nil } -func (d *SyncServerDatabase) getStateDeltas( +func (d *SyncServerDatasource) getStateDeltas( ctx context.Context, device *authtypes.Device, txn *sql.Tx, - fromPos, toPos types.StreamPosition, userID string, + fromPos, toPos int64, userID string, ) ([]stateDelta, error) { // Implement membership change algorithm: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L821 // - Get membership list changes for this user in this sync response @@ -600,7 +757,7 @@ func (d *SyncServerDatabase) getStateDeltas( } s := make([]streamEvent, len(allState)) for i := 0; i < len(s); i++ { - s[i] = streamEvent{Event: allState[i], streamPosition: types.StreamPosition(0)} + s[i] = streamEvent{Event: allState[i], streamPosition: 0} } state[roomID] = s continue // we'll add this room in when we do joined rooms diff --git a/syncapi/sync/notifier.go b/syncapi/sync/notifier.go index 5ed701d8e..30ac3a2e5 100644 --- a/syncapi/sync/notifier.go +++ b/syncapi/sync/notifier.go @@ -26,7 +26,7 @@ import ( ) // Notifier will wake up sleeping requests when there is some new data. -// It does not tell requests what that data is, only the stream position which +// It does not tell requests what that data is, only the sync position which // they can use to get at it. This is done to prevent races whereby we tell the caller // the event, but the token has already advanced by the time they fetch it, resulting // in missed events. @@ -35,18 +35,18 @@ type Notifier struct { roomIDToJoinedUsers map[string]userIDSet // Protects currPos and userStreams. streamLock *sync.Mutex - // The latest sync stream position - currPos types.StreamPosition + // The latest sync position + currPos types.SyncPosition // A map of user_id => UserStream which can be used to wake a given user's /sync request. userStreams map[string]*UserStream // The last time we cleaned out stale entries from the userStreams map lastCleanUpTime time.Time } -// NewNotifier creates a new notifier set to the given stream position. +// NewNotifier creates a new notifier set to the given sync position. // In order for this to be of any use, the Notifier needs to be told all rooms and // the joined users within each of them by calling Notifier.Load(*storage.SyncServerDatabase). -func NewNotifier(pos types.StreamPosition) *Notifier { +func NewNotifier(pos types.SyncPosition) *Notifier { return &Notifier{ currPos: pos, roomIDToJoinedUsers: make(map[string]userIDSet), @@ -58,20 +58,30 @@ func NewNotifier(pos types.StreamPosition) *Notifier { // OnNewEvent is called when a new event is received from the room server. Must only be // called from a single goroutine, to avoid races between updates which could set the -// current position in the stream incorrectly. -// Can be called either with a *gomatrixserverlib.Event, or with an user ID -func (n *Notifier) OnNewEvent(ev *gomatrixserverlib.Event, userID string, pos types.StreamPosition) { +// current sync position incorrectly. +// Chooses which user sync streams to update by a provided *gomatrixserverlib.Event +// (based on the users in the event's room), +// a roomID directly, or a list of user IDs, prioritised by parameter ordering. +// posUpdate contains the latest position(s) for one or more types of events. +// If a position in posUpdate is 0, it means no updates are available of that type. +// Typically a consumer supplies a posUpdate with the latest sync position for the +// event type it handles, leaving other fields as 0. +func (n *Notifier) OnNewEvent( + ev *gomatrixserverlib.Event, roomID string, userIDs []string, + posUpdate types.SyncPosition, +) { // update the current position then notify relevant /sync streams. // This needs to be done PRIOR to waking up users as they will read this value. n.streamLock.Lock() defer n.streamLock.Unlock() - n.currPos = pos + latestPos := n.currPos.WithUpdates(posUpdate) + n.currPos = latestPos n.removeEmptyUserStreams() if ev != nil { // Map this event's room_id to a list of joined users, and wake them up. - userIDs := n.joinedUsers(ev.RoomID()) + usersToNotify := n.joinedUsers(ev.RoomID()) // If this is an invite, also add in the invitee to this list. if ev.Type() == "m.room.member" && ev.StateKey() != nil { targetUserID := *ev.StateKey() @@ -84,11 +94,11 @@ func (n *Notifier) OnNewEvent(ev *gomatrixserverlib.Event, userID string, pos ty // Keep the joined user map up-to-date switch membership { case "invite": - userIDs = append(userIDs, targetUserID) + usersToNotify = append(usersToNotify, targetUserID) case "join": // Manually append the new user's ID so they get notified // along all members in the room - userIDs = append(userIDs, targetUserID) + usersToNotify = append(usersToNotify, targetUserID) n.addJoinedUser(ev.RoomID(), targetUserID) case "leave": fallthrough @@ -98,11 +108,15 @@ func (n *Notifier) OnNewEvent(ev *gomatrixserverlib.Event, userID string, pos ty } } - for _, toNotifyUserID := range userIDs { - n.wakeupUser(toNotifyUserID, pos) - } - } else if len(userID) > 0 { - n.wakeupUser(userID, pos) + n.wakeupUsers(usersToNotify, latestPos) + } else if roomID != "" { + n.wakeupUsers(n.joinedUsers(roomID), latestPos) + } else if len(userIDs) > 0 { + n.wakeupUsers(userIDs, latestPos) + } else { + log.WithFields(log.Fields{ + "posUpdate": posUpdate.String, + }).Warn("Notifier.OnNewEvent called but caller supplied no user to wake up") } } @@ -127,7 +141,7 @@ func (n *Notifier) GetListener(req syncRequest) UserStreamListener { } // Load the membership states required to notify users correctly. -func (n *Notifier) Load(ctx context.Context, db *storage.SyncServerDatabase) error { +func (n *Notifier) Load(ctx context.Context, db *storage.SyncServerDatasource) error { roomToUsers, err := db.AllJoinedUsersInRooms(ctx) if err != nil { return err @@ -136,8 +150,11 @@ func (n *Notifier) Load(ctx context.Context, db *storage.SyncServerDatabase) err return nil } -// CurrentPosition returns the current stream position -func (n *Notifier) CurrentPosition() types.StreamPosition { +// CurrentPosition returns the current sync position +func (n *Notifier) CurrentPosition() types.SyncPosition { + n.streamLock.Lock() + defer n.streamLock.Unlock() + return n.currPos } @@ -156,12 +173,13 @@ func (n *Notifier) setUsersJoinedToRooms(roomIDToUserIDs map[string][]string) { } } -func (n *Notifier) wakeupUser(userID string, newPos types.StreamPosition) { - stream := n.fetchUserStream(userID, false) - if stream == nil { - return +func (n *Notifier) wakeupUsers(userIDs []string, newPos types.SyncPosition) { + for _, userID := range userIDs { + stream := n.fetchUserStream(userID, false) + if stream != nil { + stream.Broadcast(newPos) // wake up all goroutines Wait()ing on this stream + } } - stream.Broadcast(newPos) // wakeup all goroutines Wait()ing on this stream } // fetchUserStream retrieves a stream unique to the given user. If makeIfNotExists is true, diff --git a/syncapi/sync/notifier_test.go b/syncapi/sync/notifier_test.go index 4fa543936..904315e9f 100644 --- a/syncapi/sync/notifier_test.go +++ b/syncapi/sync/notifier_test.go @@ -32,19 +32,40 @@ var ( randomMessageEvent gomatrixserverlib.Event aliceInviteBobEvent gomatrixserverlib.Event bobLeaveEvent gomatrixserverlib.Event + syncPositionVeryOld types.SyncPosition + syncPositionBefore types.SyncPosition + syncPositionAfter types.SyncPosition + syncPositionNewEDU types.SyncPosition + syncPositionAfter2 types.SyncPosition ) var ( - streamPositionVeryOld = types.StreamPosition(5) - streamPositionBefore = types.StreamPosition(11) - streamPositionAfter = types.StreamPosition(12) - streamPositionAfter2 = types.StreamPosition(13) - roomID = "!test:localhost" - alice = "@alice:localhost" - bob = "@bob:localhost" + roomID = "!test:localhost" + alice = "@alice:localhost" + bob = "@bob:localhost" ) func init() { + baseSyncPos := types.SyncPosition{ + PDUPosition: 0, + TypingPosition: 0, + } + + syncPositionVeryOld = baseSyncPos + syncPositionVeryOld.PDUPosition = 5 + + syncPositionBefore = baseSyncPos + syncPositionBefore.PDUPosition = 11 + + syncPositionAfter = baseSyncPos + syncPositionAfter.PDUPosition = 12 + + syncPositionNewEDU = syncPositionAfter + syncPositionNewEDU.TypingPosition = 1 + + syncPositionAfter2 = baseSyncPos + syncPositionAfter2.PDUPosition = 13 + var err error randomMessageEvent, err = gomatrixserverlib.NewEventFromTrustedJSON([]byte(`{ "type": "m.room.message", @@ -92,19 +113,19 @@ func init() { // Test that the current position is returned if a request is already behind. func TestImmediateNotification(t *testing.T) { - n := NewNotifier(streamPositionBefore) - pos, err := waitForEvents(n, newTestSyncRequest(alice, streamPositionVeryOld)) + n := NewNotifier(syncPositionBefore) + pos, err := waitForEvents(n, newTestSyncRequest(alice, syncPositionVeryOld)) if err != nil { t.Fatalf("TestImmediateNotification error: %s", err) } - if pos != streamPositionBefore { - t.Fatalf("TestImmediateNotification want %d, got %d", streamPositionBefore, pos) + if pos != syncPositionBefore { + t.Fatalf("TestImmediateNotification want %d, got %d", syncPositionBefore, pos) } } // Test that new events to a joined room unblocks the request. func TestNewEventAndJoinedToRoom(t *testing.T) { - n := NewNotifier(streamPositionBefore) + n := NewNotifier(syncPositionBefore) n.setUsersJoinedToRooms(map[string][]string{ roomID: {alice, bob}, }) @@ -112,12 +133,12 @@ func TestNewEventAndJoinedToRoom(t *testing.T) { var wg sync.WaitGroup wg.Add(1) go func() { - pos, err := waitForEvents(n, newTestSyncRequest(bob, streamPositionBefore)) + pos, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionBefore)) if err != nil { t.Errorf("TestNewEventAndJoinedToRoom error: %s", err) } - if pos != streamPositionAfter { - t.Errorf("TestNewEventAndJoinedToRoom want %d, got %d", streamPositionAfter, pos) + if pos != syncPositionAfter { + t.Errorf("TestNewEventAndJoinedToRoom want %d, got %d", syncPositionAfter, pos) } wg.Done() }() @@ -125,14 +146,14 @@ func TestNewEventAndJoinedToRoom(t *testing.T) { stream := n.fetchUserStream(bob, true) waitForBlocking(stream, 1) - n.OnNewEvent(&randomMessageEvent, "", streamPositionAfter) + n.OnNewEvent(&randomMessageEvent, "", nil, syncPositionAfter) wg.Wait() } // Test that an invite unblocks the request func TestNewInviteEventForUser(t *testing.T) { - n := NewNotifier(streamPositionBefore) + n := NewNotifier(syncPositionBefore) n.setUsersJoinedToRooms(map[string][]string{ roomID: {alice, bob}, }) @@ -140,12 +161,12 @@ func TestNewInviteEventForUser(t *testing.T) { var wg sync.WaitGroup wg.Add(1) go func() { - pos, err := waitForEvents(n, newTestSyncRequest(bob, streamPositionBefore)) + pos, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionBefore)) if err != nil { t.Errorf("TestNewInviteEventForUser error: %s", err) } - if pos != streamPositionAfter { - t.Errorf("TestNewInviteEventForUser want %d, got %d", streamPositionAfter, pos) + if pos != syncPositionAfter { + t.Errorf("TestNewInviteEventForUser want %d, got %d", syncPositionAfter, pos) } wg.Done() }() @@ -153,14 +174,42 @@ func TestNewInviteEventForUser(t *testing.T) { stream := n.fetchUserStream(bob, true) waitForBlocking(stream, 1) - n.OnNewEvent(&aliceInviteBobEvent, "", streamPositionAfter) + n.OnNewEvent(&aliceInviteBobEvent, "", nil, syncPositionAfter) + + wg.Wait() +} + +// Test an EDU-only update wakes up the request. +func TestEDUWakeup(t *testing.T) { + n := NewNotifier(syncPositionAfter) + n.setUsersJoinedToRooms(map[string][]string{ + roomID: {alice, bob}, + }) + + var wg sync.WaitGroup + wg.Add(1) + go func() { + pos, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionAfter)) + if err != nil { + t.Errorf("TestNewInviteEventForUser error: %s", err) + } + if pos != syncPositionNewEDU { + t.Errorf("TestNewInviteEventForUser want %d, got %d", syncPositionNewEDU, pos) + } + wg.Done() + }() + + stream := n.fetchUserStream(bob, true) + waitForBlocking(stream, 1) + + n.OnNewEvent(&aliceInviteBobEvent, "", nil, syncPositionNewEDU) wg.Wait() } // Test that all blocked requests get woken up on a new event. func TestMultipleRequestWakeup(t *testing.T) { - n := NewNotifier(streamPositionBefore) + n := NewNotifier(syncPositionBefore) n.setUsersJoinedToRooms(map[string][]string{ roomID: {alice, bob}, }) @@ -168,12 +217,12 @@ func TestMultipleRequestWakeup(t *testing.T) { var wg sync.WaitGroup wg.Add(3) poll := func() { - pos, err := waitForEvents(n, newTestSyncRequest(bob, streamPositionBefore)) + pos, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionBefore)) if err != nil { t.Errorf("TestMultipleRequestWakeup error: %s", err) } - if pos != streamPositionAfter { - t.Errorf("TestMultipleRequestWakeup want %d, got %d", streamPositionAfter, pos) + if pos != syncPositionAfter { + t.Errorf("TestMultipleRequestWakeup want %d, got %d", syncPositionAfter, pos) } wg.Done() } @@ -184,7 +233,7 @@ func TestMultipleRequestWakeup(t *testing.T) { stream := n.fetchUserStream(bob, true) waitForBlocking(stream, 3) - n.OnNewEvent(&randomMessageEvent, "", streamPositionAfter) + n.OnNewEvent(&randomMessageEvent, "", nil, syncPositionAfter) wg.Wait() @@ -198,7 +247,7 @@ func TestMultipleRequestWakeup(t *testing.T) { func TestNewEventAndWasPreviouslyJoinedToRoom(t *testing.T) { // listen as bob. Make bob leave room. Make alice send event to room. // Make sure alice gets woken up only and not bob as well. - n := NewNotifier(streamPositionBefore) + n := NewNotifier(syncPositionBefore) n.setUsersJoinedToRooms(map[string][]string{ roomID: {alice, bob}, }) @@ -208,18 +257,18 @@ func TestNewEventAndWasPreviouslyJoinedToRoom(t *testing.T) { // Make bob leave the room leaveWG.Add(1) go func() { - pos, err := waitForEvents(n, newTestSyncRequest(bob, streamPositionBefore)) + pos, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionBefore)) if err != nil { t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom error: %s", err) } - if pos != streamPositionAfter { - t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom want %d, got %d", streamPositionAfter, pos) + if pos != syncPositionAfter { + t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom want %d, got %d", syncPositionAfter, pos) } leaveWG.Done() }() bobStream := n.fetchUserStream(bob, true) waitForBlocking(bobStream, 1) - n.OnNewEvent(&bobLeaveEvent, "", streamPositionAfter) + n.OnNewEvent(&bobLeaveEvent, "", nil, syncPositionAfter) leaveWG.Wait() // send an event into the room. Make sure alice gets it. Bob should not. @@ -227,19 +276,19 @@ func TestNewEventAndWasPreviouslyJoinedToRoom(t *testing.T) { aliceStream := n.fetchUserStream(alice, true) aliceWG.Add(1) go func() { - pos, err := waitForEvents(n, newTestSyncRequest(alice, streamPositionAfter)) + pos, err := waitForEvents(n, newTestSyncRequest(alice, syncPositionAfter)) if err != nil { t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom error: %s", err) } - if pos != streamPositionAfter2 { - t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom want %d, got %d", streamPositionAfter2, pos) + if pos != syncPositionAfter2 { + t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom want %d, got %d", syncPositionAfter2, pos) } aliceWG.Done() }() go func() { // this should timeout with an error (but the main goroutine won't wait for the timeout explicitly) - _, err := waitForEvents(n, newTestSyncRequest(bob, streamPositionAfter)) + _, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionAfter)) if err == nil { t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom expect error but got nil") } @@ -248,7 +297,7 @@ func TestNewEventAndWasPreviouslyJoinedToRoom(t *testing.T) { waitForBlocking(aliceStream, 1) waitForBlocking(bobStream, 1) - n.OnNewEvent(&randomMessageEvent, "", streamPositionAfter2) + n.OnNewEvent(&randomMessageEvent, "", nil, syncPositionAfter2) aliceWG.Wait() // it's possible that at this point alice has been informed and bob is about to be informed, so wait @@ -256,18 +305,17 @@ func TestNewEventAndWasPreviouslyJoinedToRoom(t *testing.T) { time.Sleep(1 * time.Millisecond) } -// same as Notifier.WaitForEvents but with a timeout. -func waitForEvents(n *Notifier, req syncRequest) (types.StreamPosition, error) { +func waitForEvents(n *Notifier, req syncRequest) (types.SyncPosition, error) { listener := n.GetListener(req) defer listener.Close() select { case <-time.After(5 * time.Second): - return types.StreamPosition(0), fmt.Errorf( + return types.SyncPosition{}, fmt.Errorf( "waitForEvents timed out waiting for %s (pos=%d)", req.device.UserID, req.since, ) case <-listener.GetNotifyChannel(*req.since): - p := listener.GetStreamPosition() + p := listener.GetSyncPosition() return p, nil } } @@ -280,7 +328,7 @@ func waitForBlocking(s *UserStream, numBlocking uint) { } } -func newTestSyncRequest(userID string, since types.StreamPosition) syncRequest { +func newTestSyncRequest(userID string, since types.SyncPosition) syncRequest { return syncRequest{ device: authtypes.Device{UserID: userID}, timeout: 1 * time.Minute, diff --git a/syncapi/sync/request.go b/syncapi/sync/request.go index 35a15f6f9..a5d2f60f4 100644 --- a/syncapi/sync/request.go +++ b/syncapi/sync/request.go @@ -16,8 +16,10 @@ package sync import ( "context" + "errors" "net/http" "strconv" + "strings" "time" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" @@ -36,7 +38,7 @@ type syncRequest struct { device authtypes.Device limit int timeout time.Duration - since *types.StreamPosition // nil means that no since token was supplied + since *types.SyncPosition // nil means that no since token was supplied wantFullState bool log *log.Entry } @@ -73,15 +75,41 @@ func getTimeout(timeoutMS string) time.Duration { } // getSyncStreamPosition tries to parse a 'since' token taken from the API to a -// stream position. If the string is empty then (nil, nil) is returned. -func getSyncStreamPosition(since string) (*types.StreamPosition, error) { +// types.SyncPosition. If the string is empty then (nil, nil) is returned. +// There are two forms of tokens: The full length form containing all PDU and EDU +// positions separated by "_", and the short form containing only the PDU +// position. Short form can be used for, e.g., `prev_batch` tokens. +func getSyncStreamPosition(since string) (*types.SyncPosition, error) { if since == "" { return nil, nil } - i, err := strconv.Atoi(since) - if err != nil { - return nil, err + + posStrings := strings.Split(since, "_") + if len(posStrings) != 2 && len(posStrings) != 1 { + // A token can either be full length or short (PDU-only). + return nil, errors.New("malformed batch token") + } + + positions := make([]int64, len(posStrings)) + for i, posString := range posStrings { + pos, err := strconv.ParseInt(posString, 10, 64) + if err != nil { + return nil, err + } + positions[i] = pos + } + + if len(positions) == 2 { + // Full length token; construct SyncPosition with every entry in + // `positions`. These entries must have the same order with the fields + // in struct SyncPosition, so we disable the govet check below. + return &types.SyncPosition{ //nolint:govet + positions[0], positions[1], + }, nil + } else { + // Token with PDU position only + return &types.SyncPosition{ + PDUPosition: positions[0], + }, nil } - token := types.StreamPosition(i) - return &token, nil } diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index 89137eb59..a6ec6bd92 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -31,13 +31,13 @@ import ( // RequestPool manages HTTP long-poll connections for /sync type RequestPool struct { - db *storage.SyncServerDatabase + db *storage.SyncServerDatasource accountDB *accounts.Database notifier *Notifier } // NewRequestPool makes a new RequestPool -func NewRequestPool(db *storage.SyncServerDatabase, n *Notifier, adb *accounts.Database) *RequestPool { +func NewRequestPool(db *storage.SyncServerDatasource, n *Notifier, adb *accounts.Database) *RequestPool { return &RequestPool{db, adb, n} } @@ -92,11 +92,13 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *authtype // respond with, so we skip the return an go back to waiting for content to // be sent down or the request timing out. var hasTimedOut bool + sincePos := *syncReq.since for { select { // Wait for notifier to wake us up - case <-userStreamListener.GetNotifyChannel(currPos): - currPos = userStreamListener.GetStreamPosition() + case <-userStreamListener.GetNotifyChannel(sincePos): + currPos = userStreamListener.GetSyncPosition() + sincePos = currPos // Or for timeout to expire case <-timer.C: // We just need to ensure we get out of the select after reaching the @@ -128,24 +130,24 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *authtype } } -func (rp *RequestPool) currentSyncForUser(req syncRequest, currentPos types.StreamPosition) (res *types.Response, err error) { +func (rp *RequestPool) currentSyncForUser(req syncRequest, latestPos types.SyncPosition) (res *types.Response, err error) { // TODO: handle ignored users if req.since == nil { res, err = rp.db.CompleteSync(req.ctx, req.device.UserID, req.limit) } else { - res, err = rp.db.IncrementalSync(req.ctx, req.device, *req.since, currentPos, req.limit) + res, err = rp.db.IncrementalSync(req.ctx, req.device, *req.since, latestPos, req.limit) } if err != nil { return } - res, err = rp.appendAccountData(res, req.device.UserID, req, currentPos) + res, err = rp.appendAccountData(res, req.device.UserID, req, latestPos.PDUPosition) return } func (rp *RequestPool) appendAccountData( - data *types.Response, userID string, req syncRequest, currentPos types.StreamPosition, + data *types.Response, userID string, req syncRequest, currentPos int64, ) (*types.Response, error) { // TODO: Account data doesn't have a sync position of its own, meaning that // account data might be sent multiple time to the client if multiple account @@ -179,7 +181,7 @@ func (rp *RequestPool) appendAccountData( } // Sync is not initial, get all account data since the latest sync - dataTypes, err := rp.db.GetAccountDataInRange(req.ctx, userID, *req.since, currentPos) + dataTypes, err := rp.db.GetAccountDataInRange(req.ctx, userID, req.since.PDUPosition, currentPos) if err != nil { return nil, err } diff --git a/syncapi/sync/userstream.go b/syncapi/sync/userstream.go index 77d09c202..beb10e487 100644 --- a/syncapi/sync/userstream.go +++ b/syncapi/sync/userstream.go @@ -34,8 +34,8 @@ type UserStream struct { lock sync.Mutex // Closed when there is an update. signalChannel chan struct{} - // The last stream position that there may have been an update for the suser - pos types.StreamPosition + // The last sync position that there may have been an update for the user + pos types.SyncPosition // The last time when we had some listeners waiting timeOfLastChannel time.Time // The number of listeners waiting @@ -51,7 +51,7 @@ type UserStreamListener struct { } // NewUserStream creates a new user stream -func NewUserStream(userID string, currPos types.StreamPosition) *UserStream { +func NewUserStream(userID string, currPos types.SyncPosition) *UserStream { return &UserStream{ UserID: userID, timeOfLastChannel: time.Now(), @@ -84,8 +84,8 @@ func (s *UserStream) GetListener(ctx context.Context) UserStreamListener { return listener } -// Broadcast a new stream position for this user. -func (s *UserStream) Broadcast(pos types.StreamPosition) { +// Broadcast a new sync position for this user. +func (s *UserStream) Broadcast(pos types.SyncPosition) { s.lock.Lock() defer s.lock.Unlock() @@ -118,9 +118,9 @@ func (s *UserStream) TimeOfLastNonEmpty() time.Time { return s.timeOfLastChannel } -// GetStreamPosition returns last stream position which the UserStream was +// GetStreamPosition returns last sync position which the UserStream was // notified about -func (s *UserStreamListener) GetStreamPosition() types.StreamPosition { +func (s *UserStreamListener) GetSyncPosition() types.SyncPosition { s.userStream.lock.Lock() defer s.userStream.lock.Unlock() @@ -132,11 +132,11 @@ func (s *UserStreamListener) GetStreamPosition() types.StreamPosition { // sincePos specifies from which point we want to be notified about. If there // has already been an update after sincePos we'll return a closed channel // immediately. -func (s *UserStreamListener) GetNotifyChannel(sincePos types.StreamPosition) <-chan struct{} { +func (s *UserStreamListener) GetNotifyChannel(sincePos types.SyncPosition) <-chan struct{} { s.userStream.lock.Lock() defer s.userStream.lock.Unlock() - if sincePos < s.userStream.pos { + if s.userStream.pos.IsAfter(sincePos) { // If the listener is behind, i.e. missed a potential update, then we // want them to wake up immediately. We do this by returning a new // closed stream, which returns immediately when selected. diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go index 2db54c3ce..4738feea2 100644 --- a/syncapi/syncapi.go +++ b/syncapi/syncapi.go @@ -28,7 +28,6 @@ import ( "github.com/matrix-org/dendrite/syncapi/routing" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/sync" - "github.com/matrix-org/dendrite/syncapi/types" ) // SetupSyncAPIComponent sets up and registers HTTP handlers for the SyncAPI @@ -39,17 +38,17 @@ func SetupSyncAPIComponent( accountsDB *accounts.Database, queryAPI api.RoomserverQueryAPI, ) { - syncDB, err := storage.NewSyncServerDatabase(string(base.Cfg.Database.SyncAPI)) + syncDB, err := storage.NewSyncServerDatasource(string(base.Cfg.Database.SyncAPI)) if err != nil { logrus.WithError(err).Panicf("failed to connect to sync db") } - pos, err := syncDB.SyncStreamPosition(context.Background()) + pos, err := syncDB.SyncPosition(context.Background()) if err != nil { - logrus.WithError(err).Panicf("failed to get stream position") + logrus.WithError(err).Panicf("failed to get sync position") } - notifier := sync.NewNotifier(types.StreamPosition(pos)) + notifier := sync.NewNotifier(pos) err = notifier.Load(context.Background(), syncDB) if err != nil { logrus.WithError(err).Panicf("failed to start notifier") @@ -71,5 +70,12 @@ func SetupSyncAPIComponent( logrus.WithError(err).Panicf("failed to start client data consumer") } + typingConsumer := consumers.NewOutputTypingEventConsumer( + base.Cfg, base.KafkaConsumer, notifier, syncDB, + ) + if err = typingConsumer.Start(); err != nil { + logrus.WithError(err).Panicf("failed to start typing server consumer") + } + routing.Setup(base.APIMux, requestPool, syncDB, deviceDB) } diff --git a/syncapi/types/types.go b/syncapi/types/types.go index d0b1c38ab..af7ec865f 100644 --- a/syncapi/types/types.go +++ b/syncapi/types/types.go @@ -21,12 +21,38 @@ import ( "github.com/matrix-org/gomatrixserverlib" ) -// StreamPosition represents the offset in the sync stream a client is at. -type StreamPosition int64 +// SyncPosition contains the PDU and EDU stream sync positions for a client. +type SyncPosition struct { + // PDUPosition is the stream position for PDUs the client is at. + PDUPosition int64 + // TypingPosition is the client's position for typing notifications. + TypingPosition int64 +} // String implements the Stringer interface. -func (sp StreamPosition) String() string { - return strconv.FormatInt(int64(sp), 10) +func (sp SyncPosition) String() string { + return strconv.FormatInt(sp.PDUPosition, 10) + "_" + + strconv.FormatInt(sp.TypingPosition, 10) +} + +// IsAfter returns whether one SyncPosition refers to states newer than another SyncPosition. +func (sp SyncPosition) IsAfter(other SyncPosition) bool { + return sp.PDUPosition > other.PDUPosition || + sp.TypingPosition > other.TypingPosition +} + +// WithUpdates returns a copy of the SyncPosition with updates applied from another SyncPosition. +// If the latter SyncPosition contains a field that is not 0, it is considered an update, +// and its value will replace the corresponding value in the SyncPosition on which WithUpdates is called. +func (sp SyncPosition) WithUpdates(other SyncPosition) SyncPosition { + ret := sp + if other.PDUPosition != 0 { + ret.PDUPosition = other.PDUPosition + } + if other.TypingPosition != 0 { + ret.TypingPosition = other.TypingPosition + } + return ret } // PrevEventRef represents a reference to a previous event in a state event upgrade @@ -53,11 +79,10 @@ type Response struct { } // NewResponse creates an empty response with initialised maps. -func NewResponse(pos StreamPosition) *Response { - res := Response{} - // Make sure we send the next_batch as a string. We don't want to confuse clients by sending this - // as an integer even though (at the moment) it is. - res.NextBatch = pos.String() +func NewResponse(pos SyncPosition) *Response { + res := Response{ + NextBatch: pos.String(), + } // Pre-initialise the maps. Synapse will return {} even if there are no rooms under a specific section, // so let's do the same thing. Bonus: this means we can't get dreaded 'assignment to entry in nil map' errors. res.Rooms.Join = make(map[string]JoinResponse) diff --git a/testfile b/testfile index d90a9458c..bdd421e2d 100644 --- a/testfile +++ b/testfile @@ -1,4 +1,149 @@ GET /register yields a set of flows POST /register can create a user POST /register downcases capitals in usernames - +POST /register rejects registration of usernames with '!' +POST /register rejects registration of usernames with '"' +POST /register rejects registration of usernames with ':' +POST /register rejects registration of usernames with '?' +POST /register rejects registration of usernames with '\' +POST /register rejects registration of usernames with '@' +POST /register rejects registration of usernames with '[' +POST /register rejects registration of usernames with ']' +POST /register rejects registration of usernames with '{' +POST /register rejects registration of usernames with '|' +POST /register rejects registration of usernames with '}' +POST /register rejects registration of usernames with '£' +POST /register rejects registration of usernames with 'é' +POST /register rejects registration of usernames with '\n' +POST /register rejects registration of usernames with ''' +GET /login yields a set of flows +POST /login can log in as a user +POST /login can log in as a user with just the local part of the id +POST /login as non-existing user is rejected +POST /login wrong password is rejected +GET /events initially +GET /initialSync initially +Version responds 200 OK with valid structure +PUT /profile/:user_id/displayname sets my name +GET /profile/:user_id/displayname publicly accessible +PUT /profile/:user_id/avatar_url sets my avatar +GET /profile/:user_id/avatar_url publicly accessible +GET /device/{deviceId} gives a 404 for unknown devices +PUT /device/{deviceId} gives a 404 for unknown devices +POST /createRoom makes a public room +POST /createRoom makes a private room +POST /createRoom makes a private room with invites +POST /createRoom makes a room with a name +POST /createRoom makes a room with a topic +Can /sync newly created room +GET /rooms/:room_id/state/m.room.member/:user_id fetches my membership +GET /rooms/:room_id/state/m.room.power_levels fetches powerlevels +POST /join/:room_alias can join a room +POST /join/:room_id can join a room +POST /join/:room_id can join a room with custom content +POST /join/:room_alias can join a room with custom content +POST /rooms/:room_id/join can join a room +POST /rooms/:room_id/leave can leave a room +POST /rooms/:room_id/invite can send an invite +POST /rooms/:room_id/ban can ban a user +POST /rooms/:room_id/send/:event_type sends a message +PUT /rooms/:room_id/send/:event_type/:txn_id sends a message +PUT /rooms/:room_id/send/:event_type/:txn_id deduplicates the same txn id +GET /rooms/:room_id/state/m.room.power_levels can fetch levels +PUT /rooms/:room_id/state/m.room.power_levels can set levels +PUT power_levels should not explode if the old power levels were empty +Both GET and PUT work +POST /rooms/:room_id/read_markers can create read marker +User signups are forbidden from starting with '_' +Can logout all devices +Request to logout with invalid an access token is rejected +Request to logout without an access token is rejected +Room creation reports m.room.create to myself +Room creation reports m.room.member to myself +New room members see their own join event +Existing members see new members' join events +setting 'm.room.power_levels' respects room powerlevel +Unprivileged users can set m.room.topic if it only needs level 0 +Users cannot set ban powerlevel higher than their own +Users cannot set kick powerlevel higher than their own +Users cannot set redact powerlevel higher than their own +Can get rooms/{roomId}/members for a departed room (SPEC-216) +3pid invite join with wrong but valid signature are rejected +3pid invite join valid signature but revoked keys are rejected +3pid invite join valid signature but unreachable ID server are rejected +Room members can override their displayname on a room-specific basis +Room members can join a room with an overridden displayname +displayname updates affect room member events +avatar_url updates affect room member events +Real non-joined user cannot call /events on shared room +Real non-joined user cannot call /events on invited room +Real non-joined user cannot call /events on joined room +Real non-joined user cannot call /events on default room +Real non-joined users can get state for world_readable rooms +Real non-joined users can get individual state for world_readable rooms +Real non-joined users can get individual state for world_readable rooms after leaving +Real non-joined users cannot send messages to guest_access rooms if not joined +Real users can sync from world_readable guest_access rooms if joined +Real users can sync from default guest_access rooms if joined +Can't forget room you're still in +Can get rooms/{roomId}/members +Can create filter +Can download filter +Can sync +Can sync a joined room +Newly joined room is included in an incremental sync +User is offline if they set_presence=offline in their sync +Changes to state are included in an incremental sync +A change to displayname should appear in incremental /sync +Current state appears in timeline in private history +Current state appears in timeline in private history with many messages before +Rooms a user is invited to appear in an initial sync +Rooms a user is invited to appear in an incremental sync +Sync can be polled for updates +Sync is woken up for leaves +Newly left rooms appear in the leave section of incremental sync +We should see our own leave event, even if history_visibility is restricted (SYN-662) +We should see our own leave event when rejecting an invite, even if history_visibility is restricted (riot-web/3462) +Newly left rooms appear in the leave section of gapped sync +Previously left rooms don't appear in the leave section of sync +Left rooms appear in the leave section of full state sync +Newly banned rooms appear in the leave section of incremental sync +Newly banned rooms appear in the leave section of incremental sync +local user can join room with version 1 +User can invite local user to room with version 1 +local user can join room with version 2 +User can invite local user to room with version 2 +local user can join room with version 3 +User can invite local user to room with version 3 +local user can join room with version 4 +User can invite local user to room with version 4 +Should reject keys claiming to belong to a different user +Can add account data +Can add account data to room +Latest account data appears in v2 /sync +New account data appears in incremental v2 /sync +Checking local federation server +Inbound federation can query profile data +Outbound federation can send room-join requests +Outbound federation can send events +Inbound federation can backfill events +Backfill checks the events requested belong to the room +Can upload without a file name +Can download without a file name locally +Can upload with ASCII file name +Can send image in room message +AS cannot create users outside its own namespace +Regular users cannot register within the AS namespace +AS can't set displayname for random users +AS user (not ghost) can join room without registering, with user_id query param +Changing the actions of an unknown default rule fails with 404 +Changing the actions of an unknown rule fails with 404 +Enabling an unknown default rule fails with 404 +Trying to get push rules with unknown rule_id fails with 404 +Events come down the correct room +local user can join room with version 5 +User can invite local user to room with version 5 +Inbound federation can receive room-join requests +Typing events appear in initial sync +Typing events appear in incremental sync +Typing events appear in gapped sync diff --git a/typingserver/api/output.go b/typingserver/api/output.go index 813b9b7c7..8696acf49 100644 --- a/typingserver/api/output.go +++ b/typingserver/api/output.go @@ -12,14 +12,17 @@ package api +import "time" + // OutputTypingEvent is an entry in typing server output kafka log. // This contains the event with extra fields used to create 'm.typing' event // in clientapi & federation. type OutputTypingEvent struct { // The Event for the typing edu event. Event TypingEvent `json:"event"` - // Users typing in the room when the event was generated. - TypingUsers []string `json:"typing_users"` + // ExpireTime is the interval after which the user should no longer be + // considered typing. Only available if Event.Typing is true. + ExpireTime *time.Time } // TypingEvent represents a matrix edu event of type 'm.typing'. diff --git a/typingserver/cache/cache.go b/typingserver/cache/cache.go index 739f60a24..3f05c938e 100644 --- a/typingserver/cache/cache.go +++ b/typingserver/cache/cache.go @@ -22,25 +22,66 @@ const defaultTypingTimeout = 10 * time.Second // userSet is a map of user IDs to a timer, timer fires at expiry. type userSet map[string]*time.Timer +// TimeoutCallbackFn is a function called right after the removal of a user +// from the typing user list due to timeout. +// latestSyncPosition is the typing sync position after the removal. +type TimeoutCallbackFn func(userID, roomID string, latestSyncPosition int64) + +type roomData struct { + syncPosition int64 + userSet userSet +} + // TypingCache maintains a list of users typing in each room. type TypingCache struct { sync.RWMutex - data map[string]userSet + latestSyncPosition int64 + data map[string]*roomData + timeoutCallback TimeoutCallbackFn } -// NewTypingCache returns a new TypingCache initialized for use. +// Create a roomData with its sync position set to the latest sync position. +// Must only be called after locking the cache. +func (t *TypingCache) newRoomData() *roomData { + return &roomData{ + syncPosition: t.latestSyncPosition, + userSet: make(userSet), + } +} + +// NewTypingCache returns a new TypingCache initialised for use. func NewTypingCache() *TypingCache { - return &TypingCache{data: make(map[string]userSet)} + return &TypingCache{data: make(map[string]*roomData)} +} + +// SetTimeoutCallback sets a callback function that is called right after +// a user is removed from the typing user list due to timeout. +func (t *TypingCache) SetTimeoutCallback(fn TimeoutCallbackFn) { + t.timeoutCallback = fn } // GetTypingUsers returns the list of users typing in a room. -func (t *TypingCache) GetTypingUsers(roomID string) (users []string) { +func (t *TypingCache) GetTypingUsers(roomID string) []string { + users, _ := t.GetTypingUsersIfUpdatedAfter(roomID, 0) + // 0 should work above because the first position used will be 1. + return users +} + +// GetTypingUsersIfUpdatedAfter returns all users typing in this room with +// updated == true if the typing sync position of the room is after the given +// position. Otherwise, returns an empty slice with updated == false. +func (t *TypingCache) GetTypingUsersIfUpdatedAfter( + roomID string, position int64, +) (users []string, updated bool) { t.RLock() - usersMap, ok := t.data[roomID] - t.RUnlock() - if ok { - users = make([]string, 0, len(usersMap)) - for userID := range usersMap { + defer t.RUnlock() + + roomData, ok := t.data[roomID] + if ok && roomData.syncPosition > position { + updated = true + userSet := roomData.userSet + users = make([]string, 0, len(userSet)) + for userID := range userSet { users = append(users, userID) } } @@ -51,53 +92,84 @@ func (t *TypingCache) GetTypingUsers(roomID string) (users []string) { // AddTypingUser sets an user as typing in a room. // expire is the time when the user typing should time out. // if expire is nil, defaultTypingTimeout is assumed. -func (t *TypingCache) AddTypingUser(userID, roomID string, expire *time.Time) { +// Returns the latest sync position for typing after update. +func (t *TypingCache) AddTypingUser( + userID, roomID string, expire *time.Time, +) int64 { expireTime := getExpireTime(expire) if until := time.Until(expireTime); until > 0 { - timer := time.AfterFunc(until, t.timeoutCallback(userID, roomID)) - t.addUser(userID, roomID, timer) + timer := time.AfterFunc(until, func() { + latestSyncPosition := t.RemoveUser(userID, roomID) + if t.timeoutCallback != nil { + t.timeoutCallback(userID, roomID, latestSyncPosition) + } + }) + return t.addUser(userID, roomID, timer) } + return t.GetLatestSyncPosition() } // addUser with mutex lock & replace the previous timer. -func (t *TypingCache) addUser(userID, roomID string, expiryTimer *time.Timer) { +// Returns the latest typing sync position after update. +func (t *TypingCache) addUser( + userID, roomID string, expiryTimer *time.Timer, +) int64 { t.Lock() defer t.Unlock() + t.latestSyncPosition++ + if t.data[roomID] == nil { - t.data[roomID] = make(userSet) + t.data[roomID] = t.newRoomData() + } else { + t.data[roomID].syncPosition = t.latestSyncPosition } // Stop the timer to cancel the call to timeoutCallback - if timer, ok := t.data[roomID][userID]; ok { - // It may happen that at this stage timer fires but now we have a lock on t. - // Hence the execution of timeoutCallback will happen after we unlock. - // So we may lose a typing state, though this event is highly unlikely. - // This can be mitigated by keeping another time.Time in the map and check against it - // before removing. This however is not required in most practical scenario. + if timer, ok := t.data[roomID].userSet[userID]; ok { + // It may happen that at this stage the timer fires, but we now have a lock on + // it. Hence the execution of timeoutCallback will happen after we unlock. So + // we may lose a typing state, though this is highly unlikely. This can be + // mitigated by keeping another time.Time in the map and checking against it + // before removing, but its occurrence is so infrequent it does not seem + // worthwhile. timer.Stop() } - t.data[roomID][userID] = expiryTimer -} + t.data[roomID].userSet[userID] = expiryTimer -// Returns a function which is called after timeout happens. -// This removes the user. -func (t *TypingCache) timeoutCallback(userID, roomID string) func() { - return func() { - t.RemoveUser(userID, roomID) - } + return t.latestSyncPosition } // RemoveUser with mutex lock & stop the timer. -func (t *TypingCache) RemoveUser(userID, roomID string) { +// Returns the latest sync position for typing after update. +func (t *TypingCache) RemoveUser(userID, roomID string) int64 { t.Lock() defer t.Unlock() - if timer, ok := t.data[roomID][userID]; ok { - timer.Stop() - delete(t.data[roomID], userID) + roomData, ok := t.data[roomID] + if !ok { + return t.latestSyncPosition } + + timer, ok := roomData.userSet[userID] + if !ok { + return t.latestSyncPosition + } + + timer.Stop() + delete(roomData.userSet, userID) + + t.latestSyncPosition++ + t.data[roomID].syncPosition = t.latestSyncPosition + + return t.latestSyncPosition +} + +func (t *TypingCache) GetLatestSyncPosition() int64 { + t.Lock() + defer t.Unlock() + return t.latestSyncPosition } func getExpireTime(expire *time.Time) time.Time { diff --git a/typingserver/cache/cache_test.go b/typingserver/cache/cache_test.go index 7aa73e922..2a6ffa50e 100644 --- a/typingserver/cache/cache_test.go +++ b/typingserver/cache/cache_test.go @@ -38,7 +38,7 @@ func TestTypingCache(t *testing.T) { }) } -func testAddTypingUser(t *testing.T, tCache *TypingCache) { +func testAddTypingUser(t *testing.T, tCache *TypingCache) { // nolint: unparam present := time.Now() tests := []struct { userID string diff --git a/typingserver/input/input.go b/typingserver/input/input.go index b9968ce4c..0e2fbe51f 100644 --- a/typingserver/input/input.go +++ b/typingserver/input/input.go @@ -57,15 +57,21 @@ func (t *TypingServerInputAPI) InputTypingEvent( } func (t *TypingServerInputAPI) sendEvent(ite *api.InputTypingEvent) error { - userIDs := t.Cache.GetTypingUsers(ite.RoomID) ev := &api.TypingEvent{ Type: gomatrixserverlib.MTyping, RoomID: ite.RoomID, UserID: ite.UserID, + Typing: ite.Typing, } ote := &api.OutputTypingEvent{ - Event: *ev, - TypingUsers: userIDs, + Event: *ev, + } + + if ev.Typing { + expireTime := ite.OriginServerTS.Time().Add( + time.Duration(ite.Timeout) * time.Millisecond, + ) + ote.ExpireTime = &expireTime } eventJSON, err := json.Marshal(ote)