Some changes

.buildkite/pipeline.yaml

@@ -1,50 +1,49 @@
 steps:
   - command:
-    - "go build ./cmd/..."
+    # https://github.com/golangci/golangci-lint#memory-usage-of-golangci-lint
-    label: ":hammer-and-wrench: Build / :go: 1.11"
+    - "GOGC=20 ./scripts/find-lint.sh"
-    env:
+    label: "\U0001F9F9 Lint / :go: 1.12"
-      GOGC: "400"
+    agents:
-      DENDRITE_LINT_DISABLE_GC: "1"
+      # Use a larger instance as linting takes a looot of memory
+      queue: "medium"
     plugins:
       - docker#v3.0.1:
-        image: "golang:1.11-alpine"
+          image: "golang:1.12"
 
-  - command:
+  - wait
-    - "go test ./..."
-    label: ":female-scientist: Unit tests / :go: 1.11"
-    env:
-      GOGC: "400"
-      DENDRITE_LINT_DISABLE_GC: "1"
-    plugins:
-      - docker#v3.0.1:
-        image: "golang:1.11-alpine"
 
   - command:
     - "go build ./cmd/..."
-    label: ":hammer-and-wrench: Build / :go: 1.12"
+    label: "\U0001F528 Build / :go: 1.11"
-    env:
-      GOGC: "400"
-      DENDRITE_LINT_DISABLE_GC: "1"
     plugins:
       - docker#v3.0.1:
-        image: "golang:1.12-alpine"
+          image: "golang:1.11"
+    retry:
+      automatic:
+        - exit_status: 128
+          limit: 3
+
+  - command:
+    - "go build ./cmd/..."
+    label: "\U0001F528 Build / :go: 1.12"
+    plugins:
+      - docker#v3.0.1:
+          image: "golang:1.12"
+    retry:
+      automatic:
+        - exit_status: 128
+          limit: 3
 
   - command:
     - "go test ./..."
-    label: ":female-scientist: Unit tests / :go: 1.12"
+    label: "\U0001F9EA Unit tests / :go: 1.11"
-    env:
-      GOGC: "400"
-      DENDRITE_LINT_DISABLE_GC: "1"
     plugins:
       - docker#v3.0.1:
-        image: "golang:1.12-alpine"
+          image: "golang:1.11"
 
   - command:
-    - "./scripts/find-lint.sh"
+    - "go test ./..."
-    label: ":lower_left_crayon: Lint / :go: 1.12"
+    label: "\U0001F9EA Unit tests / :go: 1.12"
-    env:
-      GOGC: "400"
-      DENDRITE_LINT_DISABLE_GC: "1"
     plugins:
       - docker#v3.0.1:
-        image: "golang:1.12-alpine"
+          image: "golang:1.12"

.golangci.yml

@@ -0,0 +1,280 @@
+# Config file for golangci-lint
+
+# options for analysis running
+run:
+  # default concurrency is a available CPU number
+  concurrency: 4
+
+  # timeout for analysis, e.g. 30s, 5m, default is 1m
+  deadline: 30m
+
+  # exit code when at least one issue was found, default is 1
+  issues-exit-code: 1
+
+  # include test files or not, default is true
+  tests: true
+
+  # list of build tags, all linters use it. Default is empty list.
+  #build-tags:
+  #  - mytag
+
+  # which dirs to skip: they won't be analyzed;
+  # can use regexp here: generated.*, regexp is applied on full path;
+  # default value is empty list, but next dirs are always skipped independently
+  # from this option's value:
+  #   	vendor$, third_party$, testdata$, examples$, Godeps$, builtin$
+  skip-dirs:
+    - bin
+    - docs
+
+  # which files to skip: they will be analyzed, but issues from them
+  # won't be reported. Default value is empty list, but there is
+  # no need to include all autogenerated files, we confidently recognize
+  # autogenerated files. If it's not please let us know.
+  skip-files:
+    - ".*\\.md$"
+    - ".*\\.sh$"
+    - "^cmd/syncserver-integration-tests/testdata.go$"
+
+  # by default isn't set. If set we pass it to "go list -mod={option}". From "go help modules":
+  # If invoked with -mod=readonly, the go command is disallowed from the implicit
+  # automatic updating of go.mod described above. Instead, it fails when any changes
+  # to go.mod are needed. This setting is most useful to check that go.mod does
+  # not need updates, such as in a continuous integration and testing system.
+  # If invoked with -mod=vendor, the go command assumes that the vendor
+  # directory holds the correct copies of dependencies and ignores
+  # the dependency descriptions in go.mod.
+  #modules-download-mode: (release|readonly|vendor)
+
+
+# output configuration options
+output:
+  # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number"
+  format: colored-line-number
+
+  # print lines of code with issue, default is true
+  print-issued-lines: true
+
+  # print linter name in the end of issue text, default is true
+  print-linter-name: true
+
+
+# all available settings of specific linters
+linters-settings:
+  errcheck:
+    # report about not checking of errors in type assertions: `a := b.(MyStruct)`;
+    # default is false: such cases aren't reported by default.
+    check-type-assertions: false
+
+    # report about assignment of errors to blank identifier: `num, _ := strconv.Atoi(numStr)`;
+    # default is false: such cases aren't reported by default.
+    check-blank: false
+
+    # [deprecated] comma-separated list of pairs of the form pkg:regex
+    # the regex is used to ignore names within pkg. (default "fmt:.*").
+    # see https://github.com/kisielk/errcheck#the-deprecated-method for details
+    #ignore: fmt:.*,io/ioutil:^Read.*
+
+    # path to a file containing a list of functions to exclude from checking
+    # see https://github.com/kisielk/errcheck#excluding-functions for details
+    #exclude: /path/to/file.txt
+  govet:
+    # report about shadowed variables
+    check-shadowing: true
+
+    # settings per analyzer
+    settings:
+      printf: # analyzer name, run `go tool vet help` to see all analyzers
+        funcs: # run `go tool vet help printf` to see available settings for `printf` analyzer
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf
+  golint:
+    # minimal confidence for issues, default is 0.8
+    min-confidence: 0.8
+  gofmt:
+    # simplify code: gofmt with `-s` option, true by default
+    simplify: true
+  goimports:
+    # put imports beginning with prefix after 3rd-party packages;
+    # it's a comma-separated list of prefixes
+    #local-prefixes: github.com/org/project
+  gocyclo:
+    # minimal code complexity to report, 30 by default (but we recommend 10-20)
+    min-complexity: 12
+  maligned:
+    # print struct with more effective memory layout or not, false by default
+    suggest-new: true
+  dupl:
+    # tokens count to trigger issue, 150 by default
+    threshold: 100
+  goconst:
+    # minimal length of string constant, 3 by default
+    min-len: 3
+    # minimal occurrences count to trigger, 3 by default
+    min-occurrences: 3
+  depguard:
+    list-type: blacklist
+    include-go-root: false
+    packages:
+    #  - github.com/davecgh/go-spew/spew
+  misspell:
+    # Correct spellings using locale preferences for US or UK.
+    # Default is to use a neutral variety of English.
+    # Setting locale to US will correct the British spelling of 'colour' to 'color'.
+    locale: UK
+    ignore-words:
+    #  - someword
+  lll:
+    # max line length, lines longer will be reported. Default is 120.
+    # '\t' is counted as 1 character by default, and can be changed with the tab-width option
+    line-length: 96
+    # tab width in spaces. Default to 1.
+    tab-width: 1
+  unused:
+    # treat code as a program (not a library) and report unused exported identifiers; default is false.
+    # XXX: if you enable this setting, unused will report a lot of false-positives in text editors:
+    # if it's called for subdir of a project it can't find funcs usages. All text editor integrations
+    # with golangci-lint call it on a directory with the changed file.
+    check-exported: false
+  unparam:
+    # Inspect exported functions, default is false. Set to true if no external program/library imports your code.
+    # XXX: if you enable this setting, unparam will report a lot of false-positives in text editors:
+    # if it's called for subdir of a project it can't find external interfaces. All text editor integrations
+    # with golangci-lint call it on a directory with the changed file.
+    check-exported: false
+  nakedret:
+    # make an issue if func has more lines of code than this setting and it has naked returns; default is 30
+    max-func-lines: 60
+  prealloc:
+    # XXX: we don't recommend using this linter before doing performance profiling.
+    # For most programs usage of prealloc will be a premature optimization.
+
+    # Report preallocation suggestions only on simple loops that have no returns/breaks/continues/gotos in them.
+    # True by default.
+    simple: true
+    range-loops: true # Report preallocation suggestions on range loops, true by default
+    for-loops: false # Report preallocation suggestions on for loops, false by default
+  gocritic:
+    # Which checks should be enabled; can't be combined with 'disabled-checks';
+    # See https://go-critic.github.io/overview#checks-overview
+    # To check which checks are enabled run `GL_DEBUG=gocritic golangci-lint run`
+    # By default list of stable checks is used.
+    #enabled-checks:
+
+    # Which checks should be disabled; can't be combined with 'enabled-checks'; default is empty
+    #disabled-checks:
+
+    # Enable multiple checks by tags, run `GL_DEBUG=gocritic golangci-lint` run to see all tags and checks.
+    # Empty list by default. See https://github.com/go-critic/go-critic#usage -> section "Tags".
+    #enabled-tags:
+    #  - performance
+
+    settings: # settings passed to gocritic
+      captLocal: # must be valid enabled check name
+        paramsOnly: true
+      #rangeValCopy:
+      #  sizeThreshold: 32
+
+linters:
+  enable:
+    - deadcode
+    - errcheck
+    - goconst
+    - gocyclo
+    - goimports # Does everything gofmt does
+    - gosimple
+    - ineffassign
+    - megacheck
+    - misspell # Check code comments, whereas misspell in CI checks *.md files
+    - nakedret
+    - staticcheck
+    - structcheck
+    - unparam
+    - unused
+    - varcheck
+  enable-all: false
+  disable:
+    - bodyclose
+    - depguard
+    - dupl
+    - gochecknoglobals
+    - gochecknoinits
+    - gocritic
+    - gofmt
+    - golint
+    - gosec # Should turn back on soon
+    - interfacer
+    - lll
+    - maligned
+    - prealloc # Should turn back on soon
+    - scopelint
+    - stylecheck
+    - typecheck # Should turn back on soon
+    - unconvert # Should turn back on soon
+  disable-all: false
+  presets:
+  fast: false
+
+
+issues:
+  # List of regexps of issue texts to exclude, empty list by default.
+  # But independently from this option we use default exclude patterns,
+  # it can be disabled by `exclude-use-default: false`. To list all
+  # excluded by default patterns execute `golangci-lint run --help`
+  exclude:
+  # - abcdef
+
+  # Excluding configuration per-path, per-linter, per-text and per-source
+  exclude-rules:
+    # Exclude some linters from running on tests files.
+    - path: _test\.go
+      linters:
+        - gocyclo
+        - errcheck
+        - dupl
+        - gosec
+
+    # Exclude known linters from partially hard-vendored code,
+    # which is impossible to exclude via "nolint" comments.
+    - path: internal/hmac/
+      text: "weak cryptographic primitive"
+      linters:
+        - gosec
+
+    # Exclude some staticcheck messages
+    - linters:
+        - staticcheck
+      text: "SA9003:"
+
+    # Exclude lll issues for long lines with go:generate
+    - linters:
+        - lll
+      source: "^//go:generate "
+
+  # Independently from option `exclude` we use default exclude patterns,
+  # it can be disabled by this option. To list all
+  # excluded by default patterns execute `golangci-lint run --help`.
+  # Default value for this option is true.
+  exclude-use-default: false
+
+  # Maximum issues count per one linter. Set to 0 to disable. Default is 50.
+  max-issues-per-linter: 0
+
+  # Maximum count of issues with the same text. Set to 0 to disable. Default is 3.
+  max-same-issues: 0
+
+  # Show only new issues: if there are unstaged changes or untracked files,
+  # only those changes are analyzed, else only changes in HEAD~ are analyzed.
+  # It's a super-useful option for integration of golangci-lint into existing
+  # large codebase. It's not practical to fix all existing issues at the moment
+  # of integration: much better don't allow issues in new code.
+  # Default is false.
+  new: false
+
+  # Show only new issues created after git revision `REV`
+  #new-from-rev: REV
+
+  # Show only new issues created in git patch with set file path.
+  #new-from-patch: path/to/patch/file

README.md

@@ -1,4 +1,4 @@
-# Dendrite [![Build Status](https://travis-ci.org/matrix-org/dendrite.svg?branch=master)](https://travis-ci.org/matrix-org/dendrite) [![CircleCI](https://circleci.com/gh/matrix-org/dendrite.svg?style=svg)](https://circleci.com/gh/matrix-org/dendrite) [![Dendrite Dev on Matrix](https://img.shields.io/matrix/dendrite-dev:matrix.org.svg?label=%23dendrite-dev%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite-dev:matrix.org) [![Dendrite on Matrix](https://img.shields.io/matrix/dendrite:matrix.org.svg?label=%23dendrite%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite:matrix.org)
+# Dendrite [![Build Status](https://badge.buildkite.com/4be40938ab19f2bbc4a6c6724517353ee3ec1422e279faf374.svg)](https://buildkite.com/matrix-dot-org/dendrite) [![CircleCI](https://circleci.com/gh/matrix-org/dendrite.svg?style=svg)](https://circleci.com/gh/matrix-org/dendrite) [![Dendrite Dev on Matrix](https://img.shields.io/matrix/dendrite-dev:matrix.org.svg?label=%23dendrite-dev%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite-dev:matrix.org) [![Dendrite on Matrix](https://img.shields.io/matrix/dendrite:matrix.org.svg?label=%23dendrite%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite:matrix.org)
 
 Dendrite will be a matrix homeserver written in go.
 

appservice/api/query.go

@@ -134,9 +134,9 @@ func (h *httpAppServiceQueryAPI) UserIDExists(
 	return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
 }
 
-// RetreiveUserProfile is a wrapper that queries both the local database and
+// RetrieveUserProfile is a wrapper that queries both the local database and
 // application services for a given user's profile
-func RetreiveUserProfile(
+func RetrieveUserProfile(
 	ctx context.Context,
 	userID string,
 	asAPI AppServiceQueryAPI,

appservice/routing/routing.go

@@ -31,6 +31,10 @@ const pathPrefixApp = "/_matrix/app/r0"
 
 // Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client
 // to clients which need to make outbound HTTP requests.
+//
+// Due to Setup being used to call many other functions, a gocyclo nolint is
+// applied:
+// nolint: gocyclo
 func Setup(
 	apiMux *mux.Router, cfg config.Dendrite, // nolint: unparam
 	queryAPI api.RoomserverQueryAPI, aliasAPI api.RoomserverAliasAPI, // nolint: unparam

build.sh

@@ -1,3 +1,3 @@
-#!/bin/bash
+#!/bin/sh
 
 GOBIN=$PWD/`dirname $0`/bin go install -v ./cmd/...

clientapi/auth/auth.go

@@ -130,7 +130,7 @@ func VerifyUserFromRequest(
 
 	return nil, &util.JSONResponse{
 		Code: http.StatusUnauthorized,
-		JSON: jsonerror.UnknownToken("Unrecognized access token"),
+		JSON: jsonerror.UnknownToken("Unrecognized access token"), // nolint: misspell
 	}
 }
 

clientapi/auth/storage/accounts/storage.go

@@ -23,6 +23,7 @@ import (
 	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/gomatrixserverlib"
 	"golang.org/x/crypto/bcrypt"
+
 	// Import the postgres database driver.
 	_ "github.com/lib/pq"
 )

clientapi/jsonerror/jsonerror.go

@@ -87,7 +87,7 @@ func MissingToken(msg string) *MatrixError {
 }
 
 // UnknownToken is an error when the client tries to access a resource which
-// requires authentication and supplies an unrecognized token
+// requires authentication and supplies an unrecognised token
 func UnknownToken(msg string) *MatrixError {
 	return &MatrixError{"M_UNKNOWN_TOKEN", msg}
 }

clientapi/routing/createroom.go

@@ -163,7 +163,7 @@ func createRoom(
 		"roomID": roomID,
 	}).Info("Creating new room")
 
-	profile, err := appserviceAPI.RetreiveUserProfile(req.Context(), userID, asAPI, accountDB)
+	profile, err := appserviceAPI.RetrieveUserProfile(req.Context(), userID, asAPI, accountDB)
 	if err != nil {
 		return httputil.LogThenError(req, err)
 	}

clientapi/routing/directory.go

@@ -117,6 +117,9 @@ func SetLocalAlias(
 	// 1. The new method for checking for things matching an AS's namespace
 	// 2. Using an overall Regex object for all AS's just like we did for usernames
 	for _, appservice := range cfg.Derived.ApplicationServices {
+		// Don't prevent AS from creating aliases in its own namespace
+		// Note that Dendrite uses SenderLocalpart as UserID for AS users
+		if device.UserID != appservice.SenderLocalpart {
 			if aliasNamespaces, ok := appservice.NamespaceMap["aliases"]; ok {
 				for _, namespace := range aliasNamespaces {
 					if namespace.Exclusive && namespace.RegexpObject.MatchString(alias) {
@@ -128,6 +131,7 @@ func SetLocalAlias(
 				}
 			}
 		}
+	}
 
 	var r struct {
 		RoomID string `json:"room_id"`

clientapi/routing/filter.go

@@ -62,7 +62,7 @@ func GetFilter(
 	filter := gomatrix.Filter{}
 	err = json.Unmarshal(res, &filter)
 	if err != nil {
-		httputil.LogThenError(req, err)
+		return httputil.LogThenError(req, err)
 	}
 
 	return util.JSONResponse{

clientapi/routing/joinroom.go

@@ -86,7 +86,10 @@ func JoinRoomByIDOrAlias(
 	}
 	return util.JSONResponse{
 		Code: http.StatusBadRequest,
-		JSON: jsonerror.BadJSON("Invalid first character for room ID or alias"),
+		JSON: jsonerror.BadJSON(
+			fmt.Sprintf("Invalid first character '%s' for room ID or alias",
+				string([]rune(roomIDOrAlias)[0])), // Wrapping with []rune makes this call UTF-8 safe
+		),
 	}
 }
 

clientapi/routing/login.go

@@ -19,6 +19,7 @@ import (
 
 	"context"
 	"database/sql"
+
 	"github.com/matrix-org/dendrite/clientapi/auth"
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
@@ -106,7 +107,7 @@ func Login(
 
 		token, err := auth.GenerateAccessToken()
 		if err != nil {
-			httputil.LogThenError(req, err)
+			return httputil.LogThenError(req, err)
 		}
 
 		dev, err := getDevice(req.Context(), r, deviceDB, acc, localpart, token)

clientapi/routing/membership.go

@@ -58,27 +58,12 @@ func SendMembership(
 		}
 	}
 
-	inviteStored, err := threepid.CheckAndProcessInvite(
+	inviteStored, jsonErrResp := checkAndProcessThreepid(
-		req.Context(), device, &body, cfg, queryAPI, accountDB, producer,
+		req, device, &body, cfg, queryAPI, accountDB, producer,
 		membership, roomID, evTime,
 	)
-	if err == threepid.ErrMissingParameter {
+	if jsonErrResp != nil {
-		return util.JSONResponse{
+		return *jsonErrResp
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON(err.Error()),
-		}
-	} else if err == threepid.ErrNotTrusted {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.NotTrusted(body.IDServer),
-		}
-	} else if err == common.ErrRoomNoExists {
-		return util.JSONResponse{
-			Code: http.StatusNotFound,
-			JSON: jsonerror.NotFound(err.Error()),
-		}
-	} else if err != nil {
-		return httputil.LogThenError(req, err)
 	}
 
 	// If an invite has been stored on an identity server, it means that a
@@ -114,9 +99,18 @@ func SendMembership(
 		return httputil.LogThenError(req, err)
 	}
 
+	var returnData interface{} = struct{}{}
+
+	// The join membership requires the room id to be sent in the response
+	if membership == "join" {
+		returnData = struct {
+			RoomID string `json:"room_id"`
+		}{roomID}
+	}
+
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: struct{}{},
+		JSON: returnData,
 	}
 }
 
@@ -182,7 +176,7 @@ func loadProfile(
 
 	var profile *authtypes.Profile
 	if serverName == cfg.Matrix.ServerName {
-		profile, err = appserviceAPI.RetreiveUserProfile(ctx, userID, asAPI, accountDB)
+		profile, err = appserviceAPI.RetrieveUserProfile(ctx, userID, asAPI, accountDB)
 	} else {
 		profile = &authtypes.Profile{}
 	}
@@ -215,3 +209,41 @@ func getMembershipStateKey(
 
 	return
 }
+
+func checkAndProcessThreepid(
+	req *http.Request,
+	device *authtypes.Device,
+	body *threepid.MembershipRequest,
+	cfg config.Dendrite,
+	queryAPI roomserverAPI.RoomserverQueryAPI,
+	accountDB *accounts.Database,
+	producer *producers.RoomserverProducer,
+	membership, roomID string,
+	evTime time.Time,
+) (inviteStored bool, errRes *util.JSONResponse) {
+
+	inviteStored, err := threepid.CheckAndProcessInvite(
+		req.Context(), device, body, cfg, queryAPI, accountDB, producer,
+		membership, roomID, evTime,
+	)
+	if err == threepid.ErrMissingParameter {
+		return inviteStored, &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON(err.Error()),
+		}
+	} else if err == threepid.ErrNotTrusted {
+		return inviteStored, &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.NotTrusted(body.IDServer),
+		}
+	} else if err == common.ErrRoomNoExists {
+		return inviteStored, &util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: jsonerror.NotFound(err.Error()),
+		}
+	} else if err != nil {
+		er := httputil.LogThenError(req, err)
+		return inviteStored, &er
+	}
+	return
+}

clientapi/routing/profile.go

@@ -43,7 +43,7 @@ func GetProfile(
 			JSON: jsonerror.NotFound("Bad method"),
 		}
 	}
-	profile, err := appserviceAPI.RetreiveUserProfile(req.Context(), userID, asAPI, accountDB)
+	profile, err := appserviceAPI.RetrieveUserProfile(req.Context(), userID, asAPI, accountDB)
 	if err != nil {
 		return httputil.LogThenError(req, err)
 	}
@@ -62,7 +62,7 @@ func GetProfile(
 func GetAvatarURL(
 	req *http.Request, accountDB *accounts.Database, userID string, asAPI appserviceAPI.AppServiceQueryAPI,
 ) util.JSONResponse {
-	profile, err := appserviceAPI.RetreiveUserProfile(req.Context(), userID, asAPI, accountDB)
+	profile, err := appserviceAPI.RetrieveUserProfile(req.Context(), userID, asAPI, accountDB)
 	if err != nil {
 		return httputil.LogThenError(req, err)
 	}
@@ -160,7 +160,7 @@ func SetAvatarURL(
 func GetDisplayName(
 	req *http.Request, accountDB *accounts.Database, userID string, asAPI appserviceAPI.AppServiceQueryAPI,
 ) util.JSONResponse {
-	profile, err := appserviceAPI.RetreiveUserProfile(req.Context(), userID, asAPI, accountDB)
+	profile, err := appserviceAPI.RetrieveUserProfile(req.Context(), userID, asAPI, accountDB)
 	if err != nil {
 		return httputil.LogThenError(req, err)
 	}

clientapi/routing/register.go

@@ -243,8 +243,8 @@ func validateRecaptcha(
 ) *util.JSONResponse {
 	if !cfg.Matrix.RecaptchaEnabled {
 		return &util.JSONResponse{
-			Code: http.StatusBadRequest,
+			Code: http.StatusConflict,
-			JSON: jsonerror.BadJSON("Captcha registration is disabled"),
+			JSON: jsonerror.Unknown("Captcha registration is disabled"),
 		}
 	}
 
@@ -279,8 +279,8 @@ func validateRecaptcha(
 	body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		return &util.JSONResponse{
-			Code: http.StatusInternalServerError,
+			Code: http.StatusGatewayTimeout,
-			JSON: jsonerror.BadJSON("Error in contacting captcha server" + err.Error()),
+			JSON: jsonerror.Unknown("Error in contacting captcha server" + err.Error()),
 		}
 	}
 	err = json.Unmarshal(body, &r)

clientapi/routing/routing.go

@@ -41,6 +41,10 @@ const pathPrefixUnstable = "/_matrix/client/unstable"
 
 // Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client
 // to clients which need to make outbound HTTP requests.
+//
+// Due to Setup being used to call many other functions, a gocyclo nolint is
+// applied:
+// nolint: gocyclo
 func Setup(
 	apiMux *mux.Router, cfg config.Dendrite,
 	producer *producers.RoomserverProducer,
@@ -90,7 +94,10 @@ func Setup(
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/join/{roomIDOrAlias}",
 		common.MakeAuthAPI("join", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return JoinRoomByIDOrAlias(
 				req, device, vars["roomIDOrAlias"], cfg, federation, producer, queryAPI, aliasAPI, keyRing, accountDB,
 			)
@@ -98,19 +105,28 @@ func Setup(
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/{membership:(?:join|kick|ban|unban|leave|invite)}",
 		common.MakeAuthAPI("membership", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return SendMembership(req, accountDB, device, vars["roomID"], vars["membership"], cfg, queryAPI, asAPI, producer)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/send/{eventType}",
 		common.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, nil, cfg, queryAPI, producer, nil)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/send/{eventType}/{txnID}",
 		common.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			txnID := vars["txnID"]
 			return SendEvent(req, device, vars["roomID"], vars["eventType"], &txnID,
 				nil, cfg, queryAPI, producer, transactionsCache)
@@ -118,7 +134,10 @@ func Setup(
 	).Methods(http.MethodPut, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/state/{eventType:[^/]+/?}",
 		common.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			emptyString := ""
 			eventType := vars["eventType"]
 			// If there's a trailing slash, remove it
@@ -130,7 +149,10 @@ func Setup(
 	).Methods(http.MethodPut, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/state/{eventType}/{stateKey}",
 		common.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			stateKey := vars["stateKey"]
 			return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, &stateKey, cfg, queryAPI, producer, nil)
 		}),
@@ -150,21 +172,30 @@ func Setup(
 
 	r0mux.Handle("/directory/room/{roomAlias}",
 		common.MakeExternalAPI("directory_room", func(req *http.Request) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return DirectoryRoom(req, vars["roomAlias"], federation, &cfg, aliasAPI)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/directory/room/{roomAlias}",
 		common.MakeAuthAPI("directory_room", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return SetLocalAlias(req, device, vars["roomAlias"], &cfg, aliasAPI)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/directory/room/{roomAlias}",
 		common.MakeAuthAPI("directory_room", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return RemoveLocalAlias(req, device, vars["roomAlias"], aliasAPI)
 		}),
 	).Methods(http.MethodDelete, http.MethodOptions)
@@ -183,7 +214,10 @@ func Setup(
 
 	r0mux.Handle("/rooms/{roomID}/typing/{userID}",
 		common.MakeAuthAPI("rooms_typing", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return SendTyping(req, device, vars["roomID"], vars["userID"], accountDB, typingProducer)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
@@ -223,14 +257,20 @@ func Setup(
 
 	r0mux.Handle("/user/{userId}/filter",
 		common.MakeAuthAPI("put_filter", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return PutFilter(req, device, accountDB, vars["userId"])
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
 	r0mux.Handle("/user/{userId}/filter/{filterId}",
 		common.MakeAuthAPI("get_filter", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return GetFilter(req, device, accountDB, vars["userId"], vars["filterId"])
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
@@ -239,21 +279,30 @@ func Setup(
 
 	r0mux.Handle("/profile/{userID}",
 		common.MakeExternalAPI("profile", func(req *http.Request) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return GetProfile(req, accountDB, vars["userID"], asAPI)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/profile/{userID}/avatar_url",
 		common.MakeExternalAPI("profile_avatar_url", func(req *http.Request) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return GetAvatarURL(req, accountDB, vars["userID"], asAPI)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/profile/{userID}/avatar_url",
 		common.MakeAuthAPI("profile_avatar_url", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return SetAvatarURL(req, accountDB, device, vars["userID"], userUpdateProducer, &cfg, producer, queryAPI)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
@@ -262,14 +311,20 @@ func Setup(
 
 	r0mux.Handle("/profile/{userID}/displayname",
 		common.MakeExternalAPI("profile_displayname", func(req *http.Request) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return GetDisplayName(req, accountDB, vars["userID"], asAPI)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/profile/{userID}/displayname",
 		common.MakeAuthAPI("profile_displayname", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return SetDisplayName(req, accountDB, device, vars["userID"], userUpdateProducer, &cfg, producer, queryAPI)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
@@ -339,28 +394,40 @@ func Setup(
 
 	r0mux.Handle("/user/{userID}/account_data/{type}",
 		common.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return SaveAccountData(req, accountDB, device, vars["userID"], "", vars["type"], syncProducer)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/user/{userID}/rooms/{roomID}/account_data/{type}",
 		common.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return SaveAccountData(req, accountDB, device, vars["userID"], vars["roomID"], vars["type"], syncProducer)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/rooms/{roomID}/members",
 		common.MakeAuthAPI("rooms_members", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return GetMemberships(req, device, vars["roomID"], false, cfg, queryAPI)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/rooms/{roomID}/joined_members",
 		common.MakeAuthAPI("rooms_members", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return GetMemberships(req, device, vars["roomID"], true, cfg, queryAPI)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
@@ -380,14 +447,20 @@ func Setup(
 
 	r0mux.Handle("/devices/{deviceID}",
 		common.MakeAuthAPI("get_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return GetDeviceByID(req, deviceDB, device, vars["deviceID"])
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/devices/{deviceID}",
 		common.MakeAuthAPI("device_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return UpdateDeviceByID(req, deviceDB, device, vars["deviceID"])
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)

cmd/client-api-proxy/main.go

@@ -17,13 +17,14 @@ package main
 import (
 	"flag"
 	"fmt"
-	log "github.com/sirupsen/logrus"
 	"net/http"
 	"net/http/httputil"
 	"net/url"
 	"os"
 	"strings"
 	"time"
+
+	log "github.com/sirupsen/logrus"
 )
 
 const usage = `Usage: %s

cmd/federation-api-proxy/main.go

@@ -17,13 +17,14 @@ package main
 import (
 	"flag"
 	"fmt"
-	log "github.com/sirupsen/logrus"
 	"net/http"
 	"net/http/httputil"
 	"net/url"
 	"os"
 	"strings"
 	"time"
+
+	log "github.com/sirupsen/logrus"
 )
 
 const usage = `Usage: %s

cmd/kafka-producer/main.go

@@ -18,9 +18,10 @@ import (
 	"bufio"
 	"flag"
 	"fmt"
-	"github.com/Shopify/sarama"
 	"os"
 	"strings"
+
+	"github.com/Shopify/sarama"
 )
 
 const usage = `Usage: %s

common/basecomponent/base.go

@@ -71,7 +71,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string) *BaseDendrite {
 		componentName: componentName,
 		tracerCloser:  closer,
 		Cfg:           cfg,
-		APIMux:        mux.NewRouter(),
+		APIMux:        mux.NewRouter().UseEncodedPath(),
 		KafkaConsumer: kafkaConsumer,
 		KafkaProducer: kafkaProducer,
 	}

common/config/config.go

@@ -585,7 +585,7 @@ func (config *Dendrite) check(monolithic bool) error {
 	}
 
 	// Due to how Golang manages its interface types, this condition is not redundant.
-	// In order to get the proper behavior, it is necessary to return an explicit nil
+	// In order to get the proper behaviour, it is necessary to return an explicit nil
 	// and not a nil configErrors.
 	// This is because the following equalities hold:
 	// error(nil) == nil

common/log.go

@@ -76,7 +76,7 @@ func SetupHookLogging(hooks []config.LogrusHook, componentName string) {
 		// Check we received a proper logging level
 		level, err := logrus.ParseLevel(hook.Level)
 		if err != nil {
-			logrus.Fatalf("Unrecognized logging level %s: %q", hook.Level, err)
+			logrus.Fatalf("Unrecognised logging level %s: %q", hook.Level, err)
 		}
 
 		// Perform a first filter on the logs according to the lowest level of all
@@ -90,7 +90,7 @@ func SetupHookLogging(hooks []config.LogrusHook, componentName string) {
 			checkFileHookParams(hook.Params)
 			setupFileHook(hook, level, componentName)
 		default:
-			logrus.Fatalf("Unrecognized logging hook type: %s", hook.Type)
+			logrus.Fatalf("Unrecognised logging hook type: %s", hook.Type)
 		}
 	}
 }

common/partition_offset_table.go

@@ -50,7 +50,6 @@ type PartitionOffsetStatements struct {
 // Prepare converts the raw SQL statements into prepared statements.
 // Takes a prefix to prepend to the table name used to store the partition offsets.
 // This allows multiple components to share the same database schema.
-// nolint: safesql
 func (s *PartitionOffsetStatements) Prepare(db *sql.DB, prefix string) (err error) {
 	_, err = db.Exec(strings.Replace(partitionOffsetsSchema, "${prefix}", prefix, -1))
 	if err != nil {

common/routing.go

@@ -0,0 +1,35 @@
+// Copyright 2019 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common
+
+import (
+	"net/url"
+)
+
+// URLDecodeMapValues is a function that iterates through each of the items in a
+// map, URL decodes the value, and returns a new map with the decoded values
+// under the same key names
+func URLDecodeMapValues(vmap map[string]string) (map[string]string, error) {
+	decoded := make(map[string]string, len(vmap))
+	for key, value := range vmap {
+		decodedVal, err := url.QueryUnescape(value)
+		if err != nil {
+			return make(map[string]string), err
+		}
+		decoded[key] = decodedVal
+	}
+
+	return decoded, nil
+}

common/test/client.go

@@ -34,7 +34,7 @@ type Request struct {
 	LastErr          *LastRequestErr
 }
 
-// LastRequestErr is a synchronized error wrapper
+// LastRequestErr is a synchronised error wrapper
 // Useful for obtaining the last error from a set of requests
 type LastRequestErr struct {
 	sync.Mutex

common/types.go

@@ -43,7 +43,7 @@ type DisplayName struct {
 
 // WeakBoolean is a type that will Unmarshal to true or false even if the encoded
 // representation is "true"/1 or "false"/0, as well as whatever other forms are
-// recognized by strconv.ParseBool
+// recognised by strconv.ParseBool
 type WeakBoolean bool
 
 // UnmarshalJSON is overridden here to allow strings vaguely representing a true

docs/sytest.md

@@ -1,8 +1,78 @@
 # SyTest
 
 Dendrite uses [SyTest](https://github.com/matrix-org/sytest) for its
-integration testing. When creating a new PR, add the test IDs that your PR
+integration testing. When creating a new PR, add the test IDs (see below) that
-should allow to pass to `testfile` in dendrite's root directory. Not all PRs
+your PR should allow to pass to `testfile` in dendrite's root directory. Not all
-need to make new tests pass. If we find your PR should be making a test pass we
+PRs need to make new tests pass. If we find your PR should be making a test pass
-may ask you to add to that file, as generally Dendrite's progress can be
+we may ask you to add to that file, as generally Dendrite's progress can be
 tracked through the amount of SyTest tests it passes.
+
+## Finding out which tests to add
+
+We recommend you run the tests locally by manually setting up SyTest or using a
+SyTest docker image. After running the tests, a script will print the tests you
+need to add to `testfile` for you.
+
+You should proceed after you see no build problems for dendrite after running:
+
+```sh
+./build.sh
+```
+
+### Manually Setting up SyTest
+
+Make sure you have Perl v5+ installed, and get SyTest with:
+
+(Note that this guide assumes your SyTest checkout is next to your
+`dendrite` checkout.)
+
+```sh
+git clone -b develop https://github.com/matrix-org/sytest
+cd sytest
+./install-deps.pl
+```
+
+Set up the database:
+
+```sh
+sudo -u postgres psql -c "CREATE USER dendrite PASSWORD 'itsasecret'"
+sudo -u postgres psql -c "CREATE DATABASE sytest_template OWNER dendrite"
+mkdir -p "server-0"
+cat > "server-0/database.yaml" << EOF
+args:
+    user: dendrite
+    database: dendrite
+    host: 127.0.0.1
+type: pg
+EOF
+```
+
+Run the tests:
+
+```sh
+./run-tests.pl -I Dendrite::Monolith -d ../dendrite/bin -W ../dendrite/testfile -O tap --all | tee results.tap
+```
+
+where `tee` lets you see the results while they're being piped to the file.
+
+Once the tests are complete, run the helper script to see if you need to add
+any newly passing test names to `testfile` in the project's root directory:
+
+```sh
+../dendrite/show-expected-fail-tests.sh results.tap ../dendrite/testfile
+```
+
+If the script prints nothing/exits with 0, then you're good to go.
+
+### Using a SyTest Docker image
+
+Ensure you have the latest image for SyTest, then run the tests:
+
+```sh
+docker pull matrixdotorg/sytest-dendrite
+docker run --rm -v /path/to/dendrite/:/src/ matrixdotorg/sytest-dendrite
+```
+
+where `/path/to/dendrite/` should be replaced with the actual path to your
+dendrite source code. The output should tell you if you need to add any tests to
+`testfile`.

federationapi/routing/profile.go

@@ -53,7 +53,7 @@ func GetProfile(
 		return httputil.LogThenError(httpReq, err)
 	}
 
-	profile, err := appserviceAPI.RetreiveUserProfile(httpReq.Context(), userID, asAPI, accountDB)
+	profile, err := appserviceAPI.RetrieveUserProfile(httpReq.Context(), userID, asAPI, accountDB)
 	if err != nil {
 		return httputil.LogThenError(httpReq, err)
 	}

federationapi/routing/routing.go

@@ -35,6 +35,10 @@ const (
 )
 
 // Setup registers HTTP handlers with the given ServeMux.
+//
+// Due to Setup being used to call many other functions, a gocyclo nolint is
+// applied:
+// nolint: gocyclo
 func Setup(
 	apiMux *mux.Router,
 	cfg config.Dendrite,
@@ -64,7 +68,10 @@ func Setup(
 	v1fedmux.Handle("/send/{txnID}/", common.MakeFedAPI(
 		"federation_send", cfg.Matrix.ServerName, keys,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse {
-			vars := mux.Vars(httpReq)
+			vars, err := common.URLDecodeMapValues(mux.Vars(httpReq))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return Send(
 				httpReq, request, gomatrixserverlib.TransactionID(vars["txnID"]),
 				cfg, query, producer, keys, federation,
@@ -75,7 +82,10 @@ func Setup(
 	v1fedmux.Handle("/invite/{roomID}/{eventID}", common.MakeFedAPI(
 		"federation_invite", cfg.Matrix.ServerName, keys,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse {
-			vars := mux.Vars(httpReq)
+			vars, err := common.URLDecodeMapValues(mux.Vars(httpReq))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return Invite(
 				httpReq, request, vars["roomID"], vars["eventID"],
 				cfg, producer, keys,
@@ -92,7 +102,10 @@ func Setup(
 	v1fedmux.Handle("/exchange_third_party_invite/{roomID}", common.MakeFedAPI(
 		"exchange_third_party_invite", cfg.Matrix.ServerName, keys,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse {
-			vars := mux.Vars(httpReq)
+			vars, err := common.URLDecodeMapValues(mux.Vars(httpReq))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return ExchangeThirdPartyInvite(
 				httpReq, request, vars["roomID"], query, cfg, federation, producer,
 			)
@@ -102,7 +115,10 @@ func Setup(
 	v1fedmux.Handle("/event/{eventID}", common.MakeFedAPI(
 		"federation_get_event", cfg.Matrix.ServerName, keys,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse {
-			vars := mux.Vars(httpReq)
+			vars, err := common.URLDecodeMapValues(mux.Vars(httpReq))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return GetEvent(
 				httpReq.Context(), request, query, vars["eventID"],
 			)
@@ -112,7 +128,10 @@ func Setup(
 	v1fedmux.Handle("/state/{roomID}", common.MakeFedAPI(
 		"federation_get_event_auth", cfg.Matrix.ServerName, keys,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse {
-			vars := mux.Vars(httpReq)
+			vars, err := common.URLDecodeMapValues(mux.Vars(httpReq))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return GetState(
 				httpReq.Context(), request, query, vars["roomID"],
 			)
@@ -122,7 +141,10 @@ func Setup(
 	v1fedmux.Handle("/state_ids/{roomID}", common.MakeFedAPI(
 		"federation_get_event_auth", cfg.Matrix.ServerName, keys,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse {
-			vars := mux.Vars(httpReq)
+			vars, err := common.URLDecodeMapValues(mux.Vars(httpReq))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return GetStateIDs(
 				httpReq.Context(), request, query, vars["roomID"],
 			)
@@ -150,7 +172,10 @@ func Setup(
 	v1fedmux.Handle("/user/devices/{userID}", common.MakeFedAPI(
 		"federation_user_devices", cfg.Matrix.ServerName, keys,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse {
-			vars := mux.Vars(httpReq)
+			vars, err := common.URLDecodeMapValues(mux.Vars(httpReq))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return GetUserDevices(
 				httpReq, deviceDB, vars["userID"],
 			)
@@ -160,7 +185,10 @@ func Setup(
 	v1fedmux.Handle("/make_join/{roomID}/{userID}", common.MakeFedAPI(
 		"federation_make_join", cfg.Matrix.ServerName, keys,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse {
-			vars := mux.Vars(httpReq)
+			vars, err := common.URLDecodeMapValues(mux.Vars(httpReq))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			roomID := vars["roomID"]
 			userID := vars["userID"]
 			return MakeJoin(
@@ -172,7 +200,10 @@ func Setup(
 	v1fedmux.Handle("/send_join/{roomID}/{userID}", common.MakeFedAPI(
 		"federation_send_join", cfg.Matrix.ServerName, keys,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse {
-			vars := mux.Vars(httpReq)
+			vars, err := common.URLDecodeMapValues(mux.Vars(httpReq))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			roomID := vars["roomID"]
 			userID := vars["userID"]
 			return SendJoin(
@@ -184,7 +215,10 @@ func Setup(
 	v1fedmux.Handle("/make_leave/{roomID}/{userID}", common.MakeFedAPI(
 		"federation_make_leave", cfg.Matrix.ServerName, keys,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse {
-			vars := mux.Vars(httpReq)
+			vars, err := common.URLDecodeMapValues(mux.Vars(httpReq))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			roomID := vars["roomID"]
 			userID := vars["userID"]
 			return MakeLeave(
@@ -196,7 +230,10 @@ func Setup(
 	v1fedmux.Handle("/send_leave/{roomID}/{userID}", common.MakeFedAPI(
 		"federation_send_leave", cfg.Matrix.ServerName, keys,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse {
-			vars := mux.Vars(httpReq)
+			vars, err := common.URLDecodeMapValues(mux.Vars(httpReq))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			roomID := vars["roomID"]
 			userID := vars["userID"]
 			return SendLeave(
@@ -215,7 +252,10 @@ func Setup(
 	v1fedmux.Handle("/get_missing_events/{roomID}", common.MakeFedAPI(
 		"federation_get_missing_events", cfg.Matrix.ServerName, keys,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse {
-			vars := mux.Vars(httpReq)
+			vars, err := common.URLDecodeMapValues(mux.Vars(httpReq))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return GetMissingEvents(httpReq, request, query, vars["roomID"])
 		},
 	)).Methods(http.MethodPost)
@@ -223,7 +263,10 @@ func Setup(
 	v1fedmux.Handle("/backfill/{roomID}/", common.MakeFedAPI(
 		"federation_backfill", cfg.Matrix.ServerName, keys,
 		func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse {
-			vars := mux.Vars(httpReq)
+			vars, err := common.URLDecodeMapValues(mux.Vars(httpReq))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return Backfill(httpReq, request, query, vars["roomID"], cfg)
 		},
 	)).Methods(http.MethodGet)

federationapi/routing/state.go

@@ -75,7 +75,8 @@ func parseEventIDParam(
 ) (eventID string, resErr *util.JSONResponse) {
 	URL, err := url.Parse(request.RequestURI())
 	if err != nil {
-		*resErr = util.ErrorResponse(err)
+		response := util.ErrorResponse(err)
+		resErr = &response
 		return
 	}
 
@@ -102,6 +103,10 @@ func getState(
 		return nil, resErr
 	}
 
+	if event.RoomID() != roomID {
+		return nil, &util.JSONResponse{Code: http.StatusNotFound, JSON: nil}
+	}
+
 	prevEventIDs := getIDsFromEventRef(event.PrevEvents())
 	authEventIDs := getIDsFromEventRef(event.AuthEvents())
 

federationapi/routing/threepid.go

@@ -194,7 +194,7 @@ func createInviteFrom3PIDInvite(
 		StateKey: &inv.MXID,
 	}
 
-	profile, err := appserviceAPI.RetreiveUserProfile(ctx, inv.MXID, asAPI, accountDB)
+	profile, err := appserviceAPI.RetrieveUserProfile(ctx, inv.MXID, asAPI, accountDB)
 	if err != nil {
 		return nil, err
 	}

federationsender/queue/queue.go

@@ -96,9 +96,11 @@ func (oqs *OutgoingQueues) SendEDU(
 	// Remove our own server from the list of destinations.
 	destinations = filterDestinations(oqs.origin, destinations)
 
+	if len(destinations) > 0 {
 		log.WithFields(log.Fields{
 			"destinations": destinations, "edu_type": e.Type,
 		}).Info("Sending EDU event")
+	}
 
 	oqs.queuesMutex.Lock()
 	defer oqs.queuesMutex.Unlock()

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a
 	github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd
 	github.com/crossdock/crossdock-go v0.0.0-20160816171116-049aabb0122b
-	github.com/davecgh/go-spew v1.1.0
+	github.com/davecgh/go-spew v1.1.1
 	github.com/eapache/go-resiliency v0.0.0-20160104191539-b86b1ec0dd42
 	github.com/eapache/go-xerial-snappy v0.0.0-20160609142408-bb955e01b934
 	github.com/eapache/queue v1.1.0
@@ -16,16 +16,21 @@ require (
 	github.com/golang/snappy v0.0.0-20170119014723-7db9049039a0
 	github.com/google/shlex v0.0.0-20150127133951-6f45313302b9
 	github.com/gorilla/context v1.1.1
-	github.com/gorilla/mux v1.3.0
+	github.com/gorilla/mux v1.7.3
 	github.com/jaegertracing/jaeger-client-go v0.0.0-20170921145708-3ad49a1d839b
 	github.com/jaegertracing/jaeger-lib v0.0.0-20170920222118-21a3da6d66fe
 	github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6
 	github.com/lib/pq v0.0.0-20170918175043-23da1db4f16d
 	github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5
+<<<<<<< HEAD
 	github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26
 	github.com/matrix-org/gomatrixserverlib v0.0.0-20181109104322-1c2cbc0872f0
+=======
+	github.com/matrix-org/gomatrix v0.0.0-20190130130140-385f072fe9af
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20190619132215-178ed5e3b8e2
+>>>>>>> e2251199a49ab0bb846c02ba37e1cd437a7f725b
 	github.com/matrix-org/naffka v0.0.0-20171115094957-662bfd0841d0
-	github.com/matrix-org/util v0.0.0-20171013132526-8b1c8ab81986
+	github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5
 	github.com/matttproud/golang_protobuf_extensions v1.0.1
 	github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5
 	github.com/nicksnyder/go-i18n v1.8.1
@@ -40,11 +45,11 @@ require (
 	github.com/prometheus/common v0.0.0-20170108231212-dd2f054febf4
 	github.com/prometheus/procfs v0.0.0-20170128160123-1878d9fbb537
 	github.com/rcrowley/go-metrics v0.0.0-20161128210544-1f30fe9094a5
-	github.com/sirupsen/logrus v0.0.0-20170822132746-89742aefa4b2
+	github.com/sirupsen/logrus v1.3.0
-	github.com/stretchr/testify v0.0.0-20170809224252-890a5c3458b4
+	github.com/stretchr/testify v1.2.2
-	github.com/tidwall/gjson v1.0.2
+	github.com/tidwall/gjson v1.1.5
-	github.com/tidwall/match v0.0.0-20171002075945-1731857f09b1
+	github.com/tidwall/match v1.0.1
-	github.com/tidwall/sjson v1.0.0
+	github.com/tidwall/sjson v1.0.3
 	github.com/uber-go/atomic v1.3.0
 	github.com/uber/jaeger-client-go v2.15.0+incompatible
 	github.com/uber/jaeger-lib v1.5.0
@@ -52,14 +57,14 @@ require (
 	go.uber.org/atomic v1.3.0
 	go.uber.org/multierr v0.0.0-20170829224307-fb7d312c2c04
 	go.uber.org/zap v1.7.1
-	golang.org/x/crypto v0.0.0-20181106171534-e4dc69e5b2fd
+	golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613
-	golang.org/x/net v0.0.0-20170927055102-0a9397675ba3
+	golang.org/x/net v0.0.0-20190301231341-16b79f2e4e95
-	golang.org/x/sys v0.0.0-20171012164349-43eea11bc926
+	golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33
 	gopkg.in/Shopify/sarama.v1 v1.11.0
 	gopkg.in/airbrake/gobrake.v2 v2.0.9
 	gopkg.in/alecthomas/kingpin.v3-unstable v3.0.0-20170727041045-23bcc3c4eae3
 	gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2
 	gopkg.in/h2non/bimg.v1 v1.0.18
-	gopkg.in/macaroon.v2 v2.0.0
+	gopkg.in/macaroon.v2 v2.1.0
-	gopkg.in/yaml.v2 v2.0.0-20171116090243-287cf08546ab
+	gopkg.in/yaml.v2 v2.2.2
 )

go.sum

@@ -10,40 +10,62 @@ github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE
 github.com/crossdock/crossdock-go v0.0.0-20160816171116-049aabb0122b/go.mod h1:v9FBN7gdVTpiD/+LZ7Po0UKvROyT87uLVxTHVky/dlQ=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/eapache/go-resiliency v0.0.0-20160104191539-b86b1ec0dd42 h1:f8ERmXYuaC+kCSv2w+y3rBK/oVu6If4DEm3jywJJ0hc=
 github.com/eapache/go-resiliency v0.0.0-20160104191539-b86b1ec0dd42/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
 github.com/eapache/go-xerial-snappy v0.0.0-20160609142408-bb955e01b934 h1:oGLoaVIefp3tiOgi7+KInR/nNPvEpPM6GFo+El7fd14=
 github.com/eapache/go-xerial-snappy v0.0.0-20160609142408-bb955e01b934/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
 github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
+github.com/frankban/quicktest v1.0.0/go.mod h1:R98jIehRai+d1/3Hv2//jOVCTJhW1VBavT6B6CuGq2k=
 github.com/golang/protobuf v0.0.0-20161117033126-8ee79997227b h1:fE/yi9pibxGEc0gSJuEShcsBXE2d5FW3OudsjE9tKzQ=
 github.com/golang/protobuf v0.0.0-20161117033126-8ee79997227b/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/snappy v0.0.0-20170119014723-7db9049039a0 h1:FMElzTwkd/2jQ2QzLEzt97JRgvFhYhnYiaQSwZ7tuyU=
 github.com/golang/snappy v0.0.0-20170119014723-7db9049039a0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/shlex v0.0.0-20150127133951-6f45313302b9/go.mod h1:RpwtwJQFrIEPstU94h88MWPXP2ektJZ8cZ0YntAmXiE=
 github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
 github.com/gorilla/mux v1.3.0 h1:HwSEKGN6U5T2aAQTfu5pW8fiwjSp3IgwdRbkICydk/c=
 github.com/gorilla/mux v1.3.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
+github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI=
 github.com/jaegertracing/jaeger-client-go v0.0.0-20170921145708-3ad49a1d839b/go.mod h1:HWG7INeOG1ZE17I/S8eeb+svquXmBS/hf1Obi6hJUyQ=
 github.com/jaegertracing/jaeger-lib v0.0.0-20170920222118-21a3da6d66fe/go.mod h1:VqeqQrZmZr9G4WdLw4ei9tAHU54iJRkfoFHvTTQn4jQ=
 github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6 h1:KAZ1BW2TCmT6PRihDPpocIy1QTtsAsrx6TneU/4+CMg=
 github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/lib/pq v0.0.0-20170918175043-23da1db4f16d h1:Hdtccv31GWxWoCzWsIhZXy5NxEktzAkA8lywhTKu8O4=
 github.com/lib/pq v0.0.0-20170918175043-23da1db4f16d/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 h1:nMX2t7hbGF0NYDYySx0pCqEKGKAeZIiSqlWSspetlhY=
 github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg=
 github.com/matrix-org/gomatrix v0.0.0-20171003113848-a7fc80c8060c h1:aZap604NyBGhAUE0CyNHz6+Pryye5A5mHnYyO4KPPW8=
 github.com/matrix-org/gomatrix v0.0.0-20171003113848-a7fc80c8060c/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
+<<<<<<< HEAD
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bhrnp3Ky1qgx/fzCtCALOoGYylh2tpS9K4=
 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
+=======
+github.com/matrix-org/gomatrix v0.0.0-20190130130140-385f072fe9af h1:piaIBNQGIHnni27xRB7VKkEwoWCgAmeuYf8pxAyG0bI=
+github.com/matrix-org/gomatrix v0.0.0-20190130130140-385f072fe9af/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0=
+>>>>>>> e2251199a49ab0bb846c02ba37e1cd437a7f725b
 github.com/matrix-org/gomatrixserverlib v0.0.0-20181109104322-1c2cbc0872f0 h1:3UzhmERBbis4ZaB3imEbZwtDjGz/oVRC2cLLEajCzJA=
 github.com/matrix-org/gomatrixserverlib v0.0.0-20181109104322-1c2cbc0872f0/go.mod h1:YHyhIQUmuXyKtoVfDUMk/DyU93Taamlu6nPZkij/JtA=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20190619132215-178ed5e3b8e2 h1:pYajAEdi3sowj4iSunqctchhcMNW3rDjeeH0T4uDkMY=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20190619132215-178ed5e3b8e2/go.mod h1:sf0RcKOdiwJeTti7A313xsaejNUGYDq02MQZ4JD4w/E=
 github.com/matrix-org/naffka v0.0.0-20171115094957-662bfd0841d0 h1:p7WTwG+aXM86+yVrYAiCMW3ZHSmotVvuRbjtt3jC+4A=
 github.com/matrix-org/naffka v0.0.0-20171115094957-662bfd0841d0/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A=
 github.com/matrix-org/util v0.0.0-20171013132526-8b1c8ab81986 h1:TiWl4hLvezAhRPM8tPcPDFTysZ7k4T/1J4GPp/iqlZo=
 github.com/matrix-org/util v0.0.0-20171013132526-8b1c8ab81986/go.mod h1:lePuOiXLNDott7NZfnQvJk0lAZ5HgvIuWGhel6J+RLA=
+github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 h1:W7l5CP4V7wPyPb4tYE11dbmeAOwtFQBTW0rf4OonOS8=
+github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5/go.mod h1:lePuOiXLNDott7NZfnQvJk0lAZ5HgvIuWGhel6J+RLA=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/miekg/dns v1.1.4/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
+github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms=
 github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 h1:BvoENQQU+fZ9uukda/RzCAL/191HHwJA5b13R6diVlY=
 github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
 github.com/nicksnyder/go-i18n v1.8.1/go.mod h1:HrK7VCrbOvQoUAQ7Vpy7i87N7JZZZ7R2xBGjv0j365Q=
@@ -69,13 +91,23 @@ github.com/rcrowley/go-metrics v0.0.0-20161128210544-1f30fe9094a5 h1:gwcdIpH6NU2
 github.com/rcrowley/go-metrics v0.0.0-20161128210544-1f30fe9094a5/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/sirupsen/logrus v0.0.0-20170822132746-89742aefa4b2 h1:+8J/sCAVv2Y9Ct1BKszDFJEVWv6Aynr2O4FYGUg6+Mc=
 github.com/sirupsen/logrus v0.0.0-20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
+github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME=
+github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v0.0.0-20170809224252-890a5c3458b4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/tidwall/gjson v1.0.2 h1:5BsM7kyEAHAUGEGDkEKO9Mdyiuw6QQ6TSDdarP0Nnmk=
 github.com/tidwall/gjson v1.0.2/go.mod h1:c/nTNbUr0E0OrXEhq1pwa8iEgc2DOt4ZZqAt1HtCkPA=
+github.com/tidwall/gjson v1.1.5 h1:QysILxBeUEY3GTLA0fQVgkQG1zme8NxGvhh2SSqWNwI=
+github.com/tidwall/gjson v1.1.5/go.mod h1:c/nTNbUr0E0OrXEhq1pwa8iEgc2DOt4ZZqAt1HtCkPA=
 github.com/tidwall/match v0.0.0-20171002075945-1731857f09b1 h1:pWIN9LOlFRCJFqWIOEbHLvY0WWJddsjH2FQ6N0HKZdU=
 github.com/tidwall/match v0.0.0-20171002075945-1731857f09b1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
+github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc=
+github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
 github.com/tidwall/sjson v1.0.0 h1:hOrzQPtGKlKAudQVmU43GkxEgG8TOgKyiKUyb7sE0rs=
 github.com/tidwall/sjson v1.0.0/go.mod h1:bURseu1nuBkFpIES5cz6zBtjmYeOQmEESshn7VpF15Y=
+github.com/tidwall/sjson v1.0.3 h1:DeF+0LZqvIt4fKYw41aPB29ZGlvwVkHKktoXJ1YW9Y8=
+github.com/tidwall/sjson v1.0.3/go.mod h1:bURseu1nuBkFpIES5cz6zBtjmYeOQmEESshn7VpF15Y=
 github.com/uber-go/atomic v1.3.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
 github.com/uber/jaeger-client-go v2.15.0+incompatible h1:NP3qsSqNxh8VYr956ur1N/1C1PjvOJnJykCzcD5QHbk=
 github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
@@ -85,18 +117,31 @@ github.com/uber/tchannel-go v0.0.0-20170927010734-b3e26487e291/go.mod h1:Rrgz1eL
 go.uber.org/atomic v1.3.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/multierr v0.0.0-20170829224307-fb7d312c2c04/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/zap v1.7.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181106171534-e4dc69e5b2fd h1:VtIkGDhk0ph3t+THbvXHfMZ8QHgsBO39Nh52+74pq7w=
 golang.org/x/crypto v0.0.0-20181106171534-e4dc69e5b2fd/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613 h1:MQ/ZZiDsUapFFiMS+vzwXkCTeEKaum+Do5rINYJDmxc=
+golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/net v0.0.0-20170927055102-0a9397675ba3 h1:tTDpczhDVjW6WN3DinzKcw5juwkDTVn22I7MNlfxSXM=
 golang.org/x/net v0.0.0-20170927055102-0a9397675ba3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190301231341-16b79f2e4e95 h1:fY7Dsw114eJN4boqzVSbpVHO6rTdhq6/GnXeu+PKnzU=
+golang.org/x/net v0.0.0-20190301231341-16b79f2e4e95/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/sys v0.0.0-20171012164349-43eea11bc926 h1:PY6OU86NqbyZiOzaPnDw6oOjAGtYQqIua16z6y9QkwE=
 golang.org/x/sys v0.0.0-20171012164349-43eea11bc926/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 h1:I6FyU15t786LL7oL/hn43zqTuEGr4PN7F4XJ1p4E3Y8=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 gopkg.in/Shopify/sarama.v1 v1.11.0 h1:/3kaCyeYaPbr59IBjeqhIcUOB1vXlIVqXAYa5g5C5F0=
 gopkg.in/Shopify/sarama.v1 v1.11.0/go.mod h1:AxnvoaevB2nBjNK17cG61A3LleFcWFwVBHBt+cot4Oc=
 gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
 gopkg.in/alecthomas/kingpin.v3-unstable v3.0.0-20170727041045-23bcc3c4eae3/go.mod h1:3HH7i1SgMqlzxCcBmUHW657sD4Kvv9sC3HpL3YukzwA=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
 gopkg.in/h2non/bimg.v1 v1.0.18/go.mod h1:PgsZL7dLwUbsGm1NYps320GxGgvQNTnecMCZqxV11So=
+gopkg.in/h2non/gock.v1 v1.0.14/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE=
 gopkg.in/macaroon.v2 v2.0.0/go.mod h1:+I6LnTMkm/uV5ew/0nsulNjL16SK4+C8yDmRUzHR17I=
+gopkg.in/macaroon.v2 v2.1.0/go.mod h1:OUb+TQP/OP0WOerC2Jp/3CwhIKyIa9kQjuc7H24e6/o=
 gopkg.in/yaml.v2 v2.0.0-20171116090243-287cf08546ab h1:yZ6iByf7GKeJ3gsd1Dr/xaj1DyJ//wxKX1Cdh8LhoAw=
 gopkg.in/yaml.v2 v2.0.0-20171116090243-287cf08546ab/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

linter-fast.json

@@ -1,16 +0,0 @@
-{
-    "Vendor": true,
-    "Cyclo": 12,
-    "Deadline": "5m",
-    "Enable": [
-        "vetshadow",
-        "deadcode",
-        "gocyclo",
-        "ineffassign",
-        "misspell",
-        "errcheck",
-        "vet",
-        "gofmt",
-        "goconst"
-    ]
-}

linter.json

@@ -1,20 +0,0 @@
-{
-    "Vendor": true,
-    "Cyclo": 12,
-    "Deadline": "5m",
-    "Enable": [
-        "vetshadow",
-        "deadcode",
-        "gocyclo",
-        "golint",
-        "varcheck",
-        "structcheck",
-        "ineffassign",
-        "misspell",
-        "unparam",
-        "errcheck",
-        "vet",
-        "gofmt",
-        "goconst"
-    ]
-}

mediaapi/routing/download.go

@@ -305,6 +305,10 @@ func (r *downloadRequest) respondFromLocalFile(
 		}).Info("Responding with file")
 		responseFile = file
 		responseMetadata = r.MediaMetadata
+
+		if len(responseMetadata.UploadName) > 0 {
+			w.Header().Set("Content-Disposition", fmt.Sprintf(`inline; filename*=utf-8"%s"`, responseMetadata.UploadName))
+		}
 	}
 
 	w.Header().Set("Content-Type", string(responseMetadata.ContentType))

mediaapi/routing/routing.go

@@ -34,6 +34,10 @@ import (
 const pathPrefixR0 = "/_matrix/media/r0"
 
 // Setup registers the media API HTTP handlers
+//
+// Due to Setup being used to call many other functions, a gocyclo nolint is
+// applied:
+// nolint: gocyclo
 func Setup(
 	apiMux *mux.Router,
 	cfg *config.Dendrite,
@@ -87,7 +91,7 @@ func makeDownloadAPI(
 		// Content-Type will be overridden in case of returning file data, else we respond with JSON-formatted errors
 		w.Header().Set("Content-Type", "application/json")
 
-		vars := mux.Vars(req)
+		vars, _ := common.URLDecodeMapValues(mux.Vars(req))
 		Download(
 			w,
 			req,

mediaapi/thumbnailer/thumbnailer_nfnt.go

@@ -20,9 +20,11 @@ import (
 	"context"
 	"image"
 	"image/draw"
+
 	// Imported for gif codec
 	_ "image/gif"
 	"image/jpeg"
+
 	// Imported for png codec
 	_ "image/png"
 	"os"
@@ -258,9 +260,6 @@ func adjustSize(dst types.Path, img image.Image, w, h int, crop bool, logger *lo
 		out = target
 	} else {
 		out = resize.Thumbnail(uint(w), uint(h), img, resize.Lanczos3)
-		if err != nil {
-			return -1, -1, err
-		}
 	}
 
 	if err = writeFile(out, string(dst)); err != nil {

publicroomsapi/routing/routing.go

@@ -30,6 +30,10 @@ import (
 const pathPrefixR0 = "/_matrix/client/r0"
 
 // Setup configures the given mux with publicroomsapi server listeners
+//
+// Due to Setup being used to call many other functions, a gocyclo nolint is
+// applied:
+// nolint: gocyclo
 func Setup(apiMux *mux.Router, deviceDB *devices.Database, publicRoomsDB *storage.PublicRoomsServerDatabase) {
 	r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter()
 
@@ -41,14 +45,20 @@ func Setup(apiMux *mux.Router, deviceDB *devices.Database, publicRoomsDB *storag
 
 	r0mux.Handle("/directory/list/room/{roomID}",
 		common.MakeExternalAPI("directory_list", func(req *http.Request) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return directory.GetVisibility(req, publicRoomsDB, vars["roomID"])
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 	// TODO: Add AS support
 	r0mux.Handle("/directory/list/room/{roomID}",
 		common.MakeAuthAPI("directory_list", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return directory.SetVisibility(req, publicRoomsDB, vars["roomID"])
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)

publicroomsapi/storage/public_rooms_table.go

@@ -134,7 +134,6 @@ type publicRoomsStatements struct {
 	updateRoomAttributeStmts                map[string]*sql.Stmt
 }
 
-// nolint: safesql
 func (s *publicRoomsStatements) prepare(db *sql.DB) (err error) {
 	_, err = db.Exec(publicRoomsSchema)
 	if err != nil {

roomserver/alias/alias.go

@@ -96,7 +96,8 @@ func (r *RoomserverAliasAPI) GetRoomIDForAlias(
 		return err
 	}
 
-	// No rooms found locally, try our application services by making a call to
+	if roomID == "" {
+		// No room found locally, try our application services by making a call to
 		// the appservice component
 		aliasReq := appserviceAPI.RoomAliasExistsRequest{Alias: request.Alias}
 		var aliasResp appserviceAPI.RoomAliasExistsResponse
@@ -104,6 +105,14 @@ func (r *RoomserverAliasAPI) GetRoomIDForAlias(
 			return err
 		}
 
+		if aliasResp.AliasExists {
+			roomID, err = r.DB.GetRoomIDForAlias(ctx, request.Alias)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
 	response.RoomID = roomID
 	return nil
 }
@@ -268,6 +277,20 @@ func (r *RoomserverAliasAPI) SetupHTTP(servMux *http.ServeMux) {
 			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
 		}),
 	)
+	servMux.Handle(
+		roomserverAPI.RoomserverGetAliasesForRoomIDPath,
+		common.MakeInternalAPI("getAliasesForRoomID", func(req *http.Request) util.JSONResponse {
+			var request roomserverAPI.GetAliasesForRoomIDRequest
+			var response roomserverAPI.GetAliasesForRoomIDResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.ErrorResponse(err)
+			}
+			if err := r.GetAliasesForRoomID(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
 	servMux.Handle(
 		roomserverAPI.RoomserverRemoveRoomAliasPath,
 		common.MakeInternalAPI("removeRoomAlias", func(req *http.Request) util.JSONResponse {

roomserver/alias/alias_test.go

@@ -0,0 +1,196 @@
+// Copyright 2019 Serra Allgood
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package alias
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"testing"
+
+	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
+	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+)
+
+type MockRoomserverAliasAPIDatabase struct {
+	mode     string
+	attempts int
+}
+
+// These methods can be essentially noop
+func (db MockRoomserverAliasAPIDatabase) SetRoomAlias(ctx context.Context, alias string, roomID string) error {
+	return nil
+}
+
+func (db MockRoomserverAliasAPIDatabase) GetAliasesForRoomID(ctx context.Context, roomID string) ([]string, error) {
+	aliases := make([]string, 0)
+	return aliases, nil
+}
+
+func (db MockRoomserverAliasAPIDatabase) RemoveRoomAlias(ctx context.Context, alias string) error {
+	return nil
+}
+
+// This method needs to change depending on test case
+func (db *MockRoomserverAliasAPIDatabase) GetRoomIDForAlias(
+	ctx context.Context,
+	alias string,
+) (string, error) {
+	switch db.mode {
+	case "empty":
+		return "", nil
+	case "error":
+		return "", fmt.Errorf("found an error from GetRoomIDForAlias")
+	case "found":
+		return "123", nil
+	case "emptyFound":
+		switch db.attempts {
+		case 0:
+			db.attempts = 1
+			return "", nil
+		case 1:
+			db.attempts = 0
+			return "123", nil
+		default:
+			return "", nil
+		}
+	default:
+		return "", fmt.Errorf("unknown option used")
+	}
+}
+
+type MockAppServiceQueryAPI struct {
+	mode string
+}
+
+// This method can be noop
+func (q MockAppServiceQueryAPI) UserIDExists(
+	ctx context.Context,
+	req *appserviceAPI.UserIDExistsRequest,
+	resp *appserviceAPI.UserIDExistsResponse,
+) error {
+	return nil
+}
+
+func (q MockAppServiceQueryAPI) RoomAliasExists(
+	ctx context.Context,
+	req *appserviceAPI.RoomAliasExistsRequest,
+	resp *appserviceAPI.RoomAliasExistsResponse,
+) error {
+	switch q.mode {
+	case "error":
+		return fmt.Errorf("found an error from RoomAliasExists")
+	case "found":
+		resp.AliasExists = true
+		return nil
+	case "empty":
+		resp.AliasExists = false
+		return nil
+	default:
+		return fmt.Errorf("Unknown option used")
+	}
+}
+
+func TestGetRoomIDForAlias(t *testing.T) {
+	type arguments struct {
+		ctx      context.Context
+		request  *roomserverAPI.GetRoomIDForAliasRequest
+		response *roomserverAPI.GetRoomIDForAliasResponse
+	}
+	args := arguments{
+		context.Background(),
+		&roomserverAPI.GetRoomIDForAliasRequest{},
+		&roomserverAPI.GetRoomIDForAliasResponse{},
+	}
+	type testCase struct {
+		name      string
+		dbMode    string
+		queryMode string
+		wantError bool
+		errorMsg  string
+		want      string
+	}
+	tt := []testCase{
+		{
+			"found local alias",
+			"found",
+			"error",
+			false,
+			"",
+			"123",
+		},
+		{
+			"found appservice alias",
+			"emptyFound",
+			"found",
+			false,
+			"",
+			"123",
+		},
+		{
+			"error returned from DB",
+			"error",
+			"",
+			true,
+			"GetRoomIDForAlias",
+			"",
+		},
+		{
+			"error returned from appserviceAPI",
+			"empty",
+			"error",
+			true,
+			"RoomAliasExists",
+			"",
+		},
+		{
+			"no errors but no alias",
+			"empty",
+			"empty",
+			false,
+			"",
+			"",
+		},
+	}
+
+	setup := func(dbMode, queryMode string) *RoomserverAliasAPI {
+		mockAliasAPIDB := &MockRoomserverAliasAPIDatabase{dbMode, 0}
+		mockAppServiceQueryAPI := MockAppServiceQueryAPI{queryMode}
+
+		return &RoomserverAliasAPI{
+			DB:            mockAliasAPIDB,
+			AppserviceAPI: mockAppServiceQueryAPI,
+		}
+	}
+
+	for _, tc := range tt {
+		t.Run(tc.name, func(t *testing.T) {
+			aliasAPI := setup(tc.dbMode, tc.queryMode)
+
+			err := aliasAPI.GetRoomIDForAlias(args.ctx, args.request, args.response)
+			if tc.wantError {
+				if err == nil {
+					t.Fatalf("Got no error; wanted error from %s", tc.errorMsg)
+				} else if !strings.Contains(err.Error(), tc.errorMsg) {
+					t.Fatalf("Got %s; wanted error from %s", err, tc.errorMsg)
+				}
+			} else if err != nil {
+				t.Fatalf("Got %s; wanted no error", err)
+			} else if args.response.RoomID != tc.want {
+				t.Errorf("Got '%s'; wanted '%s'", args.response.RoomID, tc.want)
+			}
+		})
+	}
+}

roomserver/storage/prepare.go

@@ -25,7 +25,6 @@ type statementList []struct {
 }
 
 // prepare the SQL for each statement in the list and assign the result to the prepared statement.
-// nolint: safesql
 func (s statementList) prepare(db *sql.DB) (err error) {
 	for _, statement := range s {
 		if *statement.statement, err = db.Prepare(statement.sql); err != nil {

scripts/find-lint.sh

@@ -3,44 +3,34 @@
 # Runs the linters against dendrite
 
 # The linters can take a lot of resources and are slow, so they can be
-# configured using two environment variables:
+# configured using the following environment variables:
 #
 # - `DENDRITE_LINT_CONCURRENCY` - number of concurrent linters to run,
-#   gometalinter defaults this to 8
+#   golangci-lint defaults this to NumCPU
-# - `DENDRITE_LINT_DISABLE_GC` - if set then the the go gc will be disabled
+# - `GOGC` - how often to perform garbage collection during golangci-lint runs.
-#   when running the linters, speeding them up but using much more memory.
+#   Essentially a ratio of memory/speed. See https://github.com/golangci/golangci-lint#memory-usage-of-golangci-lint
+#   for more info.
 
 
 set -eux
 
 cd `dirname $0`/..
 
-# gometalinter doesn't seem to work without this.
-# We should move from gometalinter asap as per https://github.com/matrix-org/dendrite/issues/697 so this is a temporary
-# fix.
-export GO111MODULE=off
-
 args=""
 if [ ${1:-""} = "fast" ]
-then args="--config=linter-fast.json"
+then args="--fast"
-else args="--config=linter.json"
 fi
 
-if [ -n "${DENDRITE_LINT_CONCURRENCY:-}" ]
+echo "Installing golangci-lint..."
-then args="$args --concurrency=$DENDRITE_LINT_CONCURRENCY"
-fi
 
-if [ -z "${DENDRITE_LINT_DISABLE_GC:-}" ]
+# Make a backup of go.{mod,sum} first
-then args="$args --enable-gc"
+# TODO: Once go 1.13 is out, use go get's -mod=readonly option
-fi
+# https://github.com/golang/go/issues/30667
-
+cp go.mod go.mod.bak && cp go.sum go.sum.bak
-echo "Installing lint search engine..."
+go get github.com/golangci/golangci-lint/cmd/golangci-lint
-go get github.com/alecthomas/gometalinter/
-
-gometalinter --config=linter.json ./... --install
 
 echo "Looking for lint..."
-gometalinter ./... $args
+golangci-lint run $args
 
-echo "Double checking spelling..."
+# Restore go.{mod,sum}
-misspell -error src *.md
+mv go.mod.bak go.mod && mv go.sum.bak go.sum

show-expected-fail-tests.sh

@@ -0,0 +1,45 @@
+#! /bin/bash
+
+results_file=$1
+testfile=$2
+
+fail_build=0
+
+if [ ! -f "$results_file" ]; then
+	echo "ERROR: Specified results file ${results_file} doesn't exist."
+	fail_build=1
+fi
+
+if [ ! -f "$testfile" ]; then
+	echo "ERROR: Specified testfile ${testfile} doesn't exist."
+	fail_build=1
+fi
+
+[ "$fail_build" = 0 ] || exit 1
+
+passed_but_expected_fail=$(grep ' # TODO passed but expected fail' ${results_file} | sed -E 's/^ok [0-9]+ (\(expected fail\) )?//' | sed -E 's/( \([0-9]+ subtests\))? # TODO passed but expected fail$//')
+tests_to_add=""
+already_in_testfile=""
+
+while read -r test_id; do
+	[ "${test_id}" = "" ] && continue
+	grep "${test_id}" "${testfile}" > /dev/null 2>&1
+	if [ "$?" != "0" ]; then
+		tests_to_add="${tests_to_add}${test_id}\n"
+		fail_build=1
+	else
+		already_in_testfile="${already_in_testfile}${test_id}\n"
+	fi
+done <<< "${passed_but_expected_fail}"
+
+if [ -n "${tests_to_add}" ]; then
+	echo "ERROR: The following passed tests are not present in testfile. Please append them to the file:"
+	echo -e "${tests_to_add}"
+fi
+
+if [ -n "${already_in_testfile}" ]; then
+	echo "WARN: Tests in testfile still marked as expected fail:"
+	echo -e "${already_in_testfile}"
+fi
+
+exit ${fail_build}

syncapi/consumers/clientapi.go

@@ -22,6 +22,7 @@ import (
 	"github.com/matrix-org/dendrite/common/config"
 	"github.com/matrix-org/dendrite/syncapi/storage"
 	"github.com/matrix-org/dendrite/syncapi/sync"
+	"github.com/matrix-org/dendrite/syncapi/types"
 	log "github.com/sirupsen/logrus"
 	sarama "gopkg.in/Shopify/sarama.v1"
 )
@@ -29,7 +30,7 @@ import (
 // OutputClientDataConsumer consumes events that originated in the client API server.
 type OutputClientDataConsumer struct {
 	clientAPIConsumer *common.ContinualConsumer
-	db                *storage.SyncServerDatabase
+	db                *storage.SyncServerDatasource
 	notifier          *sync.Notifier
 }
 
@@ -38,7 +39,7 @@ func NewOutputClientDataConsumer(
 	cfg *config.Dendrite,
 	kafkaConsumer sarama.Consumer,
 	n *sync.Notifier,
-	store *storage.SyncServerDatabase,
+	store *storage.SyncServerDatasource,
 ) *OutputClientDataConsumer {
 
 	consumer := common.ContinualConsumer{
@@ -78,7 +79,7 @@ func (s *OutputClientDataConsumer) onMessage(msg *sarama.ConsumerMessage) error
 		"room_id": output.RoomID,
 	}).Info("received data from client API server")
 
-	syncStreamPos, err := s.db.UpsertAccountData(
+	pduPos, err := s.db.UpsertAccountData(
 		context.TODO(), string(msg.Key), output.RoomID, output.Type,
 	)
 	if err != nil {
@@ -89,7 +90,7 @@ func (s *OutputClientDataConsumer) onMessage(msg *sarama.ConsumerMessage) error
 		}).Panicf("could not save account data")
 	}
 
-	s.notifier.OnNewEvent(nil, string(msg.Key), syncStreamPos)
+	s.notifier.OnNewEvent(nil, "", []string{string(msg.Key)}, types.SyncPosition{PDUPosition: pduPos})
 
 	return nil
 }

syncapi/consumers/roomserver.go

@@ -33,7 +33,7 @@ import (
 // OutputRoomEventConsumer consumes events that originated in the room server.
 type OutputRoomEventConsumer struct {
 	roomServerConsumer *common.ContinualConsumer
-	db                 *storage.SyncServerDatabase
+	db                 *storage.SyncServerDatasource
 	notifier           *sync.Notifier
 	query              api.RoomserverQueryAPI
 }
@@ -43,7 +43,7 @@ func NewOutputRoomEventConsumer(
 	cfg *config.Dendrite,
 	kafkaConsumer sarama.Consumer,
 	n *sync.Notifier,
-	store *storage.SyncServerDatabase,
+	store *storage.SyncServerDatasource,
 	queryAPI api.RoomserverQueryAPI,
 ) *OutputRoomEventConsumer {
 
@@ -126,7 +126,7 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent(
 		}
 	}
 
-	syncStreamPos, err := s.db.WriteEvent(
+	pduPos, err := s.db.WriteEvent(
 		ctx,
 		&ev,
 		addsStateEvents,
@@ -134,10 +134,6 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent(
 		msg.RemovesStateEventIDs,
 		msg.TransactionID,
 	)
-	if err != nil {
-		return err
-	}
-
 	if err != nil {
 		// panic rather than continue with an inconsistent database
 		log.WithFields(log.Fields{
@@ -148,7 +144,7 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent(
 		}).Panicf("roomserver output log: write event failure")
 		return nil
 	}
-	s.notifier.OnNewEvent(&ev, "", types.StreamPosition(syncStreamPos))
+	s.notifier.OnNewEvent(&ev, "", nil, types.SyncPosition{PDUPosition: pduPos})
 
 	return nil
 }
@@ -156,7 +152,7 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent(
 func (s *OutputRoomEventConsumer) onNewInviteEvent(
 	ctx context.Context, msg api.OutputNewInviteEvent,
 ) error {
-	syncStreamPos, err := s.db.AddInviteEvent(ctx, msg.Event)
+	pduPos, err := s.db.AddInviteEvent(ctx, msg.Event)
 	if err != nil {
 		// panic rather than continue with an inconsistent database
 		log.WithFields(log.Fields{
@@ -165,7 +161,7 @@ func (s *OutputRoomEventConsumer) onNewInviteEvent(
 		}).Panicf("roomserver output log: write invite failure")
 		return nil
 	}
-	s.notifier.OnNewEvent(&msg.Event, "", syncStreamPos)
+	s.notifier.OnNewEvent(&msg.Event, "", nil, types.SyncPosition{PDUPosition: pduPos})
 	return nil
 }
 

syncapi/consumers/typingserver.go

@@ -0,0 +1,96 @@
+// Copyright 2019 Alex Chen
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package consumers
+
+import (
+	"encoding/json"
+
+	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/common/config"
+	"github.com/matrix-org/dendrite/syncapi/storage"
+	"github.com/matrix-org/dendrite/syncapi/sync"
+	"github.com/matrix-org/dendrite/syncapi/types"
+	"github.com/matrix-org/dendrite/typingserver/api"
+	log "github.com/sirupsen/logrus"
+	sarama "gopkg.in/Shopify/sarama.v1"
+)
+
+// OutputTypingEventConsumer consumes events that originated in the typing server.
+type OutputTypingEventConsumer struct {
+	typingConsumer *common.ContinualConsumer
+	db             *storage.SyncServerDatasource
+	notifier       *sync.Notifier
+}
+
+// NewOutputTypingEventConsumer creates a new OutputTypingEventConsumer.
+// Call Start() to begin consuming from the typing server.
+func NewOutputTypingEventConsumer(
+	cfg *config.Dendrite,
+	kafkaConsumer sarama.Consumer,
+	n *sync.Notifier,
+	store *storage.SyncServerDatasource,
+) *OutputTypingEventConsumer {
+
+	consumer := common.ContinualConsumer{
+		Topic:          string(cfg.Kafka.Topics.OutputTypingEvent),
+		Consumer:       kafkaConsumer,
+		PartitionStore: store,
+	}
+
+	s := &OutputTypingEventConsumer{
+		typingConsumer: &consumer,
+		db:             store,
+		notifier:       n,
+	}
+
+	consumer.ProcessMessage = s.onMessage
+
+	return s
+}
+
+// Start consuming from typing api
+func (s *OutputTypingEventConsumer) Start() error {
+	s.db.SetTypingTimeoutCallback(func(userID, roomID string, latestSyncPosition int64) {
+		s.notifier.OnNewEvent(nil, roomID, nil, types.SyncPosition{TypingPosition: latestSyncPosition})
+	})
+
+	return s.typingConsumer.Start()
+}
+
+func (s *OutputTypingEventConsumer) onMessage(msg *sarama.ConsumerMessage) error {
+	var output api.OutputTypingEvent
+	if err := json.Unmarshal(msg.Value, &output); err != nil {
+		// If the message was invalid, log it and move on to the next message in the stream
+		log.WithError(err).Errorf("typing server output log: message parse failure")
+		return nil
+	}
+
+	log.WithFields(log.Fields{
+		"room_id": output.Event.RoomID,
+		"user_id": output.Event.UserID,
+		"typing":  output.Event.Typing,
+	}).Debug("received data from typing server")
+
+	var typingPos int64
+	typingEvent := output.Event
+	if typingEvent.Typing {
+		typingPos = s.db.AddTypingUser(typingEvent.UserID, typingEvent.RoomID, output.ExpireTime)
+	} else {
+		typingPos = s.db.RemoveTypingUser(typingEvent.UserID, typingEvent.RoomID)
+	}
+
+	s.notifier.OnNewEvent(nil, output.Event.RoomID, nil, types.SyncPosition{TypingPosition: typingPos})
+	return nil
+}

syncapi/routing/routing.go

@@ -30,7 +30,11 @@ import (
 const pathPrefixR0 = "/_matrix/client/r0"
 
 // Setup configures the given mux with sync-server listeners
-func Setup(apiMux *mux.Router, srp *sync.RequestPool, syncDB *storage.SyncServerDatabase, deviceDB *devices.Database) {
+//
+// Due to Setup being used to call many other functions, a gocyclo nolint is
+// applied:
+// nolint: gocyclo
+func Setup(apiMux *mux.Router, srp *sync.RequestPool, syncDB *storage.SyncServerDatasource, deviceDB *devices.Database) {
 	r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter()
 
 	authData := auth.Data{
@@ -45,17 +49,26 @@ func Setup(apiMux *mux.Router, srp *sync.RequestPool, syncDB *storage.SyncServer
 	})).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/rooms/{roomID}/state", common.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-		vars := mux.Vars(req)
+		vars, err := common.URLDecodeMapValues(mux.Vars(req))
+		if err != nil {
+			return util.ErrorResponse(err)
+		}
 		return OnIncomingStateRequest(req, syncDB, vars["roomID"])
 	})).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/rooms/{roomID}/state/{type}", common.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-		vars := mux.Vars(req)
+		vars, err := common.URLDecodeMapValues(mux.Vars(req))
+		if err != nil {
+			return util.ErrorResponse(err)
+		}
 		return OnIncomingStateTypeRequest(req, syncDB, vars["roomID"], vars["type"], "")
 	})).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", common.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-		vars := mux.Vars(req)
+		vars, err := common.URLDecodeMapValues(mux.Vars(req))
+		if err != nil {
+			return util.ErrorResponse(err)
+		}
 		return OnIncomingStateTypeRequest(req, syncDB, vars["roomID"], vars["type"], vars["stateKey"])
 	})).Methods(http.MethodGet, http.MethodOptions)
 }

syncapi/routing/state.go

@@ -40,7 +40,7 @@ type stateEventInStateResp struct {
 // TODO: Check if the user is in the room. If not, check if the room's history
 // is publicly visible. Current behaviour is returning an empty array if the
 // user cannot see the room's history.
-func OnIncomingStateRequest(req *http.Request, db *storage.SyncServerDatabase, roomID string) util.JSONResponse {
+func OnIncomingStateRequest(req *http.Request, db *storage.SyncServerDatasource, roomID string) util.JSONResponse {
 	// TODO(#287): Auth request and handle the case where the user has left (where
 	// we should return the state at the poin they left)
 
@@ -84,7 +84,7 @@ func OnIncomingStateRequest(req *http.Request, db *storage.SyncServerDatabase, r
 // /rooms/{roomID}/state/{type}/{statekey} request. It will look in current
 // state to see if there is an event with that type and state key, if there
 // is then (by default) we return the content, otherwise a 404.
-func OnIncomingStateTypeRequest(req *http.Request, db *storage.SyncServerDatabase, roomID string, evType, stateKey string) util.JSONResponse {
+func OnIncomingStateTypeRequest(req *http.Request, db *storage.SyncServerDatasource, roomID string, evType, stateKey string) util.JSONResponse {
 	// TODO(#287): Auth request and handle the case where the user has left (where
 	// we should return the state at the poin they left)
 

syncapi/storage/account_data_table.go

@@ -19,8 +19,6 @@ import (
 	"database/sql"
 
 	"github.com/matrix-org/dendrite/common"
-
-	"github.com/matrix-org/dendrite/syncapi/types"
 )
 
 const accountDataSchema = `
@@ -94,7 +92,7 @@ func (s *accountDataStatements) insertAccountData(
 func (s *accountDataStatements) selectAccountDataInRange(
 	ctx context.Context,
 	userID string,
-	oldPos, newPos types.StreamPosition,
+	oldPos, newPos int64,
 ) (data map[string][]string, err error) {
 	data = make(map[string][]string)
 

syncapi/storage/output_room_events_table.go

@@ -23,7 +23,6 @@ import (
 
 	"github.com/lib/pq"
 	"github.com/matrix-org/dendrite/common"
-	"github.com/matrix-org/dendrite/syncapi/types"
 	"github.com/matrix-org/gomatrixserverlib"
 	log "github.com/sirupsen/logrus"
 )
@@ -109,11 +108,11 @@ func (s *outputRoomEventsStatements) prepare(db *sql.DB) (err error) {
 	return
 }
 
-// selectStateInRange returns the state events between the two given stream positions, exclusive of oldPos, inclusive of newPos.
+// selectStateInRange returns the state events between the two given PDU stream positions, exclusive of oldPos, inclusive of newPos.
 // Results are bucketed based on the room ID. If the same state is overwritten multiple times between the
 // two positions, only the most recent state is returned.
 func (s *outputRoomEventsStatements) selectStateInRange(
-	ctx context.Context, txn *sql.Tx, oldPos, newPos types.StreamPosition,
+	ctx context.Context, txn *sql.Tx, oldPos, newPos int64,
 ) (map[string]map[string]bool, map[string]streamEvent, error) {
 	stmt := common.TxStmt(txn, s.selectStateInRangeStmt)
 
@@ -171,7 +170,7 @@ func (s *outputRoomEventsStatements) selectStateInRange(
 
 		eventIDToEvent[ev.EventID()] = streamEvent{
 			Event:          ev,
-			streamPosition: types.StreamPosition(streamPos),
+			streamPosition: streamPos,
 		}
 	}
 
@@ -223,7 +222,7 @@ func (s *outputRoomEventsStatements) insertEvent(
 // RecentEventsInRoom returns the most recent events in the given room, up to a maximum of 'limit'.
 func (s *outputRoomEventsStatements) selectRecentEvents(
 	ctx context.Context, txn *sql.Tx,
-	roomID string, fromPos, toPos types.StreamPosition, limit int,
+	roomID string, fromPos, toPos int64, limit int,
 ) ([]streamEvent, error) {
 	stmt := common.TxStmt(txn, s.selectRecentEventsStmt)
 	rows, err := stmt.QueryContext(ctx, roomID, fromPos, toPos, limit)
@@ -236,7 +235,7 @@ func (s *outputRoomEventsStatements) selectRecentEvents(
 		return nil, err
 	}
 	// The events need to be returned from oldest to latest, which isn't
-	// necessary the way the SQL query returns them, so a sort is necessary to
+	// necessarily the way the SQL query returns them, so a sort is necessary to
 	// ensure the events are in the right order in the slice.
 	sort.SliceStable(events, func(i int, j int) bool {
 		return events[i].streamPosition < events[j].streamPosition
@@ -286,7 +285,7 @@ func rowsToStreamEvents(rows *sql.Rows) ([]streamEvent, error) {
 
 		result = append(result, streamEvent{
 			Event:          ev,
-			streamPosition: types.StreamPosition(streamPos),
+			streamPosition: streamPos,
 			transactionID:  transactionID,
 		})
 	}

syncapi/storage/syncserver.go

@@ -17,16 +17,21 @@ package storage
 import (
 	"context"
 	"database/sql"
+	"encoding/json"
 	"fmt"
+	"strconv"
+	"time"
 
 	"github.com/sirupsen/logrus"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/roomserver/api"
+
 	// Import the postgres database driver.
 	_ "github.com/lib/pq"
 	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/syncapi/types"
+	"github.com/matrix-org/dendrite/typingserver/cache"
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
@@ -34,33 +39,35 @@ type stateDelta struct {
 	roomID      string
 	stateEvents []gomatrixserverlib.Event
 	membership  string
-	// The stream position of the latest membership event for this user, if applicable.
+	// The PDU stream position of the latest membership event for this user, if applicable.
 	// Can be 0 if there is no membership event in this delta.
-	membershipPos types.StreamPosition
+	membershipPos int64
 }
 
-// Same as gomatrixserverlib.Event but also has the stream position for this event.
+// Same as gomatrixserverlib.Event but also has the PDU stream position for this event.
 type streamEvent struct {
 	gomatrixserverlib.Event
-	streamPosition types.StreamPosition
+	streamPosition int64
 	transactionID  *api.TransactionID
 }
 
-// SyncServerDatabase represents a sync server database
+// SyncServerDatabase represents a sync server datasource which manages
-type SyncServerDatabase struct {
+// both the database for PDUs and caches for EDUs.
+type SyncServerDatasource struct {
 	db *sql.DB
 	common.PartitionOffsetStatements
 	accountData accountDataStatements
 	events      outputRoomEventsStatements
 	roomstate   currentRoomStateStatements
 	invites     inviteEventsStatements
+	typingCache *cache.TypingCache
 }
 
 // NewSyncServerDatabase creates a new sync server database
-func NewSyncServerDatabase(dataSourceName string) (*SyncServerDatabase, error) {
+func NewSyncServerDatasource(dbDataSourceName string) (*SyncServerDatasource, error) {
-	var d SyncServerDatabase
+	var d SyncServerDatasource
 	var err error
-	if d.db, err = sql.Open("postgres", dataSourceName); err != nil {
+	if d.db, err = sql.Open("postgres", dbDataSourceName); err != nil {
 		return nil, err
 	}
 	if err = d.PartitionOffsetStatements.Prepare(d.db, "syncapi"); err != nil {
@@ -78,11 +85,12 @@ func NewSyncServerDatabase(dataSourceName string) (*SyncServerDatabase, error) {
 	if err := d.invites.prepare(d.db); err != nil {
 		return nil, err
 	}
+	d.typingCache = cache.NewTypingCache()
 	return &d, nil
 }
 
 // AllJoinedUsersInRooms returns a map of room ID to a list of all joined user IDs.
-func (d *SyncServerDatabase) AllJoinedUsersInRooms(ctx context.Context) (map[string][]string, error) {
+func (d *SyncServerDatasource) AllJoinedUsersInRooms(ctx context.Context) (map[string][]string, error) {
 	return d.roomstate.selectJoinedUsers(ctx)
 }
 
@@ -91,7 +99,7 @@ func (d *SyncServerDatabase) AllJoinedUsersInRooms(ctx context.Context) (map[str
 // If an event is not found in the database then it will be omitted from the list.
 // Returns an error if there was a problem talking with the database.
 // Does not include any transaction IDs in the returned events.
-func (d *SyncServerDatabase) Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.Event, error) {
+func (d *SyncServerDatasource) Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.Event, error) {
 	streamEvents, err := d.events.selectEvents(ctx, nil, eventIDs)
 	if err != nil {
 		return nil, err
@@ -103,38 +111,38 @@ func (d *SyncServerDatabase) Events(ctx context.Context, eventIDs []string) ([]g
 }
 
 // WriteEvent into the database. It is not safe to call this function from multiple goroutines, as it would create races
-// when generating the stream position for this event. Returns the sync stream position for the inserted event.
+// when generating the sync stream position for this event. Returns the sync stream position for the inserted event.
 // Returns an error if there was a problem inserting this event.
-func (d *SyncServerDatabase) WriteEvent(
+func (d *SyncServerDatasource) WriteEvent(
 	ctx context.Context,
 	ev *gomatrixserverlib.Event,
 	addStateEvents []gomatrixserverlib.Event,
 	addStateEventIDs, removeStateEventIDs []string,
 	transactionID *api.TransactionID,
-) (streamPos types.StreamPosition, returnErr error) {
+) (pduPosition int64, returnErr error) {
 	returnErr = common.WithTransaction(d.db, func(txn *sql.Tx) error {
 		var err error
 		pos, err := d.events.insertEvent(ctx, txn, ev, addStateEventIDs, removeStateEventIDs, transactionID)
 		if err != nil {
 			return err
 		}
-		streamPos = types.StreamPosition(pos)
+		pduPosition = pos
 
 		if len(addStateEvents) == 0 && len(removeStateEventIDs) == 0 {
 			// Nothing to do, the event may have just been a message event.
 			return nil
 		}
 
-		return d.updateRoomState(ctx, txn, removeStateEventIDs, addStateEvents, streamPos)
+		return d.updateRoomState(ctx, txn, removeStateEventIDs, addStateEvents, pduPosition)
 	})
 	return
 }
 
-func (d *SyncServerDatabase) updateRoomState(
+func (d *SyncServerDatasource) updateRoomState(
 	ctx context.Context, txn *sql.Tx,
 	removedEventIDs []string,
 	addedEvents []gomatrixserverlib.Event,
-	streamPos types.StreamPosition,
+	pduPosition int64,
 ) error {
 	// remove first, then add, as we do not ever delete state, but do replace state which is a remove followed by an add.
 	for _, eventID := range removedEventIDs {
@@ -156,7 +164,7 @@ func (d *SyncServerDatabase) updateRoomState(
 			}
 			membership = &value
 		}
-		if err := d.roomstate.upsertRoomState(ctx, txn, event, membership, int64(streamPos)); err != nil {
+		if err := d.roomstate.upsertRoomState(ctx, txn, event, membership, pduPosition); err != nil {
 			return err
 		}
 	}
@@ -167,7 +175,7 @@ func (d *SyncServerDatabase) updateRoomState(
 // GetStateEvent returns the Matrix state event of a given type for a given room with a given state key
 // If no event could be found, returns nil
 // If there was an issue during the retrieval, returns an error
-func (d *SyncServerDatabase) GetStateEvent(
+func (d *SyncServerDatasource) GetStateEvent(
 	ctx context.Context, roomID, evType, stateKey string,
 ) (*gomatrixserverlib.Event, error) {
 	return d.roomstate.selectStateEvent(ctx, roomID, evType, stateKey)
@@ -176,7 +184,7 @@ func (d *SyncServerDatabase) GetStateEvent(
 // GetStateEventsForRoom fetches the state events for a given room.
 // Returns an empty slice if no state events could be found for this room.
 // Returns an error if there was an issue with the retrieval.
-func (d *SyncServerDatabase) GetStateEventsForRoom(
+func (d *SyncServerDatasource) GetStateEventsForRoom(
 	ctx context.Context, roomID string,
 ) (stateEvents []gomatrixserverlib.Event, err error) {
 	err = common.WithTransaction(d.db, func(txn *sql.Tx) error {
@@ -186,46 +194,49 @@ func (d *SyncServerDatabase) GetStateEventsForRoom(
 	return
 }
 
-// SyncStreamPosition returns the latest position in the sync stream. Returns 0 if there are no events yet.
+// SyncPosition returns the latest positions for syncing.
-func (d *SyncServerDatabase) SyncStreamPosition(ctx context.Context) (types.StreamPosition, error) {
+func (d *SyncServerDatasource) SyncPosition(ctx context.Context) (types.SyncPosition, error) {
-	return d.syncStreamPositionTx(ctx, nil)
+	return d.syncPositionTx(ctx, nil)
 }
 
-func (d *SyncServerDatabase) syncStreamPositionTx(
+func (d *SyncServerDatasource) syncPositionTx(
 	ctx context.Context, txn *sql.Tx,
-) (types.StreamPosition, error) {
+) (sp types.SyncPosition, err error) {
-	maxID, err := d.events.selectMaxEventID(ctx, txn)
+
+	maxEventID, err := d.events.selectMaxEventID(ctx, txn)
 	if err != nil {
-		return 0, err
+		return sp, err
 	}
 	maxAccountDataID, err := d.accountData.selectMaxAccountDataID(ctx, txn)
 	if err != nil {
-		return 0, err
+		return sp, err
 	}
-	if maxAccountDataID > maxID {
+	if maxAccountDataID > maxEventID {
-		maxID = maxAccountDataID
+		maxEventID = maxAccountDataID
 	}
 	maxInviteID, err := d.invites.selectMaxInviteID(ctx, txn)
 	if err != nil {
-		return 0, err
+		return sp, err
 	}
-	if maxInviteID > maxID {
+	if maxInviteID > maxEventID {
-		maxID = maxInviteID
+		maxEventID = maxInviteID
 	}
-	return types.StreamPosition(maxID), nil
+	sp.PDUPosition = maxEventID
+
+	sp.TypingPosition = d.typingCache.GetLatestSyncPosition()
+
+	return
 }
 
-// IncrementalSync returns all the data needed in order to create an incremental
+// addPDUDeltaToResponse adds all PDU deltas to a sync response.
-// sync response for the given user. Events returned will include any client
+// IDs of all rooms the user joined are returned so EDU deltas can be added for them.
-// transaction IDs associated with the given device. These transaction IDs come
+func (d *SyncServerDatasource) addPDUDeltaToResponse(
-// from when the device sent the event via an API that included a transaction
-// ID.
-func (d *SyncServerDatabase) IncrementalSync(
 	ctx context.Context,
 	device authtypes.Device,
-	fromPos, toPos types.StreamPosition,
+	fromPos, toPos int64,
 	numRecentEventsPerRoom int,
-) (*types.Response, error) {
+	res *types.Response,
+) ([]string, error) {
 	txn, err := d.db.BeginTx(ctx, &txReadOnlySnapshot)
 	if err != nil {
 		return nil, err
@@ -234,7 +245,7 @@ func (d *SyncServerDatabase) IncrementalSync(
 	defer common.EndTransaction(txn, &succeeded)
 
 	// Work out which rooms to return in the response. This is done by getting not only the currently
-	// joined rooms, but also which rooms have membership transitions for this user between the 2 stream positions.
+	// joined rooms, but also which rooms have membership transitions for this user between the 2 PDU stream positions.
 	// This works out what the 'state' key should be for each room as well as which membership block
 	// to put the room into.
 	deltas, err := d.getStateDeltas(ctx, &device, txn, fromPos, toPos, device.UserID)
@@ -242,8 +253,9 @@ func (d *SyncServerDatabase) IncrementalSync(
 		return nil, err
 	}
 
-	res := types.NewResponse(toPos)
+	joinedRoomIDs := make([]string, 0, len(deltas))
 	for _, delta := range deltas {
+		joinedRoomIDs = append(joinedRoomIDs, delta.roomID)
 		err = d.addRoomDeltaToResponse(ctx, &device, txn, fromPos, toPos, delta, numRecentEventsPerRoom, res)
 		if err != nil {
 			return nil, err
@@ -256,52 +268,151 @@ func (d *SyncServerDatabase) IncrementalSync(
 	}
 
 	succeeded = true
+	return joinedRoomIDs, nil
+}
+
+// addTypingDeltaToResponse adds all typing notifications to a sync response
+// since the specified position.
+func (d *SyncServerDatasource) addTypingDeltaToResponse(
+	since int64,
+	joinedRoomIDs []string,
+	res *types.Response,
+) error {
+	var jr types.JoinResponse
+	var ok bool
+	var err error
+	for _, roomID := range joinedRoomIDs {
+		if typingUsers, updated := d.typingCache.GetTypingUsersIfUpdatedAfter(
+			roomID, since,
+		); updated {
+			ev := gomatrixserverlib.ClientEvent{
+				Type: gomatrixserverlib.MTyping,
+			}
+			ev.Content, err = json.Marshal(map[string]interface{}{
+				"user_ids": typingUsers,
+			})
+			if err != nil {
+				return err
+			}
+
+			if jr, ok = res.Rooms.Join[roomID]; !ok {
+				jr = *types.NewJoinResponse()
+			}
+			jr.Ephemeral.Events = append(jr.Ephemeral.Events, ev)
+			res.Rooms.Join[roomID] = jr
+		}
+	}
+	return nil
+}
+
+// addEDUDeltaToResponse adds updates for EDUs of each type since fromPos if
+// the positions of that type are not equal in fromPos and toPos.
+func (d *SyncServerDatasource) addEDUDeltaToResponse(
+	fromPos, toPos types.SyncPosition,
+	joinedRoomIDs []string,
+	res *types.Response,
+) (err error) {
+
+	if fromPos.TypingPosition != toPos.TypingPosition {
+		err = d.addTypingDeltaToResponse(
+			fromPos.TypingPosition, joinedRoomIDs, res,
+		)
+	}
+
+	return
+}
+
+// IncrementalSync returns all the data needed in order to create an incremental
+// sync response for the given user. Events returned will include any client
+// transaction IDs associated with the given device. These transaction IDs come
+// from when the device sent the event via an API that included a transaction
+// ID.
+func (d *SyncServerDatasource) IncrementalSync(
+	ctx context.Context,
+	device authtypes.Device,
+	fromPos, toPos types.SyncPosition,
+	numRecentEventsPerRoom int,
+) (*types.Response, error) {
+	nextBatchPos := fromPos.WithUpdates(toPos)
+	res := types.NewResponse(nextBatchPos)
+
+	var joinedRoomIDs []string
+	var err error
+	if fromPos.PDUPosition != toPos.PDUPosition {
+		joinedRoomIDs, err = d.addPDUDeltaToResponse(
+			ctx, device, fromPos.PDUPosition, toPos.PDUPosition, numRecentEventsPerRoom, res,
+		)
+	} else {
+		joinedRoomIDs, err = d.roomstate.selectRoomIDsWithMembership(
+			ctx, nil, device.UserID, "join",
+		)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	err = d.addEDUDeltaToResponse(
+		fromPos, toPos, joinedRoomIDs, res,
+	)
+	if err != nil {
+		return nil, err
+	}
+
 	return res, nil
 }
 
-// CompleteSync a complete /sync API response for the given user.
+// getResponseWithPDUsForCompleteSync creates a response and adds all PDUs needed
-func (d *SyncServerDatabase) CompleteSync(
+// to it. It returns toPos and joinedRoomIDs for use of adding EDUs.
-	ctx context.Context, userID string, numRecentEventsPerRoom int,
+func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync(
-) (*types.Response, error) {
+	ctx context.Context,
+	userID string,
+	numRecentEventsPerRoom int,
+) (
+	res *types.Response,
+	toPos types.SyncPosition,
+	joinedRoomIDs []string,
+	err error,
+) {
 	// This needs to be all done in a transaction as we need to do multiple SELECTs, and we need to have
-	// a consistent view of the database throughout. This includes extracting the sync stream position.
+	// a consistent view of the database throughout. This includes extracting the sync position.
 	// This does have the unfortunate side-effect that all the matrixy logic resides in this function,
 	// but it's better to not hide the fact that this is being done in a transaction.
 	txn, err := d.db.BeginTx(ctx, &txReadOnlySnapshot)
 	if err != nil {
-		return nil, err
+		return
 	}
 	var succeeded bool
 	defer common.EndTransaction(txn, &succeeded)
 
-	// Get the current stream position which we will base the sync response on.
+	// Get the current sync position which we will base the sync response on.
-	pos, err := d.syncStreamPositionTx(ctx, txn)
+	toPos, err = d.syncPositionTx(ctx, txn)
 	if err != nil {
-		return nil, err
+		return
 	}
 
+	res = types.NewResponse(toPos)
+
 	// Extract room state and recent events for all rooms the user is joined to.
-	roomIDs, err := d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, "join")
+	joinedRoomIDs, err = d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, "join")
 	if err != nil {
-		return nil, err
+		return
 	}
 
 	// Build up a /sync response. Add joined rooms.
-	res := types.NewResponse(pos)
+	for _, roomID := range joinedRoomIDs {
-	for _, roomID := range roomIDs {
 		var stateEvents []gomatrixserverlib.Event
 		stateEvents, err = d.roomstate.selectCurrentState(ctx, txn, roomID)
 		if err != nil {
-			return nil, err
+			return
 		}
 		// TODO: When filters are added, we may need to call this multiple times to get enough events.
 		//       See: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L316
 		var recentStreamEvents []streamEvent
 		recentStreamEvents, err = d.events.selectRecentEvents(
-			ctx, txn, roomID, types.StreamPosition(0), pos, numRecentEventsPerRoom,
+			ctx, txn, roomID, 0, toPos.PDUPosition, numRecentEventsPerRoom,
 		)
 		if err != nil {
-			return nil, err
+			return
 		}
 
 		// We don't include a device here as we don't need to send down
@@ -310,10 +421,12 @@ func (d *SyncServerDatabase) CompleteSync(
 
 		stateEvents = removeDuplicates(stateEvents, recentEvents)
 		jr := types.NewJoinResponse()
-		if prevBatch := recentStreamEvents[0].streamPosition - 1; prevBatch > 0 {
+		if prevPDUPos := recentStreamEvents[0].streamPosition - 1; prevPDUPos > 0 {
-			jr.Timeline.PrevBatch = types.StreamPosition(prevBatch).String()
+			// Use the short form of batch token for prev_batch
+			jr.Timeline.PrevBatch = strconv.FormatInt(prevPDUPos, 10)
 		} else {
-			jr.Timeline.PrevBatch = types.StreamPosition(1).String()
+			// Use the short form of batch token for prev_batch
+			jr.Timeline.PrevBatch = "1"
 		}
 		jr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync)
 		jr.Timeline.Limited = true
@@ -321,12 +434,34 @@ func (d *SyncServerDatabase) CompleteSync(
 		res.Rooms.Join[roomID] = *jr
 	}
 
-	if err = d.addInvitesToResponse(ctx, txn, userID, 0, pos, res); err != nil {
+	if err = d.addInvitesToResponse(ctx, txn, userID, 0, toPos.PDUPosition, res); err != nil {
-		return nil, err
+		return
 	}
 
 	succeeded = true
-	return res, err
+	return res, toPos, joinedRoomIDs, err
+}
+
+// CompleteSync returns a complete /sync API response for the given user.
+func (d *SyncServerDatasource) CompleteSync(
+	ctx context.Context, userID string, numRecentEventsPerRoom int,
+) (*types.Response, error) {
+	res, toPos, joinedRoomIDs, err := d.getResponseWithPDUsForCompleteSync(
+		ctx, userID, numRecentEventsPerRoom,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	// Use a zero value SyncPosition for fromPos so all EDU states are added.
+	err = d.addEDUDeltaToResponse(
+		types.SyncPosition{}, toPos, joinedRoomIDs, res,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	return res, nil
 }
 
 var txReadOnlySnapshot = sql.TxOptions{
@@ -344,8 +479,8 @@ var txReadOnlySnapshot = sql.TxOptions{
 // Returns a map following the format data[roomID] = []dataTypes
 // If no data is retrieved, returns an empty map
 // If there was an issue with the retrieval, returns an error
-func (d *SyncServerDatabase) GetAccountDataInRange(
+func (d *SyncServerDatasource) GetAccountDataInRange(
-	ctx context.Context, userID string, oldPos, newPos types.StreamPosition,
+	ctx context.Context, userID string, oldPos, newPos int64,
 ) (map[string][]string, error) {
 	return d.accountData.selectAccountDataInRange(ctx, userID, oldPos, newPos)
 }
@@ -356,26 +491,24 @@ func (d *SyncServerDatabase) GetAccountDataInRange(
 // If no data with the given type, user ID and room ID exists in the database,
 // creates a new row, else update the existing one
 // Returns an error if there was an issue with the upsert
-func (d *SyncServerDatabase) UpsertAccountData(
+func (d *SyncServerDatasource) UpsertAccountData(
 	ctx context.Context, userID, roomID, dataType string,
-) (types.StreamPosition, error) {
+) (int64, error) {
-	pos, err := d.accountData.insertAccountData(ctx, userID, roomID, dataType)
+	return d.accountData.insertAccountData(ctx, userID, roomID, dataType)
-	return types.StreamPosition(pos), err
 }
 
 // AddInviteEvent stores a new invite event for a user.
 // If the invite was successfully stored this returns the stream ID it was stored at.
 // Returns an error if there was a problem communicating with the database.
-func (d *SyncServerDatabase) AddInviteEvent(
+func (d *SyncServerDatasource) AddInviteEvent(
 	ctx context.Context, inviteEvent gomatrixserverlib.Event,
-) (types.StreamPosition, error) {
+) (int64, error) {
-	pos, err := d.invites.insertInviteEvent(ctx, inviteEvent)
+	return d.invites.insertInviteEvent(ctx, inviteEvent)
-	return types.StreamPosition(pos), err
 }
 
 // RetireInviteEvent removes an old invite event from the database.
 // Returns an error if there was a problem communicating with the database.
-func (d *SyncServerDatabase) RetireInviteEvent(
+func (d *SyncServerDatasource) RetireInviteEvent(
 	ctx context.Context, inviteEventID string,
 ) error {
 	// TODO: Record that invite has been retired in a stream so that we can
@@ -384,10 +517,30 @@ func (d *SyncServerDatabase) RetireInviteEvent(
 	return err
 }
 
-func (d *SyncServerDatabase) addInvitesToResponse(
+func (d *SyncServerDatasource) SetTypingTimeoutCallback(fn cache.TimeoutCallbackFn) {
+	d.typingCache.SetTimeoutCallback(fn)
+}
+
+// AddTypingUser adds a typing user to the typing cache.
+// Returns the newly calculated sync position for typing notifications.
+func (d *SyncServerDatasource) AddTypingUser(
+	userID, roomID string, expireTime *time.Time,
+) int64 {
+	return d.typingCache.AddTypingUser(userID, roomID, expireTime)
+}
+
+// RemoveTypingUser removes a typing user from the typing cache.
+// Returns the newly calculated sync position for typing notifications.
+func (d *SyncServerDatasource) RemoveTypingUser(
+	userID, roomID string,
+) int64 {
+	return d.typingCache.RemoveUser(userID, roomID)
+}
+
+func (d *SyncServerDatasource) addInvitesToResponse(
 	ctx context.Context, txn *sql.Tx,
 	userID string,
-	fromPos, toPos types.StreamPosition,
+	fromPos, toPos int64,
 	res *types.Response,
 ) error {
 	invites, err := d.invites.selectInviteEventsInRange(
@@ -408,11 +561,11 @@ func (d *SyncServerDatabase) addInvitesToResponse(
 }
 
 // addRoomDeltaToResponse adds a room state delta to a sync response
-func (d *SyncServerDatabase) addRoomDeltaToResponse(
+func (d *SyncServerDatasource) addRoomDeltaToResponse(
 	ctx context.Context,
 	device *authtypes.Device,
 	txn *sql.Tx,
-	fromPos, toPos types.StreamPosition,
+	fromPos, toPos int64,
 	delta stateDelta,
 	numRecentEventsPerRoom int,
 	res *types.Response,
@@ -444,10 +597,12 @@ func (d *SyncServerDatabase) addRoomDeltaToResponse(
 	switch delta.membership {
 	case "join":
 		jr := types.NewJoinResponse()
-		if prevBatch := recentStreamEvents[0].streamPosition - 1; prevBatch > 0 {
+		if prevPDUPos := recentStreamEvents[0].streamPosition - 1; prevPDUPos > 0 {
-			jr.Timeline.PrevBatch = types.StreamPosition(prevBatch).String()
+			// Use the short form of batch token for prev_batch
+			jr.Timeline.PrevBatch = strconv.FormatInt(prevPDUPos, 10)
 		} else {
-			jr.Timeline.PrevBatch = types.StreamPosition(1).String()
+			// Use the short form of batch token for prev_batch
+			jr.Timeline.PrevBatch = "1"
 		}
 		jr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync)
 		jr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true
@@ -459,10 +614,12 @@ func (d *SyncServerDatabase) addRoomDeltaToResponse(
 		// TODO: recentEvents may contain events that this user is not allowed to see because they are
 		//       no longer in the room.
 		lr := types.NewLeaveResponse()
-		if prevBatch := recentStreamEvents[0].streamPosition - 1; prevBatch > 0 {
+		if prevPDUPos := recentStreamEvents[0].streamPosition - 1; prevPDUPos > 0 {
-			lr.Timeline.PrevBatch = types.StreamPosition(prevBatch).String()
+			// Use the short form of batch token for prev_batch
+			lr.Timeline.PrevBatch = strconv.FormatInt(prevPDUPos, 10)
 		} else {
-			lr.Timeline.PrevBatch = types.StreamPosition(1).String()
+			// Use the short form of batch token for prev_batch
+			lr.Timeline.PrevBatch = "1"
 		}
 		lr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync)
 		lr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true
@@ -475,7 +632,7 @@ func (d *SyncServerDatabase) addRoomDeltaToResponse(
 
 // fetchStateEvents converts the set of event IDs into a set of events. It will fetch any which are missing from the database.
 // Returns a map of room ID to list of events.
-func (d *SyncServerDatabase) fetchStateEvents(
+func (d *SyncServerDatasource) fetchStateEvents(
 	ctx context.Context, txn *sql.Tx,
 	roomIDToEventIDSet map[string]map[string]bool,
 	eventIDToEvent map[string]streamEvent,
@@ -520,7 +677,7 @@ func (d *SyncServerDatabase) fetchStateEvents(
 	return stateBetween, nil
 }
 
-func (d *SyncServerDatabase) fetchMissingStateEvents(
+func (d *SyncServerDatasource) fetchMissingStateEvents(
 	ctx context.Context, txn *sql.Tx, eventIDs []string,
 ) ([]streamEvent, error) {
 	// Fetch from the events table first so we pick up the stream ID for the
@@ -559,9 +716,9 @@ func (d *SyncServerDatabase) fetchMissingStateEvents(
 	return events, nil
 }
 
-func (d *SyncServerDatabase) getStateDeltas(
+func (d *SyncServerDatasource) getStateDeltas(
 	ctx context.Context, device *authtypes.Device, txn *sql.Tx,
-	fromPos, toPos types.StreamPosition, userID string,
+	fromPos, toPos int64, userID string,
 ) ([]stateDelta, error) {
 	// Implement membership change algorithm: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L821
 	// - Get membership list changes for this user in this sync response
@@ -600,7 +757,7 @@ func (d *SyncServerDatabase) getStateDeltas(
 					}
 					s := make([]streamEvent, len(allState))
 					for i := 0; i < len(s); i++ {
-						s[i] = streamEvent{Event: allState[i], streamPosition: types.StreamPosition(0)}
+						s[i] = streamEvent{Event: allState[i], streamPosition: 0}
 					}
 					state[roomID] = s
 					continue // we'll add this room in when we do joined rooms

syncapi/sync/notifier.go

@@ -26,7 +26,7 @@ import (
 )
 
 // Notifier will wake up sleeping requests when there is some new data.
-// It does not tell requests what that data is, only the stream position which
+// It does not tell requests what that data is, only the sync position which
 // they can use to get at it. This is done to prevent races whereby we tell the caller
 // the event, but the token has already advanced by the time they fetch it, resulting
 // in missed events.
@@ -35,18 +35,18 @@ type Notifier struct {
 	roomIDToJoinedUsers map[string]userIDSet
 	// Protects currPos and userStreams.
 	streamLock *sync.Mutex
-	// The latest sync stream position
+	// The latest sync position
-	currPos types.StreamPosition
+	currPos types.SyncPosition
 	// A map of user_id => UserStream which can be used to wake a given user's /sync request.
 	userStreams map[string]*UserStream
 	// The last time we cleaned out stale entries from the userStreams map
 	lastCleanUpTime time.Time
 }
 
-// NewNotifier creates a new notifier set to the given stream position.
+// NewNotifier creates a new notifier set to the given sync position.
 // In order for this to be of any use, the Notifier needs to be told all rooms and
 // the joined users within each of them by calling Notifier.Load(*storage.SyncServerDatabase).
-func NewNotifier(pos types.StreamPosition) *Notifier {
+func NewNotifier(pos types.SyncPosition) *Notifier {
 	return &Notifier{
 		currPos:             pos,
 		roomIDToJoinedUsers: make(map[string]userIDSet),
@@ -58,20 +58,30 @@ func NewNotifier(pos types.StreamPosition) *Notifier {
 
 // OnNewEvent is called when a new event is received from the room server. Must only be
 // called from a single goroutine, to avoid races between updates which could set the
-// current position in the stream incorrectly.
+// current sync position incorrectly.
-// Can be called either with a *gomatrixserverlib.Event, or with an user ID
+// Chooses which user sync streams to update by a provided *gomatrixserverlib.Event
-func (n *Notifier) OnNewEvent(ev *gomatrixserverlib.Event, userID string, pos types.StreamPosition) {
+// (based on the users in the event's room),
+// a roomID directly, or a list of user IDs, prioritised by parameter ordering.
+// posUpdate contains the latest position(s) for one or more types of events.
+// If a position in posUpdate is 0, it means no updates are available of that type.
+// Typically a consumer supplies a posUpdate with the latest sync position for the
+// event type it handles, leaving other fields as 0.
+func (n *Notifier) OnNewEvent(
+	ev *gomatrixserverlib.Event, roomID string, userIDs []string,
+	posUpdate types.SyncPosition,
+) {
 	// update the current position then notify relevant /sync streams.
 	// This needs to be done PRIOR to waking up users as they will read this value.
 	n.streamLock.Lock()
 	defer n.streamLock.Unlock()
-	n.currPos = pos
+	latestPos := n.currPos.WithUpdates(posUpdate)
+	n.currPos = latestPos
 
 	n.removeEmptyUserStreams()
 
 	if ev != nil {
 		// Map this event's room_id to a list of joined users, and wake them up.
-		userIDs := n.joinedUsers(ev.RoomID())
+		usersToNotify := n.joinedUsers(ev.RoomID())
 		// If this is an invite, also add in the invitee to this list.
 		if ev.Type() == "m.room.member" && ev.StateKey() != nil {
 			targetUserID := *ev.StateKey()
@@ -84,11 +94,11 @@ func (n *Notifier) OnNewEvent(ev *gomatrixserverlib.Event, userID string, pos ty
 				// Keep the joined user map up-to-date
 				switch membership {
 				case "invite":
-					userIDs = append(userIDs, targetUserID)
+					usersToNotify = append(usersToNotify, targetUserID)
 				case "join":
 					// Manually append the new user's ID so they get notified
 					// along all members in the room
-					userIDs = append(userIDs, targetUserID)
+					usersToNotify = append(usersToNotify, targetUserID)
 					n.addJoinedUser(ev.RoomID(), targetUserID)
 				case "leave":
 					fallthrough
@@ -98,11 +108,15 @@ func (n *Notifier) OnNewEvent(ev *gomatrixserverlib.Event, userID string, pos ty
 			}
 		}
 
-		for _, toNotifyUserID := range userIDs {
+		n.wakeupUsers(usersToNotify, latestPos)
-			n.wakeupUser(toNotifyUserID, pos)
+	} else if roomID != "" {
-		}
+		n.wakeupUsers(n.joinedUsers(roomID), latestPos)
-	} else if len(userID) > 0 {
+	} else if len(userIDs) > 0 {
-		n.wakeupUser(userID, pos)
+		n.wakeupUsers(userIDs, latestPos)
+	} else {
+		log.WithFields(log.Fields{
+			"posUpdate": posUpdate.String,
+		}).Warn("Notifier.OnNewEvent called but caller supplied no user to wake up")
 	}
 }
 
@@ -127,7 +141,7 @@ func (n *Notifier) GetListener(req syncRequest) UserStreamListener {
 }
 
 // Load the membership states required to notify users correctly.
-func (n *Notifier) Load(ctx context.Context, db *storage.SyncServerDatabase) error {
+func (n *Notifier) Load(ctx context.Context, db *storage.SyncServerDatasource) error {
 	roomToUsers, err := db.AllJoinedUsersInRooms(ctx)
 	if err != nil {
 		return err
@@ -136,8 +150,11 @@ func (n *Notifier) Load(ctx context.Context, db *storage.SyncServerDatabase) err
 	return nil
 }
 
-// CurrentPosition returns the current stream position
+// CurrentPosition returns the current sync position
-func (n *Notifier) CurrentPosition() types.StreamPosition {
+func (n *Notifier) CurrentPosition() types.SyncPosition {
+	n.streamLock.Lock()
+	defer n.streamLock.Unlock()
+
 	return n.currPos
 }
 
@@ -156,13 +173,14 @@ func (n *Notifier) setUsersJoinedToRooms(roomIDToUserIDs map[string][]string) {
 	}
 }
 
-func (n *Notifier) wakeupUser(userID string, newPos types.StreamPosition) {
+func (n *Notifier) wakeupUsers(userIDs []string, newPos types.SyncPosition) {
+	for _, userID := range userIDs {
 		stream := n.fetchUserStream(userID, false)
-	if stream == nil {
+		if stream != nil {
-		return
-	}
 			stream.Broadcast(newPos) // wake up all goroutines Wait()ing on this stream
 		}
+	}
+}
 
 // fetchUserStream retrieves a stream unique to the given user. If makeIfNotExists is true,
 // a stream will be made for this user if one doesn't exist and it will be returned. This

syncapi/sync/notifier_test.go

@@ -32,19 +32,40 @@ var (
 	randomMessageEvent  gomatrixserverlib.Event
 	aliceInviteBobEvent gomatrixserverlib.Event
 	bobLeaveEvent       gomatrixserverlib.Event
+	syncPositionVeryOld types.SyncPosition
+	syncPositionBefore  types.SyncPosition
+	syncPositionAfter   types.SyncPosition
+	syncPositionNewEDU  types.SyncPosition
+	syncPositionAfter2  types.SyncPosition
 )
 
 var (
-	streamPositionVeryOld = types.StreamPosition(5)
-	streamPositionBefore  = types.StreamPosition(11)
-	streamPositionAfter   = types.StreamPosition(12)
-	streamPositionAfter2  = types.StreamPosition(13)
 	roomID = "!test:localhost"
 	alice  = "@alice:localhost"
 	bob    = "@bob:localhost"
 )
 
 func init() {
+	baseSyncPos := types.SyncPosition{
+		PDUPosition:    0,
+		TypingPosition: 0,
+	}
+
+	syncPositionVeryOld = baseSyncPos
+	syncPositionVeryOld.PDUPosition = 5
+
+	syncPositionBefore = baseSyncPos
+	syncPositionBefore.PDUPosition = 11
+
+	syncPositionAfter = baseSyncPos
+	syncPositionAfter.PDUPosition = 12
+
+	syncPositionNewEDU = syncPositionAfter
+	syncPositionNewEDU.TypingPosition = 1
+
+	syncPositionAfter2 = baseSyncPos
+	syncPositionAfter2.PDUPosition = 13
+
 	var err error
 	randomMessageEvent, err = gomatrixserverlib.NewEventFromTrustedJSON([]byte(`{
 		"type": "m.room.message",
@@ -92,19 +113,19 @@ func init() {
 
 // Test that the current position is returned if a request is already behind.
 func TestImmediateNotification(t *testing.T) {
-	n := NewNotifier(streamPositionBefore)
+	n := NewNotifier(syncPositionBefore)
-	pos, err := waitForEvents(n, newTestSyncRequest(alice, streamPositionVeryOld))
+	pos, err := waitForEvents(n, newTestSyncRequest(alice, syncPositionVeryOld))
 	if err != nil {
 		t.Fatalf("TestImmediateNotification error: %s", err)
 	}
-	if pos != streamPositionBefore {
+	if pos != syncPositionBefore {
-		t.Fatalf("TestImmediateNotification want %d, got %d", streamPositionBefore, pos)
+		t.Fatalf("TestImmediateNotification want %d, got %d", syncPositionBefore, pos)
 	}
 }
 
 // Test that new events to a joined room unblocks the request.
 func TestNewEventAndJoinedToRoom(t *testing.T) {
-	n := NewNotifier(streamPositionBefore)
+	n := NewNotifier(syncPositionBefore)
 	n.setUsersJoinedToRooms(map[string][]string{
 		roomID: {alice, bob},
 	})
@@ -112,12 +133,12 @@ func TestNewEventAndJoinedToRoom(t *testing.T) {
 	var wg sync.WaitGroup
 	wg.Add(1)
 	go func() {
-		pos, err := waitForEvents(n, newTestSyncRequest(bob, streamPositionBefore))
+		pos, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionBefore))
 		if err != nil {
 			t.Errorf("TestNewEventAndJoinedToRoom error: %s", err)
 		}
-		if pos != streamPositionAfter {
+		if pos != syncPositionAfter {
-			t.Errorf("TestNewEventAndJoinedToRoom want %d, got %d", streamPositionAfter, pos)
+			t.Errorf("TestNewEventAndJoinedToRoom want %d, got %d", syncPositionAfter, pos)
 		}
 		wg.Done()
 	}()
@@ -125,14 +146,14 @@ func TestNewEventAndJoinedToRoom(t *testing.T) {
 	stream := n.fetchUserStream(bob, true)
 	waitForBlocking(stream, 1)
 
-	n.OnNewEvent(&randomMessageEvent, "", streamPositionAfter)
+	n.OnNewEvent(&randomMessageEvent, "", nil, syncPositionAfter)
 
 	wg.Wait()
 }
 
 // Test that an invite unblocks the request
 func TestNewInviteEventForUser(t *testing.T) {
-	n := NewNotifier(streamPositionBefore)
+	n := NewNotifier(syncPositionBefore)
 	n.setUsersJoinedToRooms(map[string][]string{
 		roomID: {alice, bob},
 	})
@@ -140,12 +161,12 @@ func TestNewInviteEventForUser(t *testing.T) {
 	var wg sync.WaitGroup
 	wg.Add(1)
 	go func() {
-		pos, err := waitForEvents(n, newTestSyncRequest(bob, streamPositionBefore))
+		pos, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionBefore))
 		if err != nil {
 			t.Errorf("TestNewInviteEventForUser error: %s", err)
 		}
-		if pos != streamPositionAfter {
+		if pos != syncPositionAfter {
-			t.Errorf("TestNewInviteEventForUser want %d, got %d", streamPositionAfter, pos)
+			t.Errorf("TestNewInviteEventForUser want %d, got %d", syncPositionAfter, pos)
 		}
 		wg.Done()
 	}()
@@ -153,14 +174,42 @@ func TestNewInviteEventForUser(t *testing.T) {
 	stream := n.fetchUserStream(bob, true)
 	waitForBlocking(stream, 1)
 
-	n.OnNewEvent(&aliceInviteBobEvent, "", streamPositionAfter)
+	n.OnNewEvent(&aliceInviteBobEvent, "", nil, syncPositionAfter)
+
+	wg.Wait()
+}
+
+// Test an EDU-only update wakes up the request.
+func TestEDUWakeup(t *testing.T) {
+	n := NewNotifier(syncPositionAfter)
+	n.setUsersJoinedToRooms(map[string][]string{
+		roomID: {alice, bob},
+	})
+
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		pos, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionAfter))
+		if err != nil {
+			t.Errorf("TestNewInviteEventForUser error: %s", err)
+		}
+		if pos != syncPositionNewEDU {
+			t.Errorf("TestNewInviteEventForUser want %d, got %d", syncPositionNewEDU, pos)
+		}
+		wg.Done()
+	}()
+
+	stream := n.fetchUserStream(bob, true)
+	waitForBlocking(stream, 1)
+
+	n.OnNewEvent(&aliceInviteBobEvent, "", nil, syncPositionNewEDU)
 
 	wg.Wait()
 }
 
 // Test that all blocked requests get woken up on a new event.
 func TestMultipleRequestWakeup(t *testing.T) {
-	n := NewNotifier(streamPositionBefore)
+	n := NewNotifier(syncPositionBefore)
 	n.setUsersJoinedToRooms(map[string][]string{
 		roomID: {alice, bob},
 	})
@@ -168,12 +217,12 @@ func TestMultipleRequestWakeup(t *testing.T) {
 	var wg sync.WaitGroup
 	wg.Add(3)
 	poll := func() {
-		pos, err := waitForEvents(n, newTestSyncRequest(bob, streamPositionBefore))
+		pos, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionBefore))
 		if err != nil {
 			t.Errorf("TestMultipleRequestWakeup error: %s", err)
 		}
-		if pos != streamPositionAfter {
+		if pos != syncPositionAfter {
-			t.Errorf("TestMultipleRequestWakeup want %d, got %d", streamPositionAfter, pos)
+			t.Errorf("TestMultipleRequestWakeup want %d, got %d", syncPositionAfter, pos)
 		}
 		wg.Done()
 	}
@@ -184,7 +233,7 @@ func TestMultipleRequestWakeup(t *testing.T) {
 	stream := n.fetchUserStream(bob, true)
 	waitForBlocking(stream, 3)
 
-	n.OnNewEvent(&randomMessageEvent, "", streamPositionAfter)
+	n.OnNewEvent(&randomMessageEvent, "", nil, syncPositionAfter)
 
 	wg.Wait()
 
@@ -198,7 +247,7 @@ func TestMultipleRequestWakeup(t *testing.T) {
 func TestNewEventAndWasPreviouslyJoinedToRoom(t *testing.T) {
 	// listen as bob. Make bob leave room. Make alice send event to room.
 	// Make sure alice gets woken up only and not bob as well.
-	n := NewNotifier(streamPositionBefore)
+	n := NewNotifier(syncPositionBefore)
 	n.setUsersJoinedToRooms(map[string][]string{
 		roomID: {alice, bob},
 	})
@@ -208,18 +257,18 @@ func TestNewEventAndWasPreviouslyJoinedToRoom(t *testing.T) {
 	// Make bob leave the room
 	leaveWG.Add(1)
 	go func() {
-		pos, err := waitForEvents(n, newTestSyncRequest(bob, streamPositionBefore))
+		pos, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionBefore))
 		if err != nil {
 			t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom error: %s", err)
 		}
-		if pos != streamPositionAfter {
+		if pos != syncPositionAfter {
-			t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom want %d, got %d", streamPositionAfter, pos)
+			t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom want %d, got %d", syncPositionAfter, pos)
 		}
 		leaveWG.Done()
 	}()
 	bobStream := n.fetchUserStream(bob, true)
 	waitForBlocking(bobStream, 1)
-	n.OnNewEvent(&bobLeaveEvent, "", streamPositionAfter)
+	n.OnNewEvent(&bobLeaveEvent, "", nil, syncPositionAfter)
 	leaveWG.Wait()
 
 	// send an event into the room. Make sure alice gets it. Bob should not.
@@ -227,19 +276,19 @@ func TestNewEventAndWasPreviouslyJoinedToRoom(t *testing.T) {
 	aliceStream := n.fetchUserStream(alice, true)
 	aliceWG.Add(1)
 	go func() {
-		pos, err := waitForEvents(n, newTestSyncRequest(alice, streamPositionAfter))
+		pos, err := waitForEvents(n, newTestSyncRequest(alice, syncPositionAfter))
 		if err != nil {
 			t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom error: %s", err)
 		}
-		if pos != streamPositionAfter2 {
+		if pos != syncPositionAfter2 {
-			t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom want %d, got %d", streamPositionAfter2, pos)
+			t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom want %d, got %d", syncPositionAfter2, pos)
 		}
 		aliceWG.Done()
 	}()
 
 	go func() {
 		// this should timeout with an error (but the main goroutine won't wait for the timeout explicitly)
-		_, err := waitForEvents(n, newTestSyncRequest(bob, streamPositionAfter))
+		_, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionAfter))
 		if err == nil {
 			t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom expect error but got nil")
 		}
@@ -248,7 +297,7 @@ func TestNewEventAndWasPreviouslyJoinedToRoom(t *testing.T) {
 	waitForBlocking(aliceStream, 1)
 	waitForBlocking(bobStream, 1)
 
-	n.OnNewEvent(&randomMessageEvent, "", streamPositionAfter2)
+	n.OnNewEvent(&randomMessageEvent, "", nil, syncPositionAfter2)
 	aliceWG.Wait()
 
 	// it's possible that at this point alice has been informed and bob is about to be informed, so wait
@@ -256,18 +305,17 @@ func TestNewEventAndWasPreviouslyJoinedToRoom(t *testing.T) {
 	time.Sleep(1 * time.Millisecond)
 }
 
-// same as Notifier.WaitForEvents but with a timeout.
+func waitForEvents(n *Notifier, req syncRequest) (types.SyncPosition, error) {
-func waitForEvents(n *Notifier, req syncRequest) (types.StreamPosition, error) {
 	listener := n.GetListener(req)
 	defer listener.Close()
 
 	select {
 	case <-time.After(5 * time.Second):
-		return types.StreamPosition(0), fmt.Errorf(
+		return types.SyncPosition{}, fmt.Errorf(
 			"waitForEvents timed out waiting for %s (pos=%d)", req.device.UserID, req.since,
 		)
 	case <-listener.GetNotifyChannel(*req.since):
-		p := listener.GetStreamPosition()
+		p := listener.GetSyncPosition()
 		return p, nil
 	}
 }
@@ -280,7 +328,7 @@ func waitForBlocking(s *UserStream, numBlocking uint) {
 	}
 }
 
-func newTestSyncRequest(userID string, since types.StreamPosition) syncRequest {
+func newTestSyncRequest(userID string, since types.SyncPosition) syncRequest {
 	return syncRequest{
 		device:        authtypes.Device{UserID: userID},
 		timeout:       1 * time.Minute,

syncapi/sync/request.go

@@ -16,8 +16,10 @@ package sync
 
 import (
 	"context"
+	"errors"
 	"net/http"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
@@ -36,7 +38,7 @@ type syncRequest struct {
 	device        authtypes.Device
 	limit         int
 	timeout       time.Duration
-	since         *types.StreamPosition // nil means that no since token was supplied
+	since         *types.SyncPosition // nil means that no since token was supplied
 	wantFullState bool
 	log           *log.Entry
 }
@@ -73,15 +75,41 @@ func getTimeout(timeoutMS string) time.Duration {
 }
 
 // getSyncStreamPosition tries to parse a 'since' token taken from the API to a
-// stream position. If the string is empty then (nil, nil) is returned.
+// types.SyncPosition. If the string is empty then (nil, nil) is returned.
-func getSyncStreamPosition(since string) (*types.StreamPosition, error) {
+// There are two forms of tokens: The full length form containing all PDU and EDU
+// positions separated by "_", and the short form containing only the PDU
+// position. Short form can be used for, e.g., `prev_batch` tokens.
+func getSyncStreamPosition(since string) (*types.SyncPosition, error) {
 	if since == "" {
 		return nil, nil
 	}
-	i, err := strconv.Atoi(since)
+
+	posStrings := strings.Split(since, "_")
+	if len(posStrings) != 2 && len(posStrings) != 1 {
+		// A token can either be full length or short (PDU-only).
+		return nil, errors.New("malformed batch token")
+	}
+
+	positions := make([]int64, len(posStrings))
+	for i, posString := range posStrings {
+		pos, err := strconv.ParseInt(posString, 10, 64)
 		if err != nil {
 			return nil, err
 		}
-	token := types.StreamPosition(i)
+		positions[i] = pos
-	return &token, nil
+	}
+
+	if len(positions) == 2 {
+		// Full length token; construct SyncPosition with every entry in
+		// `positions`. These entries must have the same order with the fields
+		// in struct SyncPosition, so we disable the govet check below.
+		return &types.SyncPosition{ //nolint:govet
+			positions[0], positions[1],
+		}, nil
+	} else {
+		// Token with PDU position only
+		return &types.SyncPosition{
+			PDUPosition: positions[0],
+		}, nil
+	}
 }

syncapi/sync/requestpool.go

@@ -31,13 +31,13 @@ import (
 
 // RequestPool manages HTTP long-poll connections for /sync
 type RequestPool struct {
-	db        *storage.SyncServerDatabase
+	db        *storage.SyncServerDatasource
 	accountDB *accounts.Database
 	notifier  *Notifier
 }
 
 // NewRequestPool makes a new RequestPool
-func NewRequestPool(db *storage.SyncServerDatabase, n *Notifier, adb *accounts.Database) *RequestPool {
+func NewRequestPool(db *storage.SyncServerDatasource, n *Notifier, adb *accounts.Database) *RequestPool {
 	return &RequestPool{db, adb, n}
 }
 
@@ -92,11 +92,13 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *authtype
 	// respond with, so we skip the return an go back to waiting for content to
 	// be sent down or the request timing out.
 	var hasTimedOut bool
+	sincePos := *syncReq.since
 	for {
 		select {
 		// Wait for notifier to wake us up
-		case <-userStreamListener.GetNotifyChannel(currPos):
+		case <-userStreamListener.GetNotifyChannel(sincePos):
-			currPos = userStreamListener.GetStreamPosition()
+			currPos = userStreamListener.GetSyncPosition()
+			sincePos = currPos
 		// Or for timeout to expire
 		case <-timer.C:
 			// We just need to ensure we get out of the select after reaching the
@@ -128,24 +130,24 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *authtype
 	}
 }
 
-func (rp *RequestPool) currentSyncForUser(req syncRequest, currentPos types.StreamPosition) (res *types.Response, err error) {
+func (rp *RequestPool) currentSyncForUser(req syncRequest, latestPos types.SyncPosition) (res *types.Response, err error) {
 	// TODO: handle ignored users
 	if req.since == nil {
 		res, err = rp.db.CompleteSync(req.ctx, req.device.UserID, req.limit)
 	} else {
-		res, err = rp.db.IncrementalSync(req.ctx, req.device, *req.since, currentPos, req.limit)
+		res, err = rp.db.IncrementalSync(req.ctx, req.device, *req.since, latestPos, req.limit)
 	}
 
 	if err != nil {
 		return
 	}
 
-	res, err = rp.appendAccountData(res, req.device.UserID, req, currentPos)
+	res, err = rp.appendAccountData(res, req.device.UserID, req, latestPos.PDUPosition)
 	return
 }
 
 func (rp *RequestPool) appendAccountData(
-	data *types.Response, userID string, req syncRequest, currentPos types.StreamPosition,
+	data *types.Response, userID string, req syncRequest, currentPos int64,
 ) (*types.Response, error) {
 	// TODO: Account data doesn't have a sync position of its own, meaning that
 	// account data might be sent multiple time to the client if multiple account
@@ -179,7 +181,7 @@ func (rp *RequestPool) appendAccountData(
 	}
 
 	// Sync is not initial, get all account data since the latest sync
-	dataTypes, err := rp.db.GetAccountDataInRange(req.ctx, userID, *req.since, currentPos)
+	dataTypes, err := rp.db.GetAccountDataInRange(req.ctx, userID, req.since.PDUPosition, currentPos)
 	if err != nil {
 		return nil, err
 	}

syncapi/sync/userstream.go

@@ -34,8 +34,8 @@ type UserStream struct {
 	lock sync.Mutex
 	// Closed when there is an update.
 	signalChannel chan struct{}
-	// The last stream position that there may have been an update for the suser
+	// The last sync position that there may have been an update for the user
-	pos types.StreamPosition
+	pos types.SyncPosition
 	// The last time when we had some listeners waiting
 	timeOfLastChannel time.Time
 	// The number of listeners waiting
@@ -51,7 +51,7 @@ type UserStreamListener struct {
 }
 
 // NewUserStream creates a new user stream
-func NewUserStream(userID string, currPos types.StreamPosition) *UserStream {
+func NewUserStream(userID string, currPos types.SyncPosition) *UserStream {
 	return &UserStream{
 		UserID:            userID,
 		timeOfLastChannel: time.Now(),
@@ -84,8 +84,8 @@ func (s *UserStream) GetListener(ctx context.Context) UserStreamListener {
 	return listener
 }
 
-// Broadcast a new stream position for this user.
+// Broadcast a new sync position for this user.
-func (s *UserStream) Broadcast(pos types.StreamPosition) {
+func (s *UserStream) Broadcast(pos types.SyncPosition) {
 	s.lock.Lock()
 	defer s.lock.Unlock()
 
@@ -118,9 +118,9 @@ func (s *UserStream) TimeOfLastNonEmpty() time.Time {
 	return s.timeOfLastChannel
 }
 
-// GetStreamPosition returns last stream position which the UserStream was
+// GetStreamPosition returns last sync position which the UserStream was
 // notified about
-func (s *UserStreamListener) GetStreamPosition() types.StreamPosition {
+func (s *UserStreamListener) GetSyncPosition() types.SyncPosition {
 	s.userStream.lock.Lock()
 	defer s.userStream.lock.Unlock()
 
@@ -132,11 +132,11 @@ func (s *UserStreamListener) GetStreamPosition() types.StreamPosition {
 // sincePos specifies from which point we want to be notified about. If there
 // has already been an update after sincePos we'll return a closed channel
 // immediately.
-func (s *UserStreamListener) GetNotifyChannel(sincePos types.StreamPosition) <-chan struct{} {
+func (s *UserStreamListener) GetNotifyChannel(sincePos types.SyncPosition) <-chan struct{} {
 	s.userStream.lock.Lock()
 	defer s.userStream.lock.Unlock()
 
-	if sincePos < s.userStream.pos {
+	if s.userStream.pos.IsAfter(sincePos) {
 		// If the listener is behind, i.e. missed a potential update, then we
 		// want them to wake up immediately. We do this by returning a new
 		// closed stream, which returns immediately when selected.

syncapi/syncapi.go

@@ -28,7 +28,6 @@ import (
 	"github.com/matrix-org/dendrite/syncapi/routing"
 	"github.com/matrix-org/dendrite/syncapi/storage"
 	"github.com/matrix-org/dendrite/syncapi/sync"
-	"github.com/matrix-org/dendrite/syncapi/types"
 )
 
 // SetupSyncAPIComponent sets up and registers HTTP handlers for the SyncAPI
@@ -39,17 +38,17 @@ func SetupSyncAPIComponent(
 	accountsDB *accounts.Database,
 	queryAPI api.RoomserverQueryAPI,
 ) {
-	syncDB, err := storage.NewSyncServerDatabase(string(base.Cfg.Database.SyncAPI))
+	syncDB, err := storage.NewSyncServerDatasource(string(base.Cfg.Database.SyncAPI))
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to connect to sync db")
 	}
 
-	pos, err := syncDB.SyncStreamPosition(context.Background())
+	pos, err := syncDB.SyncPosition(context.Background())
 	if err != nil {
-		logrus.WithError(err).Panicf("failed to get stream position")
+		logrus.WithError(err).Panicf("failed to get sync position")
 	}
 
-	notifier := sync.NewNotifier(types.StreamPosition(pos))
+	notifier := sync.NewNotifier(pos)
 	err = notifier.Load(context.Background(), syncDB)
 	if err != nil {
 		logrus.WithError(err).Panicf("failed to start notifier")
@@ -71,5 +70,12 @@ func SetupSyncAPIComponent(
 		logrus.WithError(err).Panicf("failed to start client data consumer")
 	}
 
+	typingConsumer := consumers.NewOutputTypingEventConsumer(
+		base.Cfg, base.KafkaConsumer, notifier, syncDB,
+	)
+	if err = typingConsumer.Start(); err != nil {
+		logrus.WithError(err).Panicf("failed to start typing server consumer")
+	}
+
 	routing.Setup(base.APIMux, requestPool, syncDB, deviceDB)
 }

syncapi/types/types.go

@@ -21,12 +21,38 @@ import (
 	"github.com/matrix-org/gomatrixserverlib"
 )
 
-// StreamPosition represents the offset in the sync stream a client is at.
+// SyncPosition contains the PDU and EDU stream sync positions for a client.
-type StreamPosition int64
+type SyncPosition struct {
+	// PDUPosition is the stream position for PDUs the client is at.
+	PDUPosition int64
+	// TypingPosition is the client's position for typing notifications.
+	TypingPosition int64
+}
 
 // String implements the Stringer interface.
-func (sp StreamPosition) String() string {
+func (sp SyncPosition) String() string {
-	return strconv.FormatInt(int64(sp), 10)
+	return strconv.FormatInt(sp.PDUPosition, 10) + "_" +
+		strconv.FormatInt(sp.TypingPosition, 10)
+}
+
+// IsAfter returns whether one SyncPosition refers to states newer than another SyncPosition.
+func (sp SyncPosition) IsAfter(other SyncPosition) bool {
+	return sp.PDUPosition > other.PDUPosition ||
+		sp.TypingPosition > other.TypingPosition
+}
+
+// WithUpdates returns a copy of the SyncPosition with updates applied from another SyncPosition.
+// If the latter SyncPosition contains a field that is not 0, it is considered an update,
+// and its value will replace the corresponding value in the SyncPosition on which WithUpdates is called.
+func (sp SyncPosition) WithUpdates(other SyncPosition) SyncPosition {
+	ret := sp
+	if other.PDUPosition != 0 {
+		ret.PDUPosition = other.PDUPosition
+	}
+	if other.TypingPosition != 0 {
+		ret.TypingPosition = other.TypingPosition
+	}
+	return ret
 }
 
 // PrevEventRef represents a reference to a previous event in a state event upgrade
@@ -53,11 +79,10 @@ type Response struct {
 }
 
 // NewResponse creates an empty response with initialised maps.
-func NewResponse(pos StreamPosition) *Response {
+func NewResponse(pos SyncPosition) *Response {
-	res := Response{}
+	res := Response{
-	// Make sure we send the next_batch as a string. We don't want to confuse clients by sending this
+		NextBatch: pos.String(),
-	// as an integer even though (at the moment) it is.
+	}
-	res.NextBatch = pos.String()
 	// Pre-initialise the maps. Synapse will return {} even if there are no rooms under a specific section,
 	// so let's do the same thing. Bonus: this means we can't get dreaded 'assignment to entry in nil map' errors.
 	res.Rooms.Join = make(map[string]JoinResponse)

testfile

@@ -1,4 +1,149 @@
 GET /register yields a set of flows
 POST /register can create a user
 POST /register downcases capitals in usernames
-
+POST /register rejects registration of usernames with '!'
+POST /register rejects registration of usernames with '"'
+POST /register rejects registration of usernames with ':'
+POST /register rejects registration of usernames with '?'
+POST /register rejects registration of usernames with '\'
+POST /register rejects registration of usernames with '@'
+POST /register rejects registration of usernames with '['
+POST /register rejects registration of usernames with ']'
+POST /register rejects registration of usernames with '{'
+POST /register rejects registration of usernames with '|'
+POST /register rejects registration of usernames with '}'
+POST /register rejects registration of usernames with '£'
+POST /register rejects registration of usernames with 'é'
+POST /register rejects registration of usernames with '\n'
+POST /register rejects registration of usernames with '''
+GET /login yields a set of flows
+POST /login can log in as a user
+POST /login can log in as a user with just the local part of the id
+POST /login as non-existing user is rejected
+POST /login wrong password is rejected
+GET /events initially
+GET /initialSync initially
+Version responds 200 OK with valid structure
+PUT /profile/:user_id/displayname sets my name
+GET /profile/:user_id/displayname publicly accessible
+PUT /profile/:user_id/avatar_url sets my avatar
+GET /profile/:user_id/avatar_url publicly accessible
+GET /device/{deviceId} gives a 404 for unknown devices
+PUT /device/{deviceId} gives a 404 for unknown devices
+POST /createRoom makes a public room
+POST /createRoom makes a private room
+POST /createRoom makes a private room with invites
+POST /createRoom makes a room with a name
+POST /createRoom makes a room with a topic
+Can /sync newly created room
+GET /rooms/:room_id/state/m.room.member/:user_id fetches my membership
+GET /rooms/:room_id/state/m.room.power_levels fetches powerlevels
+POST /join/:room_alias can join a room
+POST /join/:room_id can join a room
+POST /join/:room_id can join a room with custom content
+POST /join/:room_alias can join a room with custom content
+POST /rooms/:room_id/join can join a room
+POST /rooms/:room_id/leave can leave a room
+POST /rooms/:room_id/invite can send an invite
+POST /rooms/:room_id/ban can ban a user
+POST /rooms/:room_id/send/:event_type sends a message
+PUT /rooms/:room_id/send/:event_type/:txn_id sends a message
+PUT /rooms/:room_id/send/:event_type/:txn_id deduplicates the same txn id
+GET /rooms/:room_id/state/m.room.power_levels can fetch levels
+PUT /rooms/:room_id/state/m.room.power_levels can set levels
+PUT power_levels should not explode if the old power levels were empty
+Both GET and PUT work
+POST /rooms/:room_id/read_markers can create read marker
+User signups are forbidden from starting with '_'
+Can logout all devices
+Request to logout with invalid an access token is rejected
+Request to logout without an access token is rejected
+Room creation reports m.room.create to myself
+Room creation reports m.room.member to myself
+New room members see their own join event
+Existing members see new members' join events
+setting 'm.room.power_levels' respects room powerlevel
+Unprivileged users can set m.room.topic if it only needs level 0
+Users cannot set ban powerlevel higher than their own
+Users cannot set kick powerlevel higher than their own
+Users cannot set redact powerlevel higher than their own
+Can get rooms/{roomId}/members for a departed room (SPEC-216)
+3pid invite join with wrong but valid signature are rejected
+3pid invite join valid signature but revoked keys are rejected
+3pid invite join valid signature but unreachable ID server are rejected
+Room members can override their displayname on a room-specific basis
+Room members can join a room with an overridden displayname
+displayname updates affect room member events
+avatar_url updates affect room member events
+Real non-joined user cannot call /events on shared room
+Real non-joined user cannot call /events on invited room
+Real non-joined user cannot call /events on joined room
+Real non-joined user cannot call /events on default room
+Real non-joined users can get state for world_readable rooms
+Real non-joined users can get individual state for world_readable rooms
+Real non-joined users can get individual state for world_readable rooms after leaving
+Real non-joined users cannot send messages to guest_access rooms if not joined
+Real users can sync from world_readable guest_access rooms if joined
+Real users can sync from default guest_access rooms if joined
+Can't forget room you're still in
+Can get rooms/{roomId}/members
+Can create filter
+Can download filter
+Can sync
+Can sync a joined room
+Newly joined room is included in an incremental sync
+User is offline if they set_presence=offline in their sync
+Changes to state are included in an incremental sync
+A change to displayname should appear in incremental /sync
+Current state appears in timeline in private history
+Current state appears in timeline in private history with many messages before
+Rooms a user is invited to appear in an initial sync
+Rooms a user is invited to appear in an incremental sync
+Sync can be polled for updates
+Sync is woken up for leaves
+Newly left rooms appear in the leave section of incremental sync
+We should see our own leave event, even if history_visibility is restricted (SYN-662)
+We should see our own leave event when rejecting an invite, even if history_visibility is restricted (riot-web/3462)
+Newly left rooms appear in the leave section of gapped sync
+Previously left rooms don't appear in the leave section of sync
+Left rooms appear in the leave section of full state sync
+Newly banned rooms appear in the leave section of incremental sync
+Newly banned rooms appear in the leave section of incremental sync
+local user can join room with version 1
+User can invite local user to room with version 1
+local user can join room with version 2
+User can invite local user to room with version 2
+local user can join room with version 3
+User can invite local user to room with version 3
+local user can join room with version 4
+User can invite local user to room with version 4
+Should reject keys claiming to belong to a different user
+Can add account data
+Can add account data to room
+Latest account data appears in v2 /sync
+New account data appears in incremental v2 /sync
+Checking local federation server
+Inbound federation can query profile data
+Outbound federation can send room-join requests
+Outbound federation can send events
+Inbound federation can backfill events
+Backfill checks the events requested belong to the room
+Can upload without a file name
+Can download without a file name locally
+Can upload with ASCII file name
+Can send image in room message
+AS cannot create users outside its own namespace
+Regular users cannot register within the AS namespace
+AS can't set displayname for random users
+AS user (not ghost) can join room without registering, with user_id query param
+Changing the actions of an unknown default rule fails with 404
+Changing the actions of an unknown rule fails with 404
+Enabling an unknown default rule fails with 404
+Trying to get push rules with unknown rule_id fails with 404
+Events come down the correct room
+local user can join room with version 5
+User can invite local user to room with version 5
+Inbound federation can receive room-join requests
+Typing events appear in initial sync
+Typing events appear in incremental sync
+Typing events appear in gapped sync

typingserver/api/output.go

@@ -12,14 +12,17 @@
 
 package api
 
+import "time"
+
 // OutputTypingEvent is an entry in typing server output kafka log.
 // This contains the event with extra fields used to create 'm.typing' event
 // in clientapi & federation.
 type OutputTypingEvent struct {
 	// The Event for the typing edu event.
 	Event TypingEvent `json:"event"`
-	// Users typing in the room when the event was generated.
+	// ExpireTime is the interval after which the user should no longer be
-	TypingUsers []string `json:"typing_users"`
+	// considered typing. Only available if Event.Typing is true.
+	ExpireTime *time.Time
 }
 
 // TypingEvent represents a matrix edu event of type 'm.typing'.

typingserver/cache/cache.go

@@ -22,25 +22,66 @@ const defaultTypingTimeout = 10 * time.Second
 // userSet is a map of user IDs to a timer, timer fires at expiry.
 type userSet map[string]*time.Timer
 
+// TimeoutCallbackFn is a function called right after the removal of a user
+// from the typing user list due to timeout.
+// latestSyncPosition is the typing sync position after the removal.
+type TimeoutCallbackFn func(userID, roomID string, latestSyncPosition int64)
+
+type roomData struct {
+	syncPosition int64
+	userSet      userSet
+}
+
 // TypingCache maintains a list of users typing in each room.
 type TypingCache struct {
 	sync.RWMutex
-	data map[string]userSet
+	latestSyncPosition int64
+	data               map[string]*roomData
+	timeoutCallback    TimeoutCallbackFn
 }
 
-// NewTypingCache returns a new TypingCache initialized for use.
+// Create a roomData with its sync position set to the latest sync position.
+// Must only be called after locking the cache.
+func (t *TypingCache) newRoomData() *roomData {
+	return &roomData{
+		syncPosition: t.latestSyncPosition,
+		userSet:      make(userSet),
+	}
+}
+
+// NewTypingCache returns a new TypingCache initialised for use.
 func NewTypingCache() *TypingCache {
-	return &TypingCache{data: make(map[string]userSet)}
+	return &TypingCache{data: make(map[string]*roomData)}
+}
+
+// SetTimeoutCallback sets a callback function that is called right after
+// a user is removed from the typing user list due to timeout.
+func (t *TypingCache) SetTimeoutCallback(fn TimeoutCallbackFn) {
+	t.timeoutCallback = fn
 }
 
 // GetTypingUsers returns the list of users typing in a room.
-func (t *TypingCache) GetTypingUsers(roomID string) (users []string) {
+func (t *TypingCache) GetTypingUsers(roomID string) []string {
+	users, _ := t.GetTypingUsersIfUpdatedAfter(roomID, 0)
+	// 0 should work above because the first position used will be 1.
+	return users
+}
+
+// GetTypingUsersIfUpdatedAfter returns all users typing in this room with
+// updated == true if the typing sync position of the room is after the given
+// position. Otherwise, returns an empty slice with updated == false.
+func (t *TypingCache) GetTypingUsersIfUpdatedAfter(
+	roomID string, position int64,
+) (users []string, updated bool) {
 	t.RLock()
-	usersMap, ok := t.data[roomID]
+	defer t.RUnlock()
-	t.RUnlock()
+
-	if ok {
+	roomData, ok := t.data[roomID]
-		users = make([]string, 0, len(usersMap))
+	if ok && roomData.syncPosition > position {
-		for userID := range usersMap {
+		updated = true
+		userSet := roomData.userSet
+		users = make([]string, 0, len(userSet))
+		for userID := range userSet {
 			users = append(users, userID)
 		}
 	}
@@ -51,53 +92,84 @@ func (t *TypingCache) GetTypingUsers(roomID string) (users []string) {
 // AddTypingUser sets an user as typing in a room.
 // expire is the time when the user typing should time out.
 // if expire is nil, defaultTypingTimeout is assumed.
-func (t *TypingCache) AddTypingUser(userID, roomID string, expire *time.Time) {
+// Returns the latest sync position for typing after update.
+func (t *TypingCache) AddTypingUser(
+	userID, roomID string, expire *time.Time,
+) int64 {
 	expireTime := getExpireTime(expire)
 	if until := time.Until(expireTime); until > 0 {
-		timer := time.AfterFunc(until, t.timeoutCallback(userID, roomID))
+		timer := time.AfterFunc(until, func() {
-		t.addUser(userID, roomID, timer)
+			latestSyncPosition := t.RemoveUser(userID, roomID)
+			if t.timeoutCallback != nil {
+				t.timeoutCallback(userID, roomID, latestSyncPosition)
 			}
+		})
+		return t.addUser(userID, roomID, timer)
+	}
+	return t.GetLatestSyncPosition()
 }
 
 // addUser with mutex lock & replace the previous timer.
-func (t *TypingCache) addUser(userID, roomID string, expiryTimer *time.Timer) {
+// Returns the latest typing sync position after update.
+func (t *TypingCache) addUser(
+	userID, roomID string, expiryTimer *time.Timer,
+) int64 {
 	t.Lock()
 	defer t.Unlock()
 
+	t.latestSyncPosition++
+
 	if t.data[roomID] == nil {
-		t.data[roomID] = make(userSet)
+		t.data[roomID] = t.newRoomData()
+	} else {
+		t.data[roomID].syncPosition = t.latestSyncPosition
 	}
 
 	// Stop the timer to cancel the call to timeoutCallback
-	if timer, ok := t.data[roomID][userID]; ok {
+	if timer, ok := t.data[roomID].userSet[userID]; ok {
-		// It may happen that at this stage timer fires but now we have a lock on t.
+		// It may happen that at this stage the timer fires, but we now have a lock on
-		// Hence the execution of timeoutCallback will happen after we unlock.
+		// it. Hence the execution of timeoutCallback will happen after we unlock. So
-		// So we may lose a typing state, though this event is highly unlikely.
+		// we may lose a typing state, though this is highly unlikely. This can be
-		// This can be mitigated by keeping another time.Time in the map and check against it
+		// mitigated by keeping another time.Time in the map and checking against it
-		// before removing. This however is not required in most practical scenario.
+		// before removing, but its occurrence is so infrequent it does not seem
+		// worthwhile.
 		timer.Stop()
 	}
 
-	t.data[roomID][userID] = expiryTimer
+	t.data[roomID].userSet[userID] = expiryTimer
-}
 
-// Returns a function which is called after timeout happens.
+	return t.latestSyncPosition
-// This removes the user.
-func (t *TypingCache) timeoutCallback(userID, roomID string) func() {
-	return func() {
-		t.RemoveUser(userID, roomID)
-	}
 }
 
 // RemoveUser with mutex lock & stop the timer.
-func (t *TypingCache) RemoveUser(userID, roomID string) {
+// Returns the latest sync position for typing after update.
+func (t *TypingCache) RemoveUser(userID, roomID string) int64 {
 	t.Lock()
 	defer t.Unlock()
 
-	if timer, ok := t.data[roomID][userID]; ok {
+	roomData, ok := t.data[roomID]
-		timer.Stop()
+	if !ok {
-		delete(t.data[roomID], userID)
+		return t.latestSyncPosition
 	}
+
+	timer, ok := roomData.userSet[userID]
+	if !ok {
+		return t.latestSyncPosition
+	}
+
+	timer.Stop()
+	delete(roomData.userSet, userID)
+
+	t.latestSyncPosition++
+	t.data[roomID].syncPosition = t.latestSyncPosition
+
+	return t.latestSyncPosition
+}
+
+func (t *TypingCache) GetLatestSyncPosition() int64 {
+	t.Lock()
+	defer t.Unlock()
+	return t.latestSyncPosition
 }
 
 func getExpireTime(expire *time.Time) time.Time {

typingserver/cache/cache_test.go

@@ -38,7 +38,7 @@ func TestTypingCache(t *testing.T) {
 	})
 }
 
-func testAddTypingUser(t *testing.T, tCache *TypingCache) {
+func testAddTypingUser(t *testing.T, tCache *TypingCache) { // nolint: unparam
 	present := time.Now()
 	tests := []struct {
 		userID string

typingserver/input/input.go

@@ -57,15 +57,21 @@ func (t *TypingServerInputAPI) InputTypingEvent(
 }
 
 func (t *TypingServerInputAPI) sendEvent(ite *api.InputTypingEvent) error {
-	userIDs := t.Cache.GetTypingUsers(ite.RoomID)
 	ev := &api.TypingEvent{
 		Type:   gomatrixserverlib.MTyping,
 		RoomID: ite.RoomID,
 		UserID: ite.UserID,
+		Typing: ite.Typing,
 	}
 	ote := &api.OutputTypingEvent{
 		Event: *ev,
-		TypingUsers: userIDs,
+	}
+
+	if ev.Typing {
+		expireTime := ite.OriginServerTS.Time().Add(
+			time.Duration(ite.Timeout) * time.Millisecond,
+		)
+		ote.ExpireTime = &expireTime
 	}
 
 	eventJSON, err := json.Marshal(ote)