Merge branch 'master' into filters_as_structs

.buildkite/pipeline.yaml

@@ -0,0 +1,49 @@
+steps:
+  - command:
+    # https://github.com/golangci/golangci-lint#memory-usage-of-golangci-lint
+    - "GOGC=20 ./scripts/find-lint.sh"
+    label: "\U0001F9F9 Lint / :go: 1.12"
+    agents:
+      # Use a larger instance as linting takes a looot of memory
+      queue: "medium"
+    plugins:
+      - docker#v3.0.1:
+          image: "golang:1.12"
+
+  - wait
+
+  - command:
+    - "go build ./cmd/..."
+    label: "\U0001F528 Build / :go: 1.11"
+    plugins:
+      - docker#v3.0.1:
+          image: "golang:1.11"
+    retry:
+      automatic:
+        - exit_status: 128
+          limit: 3
+
+  - command:
+    - "go build ./cmd/..."
+    label: "\U0001F528 Build / :go: 1.12"
+    plugins:
+      - docker#v3.0.1:
+          image: "golang:1.12"
+    retry:
+      automatic:
+        - exit_status: 128
+          limit: 3
+
+  - command:
+    - "go test ./..."
+    label: "\U0001F9EA Unit tests / :go: 1.11"
+    plugins:
+      - docker#v3.0.1:
+          image: "golang:1.11"
+
+  - command:
+    - "go test ./..."
+    label: "\U0001F9EA Unit tests / :go: 1.12"
+    plugins:
+      - docker#v3.0.1:
+          image: "golang:1.12"

.circleci/config.yml

@@ -0,0 +1,32 @@
+version: 2
+jobs:
+  dendrite:
+    docker:
+      - image: matrixdotorg/sytest-dendrite
+    working_directory: /src
+    steps:
+      - checkout
+
+      # Set up dendrite
+      - run: 
+          name: Build Dendrite
+          command: ./build.sh
+      - run: 
+          name: Copy dummy keys to root
+          command: |
+            mv .circleci/matrix_key.pem .
+            mv .circleci/server.key .
+      - run:
+          name: Run sytest with whitelisted tests
+          command: /dendrite_sytest.sh
+
+      - store_artifacts:
+          path: /logs
+          destination: logs
+      - store_test_results:
+          path: /logs
+workflows:
+  version: 2
+  build:
+    jobs:
+      - dendrite

.circleci/matrix_key.pem

@@ -0,0 +1,5 @@
+-----BEGIN MATRIX PRIVATE KEY-----
+Key-ID: ed25519:zXtB
+
+jDyHsx0EXbAfvM32yBEKQfIy1FHrmwtB1uMAbm5INBg=
+-----END MATRIX PRIVATE KEY-----

.circleci/server.key

@@ -0,0 +1,52 @@
+-----BEGIN PRIVATE KEY-----
+MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQCanRCqP11MLIQh
+nC26+A1oyBsFfH7auZ3pqE/WFDrCCIoc7ek7cF3fZU7q8OYI+Q9L5V8fobuLb6FB
+iXD5zZ6pBAI0VNjAS8yi8VluXIv6pJKsVY3k2hGiU7xRoEhkzckZBaEiruspQbcX
+ziNoWoueVBB1a4Eproqzy225cTcoprHsJIPXj0HpW/jKcmahmlM/OrqRAxTwxpb/
+moI6MWIeN4n7h55N6dU1ScVvBS7gZpZQ28d8akuvG3m8kE8q1OPFYGvrNeowD4sp
+qDPFijhbygwpzDQlAWriPcqV9KhuGRnYRGTGvuluOttmpgNhNFVxVAlwZJuMVAMU
+Jhek66ntKsxWkF5LsO8ls20hmHyyAsL7+rb2ZjuRtEwE8SwOstU2AIIXoSTtqXjX
+zC8Ew0VB9MCInJoJC/+iKTLoDqXRZeDKGFx1A2F3Y+Er+Z41HcwgqKRsPqZ066yR
+6iKAb5rzJutnEARtbSrNipy9nHE5hIgKJzgOnggcegypcAj3nqbfFFCZA2CFNXoG
+XFkmBHEpz38pPLI5z6HpeZRRySoIyahk9IfSwM3aB1aUi//8CcpAodGvYGNQkQ3W
+HkrZmM4MtC25I5RyMpYJQWKFpx1cOVPf2ASqaJ+IX1JJTv9dSdYHY/rxsxaiXiry
++uI7UITRvUKgAOrExfSAXco73bgUFwIDAQABAoICAQCP9QX7PhxEPH6aPKxnlWYG
+1aozJYOHa6QYVlpfXV6IIyNVZD7w1OLSiaU9IydL23nelKZI8XGJllpyhuHl9Qlx
+HQZga0+VW/4hCM7X7tt2d50JUG9ZUaFxnr2M0swU73X6Ej/B51OVilZLl+dn1kaB
+GIxqh7ovcRA774EuVLei5fJriGQpZH1eJgAznujoNqSkDq5/Lntk48LcIqR2Qly0
+/ck/pTpEGSAnCZUGlbDbxyjWCIxozx/A3rguVb8ghi+9KtXQntZ6AT71fmMV3mgz
+LqC8miFDA1rdY+MoVDAusrhZoPSkCEWYGL0HijNDYlLbvf874rDhq6diL0V8jOAd
+PGOx5BY6VUWbSQAUtKpMuNSL6tidkOACGPwbuH7OIaG+yGZ0/Oiy3fureiAEg5VU
+piyp6F7p1g0vgQEnj4CHiCQlX48bjC/mm8758DeaH8H5T++A8MOgRhgFVb9f01R+
+NMzszMziuVNDYe01cwdY1TXUx5b0o+opsbPm6sNp/7afL9Hou1epP9zQC0I8ulfP
+fgrKTddMwlNjoBuDMQ8GqoK275YU4wtyhUMfjr3xQ0JwP46cZbhhc4nh6qcRSNTf
+yVuKv/pT/bJcSmg5JOCS8qdK0BQhAvUin9HvgSAV9QmZVpxzT/xhqwuRlLDKW+VR
+XyPt996f3L4CTXI9h88AQQKCAQEAycBChu3/ZKl8a90anOlv9PwmaaXfLBKH9Rkw
+aeZrMilxTJAb+LEsmtj35rF5KPeBP6ARpX5gmvKJVzCDHT9YgNs+6C3E+l2f1/3a
+TcjZKPTukT2gJdCgejhEgTzAwEse322GSptuyidtNpY7NgbAxP4VdDMOmPYbzufb
+5BqxmfiGsfXgdvQkj8/MzHuGhhft4SU6ED/Ax+EPUWVV7kBr2995kGDF5z5CuJkb
+SJjmVxAJZP/kC2Z/iPnP51G0hiCxHp7+gPY4mvvkHvhJGnGH/vutjRjoe28BENlP
+MgB68S1/U3NGSUzWv86pT1OdHd+qynWj/NzF7Gp/T/ju8VZBXwKCAQEAxDAMSOfF
+dizsU7cJbf6vxi6XJHjhwWUWD2vMznKz1D4mkByeY8aSOc8kQZsE5nd4ZgwkYTaZ
+gItjGjM5y5dpKurfKdqQ+dA6PS03h3p+tp1lZp9/dI9X/DfkTO/LUdrfkVVcbQhE
+zqc6C35qO98rhJdsRwhOF28mOc/4bbs0XjC5dEoBGyFt7Fbn2mYoCo4FSHl7WIq6
+TZR9pLAvxjqEZ6Dwrzpp9wtdLIQYPga+KVKcDT/DStThXDTCNt5PyDE9c8eImFww
+u0T87Er5hSEQgodURxDOZh+9ktIfXzMtxiAJ3iDCEPc3NNnLCWfKMhwGsVTCCXj6
+WuHTOe79tOaQSQKCAQEAqBN52PsRl4TzWNEcyLhZQxmFzuIXKJpPlctkX/VMPL/1
+2bj89JR1+pLjA9e6fnyjuqPZz6uXQ77m2DJcKNOLId6Fa9wljAbPkZu0cLTw5YQX
+8/wJHTfPWcLin2BDnG94yt5t0F3pUJTEEYPa1EmP8w1SRjn64Ue3JwpWUJREfWdk
+n4GdfLwscXrGvVvzWGc7ECR5WOwj6OEAZ+kqS5BzyvtERRm6BcoCv9Mdvb9Tthhw
+Gypri2vat/yWTbnt0QgPRtliYYG+6q8K/xoNnPAUQkLd9PxZQevaUXUY2yk3QxGK
+T7VrSsmu5qB+wM2ByU9686xJ7/DlGu4mHjPerEQVtQKCAQBcM3iSitpyP4qRjWQR
+HbDeIudFbMosaaWEedU28REynkLhV5HYsmnmYUNY0dHrvhoHW419YnuhveBFX+25
+kN8MHHXk5aNcxE+akLWYJimHCVGueScdUIC5OEtDHS8guQx48PUPCOPNeyn8XNzw
+ZmG9Xqy0dWK+AK6mXOcUKvbhjWSbEmySo5NVj0JHkdsfmr9A4Fbntcr4yuCBlYve
+TYIMccark3hZci3HzgzWmbSlFv3f/Cd787A19VWRE8nK+9k1oIDBmhIM8M8s/c9m
+kbOApLkm7O8Tb7dYWQgFZbgNdOEuU5bhAk4fuHuDYBPWmPVMQdkvOnvuWlM61ubF
+LdaBAoIBACDpbb5AQIYsWWOnoXuuGh+YY4kmnaBFpsbgEYkZSy92AaLr4Ibf49WN
+oqNDX73YaJlURaGPYMC9J2Huq7TZcewH3SwkVA3N5UmDoijkM4juRfADAfVIMxB5
++9paWeEfnYC/o377FTJIJ9hHJWIaWSoiJZLYDBmoYdxmk8DSHAJCeWsjYDzPybsH
+7RyMPIa1u7lVdgOPEOBi1OIg7ASLxGKiHQtrYHq99GcaVvU/UxoNRMcSnPfY3G8R
+pGah+EndSCb2F20ouDyvlKfOylAltH2BeNc3B4PeP7ZhlVr7bfyOAfC2Z7FNDm3J
++yaBExKfroZjsksctNAcAbgpuvhLLG8=
+-----END PRIVATE KEY-----

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,6 @@
+### Pull Request Checklist
+
+<!-- Please read CONTRIBUTING.md before submitting your pull request -->
+
+* [ ] I have added any new tests that need to pass to `testfile` as specified in [docs/sytest.md](https://github.com/matrix-org/dendrite/blob/master/docs/sytest.md)
+* [ ] Pull request includes a [sign off](https://github.com/matrix-org/dendrite/blob/master/CONTRIBUTING.md#sign-off)

.gitignore

@@ -18,6 +18,7 @@
 /_obj
 /_test
 /vendor/bin
+/docker/build
 
 # Architecture specific extensions/prefixes
 *.[568vq]
@@ -39,3 +40,6 @@ _testmain.go
 *.pem
 *.key
 *.crt
+
+# Default configuration file
+dendrite.yaml

.golangci.yml

@@ -0,0 +1,280 @@
+# Config file for golangci-lint
+
+# options for analysis running
+run:
+  # default concurrency is a available CPU number
+  concurrency: 4
+
+  # timeout for analysis, e.g. 30s, 5m, default is 1m
+  deadline: 30m
+
+  # exit code when at least one issue was found, default is 1
+  issues-exit-code: 1
+
+  # include test files or not, default is true
+  tests: true
+
+  # list of build tags, all linters use it. Default is empty list.
+  #build-tags:
+  #  - mytag
+
+  # which dirs to skip: they won't be analyzed;
+  # can use regexp here: generated.*, regexp is applied on full path;
+  # default value is empty list, but next dirs are always skipped independently
+  # from this option's value:
+  #   	vendor$, third_party$, testdata$, examples$, Godeps$, builtin$
+  skip-dirs:
+    - bin
+    - docs
+
+  # which files to skip: they will be analyzed, but issues from them
+  # won't be reported. Default value is empty list, but there is
+  # no need to include all autogenerated files, we confidently recognize
+  # autogenerated files. If it's not please let us know.
+  skip-files:
+    - ".*\\.md$"
+    - ".*\\.sh$"
+    - "^cmd/syncserver-integration-tests/testdata.go$"
+
+  # by default isn't set. If set we pass it to "go list -mod={option}". From "go help modules":
+  # If invoked with -mod=readonly, the go command is disallowed from the implicit
+  # automatic updating of go.mod described above. Instead, it fails when any changes
+  # to go.mod are needed. This setting is most useful to check that go.mod does
+  # not need updates, such as in a continuous integration and testing system.
+  # If invoked with -mod=vendor, the go command assumes that the vendor
+  # directory holds the correct copies of dependencies and ignores
+  # the dependency descriptions in go.mod.
+  #modules-download-mode: (release|readonly|vendor)
+
+
+# output configuration options
+output:
+  # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number"
+  format: colored-line-number
+
+  # print lines of code with issue, default is true
+  print-issued-lines: true
+
+  # print linter name in the end of issue text, default is true
+  print-linter-name: true
+
+
+# all available settings of specific linters
+linters-settings:
+  errcheck:
+    # report about not checking of errors in type assertions: `a := b.(MyStruct)`;
+    # default is false: such cases aren't reported by default.
+    check-type-assertions: false
+
+    # report about assignment of errors to blank identifier: `num, _ := strconv.Atoi(numStr)`;
+    # default is false: such cases aren't reported by default.
+    check-blank: false
+
+    # [deprecated] comma-separated list of pairs of the form pkg:regex
+    # the regex is used to ignore names within pkg. (default "fmt:.*").
+    # see https://github.com/kisielk/errcheck#the-deprecated-method for details
+    #ignore: fmt:.*,io/ioutil:^Read.*
+
+    # path to a file containing a list of functions to exclude from checking
+    # see https://github.com/kisielk/errcheck#excluding-functions for details
+    #exclude: /path/to/file.txt
+  govet:
+    # report about shadowed variables
+    check-shadowing: true
+
+    # settings per analyzer
+    settings:
+      printf: # analyzer name, run `go tool vet help` to see all analyzers
+        funcs: # run `go tool vet help printf` to see available settings for `printf` analyzer
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf
+  golint:
+    # minimal confidence for issues, default is 0.8
+    min-confidence: 0.8
+  gofmt:
+    # simplify code: gofmt with `-s` option, true by default
+    simplify: true
+  goimports:
+    # put imports beginning with prefix after 3rd-party packages;
+    # it's a comma-separated list of prefixes
+    #local-prefixes: github.com/org/project
+  gocyclo:
+    # minimal code complexity to report, 30 by default (but we recommend 10-20)
+    min-complexity: 12
+  maligned:
+    # print struct with more effective memory layout or not, false by default
+    suggest-new: true
+  dupl:
+    # tokens count to trigger issue, 150 by default
+    threshold: 100
+  goconst:
+    # minimal length of string constant, 3 by default
+    min-len: 3
+    # minimal occurrences count to trigger, 3 by default
+    min-occurrences: 3
+  depguard:
+    list-type: blacklist
+    include-go-root: false
+    packages:
+    #  - github.com/davecgh/go-spew/spew
+  misspell:
+    # Correct spellings using locale preferences for US or UK.
+    # Default is to use a neutral variety of English.
+    # Setting locale to US will correct the British spelling of 'colour' to 'color'.
+    locale: UK
+    ignore-words:
+    #  - someword
+  lll:
+    # max line length, lines longer will be reported. Default is 120.
+    # '\t' is counted as 1 character by default, and can be changed with the tab-width option
+    line-length: 96
+    # tab width in spaces. Default to 1.
+    tab-width: 1
+  unused:
+    # treat code as a program (not a library) and report unused exported identifiers; default is false.
+    # XXX: if you enable this setting, unused will report a lot of false-positives in text editors:
+    # if it's called for subdir of a project it can't find funcs usages. All text editor integrations
+    # with golangci-lint call it on a directory with the changed file.
+    check-exported: false
+  unparam:
+    # Inspect exported functions, default is false. Set to true if no external program/library imports your code.
+    # XXX: if you enable this setting, unparam will report a lot of false-positives in text editors:
+    # if it's called for subdir of a project it can't find external interfaces. All text editor integrations
+    # with golangci-lint call it on a directory with the changed file.
+    check-exported: false
+  nakedret:
+    # make an issue if func has more lines of code than this setting and it has naked returns; default is 30
+    max-func-lines: 60
+  prealloc:
+    # XXX: we don't recommend using this linter before doing performance profiling.
+    # For most programs usage of prealloc will be a premature optimization.
+
+    # Report preallocation suggestions only on simple loops that have no returns/breaks/continues/gotos in them.
+    # True by default.
+    simple: true
+    range-loops: true # Report preallocation suggestions on range loops, true by default
+    for-loops: false # Report preallocation suggestions on for loops, false by default
+  gocritic:
+    # Which checks should be enabled; can't be combined with 'disabled-checks';
+    # See https://go-critic.github.io/overview#checks-overview
+    # To check which checks are enabled run `GL_DEBUG=gocritic golangci-lint run`
+    # By default list of stable checks is used.
+    #enabled-checks:
+
+    # Which checks should be disabled; can't be combined with 'enabled-checks'; default is empty
+    #disabled-checks:
+
+    # Enable multiple checks by tags, run `GL_DEBUG=gocritic golangci-lint` run to see all tags and checks.
+    # Empty list by default. See https://github.com/go-critic/go-critic#usage -> section "Tags".
+    #enabled-tags:
+    #  - performance
+
+    settings: # settings passed to gocritic
+      captLocal: # must be valid enabled check name
+        paramsOnly: true
+      #rangeValCopy:
+      #  sizeThreshold: 32
+
+linters:
+  enable:
+    - deadcode
+    - errcheck
+    - goconst
+    - gocyclo
+    - goimports # Does everything gofmt does
+    - gosimple
+    - ineffassign
+    - megacheck
+    - misspell # Check code comments, whereas misspell in CI checks *.md files
+    - nakedret
+    - staticcheck
+    - structcheck
+    - unparam
+    - unused
+    - varcheck
+  enable-all: false
+  disable:
+    - bodyclose
+    - depguard
+    - dupl
+    - gochecknoglobals
+    - gochecknoinits
+    - gocritic
+    - gofmt
+    - golint
+    - gosec # Should turn back on soon
+    - interfacer
+    - lll
+    - maligned
+    - prealloc # Should turn back on soon
+    - scopelint
+    - stylecheck
+    - typecheck # Should turn back on soon
+    - unconvert # Should turn back on soon
+  disable-all: false
+  presets:
+  fast: false
+
+
+issues:
+  # List of regexps of issue texts to exclude, empty list by default.
+  # But independently from this option we use default exclude patterns,
+  # it can be disabled by `exclude-use-default: false`. To list all
+  # excluded by default patterns execute `golangci-lint run --help`
+  exclude:
+  # - abcdef
+
+  # Excluding configuration per-path, per-linter, per-text and per-source
+  exclude-rules:
+    # Exclude some linters from running on tests files.
+    - path: _test\.go
+      linters:
+        - gocyclo
+        - errcheck
+        - dupl
+        - gosec
+
+    # Exclude known linters from partially hard-vendored code,
+    # which is impossible to exclude via "nolint" comments.
+    - path: internal/hmac/
+      text: "weak cryptographic primitive"
+      linters:
+        - gosec
+
+    # Exclude some staticcheck messages
+    - linters:
+        - staticcheck
+      text: "SA9003:"
+
+    # Exclude lll issues for long lines with go:generate
+    - linters:
+        - lll
+      source: "^//go:generate "
+
+  # Independently from option `exclude` we use default exclude patterns,
+  # it can be disabled by this option. To list all
+  # excluded by default patterns execute `golangci-lint run --help`.
+  # Default value for this option is true.
+  exclude-use-default: false
+
+  # Maximum issues count per one linter. Set to 0 to disable. Default is 50.
+  max-issues-per-linter: 0
+
+  # Maximum count of issues with the same text. Set to 0 to disable. Default is 3.
+  max-same-issues: 0
+
+  # Show only new issues: if there are unstaged changes or untracked files,
+  # only those changes are analyzed, else only changes in HEAD~ are analyzed.
+  # It's a super-useful option for integration of golangci-lint into existing
+  # large codebase. It's not practical to fix all existing issues at the moment
+  # of integration: much better don't allow issues in new code.
+  # Default is false.
+  new: false
+
+  # Show only new issues created after git revision `REV`
+  #new-from-rev: REV
+
+  # Show only new issues created in git patch with set file path.
+  #new-from-patch: path/to/patch/file

.travis.yml

@@ -1,34 +0,0 @@
-language: go
-go:
- - 1.8.x
- - 1.9.x
-
-env:
-- TEST_SUITE="lint"
-- TEST_SUITE="unit-test"
-- TEST_SUITE="integ-test"
-
-sudo: false
-
-# Use trusty for postgres 9.5 support
-dist: trusty
-
-addons:
-  postgresql: "9.5"
-
-services:
-  - postgresql
-
-cache:
-  directories:
-    - .downloads
-
-install:
- - go get github.com/constabulary/gb/...
-
-script:
- - ./scripts/travis-test.sh
-
-# we only need the latest git commit
-git:
-  depth: 1

CONTRIBUTING.md

@@ -12,11 +12,13 @@ See [INSTALL.md](INSTALL.md) for instructions on setting up a running dev
 instance of dendrite, and [CODE_STYLE.md](CODE_STYLE.md) for the code style
 guide.
 
-We use `gb` for managing our dependencies, so `gb build` and `gb test` is how
+As of May 2019, we're not using `gb` anymore, which is the tool we had been
-to build dendrite and run the unit tests respectively. Be aware that a list of
+using for managing our dependencies. We're now using Go modules. To build
-all dendrite packages is the expected output for all tests succeeding with `gb
+Dendrite, run the `build.sh` script at the root of this repository (which runs
-test`. There are also [scripts](scripts) for [linting](scripts/find-lint.sh)
+`go install` under the hood), and to run unit tests, run `go test ./...` (which
-and doing a [build/test/lint run](scripts/build-test-lint.sh).
+should pick up any unit test and run it). There are also [scripts](scripts) for
+[linting](scripts/find-lint.sh) and doing a [build/test/lint
+run](scripts/build-test-lint.sh).
 
 
 ## Picking Things To Do
@@ -33,28 +35,15 @@ nonetheless fairly well-contained.
 We ask people who are familiar with Dendrite to leave the [good first issue](https://github.com/matrix-org/dendrite/labels/good%20first%20issue)
 issues so that there is always a way for new people to come and get involved.
 
-## Contributing to dependencies
-
-Dependencies are located in `vendor/src` and are managed by `gb`. If you need
-to make some changes in those directories, you first need to open a PR in the
-dependency repository. Once your PR is merged, you need to run `gb vendor
-update $repo_url` (example: `gb vendor update github.com/matrix-org/gomatrix`)
-in the dendrite repository to update the dependency.
-
-You can then create a commit containing only the modified vendor files (along
-with the `vendor/manifest` file), name it with the command you just ran (ie
-`gb vendor update github.com/matrix-org/gomatrix`), and open a PR on Dendrite.
-
 ## Getting Help
 
 For questions related to developing on Dendrite we have a dedicated room on
-Matrix [#dendrite-dev:matrix.org](https://riot.im/develop/#/room/#dendrite-dev:matrix.org)
+Matrix [#dendrite-dev:matrix.org](https://matrix.to/#/#dendrite-dev:matrix.org)
 where we're happy to help.
 
-For more general questions please use [#dendrite:matrix.org](https://riot.im/develop/#/room/#dendrite:matrix.org).
+For more general questions please use [#dendrite:matrix.org](https://matrix.to/#/#dendrite:matrix.org).
 
 ## Sign off
 
 We ask that everyone who contributes to the project signs off their
 contributions, in accordance with the [DCO](https://github.com/matrix-org/matrix-doc/blob/master/CONTRIBUTING.rst#sign-off).
-

INSTALL.md

@@ -12,7 +12,7 @@ Dendrite can be run in one of two configurations:
 
 ## Requirements
 
- - Go 1.8+
+ - Go 1.11+
  - Postgres 9.5+
  - For Kafka (optional if using the monolith server):
    - Unix-based system (https://kafka.apache.org/documentation/#os)
@@ -22,7 +22,7 @@ Dendrite can be run in one of two configurations:
 
 ## Setting up a development environment
 
-Assumes Go 1.8 and JDK 1.8 are already installed and are on PATH.
+Assumes Go 1.10+ and JDK 1.8+ are already installed and are on PATH.
 
 ```bash
 # Get the code
@@ -30,16 +30,15 @@ git clone https://github.com/matrix-org/dendrite
 cd dendrite
 
 # Build it
-go get github.com/constabulary/gb/...
+./build.sh
-gb build
 ```
 
 If using Kafka, install and start it (c.f. [scripts/install-local-kafka.sh](scripts/install-local-kafka.sh)):
 ```bash
-MIRROR=http://apache.mirror.anlx.net/kafka/0.10.2.0/kafka_2.11-0.10.2.0.tgz
+KAFKA_URL=http://archive.apache.org/dist/kafka/2.1.0/kafka_2.11-2.1.0.tgz
 
 # Only download the kafka if it isn't already downloaded.
-test -f kafka.tgz || wget $MIRROR -O kafka.tgz
+test -f kafka.tgz || wget $KAFKA_URL -O kafka.tgz
 # Unpack the kafka over the top of any existing installation
 mkdir -p kafka && tar xzf kafka.tgz -C kafka --strip-components 1
 
@@ -72,7 +71,7 @@ Dendrite requires a postgres database engine, version 9.5 or later.
   ```
 * Create databases:
   ```bash
-  for i in account device mediaapi syncapi roomserver serverkey federationsender publicroomsapi naffka; do
+  for i in account device mediaapi syncapi roomserver serverkey federationsender publicroomsapi appservice naffka; do
       sudo -u postgres createdb -O dendrite dendrite_$i
   done
   ```
@@ -95,13 +94,15 @@ test -f matrix_key.pem || ./bin/generate-keys -private-key matrix_key.pem
 
 Create config file, based on `dendrite-config.yaml`. Call it `dendrite.yaml`. Things that will need editing include *at least*:
 * `server_name`
-* `database/*`
+* `database/*` (All lines in the database section must have the username and password of the user created with the `createuser` command above. eg:`dendrite:password@localhost`)
 
 
 ## Starting a monolith server
 
 It is possible to use 'naffka' as an in-process replacement to Kafka when using
-the monolith server. To do this, set `use_naffka: true` in `dendrite.yaml`.
+the monolith server. To do this, set `use_naffka: true` in `dendrite.yaml` and uncomment
+the necessary line related to naffka in the `database` section. Be sure to update the 
+database username and password if needed.
 
 The monolith server can be started as shown below. By default it listens for
 HTTP connections on port 8008, so point your client at
@@ -253,3 +254,14 @@ you want to support federation.
 ```bash
 ./bin/dendrite-federation-sender-server --config dendrite.yaml
 ```
+
+### Run an appservice server 
+
+This sends events from the network to [application
+services](https://matrix.org/docs/spec/application_service/unstable.html)
+running locally.  This is only required if you want to support running
+application services on your homeserver.
+
+```bash
+./bin/dendrite-appservice-server --config dendrite.yaml
+```

README.md

@@ -1,4 +1,4 @@
-# Dendrite [![Build Status](https://travis-ci.org/matrix-org/dendrite.svg?branch=master)](https://travis-ci.org/matrix-org/dendrite)
+# Dendrite [![Build Status](https://badge.buildkite.com/4be40938ab19f2bbc4a6c6724517353ee3ec1422e279faf374.svg)](https://buildkite.com/matrix-dot-org/dendrite) [![CircleCI](https://circleci.com/gh/matrix-org/dendrite.svg?style=svg)](https://circleci.com/gh/matrix-org/dendrite) [![Dendrite Dev on Matrix](https://img.shields.io/matrix/dendrite-dev:matrix.org.svg?label=%23dendrite-dev%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite-dev:matrix.org) [![Dendrite on Matrix](https://img.shields.io/matrix/dendrite:matrix.org.svg?label=%23dendrite%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite:matrix.org)
 
 Dendrite will be a matrix homeserver written in go.
 
@@ -17,9 +17,11 @@ We aim to try and make it as easy as possible to jump in.
 # Discussion
 
 For questions about Dendrite we have a dedicated room on Matrix
-[#dendrite:matrix.org](https://riot.im/develop/#/room/#dendrite:matrix.org).
+[#dendrite:matrix.org](https://matrix.to/#/#dendrite:matrix.org).
+Development discussion should happen in
+[#dendrite-dev:matrix.org](https://matrix.to/#/#dendrite-dev:matrix.org).
 
 # Progress
 
 There's plenty still to do to make Dendrite usable! We're tracking progress in
-a [spreadsheet](https://docs.google.com/spreadsheets/d/1tkMNpIpPjvuDJWjPFbw_xzNzOHBA-Hp50Rkpcr43xTw).
+a [project board](https://github.com/matrix-org/dendrite/projects/2).

WIRING.md

@@ -36,7 +36,11 @@ Diagram:
     |            |                          |        |                            |          |    |          |
     |            |                          |        |>==========================>|          |    |          |
     |            |                          |        |                            +----------+    |          |
-    |            |                          |        |                                            |          |
+    |            |                          |        |                     +---+                  |          |
+    |            |                          |        |       +-------------| R |                  |          |
+    |            |                          |        |>=====>| Application +---+                  |          |
+    |            |                          |        |       | Services     |                     |          |
+    |            |                          |        |       +--------------+                     |          |
     |            |                          |        |                                     +---+  |          |
     |            |                          |        |                            +--------| R |  |          |
     |            |                          |        |                            | Client +---+  |          |
@@ -190,3 +194,36 @@ choke-point to implement ratelimiting and backoff correctly.
  * Reads new events and the current state of the rooms from logs writeen by the Room Server.
  * Reads the position of the read marker from the Receipts Server.
  * Makes outbound HTTP hits to the push server for the client device.
+
+## Application Service
+
+ * Receives events from the Room Server.
+ * Filters events and sends them to each registered application service.
+ * Runs a separate goroutine for each application service.
+
+# Internal Component API
+
+Some dendrite components use internal APIs to communicate information back
+and forth between each other. There are two implementations of each API, one
+that uses HTTP requests and one that does not. The HTTP implementation is
+used in multi-process mode, so processes on separate computers may still
+communicate, whereas in single-process or Monolith mode, the direct
+implementation is used. HTTP is preferred here to kafka streams as it allows
+for request responses.
+
+Running `dendrite-monolith-server` will set up direct connections between
+components, whereas running each individual component (which are only run in
+multi-process mode) will set up HTTP-based connections.
+
+The functions that make HTTP requests to internal APIs of a component are
+located in `/<component name>/api/<name>.go`, named according to what
+functionality they cover. Each of these requests are handled in `/<component
+name>/<name>/<name>.go`.
+
+As an example, the `appservices` component allows other Dendrite components
+to query external application services via its internal API. A component
+would call the desired function in `/appservices/api/query.go`. In
+multi-process mode, this would send an internal HTTP request, which would
+be handled by a function in `/appservices/query/query.go`. In single-process
+mode, no internal HTTP request occurs, instead functions are simply called
+directly, thus requiring no changes on the calling component's end.

appservice/README.md

@@ -2,9 +2,9 @@
 
 This component interfaces with external [Application
 Services](https://matrix.org/docs/spec/application_service/unstable.html).
-This includes any HTTP endpoints that Application Services call, as well as talking
+This includes any HTTP endpoints that application services call, as well as talking
-to any HTTP endpoints that Application Services provide themselves.
+to any HTTP endpoints that application services provide themselves.
 
 ## Consumers
 
-This component consumes and filters events from the Roomserver Kafka stream, passing on any necessary events to subscribing Application Services.
+This component consumes and filters events from the Roomserver Kafka stream, passing on any necessary events to subscribing application services.

appservice/api/query.go

@@ -0,0 +1,178 @@
+// Copyright 2018 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package api contains methods used by dendrite components in multi-process
+// mode to send requests to the appservice component, typically in order to ask
+// an application service for some information.
+package api
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"net/http"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
+	"github.com/matrix-org/gomatrixserverlib"
+
+	commonHTTP "github.com/matrix-org/dendrite/common/http"
+	opentracing "github.com/opentracing/opentracing-go"
+)
+
+// RoomAliasExistsRequest is a request to an application service
+// about whether a room alias exists
+type RoomAliasExistsRequest struct {
+	// Alias we want to lookup
+	Alias string `json:"alias"`
+}
+
+// RoomAliasExistsResponse is a response from an application service
+// about whether a room alias exists
+type RoomAliasExistsResponse struct {
+	AliasExists bool `json:"exists"`
+}
+
+// UserIDExistsRequest is a request to an application service about whether a
+// user ID exists
+type UserIDExistsRequest struct {
+	// UserID we want to lookup
+	UserID string `json:"user_id"`
+}
+
+// UserIDExistsRequestAccessToken is a request to an application service
+// about whether a user ID exists. Includes an access token
+type UserIDExistsRequestAccessToken struct {
+	// UserID we want to lookup
+	UserID      string `json:"user_id"`
+	AccessToken string `json:"access_token"`
+}
+
+// UserIDExistsResponse is a response from an application service about
+// whether a user ID exists
+type UserIDExistsResponse struct {
+	UserIDExists bool `json:"exists"`
+}
+
+// AppServiceQueryAPI is used to query user and room alias data from application
+// services
+type AppServiceQueryAPI interface {
+	// Check whether a room alias exists within any application service namespaces
+	RoomAliasExists(
+		ctx context.Context,
+		req *RoomAliasExistsRequest,
+		resp *RoomAliasExistsResponse,
+	) error
+	// Check whether a user ID exists within any application service namespaces
+	UserIDExists(
+		ctx context.Context,
+		req *UserIDExistsRequest,
+		resp *UserIDExistsResponse,
+	) error
+}
+
+// AppServiceRoomAliasExistsPath is the HTTP path for the RoomAliasExists API
+const AppServiceRoomAliasExistsPath = "/api/appservice/RoomAliasExists"
+
+// AppServiceUserIDExistsPath is the HTTP path for the UserIDExists API
+const AppServiceUserIDExistsPath = "/api/appservice/UserIDExists"
+
+// httpAppServiceQueryAPI contains the URL to an appservice query API and a
+// reference to a httpClient used to reach it
+type httpAppServiceQueryAPI struct {
+	appserviceURL string
+	httpClient    *http.Client
+}
+
+// NewAppServiceQueryAPIHTTP creates a AppServiceQueryAPI implemented by talking
+// to a HTTP POST API.
+// If httpClient is nil then it uses http.DefaultClient
+func NewAppServiceQueryAPIHTTP(
+	appserviceURL string,
+	httpClient *http.Client,
+) AppServiceQueryAPI {
+	if httpClient == nil {
+		httpClient = http.DefaultClient
+	}
+	return &httpAppServiceQueryAPI{appserviceURL, httpClient}
+}
+
+// RoomAliasExists implements AppServiceQueryAPI
+func (h *httpAppServiceQueryAPI) RoomAliasExists(
+	ctx context.Context,
+	request *RoomAliasExistsRequest,
+	response *RoomAliasExistsResponse,
+) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "appserviceRoomAliasExists")
+	defer span.Finish()
+
+	apiURL := h.appserviceURL + AppServiceRoomAliasExistsPath
+	return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+}
+
+// UserIDExists implements AppServiceQueryAPI
+func (h *httpAppServiceQueryAPI) UserIDExists(
+	ctx context.Context,
+	request *UserIDExistsRequest,
+	response *UserIDExistsResponse,
+) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "appserviceUserIDExists")
+	defer span.Finish()
+
+	apiURL := h.appserviceURL + AppServiceUserIDExistsPath
+	return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
+}
+
+// RetrieveUserProfile is a wrapper that queries both the local database and
+// application services for a given user's profile
+func RetrieveUserProfile(
+	ctx context.Context,
+	userID string,
+	asAPI AppServiceQueryAPI,
+	accountDB *accounts.Database,
+) (*authtypes.Profile, error) {
+	localpart, _, err := gomatrixserverlib.SplitID('@', userID)
+	if err != nil {
+		return nil, err
+	}
+
+	// Try to query the user from the local database
+	profile, err := accountDB.GetProfileByLocalpart(ctx, localpart)
+	if err != nil && err != sql.ErrNoRows {
+		return nil, err
+	} else if profile != nil {
+		return profile, nil
+	}
+
+	// Query the appservice component for the existence of an AS user
+	userReq := UserIDExistsRequest{UserID: userID}
+	var userResp UserIDExistsResponse
+	if err = asAPI.UserIDExists(ctx, &userReq, &userResp); err != nil {
+		return nil, err
+	}
+
+	// If no user exists, return
+	if !userResp.UserIDExists {
+		return nil, errors.New("no known profile for given user ID")
+	}
+
+	// Try to query the user from the local database again
+	profile, err = accountDB.GetProfileByLocalpart(ctx, localpart)
+	if err != nil {
+		return nil, err
+	}
+
+	// profile should not be nil at this point
+	return profile, nil
+}

appservice/appservice.go

@@ -0,0 +1,132 @@
+// Copyright 2018 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package appservice
+
+import (
+	"context"
+	"net/http"
+	"sync"
+	"time"
+
+	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
+	"github.com/matrix-org/dendrite/appservice/consumers"
+	"github.com/matrix-org/dendrite/appservice/query"
+	"github.com/matrix-org/dendrite/appservice/routing"
+	"github.com/matrix-org/dendrite/appservice/storage"
+	"github.com/matrix-org/dendrite/appservice/types"
+	"github.com/matrix-org/dendrite/appservice/workers"
+	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
+	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
+	"github.com/matrix-org/dendrite/common/basecomponent"
+	"github.com/matrix-org/dendrite/common/config"
+	"github.com/matrix-org/dendrite/common/transactions"
+	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/sirupsen/logrus"
+)
+
+// SetupAppServiceAPIComponent sets up and registers HTTP handlers for the AppServices
+// component.
+func SetupAppServiceAPIComponent(
+	base *basecomponent.BaseDendrite,
+	accountsDB *accounts.Database,
+	deviceDB *devices.Database,
+	federation *gomatrixserverlib.FederationClient,
+	roomserverAliasAPI roomserverAPI.RoomserverAliasAPI,
+	roomserverQueryAPI roomserverAPI.RoomserverQueryAPI,
+	transactionsCache *transactions.Cache,
+) appserviceAPI.AppServiceQueryAPI {
+	// Create a connection to the appservice postgres DB
+	appserviceDB, err := storage.NewDatabase(string(base.Cfg.Database.AppService))
+	if err != nil {
+		logrus.WithError(err).Panicf("failed to connect to appservice db")
+	}
+
+	// Wrap application services in a type that relates the application service and
+	// a sync.Cond object that can be used to notify workers when there are new
+	// events to be sent out.
+	workerStates := make([]types.ApplicationServiceWorkerState, len(base.Cfg.Derived.ApplicationServices))
+	for i, appservice := range base.Cfg.Derived.ApplicationServices {
+		m := sync.Mutex{}
+		ws := types.ApplicationServiceWorkerState{
+			AppService: appservice,
+			Cond:       sync.NewCond(&m),
+		}
+		workerStates[i] = ws
+
+		// Create bot account for this AS if it doesn't already exist
+		if err = generateAppServiceAccount(accountsDB, deviceDB, appservice); err != nil {
+			logrus.WithFields(logrus.Fields{
+				"appservice": appservice.ID,
+			}).WithError(err).Panicf("failed to generate bot account for appservice")
+		}
+	}
+
+	// Create appserivce query API with an HTTP client that will be used for all
+	// outbound and inbound requests (inbound only for the internal API)
+	appserviceQueryAPI := query.AppServiceQueryAPI{
+		HTTPClient: &http.Client{
+			Timeout: time.Second * 30,
+		},
+		Cfg: base.Cfg,
+	}
+
+	appserviceQueryAPI.SetupHTTP(http.DefaultServeMux)
+
+	consumer := consumers.NewOutputRoomEventConsumer(
+		base.Cfg, base.KafkaConsumer, accountsDB, appserviceDB,
+		roomserverQueryAPI, roomserverAliasAPI, workerStates,
+	)
+	if err := consumer.Start(); err != nil {
+		logrus.WithError(err).Panicf("failed to start appservice roomserver consumer")
+	}
+
+	// Create application service transaction workers
+	if err := workers.SetupTransactionWorkers(appserviceDB, workerStates); err != nil {
+		logrus.WithError(err).Panicf("failed to start app service transaction workers")
+	}
+
+	// Set up HTTP Endpoints
+	routing.Setup(
+		base.APIMux, *base.Cfg, roomserverQueryAPI, roomserverAliasAPI,
+		accountsDB, federation, transactionsCache,
+	)
+
+	return &appserviceQueryAPI
+}
+
+// generateAppServiceAccounts creates a dummy account based off the
+// `sender_localpart` field of each application service if it doesn't
+// exist already
+func generateAppServiceAccount(
+	accountsDB *accounts.Database,
+	deviceDB *devices.Database,
+	as config.ApplicationService,
+) error {
+	ctx := context.Background()
+
+	// Create an account for the application service
+	acc, err := accountsDB.CreateAccount(ctx, as.SenderLocalpart, "", as.ID)
+	if err != nil {
+		return err
+	} else if acc == nil {
+		// This account already exists
+		return nil
+	}
+
+	// Create a dummy device with a dummy token for the application service
+	_, err = deviceDB.CreateDevice(ctx, as.SenderLocalpart, nil, as.ASToken, &as.SenderLocalpart)
+	return err
+}

appservice/consumers/roomserver.go

@@ -17,8 +17,9 @@ package consumers
 import (
 	"context"
 	"encoding/json"
-	"fmt"
 
+	"github.com/matrix-org/dendrite/appservice/storage"
+	"github.com/matrix-org/dendrite/appservice/types"
 	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/common/config"
@@ -29,29 +30,28 @@ import (
 	sarama "gopkg.in/Shopify/sarama.v1"
 )
 
-var (
-	appServices []config.ApplicationService
-)
-
 // OutputRoomEventConsumer consumes events that originated in the room server.
 type OutputRoomEventConsumer struct {
 	roomServerConsumer *common.ContinualConsumer
 	db                 *accounts.Database
+	asDB               *storage.Database
 	query              api.RoomserverQueryAPI
 	alias              api.RoomserverAliasAPI
 	serverName         string
+	workerStates       []types.ApplicationServiceWorkerState
 }
 
-// NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call Start() to begin consuming from room servers.
+// NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call
+// Start() to begin consuming from room servers.
 func NewOutputRoomEventConsumer(
 	cfg *config.Dendrite,
 	kafkaConsumer sarama.Consumer,
 	store *accounts.Database,
+	appserviceDB *storage.Database,
 	queryAPI api.RoomserverQueryAPI,
 	aliasAPI api.RoomserverAliasAPI,
+	workerStates []types.ApplicationServiceWorkerState,
 ) *OutputRoomEventConsumer {
-	appServices = cfg.Derived.ApplicationServices
-
 	consumer := common.ContinualConsumer{
 		Topic:          string(cfg.Kafka.Topics.OutputRoomEvent),
 		Consumer:       kafkaConsumer,
@@ -60,9 +60,11 @@ func NewOutputRoomEventConsumer(
 	s := &OutputRoomEventConsumer{
 		roomServerConsumer: &consumer,
 		db:                 store,
+		asDB:               appserviceDB,
 		query:              queryAPI,
 		alias:              aliasAPI,
 		serverName:         string(cfg.Matrix.ServerName),
+		workerStates:       workerStates,
 	}
 	consumer.ProcessMessage = s.onMessage
 
@@ -74,9 +76,8 @@ func (s *OutputRoomEventConsumer) Start() error {
 	return s.roomServerConsumer.Start()
 }
 
-// onMessage is called when the sync server receives a new event from the room server output log.
+// onMessage is called when the appservice component receives a new event from
-// It is not safe for this function to be called from multiple goroutines, or else the
+// the room server output log.
-// sync stream position may race and be incorrectly calculated.
 func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error {
 	// Parse out the event JSON
 	var output api.OutputEvent
@@ -98,50 +99,37 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error {
 		"event_id": ev.EventID(),
 		"room_id":  ev.RoomID(),
 		"type":     ev.Type(),
-	}).Info("appservice received event from roomserver")
+	}).Info("appservice received an event from roomserver")
 
-	events, err := s.lookupStateEvents(output.NewRoomEvent.AddsStateEventIDs, ev)
+	missingEvents, err := s.lookupMissingStateEvents(output.NewRoomEvent.AddsStateEventIDs, ev)
 	if err != nil {
 		return err
 	}
+	events := append(missingEvents, ev)
 
-	// Create a context to thread through the whole filtering process
+	// Send event to any relevant application services
-	ctx := context.TODO()
+	return s.filterRoomserverEvents(context.TODO(), events)
-
-	if err = s.db.UpdateMemberships(ctx, events, output.NewRoomEvent.RemovesStateEventIDs); err != nil {
-		return err
-	}
-
-	// Check if any events need to passed on to external application services
-	return s.filterRoomserverEvents(ctx, append(events, ev))
 }
 
-// lookupStateEvents looks up the state events that are added by a new event.
+// lookupMissingStateEvents looks up the state events that are added by a new event,
-func (s *OutputRoomEventConsumer) lookupStateEvents(
+// and returns any not already present.
+func (s *OutputRoomEventConsumer) lookupMissingStateEvents(
 	addsStateEventIDs []string, event gomatrixserverlib.Event,
 ) ([]gomatrixserverlib.Event, error) {
 	// Fast path if there aren't any new state events.
 	if len(addsStateEventIDs) == 0 {
-		// If the event is a membership update (e.g. for a profile update), it won't
+		return []gomatrixserverlib.Event{}, nil
-		// show up in AddsStateEventIDs, so we need to add it manually
-		if event.Type() == "m.room.member" {
-			return []gomatrixserverlib.Event{event}, nil
-		}
-		return nil, nil
 	}
 
 	// Fast path if the only state event added is the event itself.
 	if len(addsStateEventIDs) == 1 && addsStateEventIDs[0] == event.EventID() {
-		return []gomatrixserverlib.Event{event}, nil
+		return []gomatrixserverlib.Event{}, nil
 	}
 
 	result := []gomatrixserverlib.Event{}
 	missing := []string{}
 	for _, id := range addsStateEventIDs {
-		// Append the current event in the results if its ID is in the events list
+		if id != event.EventID() {
-		if id == event.EventID() {
-			result = append(result, event)
-		} else {
 			// If the event isn't the current one, add it to the list of events
 			// to retrieve from the roomserver
 			missing = append(missing, id)
@@ -165,13 +153,22 @@ func (s *OutputRoomEventConsumer) lookupStateEvents(
 // each namespace of each registered application service, and if there is a
 // match, adds the event to the queue for events to be sent to a particular
 // application service.
-func (s *OutputRoomEventConsumer) filterRoomserverEvents(ctx context.Context, events []gomatrixserverlib.Event) error {
+func (s *OutputRoomEventConsumer) filterRoomserverEvents(
+	ctx context.Context,
+	events []gomatrixserverlib.Event,
+) error {
+	for _, ws := range s.workerStates {
 		for _, event := range events {
-		for _, appservice := range appServices {
 			// Check if this event is interesting to this application service
-			if s.appserviceIsInterestedInEvent(ctx, event, appservice) {
+			if s.appserviceIsInterestedInEvent(ctx, event, ws.AppService) {
-				// TODO: Queue this event to be sent off to the application service
+				// Queue this event to be sent off to the application service
-				fmt.Println(appservice.ID, "was interested in", event.Sender(), event.Type(), event.RoomID())
+				if err := s.asDB.StoreEvent(ctx, ws.AppService.ID, &event); err != nil {
+					log.WithError(err).Warn("failed to insert incoming event into appservices database")
+				} else {
+					// Tell our worker to send out new messages by updating remaining message
+					// count and waking them up with a broadcast
+					ws.NotifyNewEvents()
+				}
 			}
 		}
 	}
@@ -182,31 +179,27 @@ func (s *OutputRoomEventConsumer) filterRoomserverEvents(ctx context.Context, ev
 // appserviceIsInterestedInEvent returns a boolean depending on whether a given
 // event falls within one of a given application service's namespaces.
 func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Context, event gomatrixserverlib.Event, appservice config.ApplicationService) bool {
-	// Check sender of the event
+	// No reason to queue events if they'll never be sent to the application
-	for _, userNamespace := range appservice.NamespaceMap["users"] {
+	// service
-		if userNamespace.RegexpObject.MatchString(event.Sender()) {
+	if appservice.URL == "" {
-			return true
+		return false
-		}
 	}
 
-	// Check room id of the event
+	// Check Room ID and Sender of the event
-	for _, roomNamespace := range appservice.NamespaceMap["rooms"] {
+	if appservice.IsInterestedInUserID(event.Sender()) ||
-		if roomNamespace.RegexpObject.MatchString(event.RoomID()) {
+		appservice.IsInterestedInRoomID(event.RoomID()) {
 		return true
 	}
-	}
 
 	// Check all known room aliases of the room the event came from
 	queryReq := api.GetAliasesForRoomIDRequest{RoomID: event.RoomID()}
 	var queryRes api.GetAliasesForRoomIDResponse
 	if err := s.alias.GetAliasesForRoomID(ctx, &queryReq, &queryRes); err == nil {
 		for _, alias := range queryRes.Aliases {
-			for _, aliasNamespace := range appservice.NamespaceMap["aliases"] {
+			if appservice.IsInterestedInRoomAlias(alias) {
-				if aliasNamespace.RegexpObject.MatchString(alias) {
 				return true
 			}
 		}
-		}
 	} else {
 		log.WithFields(log.Fields{
 			"room_id": event.RoomID(),

appservice/query/query.go

@@ -0,0 +1,214 @@
+// Copyright 2018 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package query handles requests from other internal dendrite components when
+// they interact with the AppServiceQueryAPI.
+package query
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/url"
+	"time"
+
+	"github.com/matrix-org/dendrite/appservice/api"
+	"github.com/matrix-org/dendrite/common"
+	"github.com/matrix-org/dendrite/common/config"
+	"github.com/matrix-org/util"
+	opentracing "github.com/opentracing/opentracing-go"
+	log "github.com/sirupsen/logrus"
+)
+
+const roomAliasExistsPath = "/rooms/"
+const userIDExistsPath = "/users/"
+
+// AppServiceQueryAPI is an implementation of api.AppServiceQueryAPI
+type AppServiceQueryAPI struct {
+	HTTPClient *http.Client
+	Cfg        *config.Dendrite
+}
+
+// RoomAliasExists performs a request to '/room/{roomAlias}' on all known
+// handling application services until one admits to owning the room
+func (a *AppServiceQueryAPI) RoomAliasExists(
+	ctx context.Context,
+	request *api.RoomAliasExistsRequest,
+	response *api.RoomAliasExistsResponse,
+) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "ApplicationServiceRoomAlias")
+	defer span.Finish()
+
+	// Create an HTTP client if one does not already exist
+	if a.HTTPClient == nil {
+		a.HTTPClient = makeHTTPClient()
+	}
+
+	// Determine which application service should handle this request
+	for _, appservice := range a.Cfg.Derived.ApplicationServices {
+		if appservice.URL != "" && appservice.IsInterestedInRoomAlias(request.Alias) {
+			// The full path to the rooms API, includes hs token
+			URL, err := url.Parse(appservice.URL + roomAliasExistsPath)
+			URL.Path += request.Alias
+			apiURL := URL.String() + "?access_token=" + appservice.HSToken
+
+			// Send a request to each application service. If one responds that it has
+			// created the room, immediately return.
+			req, err := http.NewRequest(http.MethodGet, apiURL, nil)
+			if err != nil {
+				return err
+			}
+			req = req.WithContext(ctx)
+
+			resp, err := a.HTTPClient.Do(req)
+			if resp != nil {
+				defer func() {
+					err = resp.Body.Close()
+					if err != nil {
+						log.WithFields(log.Fields{
+							"appservice_id": appservice.ID,
+							"status_code":   resp.StatusCode,
+						}).WithError(err).Error("Unable to close application service response body")
+					}
+				}()
+			}
+			if err != nil {
+				log.WithError(err).Errorf("Issue querying room alias on application service %s", appservice.ID)
+				return err
+			}
+			switch resp.StatusCode {
+			case http.StatusOK:
+				// OK received from appservice. Room exists
+				response.AliasExists = true
+				return nil
+			case http.StatusNotFound:
+				// Room does not exist
+			default:
+				// Application service reported an error. Warn
+				log.WithFields(log.Fields{
+					"appservice_id": appservice.ID,
+					"status_code":   resp.StatusCode,
+				}).Warn("Application service responded with non-OK status code")
+			}
+		}
+	}
+
+	response.AliasExists = false
+	return nil
+}
+
+// UserIDExists performs a request to '/users/{userID}' on all known
+// handling application services until one admits to owning the user ID
+func (a *AppServiceQueryAPI) UserIDExists(
+	ctx context.Context,
+	request *api.UserIDExistsRequest,
+	response *api.UserIDExistsResponse,
+) error {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "ApplicationServiceUserID")
+	defer span.Finish()
+
+	// Create an HTTP client if one does not already exist
+	if a.HTTPClient == nil {
+		a.HTTPClient = makeHTTPClient()
+	}
+
+	// Determine which application service should handle this request
+	for _, appservice := range a.Cfg.Derived.ApplicationServices {
+		if appservice.URL != "" && appservice.IsInterestedInUserID(request.UserID) {
+			// The full path to the rooms API, includes hs token
+			URL, err := url.Parse(appservice.URL + userIDExistsPath)
+			URL.Path += request.UserID
+			apiURL := URL.String() + "?access_token=" + appservice.HSToken
+
+			// Send a request to each application service. If one responds that it has
+			// created the user, immediately return.
+			req, err := http.NewRequest(http.MethodGet, apiURL, nil)
+			if err != nil {
+				return err
+			}
+			resp, err := a.HTTPClient.Do(req.WithContext(ctx))
+			if resp != nil {
+				defer func() {
+					err = resp.Body.Close()
+					if err != nil {
+						log.WithFields(log.Fields{
+							"appservice_id": appservice.ID,
+							"status_code":   resp.StatusCode,
+						}).Error("Unable to close application service response body")
+					}
+				}()
+			}
+			if err != nil {
+				log.WithFields(log.Fields{
+					"appservice_id": appservice.ID,
+				}).WithError(err).Error("issue querying user ID on application service")
+				return err
+			}
+			if resp.StatusCode == http.StatusOK {
+				// StatusOK received from appservice. User ID exists
+				response.UserIDExists = true
+				return nil
+			}
+
+			// Log non OK
+			log.WithFields(log.Fields{
+				"appservice_id": appservice.ID,
+				"status_code":   resp.StatusCode,
+			}).Warn("application service responded with non-OK status code")
+		}
+	}
+
+	response.UserIDExists = false
+	return nil
+}
+
+// makeHTTPClient creates an HTTP client with certain options that will be used for all query requests to application services
+func makeHTTPClient() *http.Client {
+	return &http.Client{
+		Timeout: time.Second * 30,
+	}
+}
+
+// SetupHTTP adds the AppServiceQueryPAI handlers to the http.ServeMux. This
+// handles and muxes incoming api requests the to internal AppServiceQueryAPI.
+func (a *AppServiceQueryAPI) SetupHTTP(servMux *http.ServeMux) {
+	servMux.Handle(
+		api.AppServiceRoomAliasExistsPath,
+		common.MakeInternalAPI("appserviceRoomAliasExists", func(req *http.Request) util.JSONResponse {
+			var request api.RoomAliasExistsRequest
+			var response api.RoomAliasExistsResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.ErrorResponse(err)
+			}
+			if err := a.RoomAliasExists(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+	servMux.Handle(
+		api.AppServiceUserIDExistsPath,
+		common.MakeInternalAPI("appserviceUserIDExists", func(req *http.Request) util.JSONResponse {
+			var request api.UserIDExistsRequest
+			var response api.UserIDExistsResponse
+			if err := json.NewDecoder(req.Body).Decode(&request); err != nil {
+				return util.ErrorResponse(err)
+			}
+			if err := a.UserIDExists(req.Context(), &request, &response); err != nil {
+				return util.ErrorResponse(err)
+			}
+			return util.JSONResponse{Code: http.StatusOK, JSON: &response}
+		}),
+	)
+}

appservice/routing/routing.go

@@ -31,6 +31,10 @@ const pathPrefixApp = "/_matrix/app/r0"
 
 // Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client
 // to clients which need to make outbound HTTP requests.
+//
+// Due to Setup being used to call many other functions, a gocyclo nolint is
+// applied:
+// nolint: gocyclo
 func Setup(
 	apiMux *mux.Router, cfg config.Dendrite, // nolint: unparam
 	queryAPI api.RoomserverQueryAPI, aliasAPI api.RoomserverAliasAPI, // nolint: unparam

appservice/storage/appservice_events_table.go

@@ -0,0 +1,248 @@
+// Copyright 2018 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package storage
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+	"time"
+
+	"github.com/matrix-org/gomatrixserverlib"
+	log "github.com/sirupsen/logrus"
+)
+
+const appserviceEventsSchema = `
+-- Stores events to be sent to application services
+CREATE TABLE IF NOT EXISTS appservice_events (
+	-- An auto-incrementing id unique to each event in the table
+	id BIGSERIAL NOT NULL PRIMARY KEY,
+	-- The ID of the application service the event will be sent to
+	as_id TEXT NOT NULL,
+	-- JSON representation of the event
+	event_json TEXT NOT NULL,
+	-- The ID of the transaction that this event is a part of
+	txn_id BIGINT NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS appservice_events_as_id ON appservice_events(as_id);
+`
+
+const selectEventsByApplicationServiceIDSQL = "" +
+	"SELECT id, event_json, txn_id " +
+	"FROM appservice_events WHERE as_id = $1 ORDER BY txn_id DESC, id ASC"
+
+const countEventsByApplicationServiceIDSQL = "" +
+	"SELECT COUNT(id) FROM appservice_events WHERE as_id = $1"
+
+const insertEventSQL = "" +
+	"INSERT INTO appservice_events(as_id, event_json, txn_id) " +
+	"VALUES ($1, $2, $3)"
+
+const updateTxnIDForEventsSQL = "" +
+	"UPDATE appservice_events SET txn_id = $1 WHERE as_id = $2 AND id <= $3"
+
+const deleteEventsBeforeAndIncludingIDSQL = "" +
+	"DELETE FROM appservice_events WHERE as_id = $1 AND id <= $2"
+
+const (
+	// A transaction ID number that no transaction should ever have. Used for
+	// checking again the default value.
+	invalidTxnID = -2
+)
+
+type eventsStatements struct {
+	selectEventsByApplicationServiceIDStmt *sql.Stmt
+	countEventsByApplicationServiceIDStmt  *sql.Stmt
+	insertEventStmt                        *sql.Stmt
+	updateTxnIDForEventsStmt               *sql.Stmt
+	deleteEventsBeforeAndIncludingIDStmt   *sql.Stmt
+}
+
+func (s *eventsStatements) prepare(db *sql.DB) (err error) {
+	_, err = db.Exec(appserviceEventsSchema)
+	if err != nil {
+		return
+	}
+
+	if s.selectEventsByApplicationServiceIDStmt, err = db.Prepare(selectEventsByApplicationServiceIDSQL); err != nil {
+		return
+	}
+	if s.countEventsByApplicationServiceIDStmt, err = db.Prepare(countEventsByApplicationServiceIDSQL); err != nil {
+		return
+	}
+	if s.insertEventStmt, err = db.Prepare(insertEventSQL); err != nil {
+		return
+	}
+	if s.updateTxnIDForEventsStmt, err = db.Prepare(updateTxnIDForEventsSQL); err != nil {
+		return
+	}
+	if s.deleteEventsBeforeAndIncludingIDStmt, err = db.Prepare(deleteEventsBeforeAndIncludingIDSQL); err != nil {
+		return
+	}
+
+	return
+}
+
+// selectEventsByApplicationServiceID takes in an application service ID and
+// returns a slice of events that need to be sent to that application service,
+// as well as an int later used to remove these same events from the database
+// once successfully sent to an application service.
+func (s *eventsStatements) selectEventsByApplicationServiceID(
+	ctx context.Context,
+	applicationServiceID string,
+	limit int,
+) (
+	txnID, maxID int,
+	events []gomatrixserverlib.Event,
+	eventsRemaining bool,
+	err error,
+) {
+	// Retrieve events from the database. Unsuccessfully sent events first
+	eventRows, err := s.selectEventsByApplicationServiceIDStmt.QueryContext(ctx, applicationServiceID)
+	if err != nil {
+		return
+	}
+	defer func() {
+		err = eventRows.Close()
+		if err != nil {
+			log.WithFields(log.Fields{
+				"appservice": applicationServiceID,
+			}).WithError(err).Fatalf("appservice unable to select new events to send")
+		}
+	}()
+	events, maxID, txnID, eventsRemaining, err = retrieveEvents(eventRows, limit)
+	if err != nil {
+		return
+	}
+
+	return
+}
+
+func retrieveEvents(eventRows *sql.Rows, limit int) (events []gomatrixserverlib.Event, maxID, txnID int, eventsRemaining bool, err error) {
+	// Get current time for use in calculating event age
+	nowMilli := time.Now().UnixNano() / int64(time.Millisecond)
+
+	// Iterate through each row and store event contents
+	// If txn_id changes dramatically, we've switched from collecting old events to
+	// new ones. Send back those events first.
+	lastTxnID := invalidTxnID
+	for eventsProcessed := 0; eventRows.Next(); {
+		var event gomatrixserverlib.Event
+		var eventJSON []byte
+		var id int
+		err = eventRows.Scan(
+			&id,
+			&eventJSON,
+			&txnID,
+		)
+		if err != nil {
+			return nil, 0, 0, false, err
+		}
+
+		// Unmarshal eventJSON
+		if err = json.Unmarshal(eventJSON, &event); err != nil {
+			return nil, 0, 0, false, err
+		}
+
+		// If txnID has changed on this event from the previous event, then we've
+		// reached the end of a transaction's events. Return only those events.
+		if lastTxnID > invalidTxnID && lastTxnID != txnID {
+			return events, maxID, lastTxnID, true, nil
+		}
+		lastTxnID = txnID
+
+		// Limit events that aren't part of an old transaction
+		if txnID == -1 {
+			// Return if we've hit the limit
+			if eventsProcessed++; eventsProcessed > limit {
+				return events, maxID, lastTxnID, true, nil
+			}
+		}
+
+		if id > maxID {
+			maxID = id
+		}
+
+		// Portion of the event that is unsigned due to rapid change
+		// TODO: Consider removing age as not many app services use it
+		if err = event.SetUnsignedField("age", nowMilli-int64(event.OriginServerTS())); err != nil {
+			return nil, 0, 0, false, err
+		}
+
+		events = append(events, event)
+	}
+
+	return
+}
+
+// countEventsByApplicationServiceID inserts an event mapped to its corresponding application service
+// IDs into the db.
+func (s *eventsStatements) countEventsByApplicationServiceID(
+	ctx context.Context,
+	appServiceID string,
+) (int, error) {
+	var count int
+	err := s.countEventsByApplicationServiceIDStmt.QueryRowContext(ctx, appServiceID).Scan(&count)
+	if err != nil && err != sql.ErrNoRows {
+		return 0, err
+	}
+
+	return count, nil
+}
+
+// insertEvent inserts an event mapped to its corresponding application service
+// IDs into the db.
+func (s *eventsStatements) insertEvent(
+	ctx context.Context,
+	appServiceID string,
+	event *gomatrixserverlib.Event,
+) (err error) {
+	// Convert event to JSON before inserting
+	eventJSON, err := json.Marshal(event)
+	if err != nil {
+		return err
+	}
+
+	_, err = s.insertEventStmt.ExecContext(
+		ctx,
+		appServiceID,
+		eventJSON,
+		-1, // No transaction ID yet
+	)
+	return
+}
+
+// updateTxnIDForEvents sets the transactionID for a collection of events. Done
+// before sending them to an AppService. Referenced before sending to make sure
+// we aren't constructing multiple transactions with the same events.
+func (s *eventsStatements) updateTxnIDForEvents(
+	ctx context.Context,
+	appserviceID string,
+	maxID, txnID int,
+) (err error) {
+	_, err = s.updateTxnIDForEventsStmt.ExecContext(ctx, txnID, appserviceID, maxID)
+	return
+}
+
+// deleteEventsBeforeAndIncludingID removes events matching given IDs from the database.
+func (s *eventsStatements) deleteEventsBeforeAndIncludingID(
+	ctx context.Context,
+	appserviceID string,
+	eventTableID int,
+) (err error) {
+	_, err = s.deleteEventsBeforeAndIncludingIDStmt.ExecContext(ctx, appserviceID, eventTableID)
+	return
+}

appservice/storage/storage.go

@@ -0,0 +1,110 @@
+// Copyright 2018 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package storage
+
+import (
+	"context"
+	"database/sql"
+
+	// Import postgres database driver
+	_ "github.com/lib/pq"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+// Database stores events intended to be later sent to application services
+type Database struct {
+	events eventsStatements
+	txnID  txnStatements
+	db     *sql.DB
+}
+
+// NewDatabase opens a new database
+func NewDatabase(dataSourceName string) (*Database, error) {
+	var result Database
+	var err error
+	if result.db, err = sql.Open("postgres", dataSourceName); err != nil {
+		return nil, err
+	}
+	if err = result.prepare(); err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+func (d *Database) prepare() error {
+	if err := d.events.prepare(d.db); err != nil {
+		return err
+	}
+
+	return d.txnID.prepare(d.db)
+}
+
+// StoreEvent takes in a gomatrixserverlib.Event and stores it in the database
+// for a transaction worker to pull and later send to an application service.
+func (d *Database) StoreEvent(
+	ctx context.Context,
+	appServiceID string,
+	event *gomatrixserverlib.Event,
+) error {
+	return d.events.insertEvent(ctx, appServiceID, event)
+}
+
+// GetEventsWithAppServiceID returns a slice of events and their IDs intended to
+// be sent to an application service given its ID.
+func (d *Database) GetEventsWithAppServiceID(
+	ctx context.Context,
+	appServiceID string,
+	limit int,
+) (int, int, []gomatrixserverlib.Event, bool, error) {
+	return d.events.selectEventsByApplicationServiceID(ctx, appServiceID, limit)
+}
+
+// CountEventsWithAppServiceID returns the number of events destined for an
+// application service given its ID.
+func (d *Database) CountEventsWithAppServiceID(
+	ctx context.Context,
+	appServiceID string,
+) (int, error) {
+	return d.events.countEventsByApplicationServiceID(ctx, appServiceID)
+}
+
+// UpdateTxnIDForEvents takes in an application service ID and a
+// and stores them in the DB, unless the pair already exists, in
+// which case it updates them.
+func (d *Database) UpdateTxnIDForEvents(
+	ctx context.Context,
+	appserviceID string,
+	maxID, txnID int,
+) error {
+	return d.events.updateTxnIDForEvents(ctx, appserviceID, maxID, txnID)
+}
+
+// RemoveEventsBeforeAndIncludingID removes all events from the database that
+// are less than or equal to a given maximum ID. IDs here are implemented as a
+// serial, thus this should always delete events in chronological order.
+func (d *Database) RemoveEventsBeforeAndIncludingID(
+	ctx context.Context,
+	appserviceID string,
+	eventTableID int,
+) error {
+	return d.events.deleteEventsBeforeAndIncludingID(ctx, appserviceID, eventTableID)
+}
+
+// GetLatestTxnID returns the latest available transaction id
+func (d *Database) GetLatestTxnID(
+	ctx context.Context,
+) (int, error) {
+	return d.txnID.selectTxnID(ctx)
+}

appservice/storage/txn_id_counter_table.go

@@ -0,0 +1,52 @@
+// Copyright 2018 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package storage
+
+import (
+	"context"
+	"database/sql"
+)
+
+const txnIDSchema = `
+-- Keeps a count of the current transaction ID
+CREATE SEQUENCE IF NOT EXISTS txn_id_counter START 1;
+`
+
+const selectTxnIDSQL = "SELECT nextval('txn_id_counter')"
+
+type txnStatements struct {
+	selectTxnIDStmt *sql.Stmt
+}
+
+func (s *txnStatements) prepare(db *sql.DB) (err error) {
+	_, err = db.Exec(txnIDSchema)
+	if err != nil {
+		return
+	}
+
+	if s.selectTxnIDStmt, err = db.Prepare(selectTxnIDSQL); err != nil {
+		return
+	}
+
+	return
+}
+
+// selectTxnID selects the latest ascending transaction ID
+func (s *txnStatements) selectTxnID(
+	ctx context.Context,
+) (txnID int, err error) {
+	err = s.selectTxnIDStmt.QueryRowContext(ctx).Scan(&txnID)
+	return
+}

appservice/types/types.go

@@ -0,0 +1,64 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package types
+
+import (
+	"sync"
+
+	"github.com/matrix-org/dendrite/common/config"
+)
+
+const (
+	// AppServiceDeviceID is the AS dummy device ID
+	AppServiceDeviceID = "AS_Device"
+)
+
+// ApplicationServiceWorkerState is a type that couples an application service,
+// a lockable condition as well as some other state variables, allowing the
+// roomserver to notify appservice workers when there are events ready to send
+// externally to application services.
+type ApplicationServiceWorkerState struct {
+	AppService config.ApplicationService
+	Cond       *sync.Cond
+	// Events ready to be sent
+	EventsReady bool
+	// Backoff exponent (2^x secs). Max 6, aka 64s.
+	Backoff int
+}
+
+// NotifyNewEvents wakes up all waiting goroutines, notifying that events remain
+// in the event queue for this application service worker.
+func (a *ApplicationServiceWorkerState) NotifyNewEvents() {
+	a.Cond.L.Lock()
+	a.EventsReady = true
+	a.Cond.Broadcast()
+	a.Cond.L.Unlock()
+}
+
+// FinishEventProcessing marks all events of this worker as being sent to the
+// application service.
+func (a *ApplicationServiceWorkerState) FinishEventProcessing() {
+	a.Cond.L.Lock()
+	a.EventsReady = false
+	a.Cond.L.Unlock()
+}
+
+// WaitForNewEvents causes the calling goroutine to wait on the worker state's
+// condition for a broadcast or similar wakeup, if there are no events ready.
+func (a *ApplicationServiceWorkerState) WaitForNewEvents() {
+	a.Cond.L.Lock()
+	if !a.EventsReady {
+		a.Cond.Wait()
+	}
+	a.Cond.L.Unlock()
+}

appservice/workers/transaction_scheduler.go

@@ -0,0 +1,227 @@
+// Copyright 2018 Vector Creations Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package workers
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"math"
+	"net/http"
+	"time"
+
+	"github.com/matrix-org/dendrite/appservice/storage"
+	"github.com/matrix-org/dendrite/appservice/types"
+	"github.com/matrix-org/dendrite/common/config"
+	"github.com/matrix-org/gomatrixserverlib"
+	log "github.com/sirupsen/logrus"
+)
+
+var (
+	// Maximum size of events sent in each transaction.
+	transactionBatchSize = 50
+	// Timeout for sending a single transaction to an application service.
+	transactionTimeout = time.Second * 60
+)
+
+// SetupTransactionWorkers spawns a separate goroutine for each application
+// service. Each of these "workers" handle taking all events intended for their
+// app service, batch them up into a single transaction (up to a max transaction
+// size), then send that off to the AS's /transactions/{txnID} endpoint. It also
+// handles exponentially backing off in case the AS isn't currently available.
+func SetupTransactionWorkers(
+	appserviceDB *storage.Database,
+	workerStates []types.ApplicationServiceWorkerState,
+) error {
+	// Create a worker that handles transmitting events to a single homeserver
+	for _, workerState := range workerStates {
+		// Don't create a worker if this AS doesn't want to receive events
+		if workerState.AppService.URL != "" {
+			go worker(appserviceDB, workerState)
+		}
+	}
+	return nil
+}
+
+// worker is a goroutine that sends any queued events to the application service
+// it is given.
+func worker(db *storage.Database, ws types.ApplicationServiceWorkerState) {
+	log.WithFields(log.Fields{
+		"appservice": ws.AppService.ID,
+	}).Info("starting application service")
+	ctx := context.Background()
+
+	// Create a HTTP client for sending requests to app services
+	client := &http.Client{
+		Timeout: transactionTimeout,
+	}
+
+	// Initial check for any leftover events to send from last time
+	eventCount, err := db.CountEventsWithAppServiceID(ctx, ws.AppService.ID)
+	if err != nil {
+		log.WithFields(log.Fields{
+			"appservice": ws.AppService.ID,
+		}).WithError(err).Fatal("appservice worker unable to read queued events from DB")
+		return
+	}
+	if eventCount > 0 {
+		ws.NotifyNewEvents()
+	}
+
+	// Loop forever and keep waiting for more events to send
+	for {
+		// Wait for more events if we've sent all the events in the database
+		ws.WaitForNewEvents()
+
+		// Batch events up into a transaction
+		transactionJSON, txnID, maxEventID, eventsRemaining, err := createTransaction(ctx, db, ws.AppService.ID)
+		if err != nil {
+			log.WithFields(log.Fields{
+				"appservice": ws.AppService.ID,
+			}).WithError(err).Fatal("appservice worker unable to create transaction")
+
+			return
+		}
+
+		// Send the events off to the application service
+		// Backoff if the application service does not respond
+		err = send(client, ws.AppService, txnID, transactionJSON)
+		if err != nil {
+			// Backoff
+			backoff(&ws, err)
+			continue
+		}
+
+		// We sent successfully, hooray!
+		ws.Backoff = 0
+
+		// Transactions have a maximum event size, so there may still be some events
+		// left over to send. Keep sending until none are left
+		if !eventsRemaining {
+			ws.FinishEventProcessing()
+		}
+
+		// Remove sent events from the DB
+		err = db.RemoveEventsBeforeAndIncludingID(ctx, ws.AppService.ID, maxEventID)
+		if err != nil {
+			log.WithFields(log.Fields{
+				"appservice": ws.AppService.ID,
+			}).WithError(err).Fatal("unable to remove appservice events from the database")
+			return
+		}
+	}
+}
+
+// backoff pauses the calling goroutine for a 2^some backoff exponent seconds
+func backoff(ws *types.ApplicationServiceWorkerState, err error) {
+	// Calculate how long to backoff for
+	backoffDuration := time.Duration(math.Pow(2, float64(ws.Backoff)))
+	backoffSeconds := time.Second * backoffDuration
+
+	log.WithFields(log.Fields{
+		"appservice": ws.AppService.ID,
+	}).WithError(err).Warnf("unable to send transactions successfully, backing off for %ds",
+		backoffDuration)
+
+	ws.Backoff++
+	if ws.Backoff > 6 {
+		ws.Backoff = 6
+	}
+
+	// Backoff
+	time.Sleep(backoffSeconds)
+}
+
+// createTransaction takes in a slice of AS events, stores them in an AS
+// transaction, and JSON-encodes the results.
+func createTransaction(
+	ctx context.Context,
+	db *storage.Database,
+	appserviceID string,
+) (
+	transactionJSON []byte,
+	txnID, maxID int,
+	eventsRemaining bool,
+	err error,
+) {
+	// Retrieve the latest events from the DB (will return old events if they weren't successfully sent)
+	txnID, maxID, events, eventsRemaining, err := db.GetEventsWithAppServiceID(ctx, appserviceID, transactionBatchSize)
+	if err != nil {
+		log.WithFields(log.Fields{
+			"appservice": appserviceID,
+		}).WithError(err).Fatalf("appservice worker unable to read queued events from DB")
+
+		return
+	}
+
+	// Check if these events do not already have a transaction ID
+	if txnID == -1 {
+		// If not, grab next available ID from the DB
+		txnID, err = db.GetLatestTxnID(ctx)
+		if err != nil {
+			return nil, 0, 0, false, err
+		}
+
+		// Mark new events with current transactionID
+		if err = db.UpdateTxnIDForEvents(ctx, appserviceID, maxID, txnID); err != nil {
+			return nil, 0, 0, false, err
+		}
+	}
+
+	// Create a transaction and store the events inside
+	transaction := gomatrixserverlib.ApplicationServiceTransaction{
+		Events: events,
+	}
+
+	transactionJSON, err = json.Marshal(transaction)
+	if err != nil {
+		return
+	}
+
+	return
+}
+
+// send sends events to an application service. Returns an error if an OK was not
+// received back from the application service or the request timed out.
+func send(
+	client *http.Client,
+	appservice config.ApplicationService,
+	txnID int,
+	transaction []byte,
+) error {
+	// POST a transaction to our AS
+	address := fmt.Sprintf("%s/transactions/%d", appservice.URL, txnID)
+	resp, err := client.Post(address, "application/json", bytes.NewBuffer(transaction))
+	if err != nil {
+		return err
+	}
+	defer func() {
+		err := resp.Body.Close()
+		if err != nil {
+			log.WithFields(log.Fields{
+				"appservice": appservice.ID,
+			}).WithError(err).Error("unable to close response body from application service")
+		}
+	}()
+
+	// Check the AS received the events correctly
+	if resp.StatusCode != http.StatusOK {
+		// TODO: Handle non-200 error codes from application services
+		return fmt.Errorf("non-OK status code %d returned from AS", resp.StatusCode)
+	}
+
+	return nil
+}

build.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+GOBIN=$PWD/`dirname $0`/bin go install -v ./cmd/...

clientapi/auth/auth.go

@@ -24,6 +24,7 @@ import (
 	"net/http"
 	"strings"
 
+	"github.com/matrix-org/dendrite/appservice/types"
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
@@ -48,26 +49,26 @@ type AccountDatabase interface {
 	GetAccountByLocalpart(ctx context.Context, localpart string) (*authtypes.Account, error)
 }
 
+// Data contains information required to authenticate a request.
+type Data struct {
+	AccountDB AccountDatabase
+	DeviceDB  DeviceDatabase
+	// AppServices is the list of all registered AS
+	AppServices []config.ApplicationService
+}
+
 // VerifyUserFromRequest authenticates the HTTP request,
-// on success returns UserID of the requester.
+// on success returns Device of the requester.
 // Finds local user or an application service user.
+// Note: For an AS user, AS dummy device is returned.
 // On failure returns an JSON error response which can be sent to the client.
 func VerifyUserFromRequest(
-	req *http.Request, accountDB AccountDatabase, deviceDB DeviceDatabase,
+	req *http.Request, data Data,
-	applicationServices []config.ApplicationService,
+) (*authtypes.Device, *util.JSONResponse) {
-) (string, *util.JSONResponse) {
-	// Try to find local user from device database
-	dev, devErr := VerifyAccessToken(req, deviceDB)
-
-	if devErr == nil {
-		return dev.UserID, nil
-	}
-
 	// Try to find the Application Service user
-	token, err := extractAccessToken(req)
+	token, err := ExtractAccessToken(req)
-
 	if err != nil {
-		return "", &util.JSONResponse{
+		return nil, &util.JSONResponse{
 			Code: http.StatusUnauthorized,
 			JSON: jsonerror.MissingToken(err.Error()),
 		}
@@ -75,7 +76,7 @@ func VerifyUserFromRequest(
 
 	// Search for app service with given access_token
 	var appService *config.ApplicationService
-	for _, as := range applicationServices {
+	for _, as := range data.AppServices {
 		if as.ASToken == token {
 			appService = &as
 			break
@@ -83,41 +84,73 @@ func VerifyUserFromRequest(
 	}
 
 	if appService != nil {
+		// Create a dummy device for AS user
+		dev := authtypes.Device{
+			// Use AS dummy device ID
+			ID: types.AppServiceDeviceID,
+			// AS dummy device has AS's token.
+			AccessToken: token,
+		}
+
 		userID := req.URL.Query().Get("user_id")
 		localpart, err := userutil.ParseUsernameParam(userID, nil)
-
 		if err != nil {
-			return "", &util.JSONResponse{
+			return nil, &util.JSONResponse{
 				Code: http.StatusBadRequest,
 				JSON: jsonerror.InvalidUsername(err.Error()),
 			}
 		}
 
+		if localpart != "" { // AS is masquerading as another user
 			// Verify that the user is registered
-		account, accountErr := accountDB.GetAccountByLocalpart(req.Context(), localpart)
+			account, err := data.AccountDB.GetAccountByLocalpart(req.Context(), localpart)
-
 			// Verify that account exists & appServiceID matches
-		if accountErr == nil && account.AppServiceID == appService.ID {
+			if err == nil && account.AppServiceID == appService.ID {
-			return userID, nil
+				// Set the userID of dummy device
+				dev.UserID = userID
+				return &dev, nil
 			}
 
-		return "", &util.JSONResponse{
+			return nil, &util.JSONResponse{
 				Code: http.StatusForbidden,
 				JSON: jsonerror.Forbidden("Application service has not registered this user"),
 			}
 		}
 
-	return "", &util.JSONResponse{
+		// AS is not masquerading as any user, so use AS's sender_localpart
+		dev.UserID = appService.SenderLocalpart
+		return &dev, nil
+	}
+
+	// Try to find local user from device database
+	dev, devErr := verifyAccessToken(req, data.DeviceDB)
+	if devErr == nil {
+		return dev, verifyUserParameters(req)
+	}
+
+	return nil, &util.JSONResponse{
 		Code: http.StatusUnauthorized,
-		JSON: jsonerror.UnknownToken("Unrecognized access token"),
+		JSON: jsonerror.UnknownToken("Unrecognized access token"), // nolint: misspell
 	}
 }
 
-// VerifyAccessToken verifies that an access token was supplied in the given HTTP request
+// verifyUserParameters ensures that a request coming from a regular user is not
+// using any query parameters reserved for an application service
+func verifyUserParameters(req *http.Request) *util.JSONResponse {
+	if req.URL.Query().Get("ts") != "" {
+		return &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.Unknown("parameter 'ts' not allowed without valid parameter 'access_token'"),
+		}
+	}
+	return nil
+}
+
+// verifyAccessToken verifies that an access token was supplied in the given HTTP request
 // and returns the device it corresponds to. Returns resErr (an error response which can be
 // sent to the client) if the token is invalid or there was a problem querying the database.
-func VerifyAccessToken(req *http.Request, deviceDB DeviceDatabase) (device *authtypes.Device, resErr *util.JSONResponse) {
+func verifyAccessToken(req *http.Request, deviceDB DeviceDatabase) (device *authtypes.Device, resErr *util.JSONResponse) {
-	token, err := extractAccessToken(req)
+	token, err := ExtractAccessToken(req)
 	if err != nil {
 		resErr = &util.JSONResponse{
 			Code: http.StatusUnauthorized,
@@ -151,9 +184,9 @@ func GenerateAccessToken() (string, error) {
 	return base64.RawURLEncoding.EncodeToString(b), nil
 }
 
-// extractAccessToken from a request, or return an error detailing what went wrong. The
+// ExtractAccessToken from a request, or return an error detailing what went wrong. The
 // error message MUST be human-readable and comprehensible to the client.
-func extractAccessToken(req *http.Request) (string, error) {
+func ExtractAccessToken(req *http.Request) (string, error) {
 	// cf https://github.com/matrix-org/synapse/blob/v0.19.2/synapse/api/auth.py#L631
 	authBearer := req.Header.Get("Authorization")
 	queryToken := req.URL.Query().Get("access_token")

clientapi/auth/storage/accounts/accounts_table.go

@@ -22,6 +22,8 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/userutil"
 	"github.com/matrix-org/gomatrixserverlib"
+
+	log "github.com/sirupsen/logrus"
 )
 
 const accountsSchema = `
@@ -33,7 +35,7 @@ CREATE TABLE IF NOT EXISTS account_accounts (
     created_ts BIGINT NOT NULL,
     -- The password hash for this account. Can be NULL if this is a passwordless account.
     password_hash TEXT,
-    -- Identifies which Application Service this account belongs to, if any.
+    -- Identifies which application service this account belongs to, if any.
     appservice_id TEXT
     -- TODO:
     -- is_guest, is_admin, upgraded_ts, devices, any email reset stuff?
@@ -121,14 +123,26 @@ func (s *accountsStatements) selectPasswordHash(
 
 func (s *accountsStatements) selectAccountByLocalpart(
 	ctx context.Context, localpart string,
-) (acc *authtypes.Account, err error) {
+) (*authtypes.Account, error) {
+	var appserviceIDPtr sql.NullString
+	var acc authtypes.Account
+
 	stmt := s.selectAccountByLocalpartStmt
-	err = stmt.QueryRowContext(ctx, localpart).Scan(&acc.Localpart, &acc.AppServiceID)
+	err := stmt.QueryRowContext(ctx, localpart).Scan(&acc.Localpart, &appserviceIDPtr)
-	if err == nil {
+	if err != nil {
+		if err != sql.ErrNoRows {
+			log.WithError(err).Error("Unable to retrieve user from the db")
+		}
+		return nil, err
+	}
+	if appserviceIDPtr.Valid {
+		acc.AppServiceID = appserviceIDPtr.String
+	}
+
 	acc.UserID = userutil.MakeUserID(localpart, s.serverName)
 	acc.ServerName = s.serverName
-	}
+
-	return
+	return &acc, nil
 }
 
 func (s *accountsStatements) selectNewNumericLocalpart(

clientapi/auth/storage/accounts/membership_table.go

@@ -48,12 +48,16 @@ const insertMembershipSQL = `
 const selectMembershipsByLocalpartSQL = "" +
 	"SELECT room_id, event_id FROM account_memberships WHERE localpart = $1"
 
+const selectMembershipInRoomByLocalpartSQL = "" +
+	"SELECT event_id FROM account_memberships WHERE localpart = $1 AND room_id = $2"
+
 const deleteMembershipsByEventIDsSQL = "" +
 	"DELETE FROM account_memberships WHERE event_id = ANY($1)"
 
 type membershipStatements struct {
 	deleteMembershipsByEventIDsStmt       *sql.Stmt
 	insertMembershipStmt                  *sql.Stmt
+	selectMembershipInRoomByLocalpartStmt *sql.Stmt
 	selectMembershipsByLocalpartStmt      *sql.Stmt
 }
 
@@ -68,6 +72,9 @@ func (s *membershipStatements) prepare(db *sql.DB) (err error) {
 	if s.insertMembershipStmt, err = db.Prepare(insertMembershipSQL); err != nil {
 		return
 	}
+	if s.selectMembershipInRoomByLocalpartStmt, err = db.Prepare(selectMembershipInRoomByLocalpartSQL); err != nil {
+		return
+	}
 	if s.selectMembershipsByLocalpartStmt, err = db.Prepare(selectMembershipsByLocalpartSQL); err != nil {
 		return
 	}
@@ -90,6 +97,16 @@ func (s *membershipStatements) deleteMembershipsByEventIDs(
 	return
 }
 
+func (s *membershipStatements) selectMembershipInRoomByLocalpart(
+	ctx context.Context, localpart, roomID string,
+) (authtypes.Membership, error) {
+	membership := authtypes.Membership{Localpart: localpart, RoomID: roomID}
+	stmt := s.selectMembershipInRoomByLocalpartStmt
+	err := stmt.QueryRowContext(ctx, localpart, roomID).Scan(&membership.EventID)
+
+	return membership, err
+}
+
 func (s *membershipStatements) selectMembershipsByLocalpart(
 	ctx context.Context, localpart string,
 ) (memberships []authtypes.Membership, err error) {

clientapi/auth/storage/accounts/storage.go

@@ -23,6 +23,7 @@ import (
 	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/gomatrixserverlib"
 	"golang.org/x/crypto/bcrypt"
+
 	// Import the postgres database driver.
 	_ "github.com/lib/pq"
 )
@@ -143,9 +144,8 @@ func (d *Database) CreateAccount(
 }
 
 // SaveMembership saves the user matching a given localpart as a member of a given
-// room. It also stores the ID of the membership event and a flag on whether the user
+// room. It also stores the ID of the membership event.
-// is still in the room.
+// If a membership already exists between the user and the room, or if the
-// If a membership already exists between the user and the room, or of the
 // insert fails, returns the SQL error
 func (d *Database) saveMembership(
 	ctx context.Context, txn *sql.Tx, localpart, roomID, eventID string,
@@ -153,8 +153,8 @@ func (d *Database) saveMembership(
 	return d.memberships.insertMembership(ctx, txn, localpart, roomID, eventID)
 }
 
-// removeMembershipsByEventIDs removes the memberships of which the `join` membership
+// removeMembershipsByEventIDs removes the memberships corresponding to the
-// event ID is included in a given array of events IDs
+// `join` membership events IDs in the eventIDs slice.
 // If the removal fails, or if there is no membership to remove, returns an error
 func (d *Database) removeMembershipsByEventIDs(
 	ctx context.Context, txn *sql.Tx, eventIDs []string,
@@ -185,6 +185,16 @@ func (d *Database) UpdateMemberships(
 	})
 }
 
+// GetMembershipInRoomByLocalpart returns the membership for an user
+// matching the given localpart if he is a member of the room matching roomID,
+// if not sql.ErrNoRows is returned.
+// If there was an issue during the retrieval, returns the SQL error
+func (d *Database) GetMembershipInRoomByLocalpart(
+	ctx context.Context, localpart, roomID string,
+) (authtypes.Membership, error) {
+	return d.memberships.selectMembershipInRoomByLocalpart(ctx, localpart, roomID)
+}
+
 // GetMembershipsByLocalpart returns an array containing the memberships for all
 // the rooms a user matching a given localpart is a member of
 // If no membership match the given localpart, returns an empty array
@@ -195,13 +205,9 @@ func (d *Database) GetMembershipsByLocalpart(
 	return d.memberships.selectMembershipsByLocalpart(ctx, localpart)
 }
 
-// newMembership will save a new membership in the database, with a flag on whether
+// newMembership saves a new membership in the database.
-// the user is still in the room. This flag is set to true if the given state
+// If the event isn't a valid m.room.member event with type `join`, does nothing.
-// event is a "join" membership event and false if the event is a "leave" or "ban"
+// If an error occurred, returns the SQL error
-// membership. If the event isn't a m.room.member event with one of these three
-// values, does nothing.
-// If the event isn't a "join" membership event, does nothing
-// If an error occurred, returns it
 func (d *Database) newMembership(
 	ctx context.Context, txn *sql.Tx, ev gomatrixserverlib.Event,
 ) error {

clientapi/auth/storage/devices/devices_table.go

@@ -169,6 +169,8 @@ func (s *devicesStatements) selectDeviceByToken(
 	return &dev, err
 }
 
+// selectDeviceByID retrieves a device from the database with the given user
+// localpart and deviceID
 func (s *devicesStatements) selectDeviceByID(
 	ctx context.Context, localpart, deviceID string,
 ) (*authtypes.Device, error) {

clientapi/auth/storage/devices/storage.go

@@ -84,7 +84,7 @@ func (d *Database) CreateDevice(
 	if deviceID != nil {
 		returnErr = common.WithTransaction(d.db, func(txn *sql.Tx) error {
 			var err error
-			// Revoke existing token for this device
+			// Revoke existing tokens for this device
 			if err = d.devices.deleteDevice(ctx, txn, *deviceID, localpart); err != nil {
 				return err
 			}
@@ -138,9 +138,9 @@ func (d *Database) UpdateDevice(
 }
 
 // RemoveDevice revokes a device by deleting the entry in the database
-// matching with the given device ID and user ID localpart
+// matching with the given device ID and user ID localpart.
 // If the device doesn't exist, it will not return an error
-// If something went wrong during the deletion, it will return the SQL error
+// If something went wrong during the deletion, it will return the SQL error.
 func (d *Database) RemoveDevice(
 	ctx context.Context, deviceID, localpart string,
 ) error {

clientapi/clientapi.go

@@ -15,6 +15,7 @@
 package clientapi
 
 import (
+	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
 	"github.com/matrix-org/dendrite/clientapi/consumers"
@@ -22,7 +23,8 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/routing"
 	"github.com/matrix-org/dendrite/common/basecomponent"
 	"github.com/matrix-org/dendrite/common/transactions"
-	"github.com/matrix-org/dendrite/roomserver/api"
+	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	typingServerAPI "github.com/matrix-org/dendrite/typingserver/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/sirupsen/logrus"
 )
@@ -35,12 +37,15 @@ func SetupClientAPIComponent(
 	accountsDB *accounts.Database,
 	federation *gomatrixserverlib.FederationClient,
 	keyRing *gomatrixserverlib.KeyRing,
-	aliasAPI api.RoomserverAliasAPI,
+	aliasAPI roomserverAPI.RoomserverAliasAPI,
-	inputAPI api.RoomserverInputAPI,
+	inputAPI roomserverAPI.RoomserverInputAPI,
-	queryAPI api.RoomserverQueryAPI,
+	queryAPI roomserverAPI.RoomserverQueryAPI,
+	typingInputAPI typingServerAPI.TypingServerInputAPI,
+	asAPI appserviceAPI.AppServiceQueryAPI,
 	transactionsCache *transactions.Cache,
 ) {
 	roomserverProducer := producers.NewRoomserverProducer(inputAPI)
+	typingProducer := producers.NewTypingServerProducer(typingInputAPI)
 
 	userUpdateProducer := &producers.UserUpdateProducer{
 		Producer: base.KafkaProducer,
@@ -60,10 +65,8 @@ func SetupClientAPIComponent(
 	}
 
 	routing.Setup(
-		base.APIMux, *base.Cfg, roomserverProducer,
+		base.APIMux, *base.Cfg, roomserverProducer, queryAPI, aliasAPI, asAPI,
-		queryAPI, aliasAPI, accountsDB, deviceDB,
+		accountsDB, deviceDB, federation, *keyRing, userUpdateProducer,
-		federation, *keyRing,
+		syncProducer, typingProducer, transactionsCache,
-		userUpdateProducer, syncProducer,
-		transactionsCache,
 	)
 }

clientapi/httputil/httputil.go

@@ -25,7 +25,6 @@ import (
 // UnmarshalJSONRequest into the given interface pointer. Returns an error JSON response if
 // there was a problem unmarshalling. Calling this function consumes the request body.
 func UnmarshalJSONRequest(req *http.Request, iface interface{}) *util.JSONResponse {
-	defer req.Body.Close() // nolint: errcheck
 	if err := json.NewDecoder(req.Body).Decode(iface); err != nil {
 		// TODO: We may want to suppress the Error() return in production? It's useful when
 		// debugging because an error will be produced for both invalid/malformed JSON AND

clientapi/httputil/parse.go

@@ -0,0 +1,39 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package httputil
+
+import (
+	"fmt"
+	"net/http"
+	"strconv"
+	"time"
+)
+
+// ParseTSParam takes a req (typically from an application service) and parses a Time object
+// from the req if it exists in the query parameters. If it doesn't exist, the
+// current time is returned.
+func ParseTSParam(req *http.Request) (time.Time, error) {
+	// Use the ts parameter's value for event time if present
+	tsStr := req.URL.Query().Get("ts")
+	if tsStr == "" {
+		return time.Now(), nil
+	}
+
+	// The parameter exists, parse into a Time object
+	ts, err := strconv.ParseInt(tsStr, 10, 64)
+	if err != nil {
+		return time.Time{}, fmt.Errorf("Param 'ts' is no valid int (%s)", err.Error())
+	}
+
+	return time.Unix(ts/1000, 0), nil
+}

clientapi/jsonerror/jsonerror.go

@@ -28,7 +28,7 @@ type MatrixError struct {
 	Err     string `json:"error"`
 }
 
-func (e *MatrixError) Error() string {
+func (e MatrixError) Error() string {
 	return fmt.Sprintf("%s: %s", e.ErrCode, e.Err)
 }
 
@@ -87,7 +87,7 @@ func MissingToken(msg string) *MatrixError {
 }
 
 // UnknownToken is an error when the client tries to access a resource which
-// requires authentication and supplies an unrecognized token
+// requires authentication and supplies an unrecognised token
 func UnknownToken(msg string) *MatrixError {
 	return &MatrixError{"M_UNKNOWN_TOKEN", msg}
 }
@@ -112,7 +112,8 @@ func UserInUse(msg string) *MatrixError {
 
 // ASExclusive is an error returned when an application service tries to
 // register an username that is outside of its registered namespace, or if a
-// user attempts to register a username within an exclusive namespace
+// user attempts to register a username or room alias within an exclusive
+// namespace.
 func ASExclusive(msg string) *MatrixError {
 	return &MatrixError{"M_EXCLUSIVE", msg}
 }

clientapi/producers/typingserver.go

@@ -0,0 +1,54 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package producers
+
+import (
+	"context"
+	"time"
+
+	"github.com/matrix-org/dendrite/typingserver/api"
+	"github.com/matrix-org/gomatrixserverlib"
+)
+
+// TypingServerProducer produces events for the typing server to consume
+type TypingServerProducer struct {
+	InputAPI api.TypingServerInputAPI
+}
+
+// NewTypingServerProducer creates a new TypingServerProducer
+func NewTypingServerProducer(inputAPI api.TypingServerInputAPI) *TypingServerProducer {
+	return &TypingServerProducer{
+		InputAPI: inputAPI,
+	}
+}
+
+// Send typing event to typing server
+func (p *TypingServerProducer) Send(
+	ctx context.Context, userID, roomID string,
+	typing bool, timeout int64,
+) error {
+	requestData := api.InputTypingEvent{
+		UserID:         userID,
+		RoomID:         roomID,
+		Typing:         typing,
+		Timeout:        timeout,
+		OriginServerTS: gomatrixserverlib.AsTimestamp(time.Now()),
+	}
+
+	var response api.InputTypingEventResponse
+	err := p.InputAPI.InputTypingEvent(
+		ctx, &api.InputTypingEventRequest{InputTypingEvent: requestData}, &response,
+	)
+
+	return err
+}

clientapi/routing/account_data.go

@@ -33,13 +33,6 @@ func SaveAccountData(
 	req *http.Request, accountDB *accounts.Database, device *authtypes.Device,
 	userID string, roomID string, dataType string, syncProducer *producers.SyncAPIProducer,
 ) util.JSONResponse {
-	if req.Method != http.MethodPut {
-		return util.JSONResponse{
-			Code: http.StatusMethodNotAllowed,
-			JSON: jsonerror.NotFound("Bad method"),
-		}
-	}
-
 	if userID != device.UserID {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,

clientapi/routing/createroom.go

@@ -20,7 +20,8 @@ import (
 	"strings"
 	"time"
 
-	"github.com/matrix-org/dendrite/roomserver/api"
+	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
+	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
@@ -71,7 +72,7 @@ func (r createRoomRequest) Validate() *util.JSONResponse {
 	if strings.ContainsAny(r.RoomAliasName, whitespace+":") {
 		return &util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON("room_alias_name cannot contain whitespace"),
+			JSON: jsonerror.BadJSON("room_alias_name cannot contain whitespace or ':'"),
 		}
 	}
 	for _, userID := range r.Invite {
@@ -88,8 +89,7 @@ func (r createRoomRequest) Validate() *util.JSONResponse {
 		}
 	}
 	switch r.Preset {
-	case presetPrivateChat, presetTrustedPrivateChat, presetPublicChat:
+	case presetPrivateChat, presetTrustedPrivateChat, presetPublicChat, "":
-		break
 	default:
 		return &util.JSONResponse{
 			Code: http.StatusBadRequest,
@@ -114,21 +114,25 @@ type fledglingEvent struct {
 }
 
 // CreateRoom implements /createRoom
-func CreateRoom(req *http.Request, device *authtypes.Device,
+func CreateRoom(
+	req *http.Request, device *authtypes.Device,
 	cfg config.Dendrite, producer *producers.RoomserverProducer,
-	accountDB *accounts.Database, aliasAPI api.RoomserverAliasAPI,
+	accountDB *accounts.Database, aliasAPI roomserverAPI.RoomserverAliasAPI,
+	asAPI appserviceAPI.AppServiceQueryAPI,
 ) util.JSONResponse {
 	// TODO (#267): Check room ID doesn't clash with an existing one, and we
 	//              probably shouldn't be using pseudo-random strings, maybe GUIDs?
 	roomID := fmt.Sprintf("!%s:%s", util.RandomString(16), cfg.Matrix.ServerName)
-	return createRoom(req, device, cfg, roomID, producer, accountDB, aliasAPI)
+	return createRoom(req, device, cfg, roomID, producer, accountDB, aliasAPI, asAPI)
 }
 
 // createRoom implements /createRoom
 // nolint: gocyclo
-func createRoom(req *http.Request, device *authtypes.Device,
+func createRoom(
+	req *http.Request, device *authtypes.Device,
 	cfg config.Dendrite, roomID string, producer *producers.RoomserverProducer,
-	accountDB *accounts.Database, aliasAPI api.RoomserverAliasAPI,
+	accountDB *accounts.Database, aliasAPI roomserverAPI.RoomserverAliasAPI,
+	asAPI appserviceAPI.AppServiceQueryAPI,
 ) util.JSONResponse {
 	logger := util.GetLogger(req.Context())
 	userID := device.UserID
@@ -143,21 +147,23 @@ func createRoom(req *http.Request, device *authtypes.Device,
 		return *resErr
 	}
 
+	evTime, err := httputil.ParseTSParam(req)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.InvalidArgumentValue(err.Error()),
+		}
+	}
 	// TODO: visibility/presets/raw initial state/creation content
-
 	// TODO: Create room alias association
+	// Make sure this doesn't fall into an application service's namespace though!
 
 	logger.WithFields(log.Fields{
 		"userID": userID,
 		"roomID": roomID,
 	}).Info("Creating new room")
 
-	localpart, _, err := gomatrixserverlib.SplitID('@', userID)
+	profile, err := appserviceAPI.RetrieveUserProfile(req.Context(), userID, asAPI, accountDB)
-	if err != nil {
-		return httputil.LogThenError(req, err)
-	}
-
-	profile, err := accountDB.GetProfileByLocalpart(req.Context(), localpart)
 	if err != nil {
 		return httputil.LogThenError(req, err)
 	}
@@ -180,6 +186,11 @@ func createRoom(req *http.Request, device *authtypes.Device,
 	case presetPublicChat:
 		joinRules = joinRulePublic
 		historyVisibility = historyVisibilityShared
+	default:
+		// Default room rules, r.Preset was previously checked for valid values so
+		// only a request with no preset should end up here.
+		joinRules = joinRuleInvite
+		historyVisibility = historyVisibilityShared
 	}
 
 	var builtEvents []gomatrixserverlib.Event
@@ -243,7 +254,7 @@ func createRoom(req *http.Request, device *authtypes.Device,
 			builder.PrevEvents = []gomatrixserverlib.EventReference{builtEvents[i-1].EventReference()}
 		}
 		var ev *gomatrixserverlib.Event
-		ev, err = buildEvent(&builder, &authEvents, cfg)
+		ev, err = buildEvent(&builder, &authEvents, cfg, evTime)
 		if err != nil {
 			return httputil.LogThenError(req, err)
 		}
@@ -273,13 +284,13 @@ func createRoom(req *http.Request, device *authtypes.Device,
 	if r.RoomAliasName != "" {
 		roomAlias = fmt.Sprintf("#%s:%s", r.RoomAliasName, cfg.Matrix.ServerName)
 
-		aliasReq := api.SetRoomAliasRequest{
+		aliasReq := roomserverAPI.SetRoomAliasRequest{
 			Alias:  roomAlias,
 			RoomID: roomID,
 			UserID: userID,
 		}
 
-		var aliasResp api.SetRoomAliasResponse
+		var aliasResp roomserverAPI.SetRoomAliasResponse
 		err = aliasAPI.SetRoomAlias(req.Context(), &aliasReq, &aliasResp)
 		if err != nil {
 			return httputil.LogThenError(req, err)
@@ -302,10 +313,12 @@ func createRoom(req *http.Request, device *authtypes.Device,
 }
 
 // buildEvent fills out auth_events for the builder then builds the event
-func buildEvent(builder *gomatrixserverlib.EventBuilder,
+func buildEvent(
+	builder *gomatrixserverlib.EventBuilder,
 	provider gomatrixserverlib.AuthEventProvider,
-	cfg config.Dendrite) (*gomatrixserverlib.Event, error) {
+	cfg config.Dendrite,
-
+	evTime time.Time,
+) (*gomatrixserverlib.Event, error) {
 	eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(builder)
 	if err != nil {
 		return nil, err
@@ -316,8 +329,7 @@ func buildEvent(builder *gomatrixserverlib.EventBuilder,
 	}
 	builder.AuthEvents = refs
 	eventID := fmt.Sprintf("$%s:%s", util.RandomString(16), cfg.Matrix.ServerName)
-	now := time.Now()
+	event, err := builder.Build(eventID, evTime, cfg.Matrix.ServerName, cfg.Matrix.KeyID, cfg.Matrix.PrivateKey)
-	event, err := builder.Build(eventID, now, cfg.Matrix.ServerName, cfg.Matrix.KeyID, cfg.Matrix.PrivateKey)
 	if err != nil {
 		return nil, fmt.Errorf("cannot build event %s : Builder failed to build. %s", builder.Type, err)
 	}

clientapi/routing/device.go

@@ -106,13 +106,6 @@ func UpdateDeviceByID(
 	req *http.Request, deviceDB *devices.Database, device *authtypes.Device,
 	deviceID string,
 ) util.JSONResponse {
-	if req.Method != http.MethodPut {
-		return util.JSONResponse{
-			Code: http.StatusMethodNotAllowed,
-			JSON: jsonerror.NotFound("Bad Method"),
-		}
-	}
-
 	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
 	if err != nil {
 		return httputil.LogThenError(req, err)

clientapi/routing/directory.go

@@ -15,13 +15,14 @@
 package routing
 
 import (
+	"fmt"
 	"net/http"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/dendrite/common/config"
-	"github.com/matrix-org/dendrite/roomserver/api"
+	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
@@ -33,7 +34,7 @@ func DirectoryRoom(
 	roomAlias string,
 	federation *gomatrixserverlib.FederationClient,
 	cfg *config.Dendrite,
-	aliasAPI api.RoomserverAliasAPI,
+	rsAPI roomserverAPI.RoomserverAliasAPI,
 ) util.JSONResponse {
 	_, domain, err := gomatrixserverlib.SplitID('#', roomAlias)
 	if err != nil {
@@ -43,50 +44,47 @@ func DirectoryRoom(
 		}
 	}
 
-	var resp gomatrixserverlib.RespDirectory
-
 	if domain == cfg.Matrix.ServerName {
-		queryReq := api.GetRoomIDForAliasRequest{Alias: roomAlias}
+		// Query the roomserver API to check if the alias exists locally
-		var queryRes api.GetRoomIDForAliasResponse
+		queryReq := roomserverAPI.GetRoomIDForAliasRequest{Alias: roomAlias}
-		if err = aliasAPI.GetRoomIDForAlias(req.Context(), &queryReq, &queryRes); err != nil {
+		var queryRes roomserverAPI.GetRoomIDForAliasResponse
+		if err = rsAPI.GetRoomIDForAlias(req.Context(), &queryReq, &queryRes); err != nil {
 			return httputil.LogThenError(req, err)
 		}
 
+		// List any roomIDs found associated with this alias
 		if len(queryRes.RoomID) > 0 {
-			// TODO: List servers that are aware of this room alias
-			resp = gomatrixserverlib.RespDirectory{
-				RoomID:  queryRes.RoomID,
-				Servers: []gomatrixserverlib.ServerName{},
-			}
-		} else {
-			// If the response doesn't contain a non-empty string, return an error
 			return util.JSONResponse{
-				Code: http.StatusNotFound,
+				Code: http.StatusOK,
-				JSON: jsonerror.NotFound("Room alias " + roomAlias + " not found."),
+				JSON: queryRes,
 			}
 		}
 	} else {
-		resp, err = federation.LookupRoomAlias(req.Context(), domain, roomAlias)
+		// Query the federation for this room alias
+		resp, err := federation.LookupRoomAlias(req.Context(), domain, roomAlias)
 		if err != nil {
-			switch x := err.(type) {
+			switch err.(type) {
 			case gomatrix.HTTPError:
-				if x.Code == http.StatusNotFound {
+			default:
-					return util.JSONResponse{
-						Code: http.StatusNotFound,
-						JSON: jsonerror.NotFound("Room alias not found"),
-					}
-				}
-			}
 				// TODO: Return 502 if the remote server errored.
 				// TODO: Return 504 if the remote server timed out.
 				return httputil.LogThenError(req, err)
 			}
 		}
-
+		if len(resp.RoomID) > 0 {
 			return util.JSONResponse{
 				Code: http.StatusOK,
 				JSON: resp,
 			}
+		}
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusNotFound,
+		JSON: jsonerror.NotFound(
+			fmt.Sprintf("Room alias %s not found", roomAlias),
+		),
+	}
 }
 
 // SetLocalAlias implements PUT /directory/room/{roomAlias}
@@ -96,7 +94,7 @@ func SetLocalAlias(
 	device *authtypes.Device,
 	alias string,
 	cfg *config.Dendrite,
-	aliasAPI api.RoomserverAliasAPI,
+	aliasAPI roomserverAPI.RoomserverAliasAPI,
 ) util.JSONResponse {
 	_, domain, err := gomatrixserverlib.SplitID('#', alias)
 	if err != nil {
@@ -113,6 +111,28 @@ func SetLocalAlias(
 		}
 	}
 
+	// Check that the alias does not fall within an exclusive namespace of an
+	// application service
+	// TODO: This code should eventually be refactored with:
+	// 1. The new method for checking for things matching an AS's namespace
+	// 2. Using an overall Regex object for all AS's just like we did for usernames
+	for _, appservice := range cfg.Derived.ApplicationServices {
+		// Don't prevent AS from creating aliases in its own namespace
+		// Note that Dendrite uses SenderLocalpart as UserID for AS users
+		if device.UserID != appservice.SenderLocalpart {
+			if aliasNamespaces, ok := appservice.NamespaceMap["aliases"]; ok {
+				for _, namespace := range aliasNamespaces {
+					if namespace.Exclusive && namespace.RegexpObject.MatchString(alias) {
+						return util.JSONResponse{
+							Code: http.StatusBadRequest,
+							JSON: jsonerror.ASExclusive("Alias is reserved by an application service"),
+						}
+					}
+				}
+			}
+		}
+	}
+
 	var r struct {
 		RoomID string `json:"room_id"`
 	}
@@ -120,12 +140,12 @@ func SetLocalAlias(
 		return *resErr
 	}
 
-	queryReq := api.SetRoomAliasRequest{
+	queryReq := roomserverAPI.SetRoomAliasRequest{
 		UserID: device.UserID,
 		RoomID: r.RoomID,
 		Alias:  alias,
 	}
-	var queryRes api.SetRoomAliasResponse
+	var queryRes roomserverAPI.SetRoomAliasResponse
 	if err := aliasAPI.SetRoomAlias(req.Context(), &queryReq, &queryRes); err != nil {
 		return httputil.LogThenError(req, err)
 	}
@@ -149,13 +169,13 @@ func RemoveLocalAlias(
 	req *http.Request,
 	device *authtypes.Device,
 	alias string,
-	aliasAPI api.RoomserverAliasAPI,
+	aliasAPI roomserverAPI.RoomserverAliasAPI,
 ) util.JSONResponse {
-	queryReq := api.RemoveRoomAliasRequest{
+	queryReq := roomserverAPI.RemoveRoomAliasRequest{
 		Alias:  alias,
 		UserID: device.UserID,
 	}
-	var queryRes api.RemoveRoomAliasResponse
+	var queryRes roomserverAPI.RemoveRoomAliasResponse
 	if err := aliasAPI.RemoveRoomAlias(req.Context(), &queryReq, &queryRes); err != nil {
 		return httputil.LogThenError(req, err)
 	}

clientapi/routing/filter.go

@@ -29,12 +29,6 @@ import (
 func GetFilter(
 	req *http.Request, device *authtypes.Device, accountDB *accounts.Database, userID string, filterID string,
 ) util.JSONResponse {
-	if req.Method != http.MethodGet {
-		return util.JSONResponse{
-			Code: http.StatusMethodNotAllowed,
-			JSON: jsonerror.NotFound("Bad method"),
-		}
-	}
 	if userID != device.UserID {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
@@ -71,12 +65,6 @@ type filterResponse struct {
 func PutFilter(
 	req *http.Request, device *authtypes.Device, accountDB *accounts.Database, userID string,
 ) util.JSONResponse {
-	if req.Method != http.MethodPost {
-		return util.JSONResponse{
-			Code: http.StatusMethodNotAllowed,
-			JSON: jsonerror.NotFound("Bad method"),
-		}
-	}
 	if userID != device.UserID {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,

clientapi/routing/joinroom.go

@@ -27,7 +27,7 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/producers"
 	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/common/config"
-	"github.com/matrix-org/dendrite/roomserver/api"
+	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
@@ -42,8 +42,8 @@ func JoinRoomByIDOrAlias(
 	cfg config.Dendrite,
 	federation *gomatrixserverlib.FederationClient,
 	producer *producers.RoomserverProducer,
-	queryAPI api.RoomserverQueryAPI,
+	queryAPI roomserverAPI.RoomserverQueryAPI,
-	aliasAPI api.RoomserverAliasAPI,
+	aliasAPI roomserverAPI.RoomserverAliasAPI,
 	keyRing gomatrixserverlib.KeyRing,
 	accountDB *accounts.Database,
 ) util.JSONResponse {
@@ -52,6 +52,14 @@ func JoinRoomByIDOrAlias(
 		return *resErr
 	}
 
+	evTime, err := httputil.ParseTSParam(req)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.InvalidArgumentValue(err.Error()),
+		}
+	}
+
 	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
 	if err != nil {
 		return httputil.LogThenError(req, err)
@@ -66,7 +74,9 @@ func JoinRoomByIDOrAlias(
 	content["displayname"] = profile.DisplayName
 	content["avatar_url"] = profile.AvatarURL
 
-	r := joinRoomReq{req, content, device.UserID, cfg, federation, producer, queryAPI, aliasAPI, keyRing}
+	r := joinRoomReq{
+		req, evTime, content, device.UserID, cfg, federation, producer, queryAPI, aliasAPI, keyRing,
+	}
 
 	if strings.HasPrefix(roomIDOrAlias, "!") {
 		return r.joinRoomByID(roomIDOrAlias)
@@ -76,19 +86,23 @@ func JoinRoomByIDOrAlias(
 	}
 	return util.JSONResponse{
 		Code: http.StatusBadRequest,
-		JSON: jsonerror.BadJSON("Invalid first character for room ID or alias"),
+		JSON: jsonerror.BadJSON(
+			fmt.Sprintf("Invalid first character '%s' for room ID or alias",
+				string([]rune(roomIDOrAlias)[0])), // Wrapping with []rune makes this call UTF-8 safe
+		),
 	}
 }
 
 type joinRoomReq struct {
 	req        *http.Request
+	evTime     time.Time
 	content    map[string]interface{}
 	userID     string
 	cfg        config.Dendrite
 	federation *gomatrixserverlib.FederationClient
 	producer   *producers.RoomserverProducer
-	queryAPI   api.RoomserverQueryAPI
+	queryAPI   roomserverAPI.RoomserverQueryAPI
-	aliasAPI   api.RoomserverAliasAPI
+	aliasAPI   roomserverAPI.RoomserverAliasAPI
 	keyRing    gomatrixserverlib.KeyRing
 }
 
@@ -100,37 +114,39 @@ func (r joinRoomReq) joinRoomByID(roomID string) util.JSONResponse {
 	// If the server is not in the room the we will need to look up the
 	// remote server the invite came from in order to request a join event
 	// from that server.
-	queryReq := api.QueryInvitesForUserRequest{
+	queryReq := roomserverAPI.QueryInvitesForUserRequest{
 		RoomID: roomID, TargetUserID: r.userID,
 	}
-	var queryRes api.QueryInvitesForUserResponse
+	var queryRes roomserverAPI.QueryInvitesForUserResponse
 	if err := r.queryAPI.QueryInvitesForUser(r.req.Context(), &queryReq, &queryRes); err != nil {
 		return httputil.LogThenError(r.req, err)
 	}
-	if len(queryRes.InviteSenderUserIDs) == 0 {
+
-		// TODO: We might need to support clients which erroneously try to join
-		// the room by ID even when they are not invited.
-		// This can be done by removing this check and falling through to
-		// joinRoomUsingServers passing an empty list since joinRoomUserServers
-		// will check if we are already in the room first.
-		return util.JSONResponse{
-			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden("You are not invited to the room"),
-		}
-	}
 	servers := []gomatrixserverlib.ServerName{}
-	seenBefore := map[gomatrixserverlib.ServerName]bool{}
+	seenInInviterIDs := map[gomatrixserverlib.ServerName]bool{}
 	for _, userID := range queryRes.InviteSenderUserIDs {
 		_, domain, err := gomatrixserverlib.SplitID('@', userID)
 		if err != nil {
 			return httputil.LogThenError(r.req, err)
 		}
-		if !seenBefore[domain] {
+		if !seenInInviterIDs[domain] {
 			servers = append(servers, domain)
-			seenBefore[domain] = true
+			seenInInviterIDs[domain] = true
 		}
 	}
 
+	// Also add the domain extracted from the roomID as a last resort to join
+	// in case the client is erroneously trying to join by ID without an invite
+	// or all previous attempts at domains extracted from the inviter IDs fail
+	// Note: It's no guarantee we'll succeed because a room isn't bound to the domain in its ID
+	_, domain, err := gomatrixserverlib.SplitID('!', roomID)
+	if err != nil {
+		return httputil.LogThenError(r.req, err)
+	}
+	if domain != r.cfg.Matrix.ServerName && !seenInInviterIDs[domain] {
+		servers = append(servers, domain)
+	}
+
 	return r.joinRoomUsingServers(roomID, servers)
 
 }
@@ -145,8 +161,8 @@ func (r joinRoomReq) joinRoomByAlias(roomAlias string) util.JSONResponse {
 		}
 	}
 	if domain == r.cfg.Matrix.ServerName {
-		queryReq := api.GetRoomIDForAliasRequest{Alias: roomAlias}
+		queryReq := roomserverAPI.GetRoomIDForAliasRequest{Alias: roomAlias}
-		var queryRes api.GetRoomIDForAliasResponse
+		var queryRes roomserverAPI.GetRoomIDForAliasResponse
 		if err = r.aliasAPI.GetRoomIDForAlias(r.req.Context(), &queryReq, &queryRes); err != nil {
 			return httputil.LogThenError(r.req, err)
 		}
@@ -214,8 +230,8 @@ func (r joinRoomReq) joinRoomUsingServers(
 		return httputil.LogThenError(r.req, err)
 	}
 
-	var queryRes api.QueryLatestEventsAndStateResponse
+	var queryRes roomserverAPI.QueryLatestEventsAndStateResponse
-	event, err := common.BuildEvent(r.req.Context(), &eb, r.cfg, r.queryAPI, &queryRes)
+	event, err := common.BuildEvent(r.req.Context(), &eb, r.cfg, r.evTime, r.queryAPI, &queryRes)
 	if err == nil {
 		if _, err = r.producer.SendEvents(r.req.Context(), []gomatrixserverlib.Event{*event}, r.cfg.Matrix.ServerName, nil); err != nil {
 			return httputil.LogThenError(r.req, err)
@@ -285,10 +301,9 @@ func (r joinRoomReq) joinRoomUsingServer(roomID string, server gomatrixserverlib
 		return nil, err
 	}
 
-	now := time.Now()
 	eventID := fmt.Sprintf("$%s:%s", util.RandomString(16), r.cfg.Matrix.ServerName)
 	event, err := respMakeJoin.JoinEvent.Build(
-		eventID, now, r.cfg.Matrix.ServerName, r.cfg.Matrix.KeyID, r.cfg.Matrix.PrivateKey,
+		eventID, r.evTime, r.cfg.Matrix.ServerName, r.cfg.Matrix.KeyID, r.cfg.Matrix.PrivateKey,
 	)
 	if err != nil {
 		res := httputil.LogThenError(r.req, err)

clientapi/routing/login.go

@@ -17,7 +17,10 @@ package routing
 import (
 	"net/http"
 
+	"context"
+
 	"github.com/matrix-org/dendrite/clientapi/auth"
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
@@ -40,7 +43,10 @@ type flow struct {
 type passwordRequest struct {
 	User     string `json:"user"`
 	Password string `json:"password"`
+	// Both DeviceID and InitialDisplayName can be omitted, or empty strings ("")
+	// Thus a pointer is needed to differentiate between the two
 	InitialDisplayName *string `json:"initial_device_display_name"`
+	DeviceID           *string `json:"device_id"`
 }
 
 type loginResponse struct {
@@ -102,13 +108,10 @@ func Login(
 
 		token, err := auth.GenerateAccessToken()
 		if err != nil {
-			httputil.LogThenError(req, err)
+			return httputil.LogThenError(req, err)
 		}
 
-		// TODO: Use the device ID in the request
+		dev, err := getDevice(req.Context(), r, deviceDB, acc, token)
-		dev, err := deviceDB.CreateDevice(
-			req.Context(), acc.Localpart, nil, token, r.InitialDisplayName,
-		)
 		if err != nil {
 			return util.JSONResponse{
 				Code: http.StatusInternalServerError,
@@ -131,3 +134,17 @@ func Login(
 		JSON: jsonerror.NotFound("Bad method"),
 	}
 }
+
+// getDevice returns a new or existing device
+func getDevice(
+	ctx context.Context,
+	r passwordRequest,
+	deviceDB *devices.Database,
+	acc *authtypes.Account,
+	token string,
+) (dev *authtypes.Device, err error) {
+	dev, err = deviceDB.CreateDevice(
+		ctx, acc.Localpart, r.DeviceID, token, r.InitialDisplayName,
+	)
+	return
+}

clientapi/routing/logout.go

@@ -20,7 +20,6 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
@@ -29,13 +28,6 @@ import (
 func Logout(
 	req *http.Request, deviceDB *devices.Database, device *authtypes.Device,
 ) util.JSONResponse {
-	if req.Method != http.MethodPost {
-		return util.JSONResponse{
-			Code: http.StatusMethodNotAllowed,
-			JSON: jsonerror.NotFound("Bad method"),
-		}
-	}
-
 	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
 	if err != nil {
 		return httputil.LogThenError(req, err)

clientapi/routing/membership.go

@@ -18,7 +18,9 @@ import (
 	"context"
 	"errors"
 	"net/http"
+	"time"
 
+	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
@@ -27,7 +29,7 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/threepid"
 	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/common/config"
-	"github.com/matrix-org/dendrite/roomserver/api"
+	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/gomatrixserverlib"
 
 	"github.com/matrix-org/util"
@@ -40,34 +42,28 @@ var errMissingUserID = errors.New("'user_id' must be supplied")
 func SendMembership(
 	req *http.Request, accountDB *accounts.Database, device *authtypes.Device,
 	roomID string, membership string, cfg config.Dendrite,
-	queryAPI api.RoomserverQueryAPI, producer *producers.RoomserverProducer,
+	queryAPI roomserverAPI.RoomserverQueryAPI, asAPI appserviceAPI.AppServiceQueryAPI,
+	producer *producers.RoomserverProducer,
 ) util.JSONResponse {
 	var body threepid.MembershipRequest
 	if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil {
 		return *reqErr
 	}
 
-	inviteStored, err := threepid.CheckAndProcessInvite(
+	evTime, err := httputil.ParseTSParam(req)
-		req.Context(),
+	if err != nil {
-		device, &body, cfg, queryAPI, accountDB, producer, membership, roomID,
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.InvalidArgumentValue(err.Error()),
+		}
+	}
+
+	inviteStored, jsonErrResp := checkAndProcessThreepid(
+		req, device, &body, cfg, queryAPI, accountDB, producer,
+		membership, roomID, evTime,
 	)
-	if err == threepid.ErrMissingParameter {
+	if jsonErrResp != nil {
-		return util.JSONResponse{
+		return *jsonErrResp
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON(err.Error()),
-		}
-	} else if err == threepid.ErrNotTrusted {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.NotTrusted(body.IDServer),
-		}
-	} else if err == common.ErrRoomNoExists {
-		return util.JSONResponse{
-			Code: http.StatusNotFound,
-			JSON: jsonerror.NotFound(err.Error()),
-		}
-	} else if err != nil {
-		return httputil.LogThenError(req, err)
 	}
 
 	// If an invite has been stored on an identity server, it means that a
@@ -81,7 +77,7 @@ func SendMembership(
 	}
 
 	event, err := buildMembershipEvent(
-		req.Context(), body, accountDB, device, membership, roomID, cfg, queryAPI,
+		req.Context(), body, accountDB, device, membership, roomID, cfg, evTime, queryAPI, asAPI,
 	)
 	if err == errMissingUserID {
 		return util.JSONResponse{
@@ -103,24 +99,35 @@ func SendMembership(
 		return httputil.LogThenError(req, err)
 	}
 
+	var returnData interface{} = struct{}{}
+
+	// The join membership requires the room id to be sent in the response
+	if membership == "join" {
+		returnData = struct {
+			RoomID string `json:"room_id"`
+		}{roomID}
+	}
+
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: struct{}{},
+		JSON: returnData,
 	}
 }
 
 func buildMembershipEvent(
 	ctx context.Context,
 	body threepid.MembershipRequest, accountDB *accounts.Database,
-	device *authtypes.Device, membership string, roomID string, cfg config.Dendrite,
+	device *authtypes.Device,
-	queryAPI api.RoomserverQueryAPI,
+	membership, roomID string,
+	cfg config.Dendrite, evTime time.Time,
+	queryAPI roomserverAPI.RoomserverQueryAPI, asAPI appserviceAPI.AppServiceQueryAPI,
 ) (*gomatrixserverlib.Event, error) {
 	stateKey, reason, err := getMembershipStateKey(body, device, membership)
 	if err != nil {
 		return nil, err
 	}
 
-	profile, err := loadProfile(ctx, stateKey, cfg, accountDB)
+	profile, err := loadProfile(ctx, stateKey, cfg, accountDB, asAPI)
 	if err != nil {
 		return nil, err
 	}
@@ -148,7 +155,7 @@ func buildMembershipEvent(
 		return nil, err
 	}
 
-	return common.BuildEvent(ctx, &builder, cfg, queryAPI, nil)
+	return common.BuildEvent(ctx, &builder, cfg, evTime, queryAPI, nil)
 }
 
 // loadProfile lookups the profile of a given user from the database and returns
@@ -156,16 +163,20 @@ func buildMembershipEvent(
 // Returns an error if the retrieval failed or if the first parameter isn't a
 // valid Matrix ID.
 func loadProfile(
-	ctx context.Context, userID string, cfg config.Dendrite, accountDB *accounts.Database,
+	ctx context.Context,
+	userID string,
+	cfg config.Dendrite,
+	accountDB *accounts.Database,
+	asAPI appserviceAPI.AppServiceQueryAPI,
 ) (*authtypes.Profile, error) {
-	localpart, serverName, err := gomatrixserverlib.SplitID('@', userID)
+	_, serverName, err := gomatrixserverlib.SplitID('@', userID)
 	if err != nil {
 		return nil, err
 	}
 
 	var profile *authtypes.Profile
 	if serverName == cfg.Matrix.ServerName {
-		profile, err = accountDB.GetProfileByLocalpart(ctx, localpart)
+		profile, err = appserviceAPI.RetrieveUserProfile(ctx, userID, asAPI, accountDB)
 	} else {
 		profile = &authtypes.Profile{}
 	}
@@ -198,3 +209,41 @@ func getMembershipStateKey(
 
 	return
 }
+
+func checkAndProcessThreepid(
+	req *http.Request,
+	device *authtypes.Device,
+	body *threepid.MembershipRequest,
+	cfg config.Dendrite,
+	queryAPI roomserverAPI.RoomserverQueryAPI,
+	accountDB *accounts.Database,
+	producer *producers.RoomserverProducer,
+	membership, roomID string,
+	evTime time.Time,
+) (inviteStored bool, errRes *util.JSONResponse) {
+
+	inviteStored, err := threepid.CheckAndProcessInvite(
+		req.Context(), device, body, cfg, queryAPI, accountDB, producer,
+		membership, roomID, evTime,
+	)
+	if err == threepid.ErrMissingParameter {
+		return inviteStored, &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON(err.Error()),
+		}
+	} else if err == threepid.ErrNotTrusted {
+		return inviteStored, &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.NotTrusted(body.IDServer),
+		}
+	} else if err == common.ErrRoomNoExists {
+		return inviteStored, &util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: jsonerror.NotFound(err.Error()),
+		}
+	} else if err != nil {
+		er := httputil.LogThenError(req, err)
+		return inviteStored, &er
+	}
+	return
+}

clientapi/routing/profile.go

@@ -17,7 +17,9 @@ package routing
 import (
 	"context"
 	"net/http"
+	"time"
 
+	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
@@ -33,23 +35,13 @@ import (
 
 // GetProfile implements GET /profile/{userID}
 func GetProfile(
-	req *http.Request, accountDB *accounts.Database, userID string,
+	req *http.Request, accountDB *accounts.Database, userID string, asAPI appserviceAPI.AppServiceQueryAPI,
 ) util.JSONResponse {
-	if req.Method != http.MethodGet {
+	profile, err := appserviceAPI.RetrieveUserProfile(req.Context(), userID, asAPI, accountDB)
-		return util.JSONResponse{
-			Code: http.StatusMethodNotAllowed,
-			JSON: jsonerror.NotFound("Bad method"),
-		}
-	}
-	localpart, _, err := gomatrixserverlib.SplitID('@', userID)
 	if err != nil {
 		return httputil.LogThenError(req, err)
 	}
 
-	profile, err := accountDB.GetProfileByLocalpart(req.Context(), localpart)
-	if err != nil {
-		return httputil.LogThenError(req, err)
-	}
 	res := common.ProfileResponse{
 		AvatarURL:   profile.AvatarURL,
 		DisplayName: profile.DisplayName,
@@ -62,17 +54,13 @@ func GetProfile(
 
 // GetAvatarURL implements GET /profile/{userID}/avatar_url
 func GetAvatarURL(
-	req *http.Request, accountDB *accounts.Database, userID string,
+	req *http.Request, accountDB *accounts.Database, userID string, asAPI appserviceAPI.AppServiceQueryAPI,
 ) util.JSONResponse {
-	localpart, _, err := gomatrixserverlib.SplitID('@', userID)
+	profile, err := appserviceAPI.RetrieveUserProfile(req.Context(), userID, asAPI, accountDB)
 	if err != nil {
 		return httputil.LogThenError(req, err)
 	}
 
-	profile, err := accountDB.GetProfileByLocalpart(req.Context(), localpart)
-	if err != nil {
-		return httputil.LogThenError(req, err)
-	}
 	res := common.AvatarURL{
 		AvatarURL: profile.AvatarURL,
 	}
@@ -113,6 +101,14 @@ func SetAvatarURL(
 		return httputil.LogThenError(req, err)
 	}
 
+	evTime, err := httputil.ParseTSParam(req)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.InvalidArgumentValue(err.Error()),
+		}
+	}
+
 	oldProfile, err := accountDB.GetProfileByLocalpart(req.Context(), localpart)
 	if err != nil {
 		return httputil.LogThenError(req, err)
@@ -133,7 +129,9 @@ func SetAvatarURL(
 		AvatarURL:   r.AvatarURL,
 	}
 
-	events, err := buildMembershipEvents(req.Context(), memberships, newProfile, userID, cfg, queryAPI)
+	events, err := buildMembershipEvents(
+		req.Context(), memberships, newProfile, userID, cfg, evTime, queryAPI,
+	)
 	if err != nil {
 		return httputil.LogThenError(req, err)
 	}
@@ -154,14 +152,9 @@ func SetAvatarURL(
 
 // GetDisplayName implements GET /profile/{userID}/displayname
 func GetDisplayName(
-	req *http.Request, accountDB *accounts.Database, userID string,
+	req *http.Request, accountDB *accounts.Database, userID string, asAPI appserviceAPI.AppServiceQueryAPI,
 ) util.JSONResponse {
-	localpart, _, err := gomatrixserverlib.SplitID('@', userID)
+	profile, err := appserviceAPI.RetrieveUserProfile(req.Context(), userID, asAPI, accountDB)
-	if err != nil {
-		return httputil.LogThenError(req, err)
-	}
-
-	profile, err := accountDB.GetProfileByLocalpart(req.Context(), localpart)
 	if err != nil {
 		return httputil.LogThenError(req, err)
 	}
@@ -205,6 +198,14 @@ func SetDisplayName(
 		return httputil.LogThenError(req, err)
 	}
 
+	evTime, err := httputil.ParseTSParam(req)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.InvalidArgumentValue(err.Error()),
+		}
+	}
+
 	oldProfile, err := accountDB.GetProfileByLocalpart(req.Context(), localpart)
 	if err != nil {
 		return httputil.LogThenError(req, err)
@@ -225,7 +226,9 @@ func SetDisplayName(
 		AvatarURL:   oldProfile.AvatarURL,
 	}
 
-	events, err := buildMembershipEvents(req.Context(), memberships, newProfile, userID, cfg, queryAPI)
+	events, err := buildMembershipEvents(
+		req.Context(), memberships, newProfile, userID, cfg, evTime, queryAPI,
+	)
 	if err != nil {
 		return httputil.LogThenError(req, err)
 	}
@@ -248,7 +251,7 @@ func buildMembershipEvents(
 	ctx context.Context,
 	memberships []authtypes.Membership,
 	newProfile authtypes.Profile, userID string, cfg *config.Dendrite,
-	queryAPI api.RoomserverQueryAPI,
+	evTime time.Time, queryAPI api.RoomserverQueryAPI,
 ) ([]gomatrixserverlib.Event, error) {
 	evs := []gomatrixserverlib.Event{}
 
@@ -271,7 +274,7 @@ func buildMembershipEvents(
 			return nil, err
 		}
 
-		event, err := common.BuildEvent(ctx, &builder, *cfg, queryAPI, nil)
+		event, err := common.BuildEvent(ctx, &builder, *cfg, evTime, queryAPI, nil)
 		if err != nil {
 			return nil, err
 		}

clientapi/routing/register.go

@@ -39,6 +39,8 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/clientapi/userutil"
+	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 	"github.com/prometheus/client_golang/prometheus"
@@ -83,6 +85,12 @@ func (d sessionsDict) GetCompletedStages(sessionID string) []authtypes.LoginType
 
 // AddCompletedStage records that a session has completed an auth stage.
 func (d *sessionsDict) AddCompletedStage(sessionID string, stage authtypes.LoginType) {
+	// Return if the stage is already present
+	for _, completedStage := range d.GetCompletedStages(sessionID) {
+		if completedStage == stage {
+			return
+		}
+	}
 	d.sessions[sessionID] = append(d.GetCompletedStages(sessionID), stage)
 }
 
@@ -113,7 +121,13 @@ type registerRequest struct {
 	// user-interactive auth params
 	Auth authDict `json:"auth"`
 
+	// Both DeviceID and InitialDisplayName can be omitted, or empty strings ("")
+	// Thus a pointer is needed to differentiate between the two
 	InitialDisplayName *string `json:"initial_device_display_name"`
+	DeviceID           *string `json:"device_id"`
+
+	// Prevent this user from logging in
+	InhibitLogin common.WeakBoolean `json:"inhibit_login"`
 
 	// Application Services place Type in the root of their registration
 	// request, whereas clients place it in the authDict struct.
@@ -162,9 +176,9 @@ func newUserInteractiveResponse(
 // http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#post-matrix-client-unstable-register
 type registerResponse struct {
 	UserID      string                       `json:"user_id"`
-	AccessToken string                       `json:"access_token"`
+	AccessToken string                       `json:"access_token,omitempty"`
 	HomeServer  gomatrixserverlib.ServerName `json:"home_server"`
-	DeviceID    string                       `json:"device_id"`
+	DeviceID    string                       `json:"device_id,omitempty"`
 }
 
 // recaptchaResponse represents the HTTP response from a Google Recaptcha server
@@ -175,8 +189,8 @@ type recaptchaResponse struct {
 	ErrorCodes  []int     `json:"error-codes"`
 }
 
-// validateUserName returns an error response if the username is invalid
+// validateUsername returns an error response if the username is invalid
-func validateUserName(username string) *util.JSONResponse {
+func validateUsername(username string) *util.JSONResponse {
 	// https://github.com/matrix-org/synapse/blob/v0.20.0/synapse/rest/client/v2_alpha/register.py#L161
 	if len(username) > maxUsernameLength {
 		return &util.JSONResponse{
@@ -186,12 +200,28 @@ func validateUserName(username string) *util.JSONResponse {
 	} else if !validUsernameRegex.MatchString(username) {
 		return &util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.InvalidUsername("User ID can only contain characters a-z, 0-9, or '_-./'"),
+			JSON: jsonerror.InvalidUsername("Username can only contain characters a-z, 0-9, or '_-./'"),
 		}
 	} else if username[0] == '_' { // Regex checks its not a zero length string
 		return &util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.InvalidUsername("User ID can't start with a '_'"),
+			JSON: jsonerror.InvalidUsername("Username cannot start with a '_'"),
+		}
+	}
+	return nil
+}
+
+// validateApplicationServiceUsername returns an error response if the username is invalid for an application service
+func validateApplicationServiceUsername(username string) *util.JSONResponse {
+	if len(username) > maxUsernameLength {
+		return &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON(fmt.Sprintf("'username' >%d characters", maxUsernameLength)),
+		}
+	} else if !validUsernameRegex.MatchString(username) {
+		return &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.InvalidUsername("Username can only contain characters a-z, 0-9, or '_-./'"),
 		}
 	}
 	return nil
@@ -222,8 +252,8 @@ func validateRecaptcha(
 ) *util.JSONResponse {
 	if !cfg.Matrix.RecaptchaEnabled {
 		return &util.JSONResponse{
-			Code: http.StatusBadRequest,
+			Code: http.StatusConflict,
-			JSON: jsonerror.BadJSON("Captcha registration is disabled"),
+			JSON: jsonerror.Unknown("Captcha registration is disabled"),
 		}
 	}
 
@@ -258,8 +288,8 @@ func validateRecaptcha(
 	body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		return &util.JSONResponse{
-			Code: http.StatusInternalServerError,
+			Code: http.StatusGatewayTimeout,
-			JSON: jsonerror.BadJSON("Error in contacting captcha server" + err.Error()),
+			JSON: jsonerror.Unknown("Error in contacting captcha server" + err.Error()),
 		}
 	}
 	err = json.Unmarshal(body, &r)
@@ -280,31 +310,31 @@ func validateRecaptcha(
 	return nil
 }
 
-// UsernameIsWithinApplicationServiceNamespace checks to see if a username falls
+// UserIDIsWithinApplicationServiceNamespace checks to see if a given userID
-// within any of the namespaces of a given Application Service. If no
+// falls within any of the namespaces of a given Application Service. If no
 // Application Service is given, it will check to see if it matches any
 // Application Service's namespace.
-func UsernameIsWithinApplicationServiceNamespace(
+func UserIDIsWithinApplicationServiceNamespace(
 	cfg *config.Dendrite,
-	username string,
+	userID string,
 	appservice *config.ApplicationService,
 ) bool {
 	if appservice != nil {
-		// Loop through given Application Service's namespaces and see if any match
+		// Loop through given application service's namespaces and see if any match
 		for _, namespace := range appservice.NamespaceMap["users"] {
 			// AS namespaces are checked for validity in config
-			if namespace.RegexpObject.MatchString(username) {
+			if namespace.RegexpObject.MatchString(userID) {
 				return true
 			}
 		}
 		return false
 	}
 
-	// Loop through all known Application Service's namespaces and see if any match
+	// Loop through all known application service's namespaces and see if any match
-	for _, knownAppservice := range cfg.Derived.ApplicationServices {
+	for _, knownAppService := range cfg.Derived.ApplicationServices {
-		for _, namespace := range knownAppservice.NamespaceMap["users"] {
+		for _, namespace := range knownAppService.NamespaceMap["users"] {
 			// AS namespaces are checked for validity in config
-			if namespace.RegexpObject.MatchString(username) {
+			if namespace.RegexpObject.MatchString(userID) {
 				return true
 			}
 		}
@@ -318,19 +348,28 @@ func UsernameMatchesMultipleExclusiveNamespaces(
 	cfg *config.Dendrite,
 	username string,
 ) bool {
+	userID := userutil.MakeUserID(username, cfg.Matrix.ServerName)
+
 	// Check namespaces and see if more than one match
 	matchCount := 0
 	for _, appservice := range cfg.Derived.ApplicationServices {
-		for _, namespaceSlice := range appservice.NamespaceMap {
+		if appservice.IsInterestedInUserID(userID) {
-			for _, namespace := range namespaceSlice {
+			if matchCount++; matchCount > 1 {
-				// Check if we have a match on this username
+				return true
-				if namespace.RegexpObject.MatchString(username) {
-					matchCount++
 			}
 		}
 	}
-	}
+	return false
-	return matchCount > 1
+}
+
+// UsernameMatchesExclusiveNamespaces will check if a given username matches any
+// application service's exclusive users namespace
+func UsernameMatchesExclusiveNamespaces(
+	cfg *config.Dendrite,
+	username string,
+) bool {
+	userID := userutil.MakeUserID(username, cfg.Matrix.ServerName)
+	return cfg.Derived.ExclusiveApplicationServicesUsernameRegexp.MatchString(userID)
 }
 
 // validateApplicationService checks if a provided application service token
@@ -339,12 +378,11 @@ func UsernameMatchesMultipleExclusiveNamespaces(
 // two requirements are met, no error will be returned.
 func validateApplicationService(
 	cfg *config.Dendrite,
-	req *http.Request,
 	username string,
+	accessToken string,
 ) (string, *util.JSONResponse) {
 	// Check if the token if the application service is valid with one we have
 	// registered in the config.
-	accessToken := req.URL.Query().Get("access_token")
 	var matchedApplicationService *config.ApplicationService
 	for _, appservice := range cfg.Derived.ApplicationServices {
 		if appservice.ASToken == accessToken {
@@ -359,25 +397,32 @@ func validateApplicationService(
 		}
 	}
 
+	userID := userutil.MakeUserID(username, cfg.Matrix.ServerName)
+
 	// Ensure the desired username is within at least one of the application service's namespaces.
-	if !UsernameIsWithinApplicationServiceNamespace(cfg, username, matchedApplicationService) {
+	if !UserIDIsWithinApplicationServiceNamespace(cfg, userID, matchedApplicationService) {
 		// If we didn't find any matches, return M_EXCLUSIVE
 		return "", &util.JSONResponse{
-			Code: http.StatusUnauthorized,
+			Code: http.StatusBadRequest,
 			JSON: jsonerror.ASExclusive(fmt.Sprintf(
 				"Supplied username %s did not match any namespaces for application service ID: %s", username, matchedApplicationService.ID)),
 		}
 	}
 
 	// Check this user does not fit multiple application service namespaces
-	if UsernameMatchesMultipleExclusiveNamespaces(cfg, username) {
+	if UsernameMatchesMultipleExclusiveNamespaces(cfg, userID) {
 		return "", &util.JSONResponse{
-			Code: http.StatusUnauthorized,
+			Code: http.StatusBadRequest,
 			JSON: jsonerror.ASExclusive(fmt.Sprintf(
 				"Supplied username %s matches multiple exclusive application service namespaces. Only 1 match allowed", username)),
 		}
 	}
 
+	// Check username application service is trying to register is valid
+	if err := validateApplicationServiceUsername(username); err != nil {
+		return "", err
+	}
+
 	// No errors, registration valid
 	return matchedApplicationService.ID, nil
 }
@@ -421,19 +466,10 @@ func Register(
 		r.Username = strconv.FormatInt(id, 10)
 	}
 
-	// If no auth type is specified by the client, send back the list of available flows
-	if r.Auth.Type == "" {
-		return util.JSONResponse{
-			Code: http.StatusUnauthorized,
-			JSON: newUserInteractiveResponse(sessionID,
-				cfg.Derived.Registration.Flows, cfg.Derived.Registration.Params),
-		}
-	}
-
 	// Squash username to all lowercase letters
 	r.Username = strings.ToLower(r.Username)
 
-	if resErr = validateUserName(r.Username); resErr != nil {
+	if resErr = validateUsername(r.Username); resErr != nil {
 		return *resErr
 	}
 	if resErr = validatePassword(r.Password); resErr != nil {
@@ -442,9 +478,9 @@ func Register(
 
 	// Make sure normal user isn't registering under an exclusive application
 	// service namespace. Skip this check if no app services are registered.
-	if r.Auth.Type != "m.login.application_service" &&
+	if r.Auth.Type != authtypes.LoginTypeApplicationService &&
 		len(cfg.Derived.ApplicationServices) != 0 &&
-		cfg.Derived.ExclusiveApplicationServicesUsernameRegexp.MatchString(r.Username) {
+		UsernameMatchesExclusiveNamespaces(cfg, r.Username) {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
 			JSON: jsonerror.ASExclusive("This username is reserved by an application service."),
@@ -463,6 +499,7 @@ func Register(
 
 // handleRegistrationFlow will direct and complete registration flow stages
 // that the client has requested.
+// nolint: gocyclo
 func handleRegistrationFlow(
 	req *http.Request,
 	r registerRequest,
@@ -508,20 +545,30 @@ func handleRegistrationFlow(
 		// Add SharedSecret to the list of completed registration stages
 		sessions.AddCompletedStage(sessionID, authtypes.LoginTypeSharedSecret)
 
-	case authtypes.LoginTypeApplicationService:
+	case "":
-		// Check Application Service register user request is valid.
+		// Extract the access token from the request, if there's one to extract
-		// The application service's ID is returned if so.
+		// (which we can know by checking whether the error is nil or not).
-		appserviceID, err := validateApplicationService(cfg, req, r.Username)
+		accessToken, err := auth.ExtractAccessToken(req)
 
-		if err != nil {
+		// A missing auth type can mean either the registration is performed by
-			return *err
+		// an AS or the request is made as the first step of a registration
+		// using the User-Interactive Authentication API. This can be determined
+		// by whether the request contains an access token.
+		if err == nil {
+			return handleApplicationServiceRegistration(
+				accessToken, err, req, r, cfg, accountDB, deviceDB,
+			)
 		}
 
-		// If no error, application service was successfully validated.
+	case authtypes.LoginTypeApplicationService:
-		// Don't need to worry about appending to registration stages as
+		// Extract the access token from the request.
-		// application service registration is entirely separate.
+		accessToken, err := auth.ExtractAccessToken(req)
-		return completeRegistration(req.Context(), accountDB, deviceDB,
+		// Let the AS registration handler handle the process from here. We
-			r.Username, "", appserviceID, r.InitialDisplayName)
+		// don't need a condition on that call since the registration is clearly
+		// stated as being AS-related.
+		return handleApplicationServiceRegistration(
+			accessToken, err, req, r, cfg, accountDB, deviceDB,
+		)
 
 	case authtypes.LoginTypeDummy:
 		// there is nothing to do
@@ -542,6 +589,50 @@ func handleRegistrationFlow(
 		req, r, sessionID, cfg, accountDB, deviceDB)
 }
 
+// handleApplicationServiceRegistration handles the registration of an
+// application service's user by validating the AS from its access token and
+// registering the user. Its two first parameters must be the two return values
+// of the auth.ExtractAccessToken function.
+// Returns an error if the access token couldn't be extracted from the request
+// at an earlier step of the registration workflow, or if the provided access
+// token doesn't belong to a valid AS, or if there was an issue completing the
+// registration process.
+func handleApplicationServiceRegistration(
+	accessToken string,
+	tokenErr error,
+	req *http.Request,
+	r registerRequest,
+	cfg *config.Dendrite,
+	accountDB *accounts.Database,
+	deviceDB *devices.Database,
+) util.JSONResponse {
+	// Check if we previously had issues extracting the access token from the
+	// request.
+	if tokenErr != nil {
+		return util.JSONResponse{
+			Code: http.StatusUnauthorized,
+			JSON: jsonerror.MissingToken(tokenErr.Error()),
+		}
+	}
+
+	// Check application service register user request is valid.
+	// The application service's ID is returned if so.
+	appserviceID, err := validateApplicationService(
+		cfg, r.Username, accessToken,
+	)
+	if err != nil {
+		return *err
+	}
+
+	// If no error, application service was successfully validated.
+	// Don't need to worry about appending to registration stages as
+	// application service registration is entirely separate.
+	return completeRegistration(
+		req.Context(), accountDB, deviceDB, r.Username, "", appserviceID,
+		r.InhibitLogin, r.InitialDisplayName, r.DeviceID,
+	)
+}
+
 // checkAndCompleteFlow checks if a given registration flow is completed given
 // a set of allowed flows. If so, registration is completed, otherwise a
 // response with
@@ -556,8 +647,10 @@ func checkAndCompleteFlow(
 ) util.JSONResponse {
 	if checkFlowCompleted(flow, cfg.Derived.Registration.Flows) {
 		// This flow was completed, registration can continue
-		return completeRegistration(req.Context(), accountDB, deviceDB,
+		return completeRegistration(
-			r.Username, r.Password, "", r.InitialDisplayName)
+			req.Context(), accountDB, deviceDB, r.Username, r.Password, "",
+			r.InhibitLogin, r.InitialDisplayName, r.DeviceID,
+		)
 	}
 
 	// There are still more stages to complete.
@@ -607,10 +700,10 @@ func LegacyRegister(
 			return util.MessageResponse(http.StatusForbidden, "HMAC incorrect")
 		}
 
-		return completeRegistration(req.Context(), accountDB, deviceDB, r.Username, r.Password, "", nil)
+		return completeRegistration(req.Context(), accountDB, deviceDB, r.Username, r.Password, "", false, nil, nil)
 	case authtypes.LoginTypeDummy:
 		// there is nothing to do
-		return completeRegistration(req.Context(), accountDB, deviceDB, r.Username, r.Password, "", nil)
+		return completeRegistration(req.Context(), accountDB, deviceDB, r.Username, r.Password, "", false, nil, nil)
 	default:
 		return util.JSONResponse{
 			Code: http.StatusNotImplemented,
@@ -630,7 +723,7 @@ func parseAndValidateLegacyLogin(req *http.Request, r *legacyRegisterRequest) *u
 	// Squash username to all lowercase letters
 	r.Username = strings.ToLower(r.Username)
 
-	if resErr = validateUserName(r.Username); resErr != nil {
+	if resErr = validateUsername(r.Username); resErr != nil {
 		return resErr
 	}
 	if resErr = validatePassword(r.Password); resErr != nil {
@@ -648,12 +741,19 @@ func parseAndValidateLegacyLogin(req *http.Request, r *legacyRegisterRequest) *u
 	return nil
 }
 
+// completeRegistration runs some rudimentary checks against the submitted
+// input, then if successful creates an account and a newly associated device
+// We pass in each individual part of the request here instead of just passing a
+// registerRequest, as this function serves requests encoded as both
+// registerRequests and legacyRegisterRequests, which share some attributes but
+// not all
 func completeRegistration(
 	ctx context.Context,
 	accountDB *accounts.Database,
 	deviceDB *devices.Database,
 	username, password, appserviceID string,
-	displayName *string,
+	inhibitLogin common.WeakBoolean,
+	displayName, deviceID *string,
 ) util.JSONResponse {
 	if username == "" {
 		return util.JSONResponse{
@@ -682,6 +782,21 @@ func completeRegistration(
 		}
 	}
 
+	// Increment prometheus counter for created users
+	amtRegUsers.Inc()
+
+	// Check whether inhibit_login option is set. If so, don't create an access
+	// token or a device for this user
+	if inhibitLogin {
+		return util.JSONResponse{
+			Code: http.StatusOK,
+			JSON: registerResponse{
+				UserID:     userutil.MakeUserID(username, acc.ServerName),
+				HomeServer: acc.ServerName,
+			},
+		}
+	}
+
 	token, err := auth.GenerateAccessToken()
 	if err != nil {
 		return util.JSONResponse{
@@ -690,8 +805,7 @@ func completeRegistration(
 		}
 	}
 
-	// // TODO: Use the device ID in the request.
+	dev, err := deviceDB.CreateDevice(ctx, username, deviceID, token, displayName)
-	dev, err := deviceDB.CreateDevice(ctx, username, nil, token, displayName)
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
@@ -699,9 +813,6 @@ func completeRegistration(
 		}
 	}
 
-	// Increment prometheus counter for created users
-	amtRegUsers.Inc()
-
 	return util.JSONResponse{
 		Code: http.StatusOK,
 		JSON: registerResponse{
@@ -815,6 +926,7 @@ type availableResponse struct {
 // RegisterAvailable checks if the username is already taken or invalid.
 func RegisterAvailable(
 	req *http.Request,
+	cfg config.Dendrite,
 	accountDB *accounts.Database,
 ) util.JSONResponse {
 	username := req.URL.Query().Get("username")
@@ -822,10 +934,21 @@ func RegisterAvailable(
 	// Squash username to all lowercase letters
 	username = strings.ToLower(username)
 
-	if err := validateUserName(username); err != nil {
+	if err := validateUsername(username); err != nil {
 		return *err
 	}
 
+	// Check if this username is reserved by an application service
+	userID := userutil.MakeUserID(username, cfg.Matrix.ServerName)
+	for _, appservice := range cfg.Derived.ApplicationServices {
+		if appservice.IsInterestedInUserID(userID) {
+			return util.JSONResponse{
+				Code: http.StatusBadRequest,
+				JSON: jsonerror.UserInUse("Desired user ID is reserved by an application service."),
+			}
+		}
+	}
+
 	availability, availabilityErr := accountDB.CheckAccountAvailability(req.Context(), username)
 	if availabilityErr != nil {
 		return util.JSONResponse{
@@ -836,7 +959,7 @@ func RegisterAvailable(
 	if !availability {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.InvalidUsername("A different user ID has already been registered for this session"),
+			JSON: jsonerror.UserInUse("Desired User ID is already taken."),
 		}
 	}
 

clientapi/routing/register_test.go

@@ -15,9 +15,11 @@
 package routing
 
 import (
+	"regexp"
 	"testing"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/common/config"
 )
 
 var (
@@ -145,3 +147,63 @@ func TestEmptyCompletedFlows(t *testing.T) {
 		t.Error("Empty Completed Flow Stages should be a empty slice: returned ", ret, ". Should be []")
 	}
 }
+
+// This method tests validation of the provided Application Service token and
+// username that they're registering
+func TestValidationOfApplicationServices(t *testing.T) {
+	// Set up application service namespaces
+	regex := "@_appservice_.*"
+	regexp, err := regexp.Compile(regex)
+	if err != nil {
+		t.Errorf("Error compiling regex: %s", regex)
+	}
+
+	fakeNamespace := config.ApplicationServiceNamespace{
+		Exclusive:    true,
+		Regex:        regex,
+		RegexpObject: regexp,
+	}
+
+	// Create a fake application service
+	fakeID := "FakeAS"
+	fakeSenderLocalpart := "_appservice_bot"
+	fakeApplicationService := config.ApplicationService{
+		ID:              fakeID,
+		URL:             "null",
+		ASToken:         "1234",
+		HSToken:         "4321",
+		SenderLocalpart: fakeSenderLocalpart,
+		NamespaceMap: map[string][]config.ApplicationServiceNamespace{
+			"users": {fakeNamespace},
+		},
+	}
+
+	// Set up a config
+	fakeConfig := config.Dendrite{}
+	fakeConfig.Matrix.ServerName = "localhost"
+	fakeConfig.Derived.ApplicationServices = []config.ApplicationService{fakeApplicationService}
+
+	// Access token is correct, user_id omitted so we are acting as SenderLocalpart
+	asID, resp := validateApplicationService(&fakeConfig, fakeSenderLocalpart, "1234")
+	if resp != nil || asID != fakeID {
+		t.Errorf("appservice should have validated and returned correct ID: %s", resp.JSON)
+	}
+
+	// Access token is incorrect, user_id omitted so we are acting as SenderLocalpart
+	asID, resp = validateApplicationService(&fakeConfig, fakeSenderLocalpart, "xxxx")
+	if resp == nil || asID == fakeID {
+		t.Errorf("access_token should have been marked as invalid")
+	}
+
+	// Access token is correct, acting as valid user_id
+	asID, resp = validateApplicationService(&fakeConfig, "_appservice_bob", "1234")
+	if resp != nil || asID != fakeID {
+		t.Errorf("access_token and user_id should've been valid: %s", resp.JSON)
+	}
+
+	// Access token is correct, acting as invalid user_id
+	asID, resp = validateApplicationService(&fakeConfig, "_something_else", "1234")
+	if resp == nil || asID == fakeID {
+		t.Errorf("user_id should not have been valid: @_something_else:localhost")
+	}
+}

clientapi/routing/routing.go

@@ -20,6 +20,8 @@ import (
 	"strings"
 
 	"github.com/gorilla/mux"
+	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
+	"github.com/matrix-org/dendrite/clientapi/auth"
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
@@ -28,7 +30,7 @@ import (
 	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/common/config"
 	"github.com/matrix-org/dendrite/common/transactions"
-	"github.com/matrix-org/dendrite/roomserver/api"
+	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
@@ -39,16 +41,23 @@ const pathPrefixUnstable = "/_matrix/client/unstable"
 
 // Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client
 // to clients which need to make outbound HTTP requests.
+//
+// Due to Setup being used to call many other functions, a gocyclo nolint is
+// applied:
+// nolint: gocyclo
 func Setup(
 	apiMux *mux.Router, cfg config.Dendrite,
-	producer *producers.RoomserverProducer, queryAPI api.RoomserverQueryAPI,
+	producer *producers.RoomserverProducer,
-	aliasAPI api.RoomserverAliasAPI,
+	queryAPI roomserverAPI.RoomserverQueryAPI,
+	aliasAPI roomserverAPI.RoomserverAliasAPI,
+	asAPI appserviceAPI.AppServiceQueryAPI,
 	accountDB *accounts.Database,
 	deviceDB *devices.Database,
 	federation *gomatrixserverlib.FederationClient,
 	keyRing gomatrixserverlib.KeyRing,
 	userUpdateProducer *producers.UserUpdateProducer,
 	syncProducer *producers.SyncAPIProducer,
+	typingProducer *producers.TypingServerProducer,
 	transactionsCache *transactions.Cache,
 ) {
 
@@ -72,42 +81,63 @@ func Setup(
 	v1mux := apiMux.PathPrefix(pathPrefixV1).Subrouter()
 	unstableMux := apiMux.PathPrefix(pathPrefixUnstable).Subrouter()
 
+	authData := auth.Data{
+		AccountDB:   accountDB,
+		DeviceDB:    deviceDB,
+		AppServices: cfg.Derived.ApplicationServices,
+	}
+
 	r0mux.Handle("/createRoom",
-		common.MakeAuthAPI("createRoom", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		common.MakeAuthAPI("createRoom", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			return CreateRoom(req, device, cfg, producer, accountDB, aliasAPI)
+			return CreateRoom(req, device, cfg, producer, accountDB, aliasAPI, asAPI)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/join/{roomIDOrAlias}",
-		common.MakeAuthAPI("join", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		common.MakeAuthAPI("join", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return JoinRoomByIDOrAlias(
 				req, device, vars["roomIDOrAlias"], cfg, federation, producer, queryAPI, aliasAPI, keyRing, accountDB,
 			)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/{membership:(?:join|kick|ban|unban|leave|invite)}",
-		common.MakeAuthAPI("membership", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		common.MakeAuthAPI("membership", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
-			return SendMembership(req, accountDB, device, vars["roomID"], vars["membership"], cfg, queryAPI, producer)
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return SendMembership(req, accountDB, device, vars["roomID"], vars["membership"], cfg, queryAPI, asAPI, producer)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/send/{eventType}",
-		common.MakeAuthAPI("send_message", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		common.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, nil, cfg, queryAPI, producer, nil)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/send/{eventType}/{txnID}",
-		common.MakeAuthAPI("send_message", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		common.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			txnID := vars["txnID"]
 			return SendEvent(req, device, vars["roomID"], vars["eventType"], &txnID,
 				nil, cfg, queryAPI, producer, transactionsCache)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/state/{eventType:[^/]+/?}",
-		common.MakeAuthAPI("send_message", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		common.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			emptyString := ""
 			eventType := vars["eventType"]
 			// If there's a trailing slash, remove it
@@ -118,8 +148,11 @@ func Setup(
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/state/{eventType}/{stateKey}",
-		common.MakeAuthAPI("send_message", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		common.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			stateKey := vars["stateKey"]
 			return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, &stateKey, cfg, queryAPI, producer, nil)
 		}),
@@ -134,42 +167,67 @@ func Setup(
 	})).Methods(http.MethodPost, http.MethodOptions)
 
 	r0mux.Handle("/register/available", common.MakeExternalAPI("registerAvailable", func(req *http.Request) util.JSONResponse {
-		return RegisterAvailable(req, accountDB)
+		return RegisterAvailable(req, cfg, accountDB)
 	})).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/directory/room/{roomAlias}",
-		common.MakeAuthAPI("directory_room", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		common.MakeExternalAPI("directory_room", func(req *http.Request) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return DirectoryRoom(req, vars["roomAlias"], federation, &cfg, aliasAPI)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/directory/room/{roomAlias}",
-		common.MakeAuthAPI("directory_room", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		common.MakeAuthAPI("directory_room", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return SetLocalAlias(req, device, vars["roomAlias"], &cfg, aliasAPI)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/directory/room/{roomAlias}",
-		common.MakeAuthAPI("directory_room", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		common.MakeAuthAPI("directory_room", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return RemoveLocalAlias(req, device, vars["roomAlias"], aliasAPI)
 		}),
 	).Methods(http.MethodDelete, http.MethodOptions)
 
 	r0mux.Handle("/logout",
-		common.MakeAuthAPI("logout", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		common.MakeAuthAPI("logout", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
 			return Logout(req, deviceDB, device)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
 	r0mux.Handle("/logout/all",
-		common.MakeAuthAPI("logout", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		common.MakeAuthAPI("logout", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
 			return LogoutAll(req, deviceDB, device)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
+	r0mux.Handle("/rooms/{roomID}/typing/{userID}",
+		common.MakeAuthAPI("rooms_typing", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return SendTyping(req, device, vars["roomID"], vars["userID"], accountDB, typingProducer)
+		}),
+	).Methods(http.MethodPut, http.MethodOptions)
+
+	r0mux.Handle("/account/whoami",
+		common.MakeAuthAPI("whoami", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+			return Whoami(req, device)
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
 	// Stub endpoints required by Riot
 
 	r0mux.Handle("/login",
@@ -198,15 +256,21 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/user/{userId}/filter",
-		common.MakeAuthAPI("put_filter", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		common.MakeAuthAPI("put_filter", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return PutFilter(req, device, accountDB, vars["userId"])
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
 	r0mux.Handle("/user/{userId}/filter/{filterId}",
-		common.MakeAuthAPI("get_filter", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		common.MakeAuthAPI("get_filter", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return GetFilter(req, device, accountDB, vars["userId"], vars["filterId"])
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
@@ -215,21 +279,30 @@ func Setup(
 
 	r0mux.Handle("/profile/{userID}",
 		common.MakeExternalAPI("profile", func(req *http.Request) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
-			return GetProfile(req, accountDB, vars["userID"])
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return GetProfile(req, accountDB, vars["userID"], asAPI)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/profile/{userID}/avatar_url",
 		common.MakeExternalAPI("profile_avatar_url", func(req *http.Request) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
-			return GetAvatarURL(req, accountDB, vars["userID"])
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return GetAvatarURL(req, accountDB, vars["userID"], asAPI)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/profile/{userID}/avatar_url",
-		common.MakeAuthAPI("profile_avatar_url", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		common.MakeAuthAPI("profile_avatar_url", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return SetAvatarURL(req, accountDB, device, vars["userID"], userUpdateProducer, &cfg, producer, queryAPI)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
@@ -238,14 +311,20 @@ func Setup(
 
 	r0mux.Handle("/profile/{userID}/displayname",
 		common.MakeExternalAPI("profile_displayname", func(req *http.Request) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
-			return GetDisplayName(req, accountDB, vars["userID"])
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return GetDisplayName(req, accountDB, vars["userID"], asAPI)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/profile/{userID}/displayname",
-		common.MakeAuthAPI("profile_displayname", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		common.MakeAuthAPI("profile_displayname", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return SetDisplayName(req, accountDB, device, vars["userID"], userUpdateProducer, &cfg, producer, queryAPI)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
@@ -253,19 +332,19 @@ func Setup(
 	// PUT requests, so we need to allow this method
 
 	r0mux.Handle("/account/3pid",
-		common.MakeAuthAPI("account_3pid", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		common.MakeAuthAPI("account_3pid", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
 			return GetAssociated3PIDs(req, accountDB, device)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/account/3pid",
-		common.MakeAuthAPI("account_3pid", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		common.MakeAuthAPI("account_3pid", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
 			return CheckAndSave3PIDAssociation(req, accountDB, device, cfg)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
 	unstableMux.Handle("/account/3pid/delete",
-		common.MakeAuthAPI("account_3pid", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		common.MakeAuthAPI("account_3pid", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
 			return Forget3PID(req, accountDB)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
@@ -288,7 +367,7 @@ func Setup(
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/voip/turnServer",
-		common.MakeAuthAPI("turn_server", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		common.MakeAuthAPI("turn_server", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
 			return RequestTurnServer(req, device, cfg)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
@@ -314,29 +393,41 @@ func Setup(
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/user/{userID}/account_data/{type}",
-		common.MakeAuthAPI("user_account_data", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		common.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return SaveAccountData(req, accountDB, device, vars["userID"], "", vars["type"], syncProducer)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/user/{userID}/rooms/{roomID}/account_data/{type}",
-		common.MakeAuthAPI("user_account_data", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		common.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return SaveAccountData(req, accountDB, device, vars["userID"], vars["roomID"], vars["type"], syncProducer)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/rooms/{roomID}/members",
-		common.MakeAuthAPI("rooms_members", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		common.MakeAuthAPI("rooms_members", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return GetMemberships(req, device, vars["roomID"], false, cfg, queryAPI)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/rooms/{roomID}/joined_members",
-		common.MakeAuthAPI("rooms_members", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		common.MakeAuthAPI("rooms_members", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return GetMemberships(req, device, vars["roomID"], true, cfg, queryAPI)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
@@ -348,29 +439,28 @@ func Setup(
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
-	r0mux.Handle("/rooms/{roomID}/typing/{userID}",
-		common.MakeExternalAPI("rooms_typing", func(req *http.Request) util.JSONResponse {
-			// TODO: handling typing
-			return util.JSONResponse{Code: http.StatusOK, JSON: struct{}{}}
-		}),
-	).Methods(http.MethodPut, http.MethodOptions)
-
 	r0mux.Handle("/devices",
-		common.MakeAuthAPI("get_devices", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		common.MakeAuthAPI("get_devices", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
 			return GetDevicesByLocalpart(req, deviceDB, device)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/devices/{deviceID}",
-		common.MakeAuthAPI("get_device", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		common.MakeAuthAPI("get_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return GetDeviceByID(req, deviceDB, device, vars["deviceID"])
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/devices/{deviceID}",
-		common.MakeAuthAPI("device_data", deviceDB, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		common.MakeAuthAPI("device_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return UpdateDeviceByID(req, deviceDB, device, vars["deviceID"])
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)

clientapi/routing/sendevent.go

@@ -55,50 +55,11 @@ func SendEvent(
 		}
 	}
 
-	// parse the incoming http request
+	e, resErr := generateSendEvent(req, device, roomID, eventType, stateKey, cfg, queryAPI)
-	userID := device.UserID
-	var r map[string]interface{} // must be a JSON object
-	resErr := httputil.UnmarshalJSONRequest(req, &r)
 	if resErr != nil {
 		return *resErr
 	}
 
-	// create the new event and set all the fields we can
-	builder := gomatrixserverlib.EventBuilder{
-		Sender:   userID,
-		RoomID:   roomID,
-		Type:     eventType,
-		StateKey: stateKey,
-	}
-	err := builder.SetContent(r)
-	if err != nil {
-		return httputil.LogThenError(req, err)
-	}
-
-	var queryRes api.QueryLatestEventsAndStateResponse
-	e, err := common.BuildEvent(req.Context(), &builder, cfg, queryAPI, &queryRes)
-	if err == common.ErrRoomNoExists {
-		return util.JSONResponse{
-			Code: http.StatusNotFound,
-			JSON: jsonerror.NotFound("Room does not exist"),
-		}
-	} else if err != nil {
-		return httputil.LogThenError(req, err)
-	}
-
-	// check to see if this user can perform this operation
-	stateEvents := make([]*gomatrixserverlib.Event, len(queryRes.StateEvents))
-	for i := range queryRes.StateEvents {
-		stateEvents[i] = &queryRes.StateEvents[i]
-	}
-	provider := gomatrixserverlib.NewAuthEvents(stateEvents)
-	if err = gomatrixserverlib.Allowed(*e, &provider); err != nil {
-		return util.JSONResponse{
-			Code: http.StatusForbidden,
-			JSON: jsonerror.Forbidden(err.Error()), // TODO: Is this error string comprehensible to the client?
-		}
-	}
-
 	var txnAndDeviceID *api.TransactionID
 	if txnID != nil {
 		txnAndDeviceID = &api.TransactionID{
@@ -127,3 +88,66 @@ func SendEvent(
 
 	return res
 }
+
+func generateSendEvent(
+	req *http.Request,
+	device *authtypes.Device,
+	roomID, eventType string, stateKey *string,
+	cfg config.Dendrite,
+	queryAPI api.RoomserverQueryAPI,
+) (*gomatrixserverlib.Event, *util.JSONResponse) {
+	// parse the incoming http request
+	userID := device.UserID
+	var r map[string]interface{} // must be a JSON object
+	resErr := httputil.UnmarshalJSONRequest(req, &r)
+	if resErr != nil {
+		return nil, resErr
+	}
+
+	evTime, err := httputil.ParseTSParam(req)
+	if err != nil {
+		return nil, &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.InvalidArgumentValue(err.Error()),
+		}
+	}
+
+	// create the new event and set all the fields we can
+	builder := gomatrixserverlib.EventBuilder{
+		Sender:   userID,
+		RoomID:   roomID,
+		Type:     eventType,
+		StateKey: stateKey,
+	}
+	err = builder.SetContent(r)
+	if err != nil {
+		resErr := httputil.LogThenError(req, err)
+		return nil, &resErr
+	}
+
+	var queryRes api.QueryLatestEventsAndStateResponse
+	e, err := common.BuildEvent(req.Context(), &builder, cfg, evTime, queryAPI, &queryRes)
+	if err == common.ErrRoomNoExists {
+		return nil, &util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: jsonerror.NotFound("Room does not exist"),
+		}
+	} else if err != nil {
+		resErr := httputil.LogThenError(req, err)
+		return nil, &resErr
+	}
+
+	// check to see if this user can perform this operation
+	stateEvents := make([]*gomatrixserverlib.Event, len(queryRes.StateEvents))
+	for i := range queryRes.StateEvents {
+		stateEvents[i] = &queryRes.StateEvents[i]
+	}
+	provider := gomatrixserverlib.NewAuthEvents(stateEvents)
+	if err = gomatrixserverlib.Allowed(*e, &provider); err != nil {
+		return nil, &util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden(err.Error()), // TODO: Is this error string comprehensible to the client?
+		}
+	}
+	return e, nil
+}

clientapi/routing/sendtyping.go

@@ -0,0 +1,80 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"database/sql"
+	"net/http"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/clientapi/producers"
+	"github.com/matrix-org/dendrite/clientapi/userutil"
+	"github.com/matrix-org/util"
+)
+
+type typingContentJSON struct {
+	Typing  bool  `json:"typing"`
+	Timeout int64 `json:"timeout"`
+}
+
+// SendTyping handles PUT /rooms/{roomID}/typing/{userID}
+// sends the typing events to client API typingProducer
+func SendTyping(
+	req *http.Request, device *authtypes.Device, roomID string,
+	userID string, accountDB *accounts.Database,
+	typingProducer *producers.TypingServerProducer,
+) util.JSONResponse {
+	if device.UserID != userID {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("Cannot set another user's typing state"),
+		}
+	}
+
+	localpart, err := userutil.ParseUsernameParam(userID, nil)
+	if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	// Verify that the user is a member of this room
+	_, err = accountDB.GetMembershipInRoomByLocalpart(req.Context(), localpart, roomID)
+	if err == sql.ErrNoRows {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("User not in this room"),
+		}
+	} else if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	// parse the incoming http request
+	var r typingContentJSON
+	resErr := httputil.UnmarshalJSONRequest(req, &r)
+	if resErr != nil {
+		return *resErr
+	}
+
+	if err = typingProducer.Send(
+		req.Context(), userID, roomID, r.Typing, r.Timeout,
+	); err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}
+}

clientapi/routing/whoami.go

@@ -0,0 +1,34 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"net/http"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/util"
+)
+
+// whoamiResponse represents an response for a `whoami` request
+type whoamiResponse struct {
+	UserID string `json:"user_id"`
+}
+
+// Whoami implements `/account/whoami` which enables client to query their account user id.
+// https://matrix.org/docs/spec/client_server/r0.3.0.html#get-matrix-client-r0-account-whoami
+func Whoami(req *http.Request, device *authtypes.Device) util.JSONResponse {
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: whoamiResponse{UserID: device.UserID},
+	}
+}

clientapi/threepid/invites.go

@@ -89,6 +89,7 @@ func CheckAndProcessInvite(
 	device *authtypes.Device, body *MembershipRequest, cfg config.Dendrite,
 	queryAPI api.RoomserverQueryAPI, db *accounts.Database,
 	producer *producers.RoomserverProducer, membership string, roomID string,
+	evTime time.Time,
 ) (inviteStoredOnIDServer bool, err error) {
 	if membership != "invite" || (body.Address == "" && body.IDServer == "" && body.Medium == "") {
 		// If none of the 3PID-specific fields are supplied, it's a standard invite
@@ -110,7 +111,9 @@ func CheckAndProcessInvite(
 		// No Matrix ID could be found for this 3PID, meaning that a
 		// "m.room.third_party_invite" have to be emitted from the data in
 		// storeInviteRes.
-		err = emit3PIDInviteEvent(ctx, body, storeInviteRes, device, roomID, cfg, queryAPI, producer)
+		err = emit3PIDInviteEvent(
+			ctx, body, storeInviteRes, device, roomID, cfg, queryAPI, producer, evTime,
+		)
 		inviteStoredOnIDServer = err == nil
 
 		return
@@ -329,6 +332,7 @@ func emit3PIDInviteEvent(
 	body *MembershipRequest, res *idServerStoreInviteResponse,
 	device *authtypes.Device, roomID string, cfg config.Dendrite,
 	queryAPI api.RoomserverQueryAPI, producer *producers.RoomserverProducer,
+	evTime time.Time,
 ) error {
 	builder := &gomatrixserverlib.EventBuilder{
 		Sender:   device.UserID,
@@ -350,7 +354,7 @@ func emit3PIDInviteEvent(
 	}
 
 	var queryRes *api.QueryLatestEventsAndStateResponse
-	event, err := common.BuildEvent(ctx, builder, cfg, queryAPI, queryRes)
+	event, err := common.BuildEvent(ctx, builder, cfg, evTime, queryAPI, queryRes)
 	if err != nil {
 		return err
 	}

cmd/client-api-proxy/main.go

@@ -17,13 +17,14 @@ package main
 import (
 	"flag"
 	"fmt"
-	log "github.com/sirupsen/logrus"
 	"net/http"
 	"net/http/httputil"
 	"net/url"
 	"os"
 	"strings"
 	"time"
+
+	log "github.com/sirupsen/logrus"
 )
 
 const usage = `Usage: %s

cmd/create-room-events/main.go

@@ -139,6 +139,6 @@ func writeEvent(event gomatrixserverlib.Event) {
 			panic(err)
 		}
 	} else {
-		panic(fmt.Errorf("Format %q is not valid, must be %q or %q", format, "InputRoomEvent", "Event"))
+		panic(fmt.Errorf("Format %q is not valid, must be %q or %q", *format, "InputRoomEvent", "Event"))
 	}
 }

cmd/dendrite-appservice-server/main.go

@@ -22,16 +22,17 @@ import (
 
 func main() {
 	cfg := basecomponent.ParseFlags()
-	base := basecomponent.NewBaseDendrite(cfg, "AppService")
+	base := basecomponent.NewBaseDendrite(cfg, "AppServiceAPI")
 
 	defer base.Close() // nolint: errcheck
 	accountDB := base.CreateAccountsDB()
+	deviceDB := base.CreateDeviceDB()
 	federation := base.CreateFederationClient()
 	alias, _, query := base.CreateHTTPRoomserverAPIs()
 	cache := transactions.New()
 
 	appservice.SetupAppServiceAPIComponent(
-		base, accountDB, federation, alias, query, cache,
+		base, accountDB, deviceDB, federation, alias, query, cache,
 	)
 
 	base.SetupAndServeHTTP(string(base.Cfg.Listen.FederationSender))

cmd/dendrite-client-api-server/main.go

@@ -19,6 +19,8 @@ import (
 	"github.com/matrix-org/dendrite/common/basecomponent"
 	"github.com/matrix-org/dendrite/common/keydb"
 	"github.com/matrix-org/dendrite/common/transactions"
+	"github.com/matrix-org/dendrite/typingserver"
+	"github.com/matrix-org/dendrite/typingserver/cache"
 )
 
 func main() {
@@ -33,12 +35,13 @@ func main() {
 	federation := base.CreateFederationClient()
 	keyRing := keydb.CreateKeyRing(federation.Client, keyDB)
 
+	asQuery := base.CreateHTTPAppServiceAPIs()
 	alias, input, query := base.CreateHTTPRoomserverAPIs()
-	cache := transactions.New()
+	typingInputAPI := typingserver.SetupTypingServerComponent(base, cache.NewTypingCache())
 
 	clientapi.SetupClientAPIComponent(
 		base, deviceDB, accountDB, federation, &keyRing,
-		alias, input, query, cache,
+		alias, input, query, typingInputAPI, asQuery, transactions.New(),
 	)
 
 	base.SetupAndServeHTTP(string(base.Cfg.Listen.ClientAPI))

cmd/dendrite-federation-api-server/main.go

@@ -26,15 +26,17 @@ func main() {
 	defer base.Close() // nolint: errcheck
 
 	accountDB := base.CreateAccountsDB()
+	deviceDB := base.CreateDeviceDB()
 	keyDB := base.CreateKeyDB()
 	federation := base.CreateFederationClient()
 	keyRing := keydb.CreateKeyRing(federation.Client, keyDB)
 
 	alias, input, query := base.CreateHTTPRoomserverAPIs()
+	asQuery := base.CreateHTTPAppServiceAPIs()
 
 	federationapi.SetupFederationAPIComponent(
-		base, accountDB, federation, &keyRing,
+		base, accountDB, deviceDB, federation, &keyRing,
-		alias, input, query,
+		alias, input, query, asQuery,
 	)
 
 	base.SetupAndServeHTTP(string(base.Cfg.Listen.FederationAPI))

cmd/dendrite-monolith-server/main.go

@@ -18,19 +18,20 @@ import (
 	"flag"
 	"net/http"
 
-	"github.com/matrix-org/dendrite/common/keydb"
-	"github.com/matrix-org/dendrite/common/transactions"
-
 	"github.com/matrix-org/dendrite/appservice"
 	"github.com/matrix-org/dendrite/clientapi"
 	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/dendrite/common/basecomponent"
+	"github.com/matrix-org/dendrite/common/keydb"
+	"github.com/matrix-org/dendrite/common/transactions"
 	"github.com/matrix-org/dendrite/federationapi"
 	"github.com/matrix-org/dendrite/federationsender"
 	"github.com/matrix-org/dendrite/mediaapi"
 	"github.com/matrix-org/dendrite/publicroomsapi"
 	"github.com/matrix-org/dendrite/roomserver"
 	"github.com/matrix-org/dendrite/syncapi"
+	"github.com/matrix-org/dendrite/typingserver"
+	"github.com/matrix-org/dendrite/typingserver/cache"
 
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/sirupsen/logrus"
@@ -55,18 +56,21 @@ func main() {
 	keyRing := keydb.CreateKeyRing(federation.Client, keyDB)
 
 	alias, input, query := roomserver.SetupRoomServerComponent(base)
+	typingInputAPI := typingserver.SetupTypingServerComponent(base, cache.NewTypingCache())
+	asQuery := appservice.SetupAppServiceAPIComponent(
+		base, accountDB, deviceDB, federation, alias, query, transactions.New(),
+	)
 
 	clientapi.SetupClientAPIComponent(
 		base, deviceDB, accountDB,
 		federation, &keyRing, alias, input, query,
-		transactions.New(),
+		typingInputAPI, asQuery, transactions.New(),
 	)
-	federationapi.SetupFederationAPIComponent(base, accountDB, federation, &keyRing, alias, input, query)
+	federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery)
 	federationsender.SetupFederationSenderComponent(base, federation, query)
 	mediaapi.SetupMediaAPIComponent(base, deviceDB)
 	publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB)
 	syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query)
-	appservice.SetupAppServiceAPIComponent(base, accountDB, federation, alias, query, transactions.New())
 
 	httpHandler := common.WrapHandlerInCORS(base.APIMux)
 

cmd/dendrite-typing-server/main.go

@@ -0,0 +1,36 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	_ "net/http/pprof"
+
+	"github.com/matrix-org/dendrite/common/basecomponent"
+	"github.com/matrix-org/dendrite/typingserver"
+	"github.com/matrix-org/dendrite/typingserver/cache"
+	"github.com/sirupsen/logrus"
+)
+
+func main() {
+	cfg := basecomponent.ParseFlags()
+	base := basecomponent.NewBaseDendrite(cfg, "TypingServerAPI")
+	defer func() {
+		if err := base.Close(); err != nil {
+			logrus.WithError(err).Warn("BaseDendrite close failed")
+		}
+	}()
+
+	typingserver.SetupTypingServerComponent(base, cache.NewTypingCache())
+
+	base.SetupAndServeHTTP(string(base.Cfg.Listen.TypingServer))
+}

cmd/federation-api-proxy/main.go

@@ -17,13 +17,14 @@ package main
 import (
 	"flag"
 	"fmt"
-	log "github.com/sirupsen/logrus"
 	"net/http"
 	"net/http/httputil"
 	"net/url"
 	"os"
 	"strings"
 	"time"
+
+	log "github.com/sirupsen/logrus"
 )
 
 const usage = `Usage: %s

cmd/kafka-producer/main.go

@@ -18,9 +18,10 @@ import (
 	"bufio"
 	"flag"
 	"fmt"
-	"github.com/Shopify/sarama"
 	"os"
 	"strings"
+
+	"github.com/Shopify/sarama"
 )
 
 const usage = `Usage: %s

cmd/mediaapi-integration-tests/main.go

@@ -41,7 +41,7 @@ var (
 	// Postgres docker container name (for running psql). If not set, psql must be in PATH.
 	postgresContainerName = os.Getenv("POSTGRES_CONTAINER")
 	// Test image to be uploaded/downloaded
-	testJPEG = test.Defaulting(os.Getenv("TEST_JPEG_PATH"), "src/github.com/matrix-org/dendrite/cmd/mediaapi-integration-tests/totem.jpg")
+	testJPEG = test.Defaulting(os.Getenv("TEST_JPEG_PATH"), "cmd/mediaapi-integration-tests/totem.jpg")
 	kafkaURI = test.Defaulting(os.Getenv("KAFKA_URIS"), "localhost:9092")
 )
 

common/basecomponent/base.go

@@ -30,8 +30,10 @@ import (
 	"github.com/gorilla/mux"
 	sarama "gopkg.in/Shopify/sarama.v1"
 
+	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/common/config"
-	"github.com/matrix-org/dendrite/roomserver/api"
+	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
+	typingServerAPI "github.com/matrix-org/dendrite/typingserver/api"
 	"github.com/sirupsen/logrus"
 )
 
@@ -69,7 +71,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string) *BaseDendrite {
 		componentName: componentName,
 		tracerCloser:  closer,
 		Cfg:           cfg,
-		APIMux:        mux.NewRouter(),
+		APIMux:        mux.NewRouter().UseEncodedPath(),
 		KafkaConsumer: kafkaConsumer,
 		KafkaProducer: kafkaProducer,
 	}
@@ -80,15 +82,31 @@ func (b *BaseDendrite) Close() error {
 	return b.tracerCloser.Close()
 }
 
-// CreateHTTPRoomserverAPIs returns the AliasAPI, InputAPI and QueryAPI to hit
+// CreateHTTPAppServiceAPIs returns the QueryAPI for hitting the appservice
+// component over HTTP.
+func (b *BaseDendrite) CreateHTTPAppServiceAPIs() appserviceAPI.AppServiceQueryAPI {
+	return appserviceAPI.NewAppServiceQueryAPIHTTP(b.Cfg.AppServiceURL(), nil)
+}
+
+// CreateHTTPRoomserverAPIs returns the AliasAPI, InputAPI and QueryAPI for hitting
 // the roomserver over HTTP.
-func (b *BaseDendrite) CreateHTTPRoomserverAPIs() (api.RoomserverAliasAPI, api.RoomserverInputAPI, api.RoomserverQueryAPI) {
+func (b *BaseDendrite) CreateHTTPRoomserverAPIs() (
-	alias := api.NewRoomserverAliasAPIHTTP(b.Cfg.RoomServerURL(), nil)
+	roomserverAPI.RoomserverAliasAPI,
-	input := api.NewRoomserverInputAPIHTTP(b.Cfg.RoomServerURL(), nil)
+	roomserverAPI.RoomserverInputAPI,
-	query := api.NewRoomserverQueryAPIHTTP(b.Cfg.RoomServerURL(), nil)
+	roomserverAPI.RoomserverQueryAPI,
+) {
+	alias := roomserverAPI.NewRoomserverAliasAPIHTTP(b.Cfg.RoomServerURL(), nil)
+	input := roomserverAPI.NewRoomserverInputAPIHTTP(b.Cfg.RoomServerURL(), nil)
+	query := roomserverAPI.NewRoomserverQueryAPIHTTP(b.Cfg.RoomServerURL(), nil)
 	return alias, input, query
 }
 
+// CreateHTTPTypingServerAPIs returns typingInputAPI for hitting the typing
+// server over HTTP
+func (b *BaseDendrite) CreateHTTPTypingServerAPIs() typingServerAPI.TypingServerInputAPI {
+	return typingServerAPI.NewTypingServerInputAPIHTTP(b.Cfg.TypingServerURL(), nil)
+}
+
 // CreateDeviceDB creates a new instance of the device database. Should only be
 // called once per component.
 func (b *BaseDendrite) CreateDeviceDB() *devices.Database {

common/config/appservice.go

@@ -21,6 +21,7 @@ import (
 	"regexp"
 	"strings"
 
+	log "github.com/sirupsen/logrus"
 	yaml "gopkg.in/yaml.v2"
 )
 
@@ -31,6 +32,14 @@ type ApplicationServiceNamespace struct {
 	Exclusive bool `yaml:"exclusive"`
 	// A regex pattern that represents the namespace
 	Regex string `yaml:"regex"`
+	// The ID of an existing group that all users of this application service will
+	// be added to. This field is only relevant to the `users` namespace.
+	// Note that users who are joined to this group through an application service
+	// are not to be listed when querying for the group's members, however the
+	// group should be listed when querying an application service user's groups.
+	// This is to prevent making spamming all users of an application service
+	// trivial.
+	GroupID string `yaml:"group_id"`
 	// Regex object representing our pattern. Saves having to recompile every time
 	RegexpObject *regexp.Regexp
 }
@@ -51,14 +60,68 @@ type ApplicationService struct {
 	// Information about an application service's namespaces. Key is either
 	// "users", "aliases" or "rooms"
 	NamespaceMap map[string][]ApplicationServiceNamespace `yaml:"namespaces"`
+	// Whether rate limiting is applied to each application service user
+	RateLimited bool `yaml:"rate_limited"`
+	// Any custom protocols that this application service provides (e.g. IRC)
+	Protocols []string `yaml:"protocols"`
 }
 
-// loadAppservices iterates through all application service config files
+// IsInterestedInRoomID returns a bool on whether an application service's
+// namespace includes the given room ID
+func (a *ApplicationService) IsInterestedInRoomID(
+	roomID string,
+) bool {
+	if namespaceSlice, ok := a.NamespaceMap["rooms"]; ok {
+		for _, namespace := range namespaceSlice {
+			if namespace.RegexpObject.MatchString(roomID) {
+				return true
+			}
+		}
+	}
+
+	return false
+}
+
+// IsInterestedInUserID returns a bool on whether an application service's
+// namespace includes the given user ID
+func (a *ApplicationService) IsInterestedInUserID(
+	userID string,
+) bool {
+	if namespaceSlice, ok := a.NamespaceMap["users"]; ok {
+		for _, namespace := range namespaceSlice {
+			if namespace.RegexpObject.MatchString(userID) {
+				return true
+			}
+		}
+	}
+
+	return false
+}
+
+// IsInterestedInRoomAlias returns a bool on whether an application service's
+// namespace includes the given room alias
+func (a *ApplicationService) IsInterestedInRoomAlias(
+	roomAlias string,
+) bool {
+	if namespaceSlice, ok := a.NamespaceMap["aliases"]; ok {
+		for _, namespace := range namespaceSlice {
+			if namespace.RegexpObject.MatchString(roomAlias) {
+				return true
+			}
+		}
+	}
+
+	return false
+}
+
+// loadAppServices iterates through all application service config files
 // and loads their data into the config object for later access.
-func loadAppservices(config *Dendrite) error {
+func loadAppServices(config *Dendrite) error {
 	for _, configPath := range config.ApplicationServices.ConfigFiles {
-		// Create a new application service
+		// Create a new application service with default options
-		var appservice ApplicationService
+		appservice := ApplicationService{
+			RateLimited: true,
+		}
 
 		// Create an absolute path from a potentially relative path
 		absPath, err := filepath.Abs(configPath)
@@ -108,8 +171,6 @@ func setupRegexps(cfg *Dendrite) (err error) {
 		}
 	}
 
-	fmt.Println(exclusiveUsernameStrings, exclusiveAliasStrings)
-
 	// Join the regexes together into one big regex.
 	// i.e. "app1.*", "app2.*" -> "(app1.*)|(app2.*)"
 	// Later we can check if a username or alias matches any exclusive regex and
@@ -161,19 +222,34 @@ func checkErrors(config *Dendrite) (err error) {
 	var idMap = make(map[string]bool)
 	var tokenMap = make(map[string]bool)
 
+	// Compile regexp object for checking groupIDs
+	groupIDRegexp := regexp.MustCompile(`\+.*:.*`)
+
 	// Check each application service for any config errors
 	for _, appservice := range config.Derived.ApplicationServices {
+		// Namespace-related checks
+		for key, namespaceSlice := range appservice.NamespaceMap {
+			for _, namespace := range namespaceSlice {
+				if err := validateNamespace(&appservice, key, &namespace, groupIDRegexp); err != nil {
+					return err
+				}
+			}
+		}
+
+		// Check if the url has trailing /'s. If so, remove them
+		appservice.URL = strings.TrimRight(appservice.URL, "/")
+
 		// Check if we've already seen this ID. No two application services
 		// can have the same ID or token.
 		if idMap[appservice.ID] {
 			return configErrors([]string{fmt.Sprintf(
-				"Application Service ID %s must be unique", appservice.ID,
+				"Application service ID %s must be unique", appservice.ID,
 			)})
 		}
 		// Check if we've already seen this token
 		if tokenMap[appservice.ASToken] {
 			return configErrors([]string{fmt.Sprintf(
-				"Application Service Token %s must be unique", appservice.ASToken,
+				"Application service Token %s must be unique", appservice.ASToken,
 			)})
 		}
 
@@ -182,33 +258,50 @@ func checkErrors(config *Dendrite) (err error) {
 		idMap[appservice.ID] = true
 		tokenMap[appservice.ASToken] = true
 
-		// Check if more than one regex exists per namespace
+		// TODO: Remove once rate_limited is implemented
-		for _, namespace := range appservice.NamespaceMap {
+		if appservice.RateLimited {
-			if len(namespace) > 1 {
+			log.Warn("WARNING: Application service option rate_limited is currently unimplemented")
-				// It's quite easy to accidentally make multiple regex objects per
-				// namespace, which often ends up in an application service receiving events
-				// it doesn't want, as an empty regex will match all events.
-				return configErrors([]string{fmt.Sprintf(
-					"Application Service namespace can only contain a single regex tuple. Check your YAML.",
-				)})
 		}
+		// TODO: Remove once protocols is implemented
+		if len(appservice.Protocols) > 0 {
+			log.Warn("WARNING: Application service option protocols is currently unimplemented")
 		}
 	}
 
+	return setupRegexps(config)
+}
+
+// validateNamespace returns nil or an error based on whether a given
+// application service namespace is valid. A namespace is valid if it has the
+// required fields, and its regex is correct.
+func validateNamespace(
+	appservice *ApplicationService,
+	key string,
+	namespace *ApplicationServiceNamespace,
+	groupIDRegexp *regexp.Regexp,
+) error {
 	// Check that namespace(s) are valid regex
-	for _, appservice := range config.Derived.ApplicationServices {
-		for _, namespaceSlice := range appservice.NamespaceMap {
-			for _, namespace := range namespaceSlice {
 	if !IsValidRegex(namespace.Regex) {
 		return configErrors([]string{fmt.Sprintf(
 			"Invalid regex string for Application Service %s", appservice.ID,
 		)})
 	}
-			}
+
+	// Check if GroupID for the users namespace is in the correct format
+	if key == "users" && namespace.GroupID != "" {
+		// TODO: Remove once group_id is implemented
+		log.Warn("WARNING: Application service option group_id is currently unimplemented")
+
+		correctFormat := groupIDRegexp.MatchString(namespace.GroupID)
+		if !correctFormat {
+			return configErrors([]string{fmt.Sprintf(
+				"Invalid user group_id field for application service %s.",
+				appservice.ID,
+			)})
 		}
 	}
 
-	return setupRegexps(config)
+	return nil
 }
 
 // IsValidRegex returns true or false based on whether the