Merge branch 'master' into viccuad/add-missing-docker-compose

.circleci/config.yml

@@ -0,0 +1,32 @@
+version: 2
+jobs:
+  dendrite:
+    docker:
+      - image: matrixdotorg/sytest-dendrite
+    working_directory: /src
+    steps:
+      - checkout
+
+      # Set up dendrite
+      - run: 
+          name: Build Dendrite
+          command: ./build.sh
+      - run: 
+          name: Copy dummy keys to root
+          command: |
+            mv .circleci/matrix_key.pem .
+            mv .circleci/server.key .
+      - run:
+          name: Run sytest with whitelisted tests
+          command: /dendrite_sytest.sh
+
+      - store_artifacts:
+          path: /logs
+          destination: logs
+      - store_test_results:
+          path: /logs
+workflows:
+  version: 2
+  build:
+    jobs:
+      - dendrite

.circleci/matrix_key.pem

@@ -0,0 +1,5 @@
+-----BEGIN MATRIX PRIVATE KEY-----
+Key-ID: ed25519:zXtB
+
+jDyHsx0EXbAfvM32yBEKQfIy1FHrmwtB1uMAbm5INBg=
+-----END MATRIX PRIVATE KEY-----

.circleci/server.key

@@ -0,0 +1,52 @@
+-----BEGIN PRIVATE KEY-----
+MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQCanRCqP11MLIQh
+nC26+A1oyBsFfH7auZ3pqE/WFDrCCIoc7ek7cF3fZU7q8OYI+Q9L5V8fobuLb6FB
+iXD5zZ6pBAI0VNjAS8yi8VluXIv6pJKsVY3k2hGiU7xRoEhkzckZBaEiruspQbcX
+ziNoWoueVBB1a4Eproqzy225cTcoprHsJIPXj0HpW/jKcmahmlM/OrqRAxTwxpb/
+moI6MWIeN4n7h55N6dU1ScVvBS7gZpZQ28d8akuvG3m8kE8q1OPFYGvrNeowD4sp
+qDPFijhbygwpzDQlAWriPcqV9KhuGRnYRGTGvuluOttmpgNhNFVxVAlwZJuMVAMU
+Jhek66ntKsxWkF5LsO8ls20hmHyyAsL7+rb2ZjuRtEwE8SwOstU2AIIXoSTtqXjX
+zC8Ew0VB9MCInJoJC/+iKTLoDqXRZeDKGFx1A2F3Y+Er+Z41HcwgqKRsPqZ066yR
+6iKAb5rzJutnEARtbSrNipy9nHE5hIgKJzgOnggcegypcAj3nqbfFFCZA2CFNXoG
+XFkmBHEpz38pPLI5z6HpeZRRySoIyahk9IfSwM3aB1aUi//8CcpAodGvYGNQkQ3W
+HkrZmM4MtC25I5RyMpYJQWKFpx1cOVPf2ASqaJ+IX1JJTv9dSdYHY/rxsxaiXiry
++uI7UITRvUKgAOrExfSAXco73bgUFwIDAQABAoICAQCP9QX7PhxEPH6aPKxnlWYG
+1aozJYOHa6QYVlpfXV6IIyNVZD7w1OLSiaU9IydL23nelKZI8XGJllpyhuHl9Qlx
+HQZga0+VW/4hCM7X7tt2d50JUG9ZUaFxnr2M0swU73X6Ej/B51OVilZLl+dn1kaB
+GIxqh7ovcRA774EuVLei5fJriGQpZH1eJgAznujoNqSkDq5/Lntk48LcIqR2Qly0
+/ck/pTpEGSAnCZUGlbDbxyjWCIxozx/A3rguVb8ghi+9KtXQntZ6AT71fmMV3mgz
+LqC8miFDA1rdY+MoVDAusrhZoPSkCEWYGL0HijNDYlLbvf874rDhq6diL0V8jOAd
+PGOx5BY6VUWbSQAUtKpMuNSL6tidkOACGPwbuH7OIaG+yGZ0/Oiy3fureiAEg5VU
+piyp6F7p1g0vgQEnj4CHiCQlX48bjC/mm8758DeaH8H5T++A8MOgRhgFVb9f01R+
+NMzszMziuVNDYe01cwdY1TXUx5b0o+opsbPm6sNp/7afL9Hou1epP9zQC0I8ulfP
+fgrKTddMwlNjoBuDMQ8GqoK275YU4wtyhUMfjr3xQ0JwP46cZbhhc4nh6qcRSNTf
+yVuKv/pT/bJcSmg5JOCS8qdK0BQhAvUin9HvgSAV9QmZVpxzT/xhqwuRlLDKW+VR
+XyPt996f3L4CTXI9h88AQQKCAQEAycBChu3/ZKl8a90anOlv9PwmaaXfLBKH9Rkw
+aeZrMilxTJAb+LEsmtj35rF5KPeBP6ARpX5gmvKJVzCDHT9YgNs+6C3E+l2f1/3a
+TcjZKPTukT2gJdCgejhEgTzAwEse322GSptuyidtNpY7NgbAxP4VdDMOmPYbzufb
+5BqxmfiGsfXgdvQkj8/MzHuGhhft4SU6ED/Ax+EPUWVV7kBr2995kGDF5z5CuJkb
+SJjmVxAJZP/kC2Z/iPnP51G0hiCxHp7+gPY4mvvkHvhJGnGH/vutjRjoe28BENlP
+MgB68S1/U3NGSUzWv86pT1OdHd+qynWj/NzF7Gp/T/ju8VZBXwKCAQEAxDAMSOfF
+dizsU7cJbf6vxi6XJHjhwWUWD2vMznKz1D4mkByeY8aSOc8kQZsE5nd4ZgwkYTaZ
+gItjGjM5y5dpKurfKdqQ+dA6PS03h3p+tp1lZp9/dI9X/DfkTO/LUdrfkVVcbQhE
+zqc6C35qO98rhJdsRwhOF28mOc/4bbs0XjC5dEoBGyFt7Fbn2mYoCo4FSHl7WIq6
+TZR9pLAvxjqEZ6Dwrzpp9wtdLIQYPga+KVKcDT/DStThXDTCNt5PyDE9c8eImFww
+u0T87Er5hSEQgodURxDOZh+9ktIfXzMtxiAJ3iDCEPc3NNnLCWfKMhwGsVTCCXj6
+WuHTOe79tOaQSQKCAQEAqBN52PsRl4TzWNEcyLhZQxmFzuIXKJpPlctkX/VMPL/1
+2bj89JR1+pLjA9e6fnyjuqPZz6uXQ77m2DJcKNOLId6Fa9wljAbPkZu0cLTw5YQX
+8/wJHTfPWcLin2BDnG94yt5t0F3pUJTEEYPa1EmP8w1SRjn64Ue3JwpWUJREfWdk
+n4GdfLwscXrGvVvzWGc7ECR5WOwj6OEAZ+kqS5BzyvtERRm6BcoCv9Mdvb9Tthhw
+Gypri2vat/yWTbnt0QgPRtliYYG+6q8K/xoNnPAUQkLd9PxZQevaUXUY2yk3QxGK
+T7VrSsmu5qB+wM2ByU9686xJ7/DlGu4mHjPerEQVtQKCAQBcM3iSitpyP4qRjWQR
+HbDeIudFbMosaaWEedU28REynkLhV5HYsmnmYUNY0dHrvhoHW419YnuhveBFX+25
+kN8MHHXk5aNcxE+akLWYJimHCVGueScdUIC5OEtDHS8guQx48PUPCOPNeyn8XNzw
+ZmG9Xqy0dWK+AK6mXOcUKvbhjWSbEmySo5NVj0JHkdsfmr9A4Fbntcr4yuCBlYve
+TYIMccark3hZci3HzgzWmbSlFv3f/Cd787A19VWRE8nK+9k1oIDBmhIM8M8s/c9m
+kbOApLkm7O8Tb7dYWQgFZbgNdOEuU5bhAk4fuHuDYBPWmPVMQdkvOnvuWlM61ubF
+LdaBAoIBACDpbb5AQIYsWWOnoXuuGh+YY4kmnaBFpsbgEYkZSy92AaLr4Ibf49WN
+oqNDX73YaJlURaGPYMC9J2Huq7TZcewH3SwkVA3N5UmDoijkM4juRfADAfVIMxB5
++9paWeEfnYC/o377FTJIJ9hHJWIaWSoiJZLYDBmoYdxmk8DSHAJCeWsjYDzPybsH
+7RyMPIa1u7lVdgOPEOBi1OIg7ASLxGKiHQtrYHq99GcaVvU/UxoNRMcSnPfY3G8R
+pGah+EndSCb2F20ouDyvlKfOylAltH2BeNc3B4PeP7ZhlVr7bfyOAfC2Z7FNDm3J
++yaBExKfroZjsksctNAcAbgpuvhLLG8=
+-----END PRIVATE KEY-----

.github/PULL_REQUEST_TEMPLATE.md

@@ -2,5 +2,5 @@
 
 <!-- Please read CONTRIBUTING.md before submitting your pull request -->
 
-* [ ] I have made sure any new dependencies have been checked into the `vendor/` directory
+* [ ] I have added any new tests that need to pass to `testfile` as specified in [docs/sytest.md](https://github.com/matrix-org/dendrite/blob/master/docs/sytest.md)
 * [ ] Pull request includes a [sign off](https://github.com/matrix-org/dendrite/blob/master/CONTRIBUTING.md#sign-off)

.gitignore

@@ -40,3 +40,6 @@ _testmain.go
 *.pem
 *.key
 *.crt
+
+# Default configuration file
+dendrite.yaml

.golangci.yml

@@ -0,0 +1,280 @@
+# Config file for golangci-lint
+
+# options for analysis running
+run:
+  # default concurrency is a available CPU number
+  concurrency: 4
+
+  # timeout for analysis, e.g. 30s, 5m, default is 1m
+  deadline: 30m
+
+  # exit code when at least one issue was found, default is 1
+  issues-exit-code: 1
+
+  # include test files or not, default is true
+  tests: true
+
+  # list of build tags, all linters use it. Default is empty list.
+  #build-tags:
+  #  - mytag
+
+  # which dirs to skip: they won't be analyzed;
+  # can use regexp here: generated.*, regexp is applied on full path;
+  # default value is empty list, but next dirs are always skipped independently
+  # from this option's value:
+  #   	vendor$, third_party$, testdata$, examples$, Godeps$, builtin$
+  skip-dirs:
+    - bin
+    - docs
+
+  # which files to skip: they will be analyzed, but issues from them
+  # won't be reported. Default value is empty list, but there is
+  # no need to include all autogenerated files, we confidently recognize
+  # autogenerated files. If it's not please let us know.
+  skip-files:
+    - ".*\\.md$"
+    - ".*\\.sh$"
+    - "^cmd/syncserver-integration-tests/testdata.go$"
+
+  # by default isn't set. If set we pass it to "go list -mod={option}". From "go help modules":
+  # If invoked with -mod=readonly, the go command is disallowed from the implicit
+  # automatic updating of go.mod described above. Instead, it fails when any changes
+  # to go.mod are needed. This setting is most useful to check that go.mod does
+  # not need updates, such as in a continuous integration and testing system.
+  # If invoked with -mod=vendor, the go command assumes that the vendor
+  # directory holds the correct copies of dependencies and ignores
+  # the dependency descriptions in go.mod.
+  #modules-download-mode: (release|readonly|vendor)
+
+
+# output configuration options
+output:
+  # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number"
+  format: colored-line-number
+
+  # print lines of code with issue, default is true
+  print-issued-lines: true
+
+  # print linter name in the end of issue text, default is true
+  print-linter-name: true
+
+
+# all available settings of specific linters
+linters-settings:
+  errcheck:
+    # report about not checking of errors in type assertions: `a := b.(MyStruct)`;
+    # default is false: such cases aren't reported by default.
+    check-type-assertions: false
+
+    # report about assignment of errors to blank identifier: `num, _ := strconv.Atoi(numStr)`;
+    # default is false: such cases aren't reported by default.
+    check-blank: false
+
+    # [deprecated] comma-separated list of pairs of the form pkg:regex
+    # the regex is used to ignore names within pkg. (default "fmt:.*").
+    # see https://github.com/kisielk/errcheck#the-deprecated-method for details
+    #ignore: fmt:.*,io/ioutil:^Read.*
+
+    # path to a file containing a list of functions to exclude from checking
+    # see https://github.com/kisielk/errcheck#excluding-functions for details
+    #exclude: /path/to/file.txt
+  govet:
+    # report about shadowed variables
+    check-shadowing: true
+
+    # settings per analyzer
+    settings:
+      printf: # analyzer name, run `go tool vet help` to see all analyzers
+        funcs: # run `go tool vet help printf` to see available settings for `printf` analyzer
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf
+          - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf
+  golint:
+    # minimal confidence for issues, default is 0.8
+    min-confidence: 0.8
+  gofmt:
+    # simplify code: gofmt with `-s` option, true by default
+    simplify: true
+  goimports:
+    # put imports beginning with prefix after 3rd-party packages;
+    # it's a comma-separated list of prefixes
+    #local-prefixes: github.com/org/project
+  gocyclo:
+    # minimal code complexity to report, 30 by default (but we recommend 10-20)
+    min-complexity: 12
+  maligned:
+    # print struct with more effective memory layout or not, false by default
+    suggest-new: true
+  dupl:
+    # tokens count to trigger issue, 150 by default
+    threshold: 100
+  goconst:
+    # minimal length of string constant, 3 by default
+    min-len: 3
+    # minimal occurrences count to trigger, 3 by default
+    min-occurrences: 3
+  depguard:
+    list-type: blacklist
+    include-go-root: false
+    packages:
+    #  - github.com/davecgh/go-spew/spew
+  misspell:
+    # Correct spellings using locale preferences for US or UK.
+    # Default is to use a neutral variety of English.
+    # Setting locale to US will correct the British spelling of 'colour' to 'color'.
+    locale: UK
+    ignore-words:
+    #  - someword
+  lll:
+    # max line length, lines longer will be reported. Default is 120.
+    # '\t' is counted as 1 character by default, and can be changed with the tab-width option
+    line-length: 96
+    # tab width in spaces. Default to 1.
+    tab-width: 1
+  unused:
+    # treat code as a program (not a library) and report unused exported identifiers; default is false.
+    # XXX: if you enable this setting, unused will report a lot of false-positives in text editors:
+    # if it's called for subdir of a project it can't find funcs usages. All text editor integrations
+    # with golangci-lint call it on a directory with the changed file.
+    check-exported: false
+  unparam:
+    # Inspect exported functions, default is false. Set to true if no external program/library imports your code.
+    # XXX: if you enable this setting, unparam will report a lot of false-positives in text editors:
+    # if it's called for subdir of a project it can't find external interfaces. All text editor integrations
+    # with golangci-lint call it on a directory with the changed file.
+    check-exported: false
+  nakedret:
+    # make an issue if func has more lines of code than this setting and it has naked returns; default is 30
+    max-func-lines: 60
+  prealloc:
+    # XXX: we don't recommend using this linter before doing performance profiling.
+    # For most programs usage of prealloc will be a premature optimization.
+
+    # Report preallocation suggestions only on simple loops that have no returns/breaks/continues/gotos in them.
+    # True by default.
+    simple: true
+    range-loops: true # Report preallocation suggestions on range loops, true by default
+    for-loops: false # Report preallocation suggestions on for loops, false by default
+  gocritic:
+    # Which checks should be enabled; can't be combined with 'disabled-checks';
+    # See https://go-critic.github.io/overview#checks-overview
+    # To check which checks are enabled run `GL_DEBUG=gocritic golangci-lint run`
+    # By default list of stable checks is used.
+    #enabled-checks:
+
+    # Which checks should be disabled; can't be combined with 'enabled-checks'; default is empty
+    #disabled-checks:
+
+    # Enable multiple checks by tags, run `GL_DEBUG=gocritic golangci-lint` run to see all tags and checks.
+    # Empty list by default. See https://github.com/go-critic/go-critic#usage -> section "Tags".
+    #enabled-tags:
+    #  - performance
+
+    settings: # settings passed to gocritic
+      captLocal: # must be valid enabled check name
+        paramsOnly: true
+      #rangeValCopy:
+      #  sizeThreshold: 32
+
+linters:
+  enable:
+    - deadcode
+    - errcheck
+    - goconst
+    - gocyclo
+    - goimports # Does everything gofmt does
+    - gosimple
+    - ineffassign
+    - megacheck
+    - misspell # Check code comments, whereas misspell in CI checks *.md files
+    - nakedret
+    - staticcheck
+    - structcheck
+    - unparam
+    - unused
+    - varcheck
+  enable-all: false
+  disable:
+    - bodyclose
+    - depguard
+    - dupl
+    - gochecknoglobals
+    - gochecknoinits
+    - gocritic
+    - gofmt
+    - golint
+    - gosec # Should turn back on soon
+    - interfacer
+    - lll
+    - maligned
+    - prealloc # Should turn back on soon
+    - scopelint
+    - stylecheck
+    - typecheck # Should turn back on soon
+    - unconvert # Should turn back on soon
+  disable-all: false
+  presets:
+  fast: false
+
+
+issues:
+  # List of regexps of issue texts to exclude, empty list by default.
+  # But independently from this option we use default exclude patterns,
+  # it can be disabled by `exclude-use-default: false`. To list all
+  # excluded by default patterns execute `golangci-lint run --help`
+  exclude:
+  # - abcdef
+
+  # Excluding configuration per-path, per-linter, per-text and per-source
+  exclude-rules:
+    # Exclude some linters from running on tests files.
+    - path: _test\.go
+      linters:
+        - gocyclo
+        - errcheck
+        - dupl
+        - gosec
+
+    # Exclude known linters from partially hard-vendored code,
+    # which is impossible to exclude via "nolint" comments.
+    - path: internal/hmac/
+      text: "weak cryptographic primitive"
+      linters:
+        - gosec
+
+    # Exclude some staticcheck messages
+    - linters:
+        - staticcheck
+      text: "SA9003:"
+
+    # Exclude lll issues for long lines with go:generate
+    - linters:
+        - lll
+      source: "^//go:generate "
+
+  # Independently from option `exclude` we use default exclude patterns,
+  # it can be disabled by this option. To list all
+  # excluded by default patterns execute `golangci-lint run --help`.
+  # Default value for this option is true.
+  exclude-use-default: false
+
+  # Maximum issues count per one linter. Set to 0 to disable. Default is 50.
+  max-issues-per-linter: 0
+
+  # Maximum count of issues with the same text. Set to 0 to disable. Default is 3.
+  max-same-issues: 0
+
+  # Show only new issues: if there are unstaged changes or untracked files,
+  # only those changes are analyzed, else only changes in HEAD~ are analyzed.
+  # It's a super-useful option for integration of golangci-lint into existing
+  # large codebase. It's not practical to fix all existing issues at the moment
+  # of integration: much better don't allow issues in new code.
+  # Default is false.
+  new: false
+
+  # Show only new issues created after git revision `REV`
+  #new-from-rev: REV
+
+  # Show only new issues created in git patch with set file path.
+  #new-from-patch: path/to/patch/file

.travis.yml

@@ -1,34 +0,0 @@
-language: go
-go:
- - 1.10.x
- - 1.11.x
-
-env:
-- TEST_SUITE="lint"
-- TEST_SUITE="unit-test"
-- TEST_SUITE="integ-test"
-
-sudo: false
-
-# Use trusty for postgres 9.5 support
-dist: trusty
-
-addons:
-  postgresql: "9.5"
-
-services:
-  - postgresql
-
-cache:
-  directories:
-    - .downloads
-
-install:
- - go get github.com/constabulary/gb/...
-
-script:
- - ./scripts/travis-test.sh
-
-# we only need the latest git commit
-git:
-  depth: 1

CONTRIBUTING.md

@@ -12,11 +12,46 @@ See [INSTALL.md](INSTALL.md) for instructions on setting up a running dev
 instance of dendrite, and [CODE_STYLE.md](CODE_STYLE.md) for the code style
 guide.
 
-We use `gb` for managing our dependencies, so `gb build` and `gb test` is how
+As of May 2019, we're not using `gb` anymore, which is the tool we had been
-to build dendrite and run the unit tests respectively. Be aware that a list of
+using for managing our dependencies. We're now using Go modules. To build
-all dendrite packages is the expected output for all tests succeeding with `gb
+Dendrite, run the `build.sh` script at the root of this repository (which runs
-test`. There are also [scripts](scripts) for [linting](scripts/find-lint.sh)
+`go install` under the hood), and to run unit tests, run `go test ./...` (which
-and doing a [build/test/lint run](scripts/build-test-lint.sh).
+should pick up any unit test and run it). There are also [scripts](scripts) for
+[linting](scripts/find-lint.sh) and doing a [build/test/lint
+run](scripts/build-test-lint.sh).
+
+## Continuous Integration
+
+When a Pull Request is submitted, continuous integration jobs are run
+automatically to ensure the code builds and is relatively well-written. Checks
+are run on [Buildkite](https://buildkite.com/matrix-dot-org/dendrite/) and
+[CircleCI](https://circleci.com/gh/matrix-org/dendrite/).
+
+If a job fails, click the "details" button and you should be taken to the job's
+logs.
+
+![Click the details button on the failing build step](docs/images/details-button-location.jpg)
+
+Scroll down to the failing step and you should see some log output. Scan
+the logs until you find what it's complaining about, fix it, submit a new
+commit, then rinse and repeat until CI passes.
+
+### Running CI Tests Locally
+
+To save waiting for CI to finish after every commit, it is ideal to run the
+checks locally before pushing, fixing errors first. This also saves other
+people time as only so many PRs can be tested at a given time.
+
+To execute what Buildkite tests, simply run `./scripts/build-test-lint.sh`.
+This script will build the code, lint it, and run `go test ./...` with race
+condition checking enabled. If something needs to be changed, fix it and then
+run the script again until it no longer complains. Be warned that the linting
+can take a significant amount of CPU and RAM.
+
+CircleCI simply runs [Sytest](https://github.com/matrix-org/sytest) with a test
+whitelist. See
+[docs/sytest.md](https://github.com/matrix-org/dendrite/blob/master/docs/sytest.md#using-a-sytest-docker-image)
+for instructions on setting it up to run locally.
 
 
 ## Picking Things To Do
@@ -33,18 +68,6 @@ nonetheless fairly well-contained.
 We ask people who are familiar with Dendrite to leave the [good first issue](https://github.com/matrix-org/dendrite/labels/good%20first%20issue)
 issues so that there is always a way for new people to come and get involved.
 
-## Contributing to dependencies
-
-Dependencies are located in `vendor/src` and are managed by `gb`. If you need
-to make some changes in those directories, you first need to open a PR in the
-dependency repository. Once your PR is merged, you need to run `gb vendor
-update $repo_url` (example: `gb vendor update github.com/matrix-org/gomatrix`)
-in the dendrite repository to update the dependency.
-
-You can then create a commit containing only the modified vendor files (along
-with the `vendor/manifest` file), name it with the command you just ran (ie
-`gb vendor update github.com/matrix-org/gomatrix`), and open a PR on Dendrite.
-
 ## Getting Help
 
 For questions related to developing on Dendrite we have a dedicated room on
@@ -57,4 +80,3 @@ For more general questions please use [#dendrite:matrix.org](https://matrix.to/#
 
 We ask that everyone who contributes to the project signs off their
 contributions, in accordance with the [DCO](https://github.com/matrix-org/matrix-doc/blob/master/CONTRIBUTING.rst#sign-off).
-

INSTALL.md

@@ -12,7 +12,7 @@ Dendrite can be run in one of two configurations:
 
 ## Requirements
 
- - Go 1.10+
+ - Go 1.11+
  - Postgres 9.5+
  - For Kafka (optional if using the monolith server):
    - Unix-based system (https://kafka.apache.org/documentation/#os)
@@ -30,16 +30,15 @@ git clone https://github.com/matrix-org/dendrite
 cd dendrite
 
 # Build it
-go get github.com/constabulary/gb/...
+./build.sh
-gb build
 ```
 
 If using Kafka, install and start it (c.f. [scripts/install-local-kafka.sh](scripts/install-local-kafka.sh)):
 ```bash
-MIRROR=http://apache.mirror.anlx.net/kafka/0.10.2.0/kafka_2.11-0.10.2.0.tgz
+KAFKA_URL=http://archive.apache.org/dist/kafka/2.1.0/kafka_2.11-2.1.0.tgz
 
 # Only download the kafka if it isn't already downloaded.
-test -f kafka.tgz || wget $MIRROR -O kafka.tgz
+test -f kafka.tgz || wget $KAFKA_URL -O kafka.tgz
 # Unpack the kafka over the top of any existing installation
 mkdir -p kafka && tar xzf kafka.tgz -C kafka --strip-components 1
 

README.md

@@ -1,4 +1,4 @@
-# Dendrite [![Build Status](https://travis-ci.org/matrix-org/dendrite.svg?branch=master)](https://travis-ci.org/matrix-org/dendrite) [![Dendrite Dev on Matrix](https://img.shields.io/matrix/dendrite-dev:matrix.org.svg?label=%23dendrite-dev%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite-dev:matrix.org) [![Dendrite on Matrix](https://img.shields.io/matrix/dendrite:matrix.org.svg?label=%23dendrite%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite:matrix.org)
+# Dendrite [![Build Status](https://badge.buildkite.com/4be40938ab19f2bbc4a6c6724517353ee3ec1422e279faf374.svg?branch=master)](https://buildkite.com/matrix-dot-org/dendrite) [![CircleCI](https://circleci.com/gh/matrix-org/dendrite.svg?style=svg)](https://circleci.com/gh/matrix-org/dendrite) [![Dendrite Dev on Matrix](https://img.shields.io/matrix/dendrite-dev:matrix.org.svg?label=%23dendrite-dev%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite-dev:matrix.org) [![Dendrite on Matrix](https://img.shields.io/matrix/dendrite:matrix.org.svg?label=%23dendrite%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite:matrix.org)
 
 Dendrite will be a matrix homeserver written in go.
 
@@ -18,8 +18,10 @@ We aim to try and make it as easy as possible to jump in.
 
 For questions about Dendrite we have a dedicated room on Matrix
 [#dendrite:matrix.org](https://matrix.to/#/#dendrite:matrix.org).
+Development discussion should happen in
+[#dendrite-dev:matrix.org](https://matrix.to/#/#dendrite-dev:matrix.org).
 
 # Progress
 
 There's plenty still to do to make Dendrite usable! We're tracking progress in
-a [spreadsheet](https://docs.google.com/spreadsheets/d/1tkMNpIpPjvuDJWjPFbw_xzNzOHBA-Hp50Rkpcr43xTw).
+a [project board](https://github.com/matrix-org/dendrite/projects/2).

appservice/api/query.go

@@ -20,13 +20,13 @@ package api
 import (
 	"context"
 	"database/sql"
-	"errors"
 	"net/http"
 
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/gomatrixserverlib"
 
+	"github.com/matrix-org/dendrite/common"
 	commonHTTP "github.com/matrix-org/dendrite/common/http"
 	opentracing "github.com/opentracing/opentracing-go"
 )
@@ -134,9 +134,9 @@ func (h *httpAppServiceQueryAPI) UserIDExists(
 	return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response)
 }
 
-// RetreiveUserProfile is a wrapper that queries both the local database and
+// RetrieveUserProfile is a wrapper that queries both the local database and
 // application services for a given user's profile
-func RetreiveUserProfile(
+func RetrieveUserProfile(
 	ctx context.Context,
 	userID string,
 	asAPI AppServiceQueryAPI,
@@ -164,7 +164,7 @@ func RetreiveUserProfile(
 
 	// If no user exists, return
 	if !userResp.UserIDExists {
-		return nil, errors.New("no known profile for given user ID")
+		return nil, common.ErrProfileNoExists
 	}
 
 	// Try to query the user from the local database again

appservice/routing/routing.go

@@ -31,6 +31,10 @@ const pathPrefixApp = "/_matrix/app/r0"
 
 // Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client
 // to clients which need to make outbound HTTP requests.
+//
+// Due to Setup being used to call many other functions, a gocyclo nolint is
+// applied:
+// nolint: gocyclo
 func Setup(
 	apiMux *mux.Router, cfg config.Dendrite, // nolint: unparam
 	queryAPI api.RoomserverQueryAPI, aliasAPI api.RoomserverAliasAPI, // nolint: unparam

build.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+GOBIN=$PWD/`dirname $0`/bin go install -v ./cmd/...

clientapi/auth/auth.go

@@ -65,12 +65,6 @@ type Data struct {
 func VerifyUserFromRequest(
 	req *http.Request, data Data,
 ) (*authtypes.Device, *util.JSONResponse) {
-	// Try to find local user from device database
-	dev, devErr := verifyAccessToken(req, data.DeviceDB)
-	if devErr == nil {
-		return dev, verifyUserParameters(req)
-	}
-
 	// Try to find the Application Service user
 	token, err := ExtractAccessToken(req)
 	if err != nil {
@@ -128,9 +122,15 @@ func VerifyUserFromRequest(
 		return &dev, nil
 	}
 
+	// Try to find local user from device database
+	dev, devErr := verifyAccessToken(req, data.DeviceDB)
+	if devErr == nil {
+		return dev, verifyUserParameters(req)
+	}
+
 	return nil, &util.JSONResponse{
 		Code: http.StatusUnauthorized,
-		JSON: jsonerror.UnknownToken("Unrecognized access token"),
+		JSON: jsonerror.UnknownToken("Unrecognized access token"), // nolint: misspell
 	}
 }
 

clientapi/auth/authtypes/profile.go

@@ -14,7 +14,7 @@
 
 package authtypes
 
-// Profile represents the profile for a Matrix account on this home server.
+// Profile represents the profile for a Matrix account.
 type Profile struct {
 	Localpart   string
 	DisplayName string

clientapi/auth/storage/accounts/filter_table.go

@@ -17,6 +17,7 @@ package accounts
 import (
 	"context"
 	"database/sql"
+	"encoding/json"
 
 	"github.com/matrix-org/gomatrixserverlib"
 )
@@ -71,25 +72,44 @@ func (s *filterStatements) prepare(db *sql.DB) (err error) {
 
 func (s *filterStatements) selectFilter(
 	ctx context.Context, localpart string, filterID string,
-) (filter []byte, err error) {
+) (*gomatrixserverlib.Filter, error) {
-	err = s.selectFilterStmt.QueryRowContext(ctx, localpart, filterID).Scan(&filter)
+	// Retrieve filter from database (stored as canonical JSON)
-	return
+	var filterData []byte
+	err := s.selectFilterStmt.QueryRowContext(ctx, localpart, filterID).Scan(&filterData)
+	if err != nil {
+		return nil, err
+	}
+
+	// Unmarshal JSON into Filter struct
+	var filter gomatrixserverlib.Filter
+	if err = json.Unmarshal(filterData, &filter); err != nil {
+		return nil, err
+	}
+	return &filter, nil
 }
 
 func (s *filterStatements) insertFilter(
-	ctx context.Context, filter []byte, localpart string,
+	ctx context.Context, filter *gomatrixserverlib.Filter, localpart string,
 ) (filterID string, err error) {
 	var existingFilterID string
 
-	// This can result in a race condition when two clients try to insert the
+	// Serialise json
-	// same filter and localpart at the same time, however this is not a
+	filterJSON, err := json.Marshal(filter)
-	// problem as both calls will result in the same filterID
+	if err != nil {
-	filterJSON, err := gomatrixserverlib.CanonicalJSON(filter)
+		return "", err
+	}
+	// Remove whitespaces and sort JSON data
+	// needed to prevent from inserting the same filter multiple times
+	filterJSON, err = gomatrixserverlib.CanonicalJSON(filterJSON)
 	if err != nil {
 		return "", err
 	}
 
-	// Check if filter already exists in the database
+	// Check if filter already exists in the database using its localpart and content
+	//
+	// This can result in a race condition when two clients try to insert the
+	// same filter and localpart at the same time, however this is not a
+	// problem as both calls will result in the same filterID
 	err = s.selectFilterIDByContentStmt.QueryRowContext(ctx,
 		localpart, filterJSON).Scan(&existingFilterID)
 	if err != nil && err != sql.ErrNoRows {

clientapi/auth/storage/accounts/storage.go

@@ -23,6 +23,7 @@ import (
 	"github.com/matrix-org/dendrite/common"
 	"github.com/matrix-org/gomatrixserverlib"
 	"golang.org/x/crypto/bcrypt"
+
 	// Import the postgres database driver.
 	_ "github.com/lib/pq"
 )
@@ -229,7 +230,7 @@ func (d *Database) newMembership(
 		}
 
 		// Only "join" membership events can be considered as new memberships
-		if membership == "join" {
+		if membership == gomatrixserverlib.Join {
 			if err := d.saveMembership(ctx, txn, localpart, roomID, eventID); err != nil {
 				return err
 			}
@@ -343,11 +344,11 @@ func (d *Database) GetThreePIDsForLocalpart(
 }
 
 // GetFilter looks up the filter associated with a given local user and filter ID.
-// Returns a filter represented as a byte slice. Otherwise returns an error if
+// Returns a filter structure. Otherwise returns an error if no such filter exists
-// no such filter exists or if there was an error talking to the database.
+// or if there was an error talking to the database.
 func (d *Database) GetFilter(
 	ctx context.Context, localpart string, filterID string,
-) ([]byte, error) {
+) (*gomatrixserverlib.Filter, error) {
 	return d.filter.selectFilter(ctx, localpart, filterID)
 }
 
@@ -355,7 +356,7 @@ func (d *Database) GetFilter(
 // Returns the filterID as a string. Otherwise returns an error if something
 // goes wrong.
 func (d *Database) PutFilter(
-	ctx context.Context, localpart string, filter []byte,
+	ctx context.Context, localpart string, filter *gomatrixserverlib.Filter,
 ) (string, error) {
 	return d.filter.insertFilter(ctx, filter, localpart)
 }

clientapi/auth/storage/devices/devices_table.go

@@ -169,6 +169,8 @@ func (s *devicesStatements) selectDeviceByToken(
 	return &dev, err
 }
 
+// selectDeviceByID retrieves a device from the database with the given user
+// localpart and deviceID
 func (s *devicesStatements) selectDeviceByID(
 	ctx context.Context, localpart, deviceID string,
 ) (*authtypes.Device, error) {

clientapi/auth/storage/devices/storage.go

@@ -84,7 +84,7 @@ func (d *Database) CreateDevice(
 	if deviceID != nil {
 		returnErr = common.WithTransaction(d.db, func(txn *sql.Tx) error {
 			var err error
-			// Revoke existing token for this device
+			// Revoke existing tokens for this device
 			if err = d.devices.deleteDevice(ctx, txn, *deviceID, localpart); err != nil {
 				return err
 			}

clientapi/jsonerror/jsonerror.go

@@ -87,7 +87,7 @@ func MissingToken(msg string) *MatrixError {
 }
 
 // UnknownToken is an error when the client tries to access a resource which
-// requires authentication and supplies an unrecognized token
+// requires authentication and supplies an unrecognised token
 func UnknownToken(msg string) *MatrixError {
 	return &MatrixError{"M_UNKNOWN_TOKEN", msg}
 }

clientapi/routing/account_data.go

@@ -33,13 +33,6 @@ func SaveAccountData(
 	req *http.Request, accountDB *accounts.Database, device *authtypes.Device,
 	userID string, roomID string, dataType string, syncProducer *producers.SyncAPIProducer,
 ) util.JSONResponse {
-	if req.Method != http.MethodPut {
-		return util.JSONResponse{
-			Code: http.StatusMethodNotAllowed,
-			JSON: jsonerror.NotFound("Bad method"),
-		}
-	}
-
 	if userID != device.UserID {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,

clientapi/routing/createroom.go

@@ -15,6 +15,7 @@
 package routing
 
 import (
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"strings"
@@ -54,10 +55,6 @@ const (
 	presetPublicChat         = "public_chat"
 )
 
-const (
-	joinRulePublic = "public"
-	joinRuleInvite = "invite"
-)
 const (
 	historyVisibilityShared = "shared"
 	// TODO: These should be implemented once history visibility is implemented
@@ -97,6 +94,27 @@ func (r createRoomRequest) Validate() *util.JSONResponse {
 		}
 	}
 
+	// Validate creation_content fields defined in the spec by marshalling the
+	// creation_content map into bytes and then unmarshalling the bytes into
+	// common.CreateContent.
+
+	creationContentBytes, err := json.Marshal(r.CreationContent)
+	if err != nil {
+		return &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("malformed creation_content"),
+		}
+	}
+
+	var CreationContent common.CreateContent
+	err = json.Unmarshal(creationContentBytes, &CreationContent)
+	if err != nil {
+		return &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON("malformed creation_content"),
+		}
+	}
+
 	return nil
 }
 
@@ -154,7 +172,17 @@ func createRoom(
 			JSON: jsonerror.InvalidArgumentValue(err.Error()),
 		}
 	}
-	// TODO: visibility/presets/raw initial state/creation content
+
+	// Clobber keys: creator, room_version
+
+	if r.CreationContent == nil {
+		r.CreationContent = make(map[string]interface{}, 2)
+	}
+
+	r.CreationContent["creator"] = userID
+	r.CreationContent["room_version"] = "1" // TODO: We set this to 1 before we support Room versioning
+
+	// TODO: visibility/presets/raw initial state
 	// TODO: Create room alias association
 	// Make sure this doesn't fall into an application service's namespace though!
 
@@ -163,13 +191,13 @@ func createRoom(
 		"roomID": roomID,
 	}).Info("Creating new room")
 
-	profile, err := appserviceAPI.RetreiveUserProfile(req.Context(), userID, asAPI, accountDB)
+	profile, err := appserviceAPI.RetrieveUserProfile(req.Context(), userID, asAPI, accountDB)
 	if err != nil {
 		return httputil.LogThenError(req, err)
 	}
 
 	membershipContent := common.MemberContent{
-		Membership:  "join",
+		Membership:  gomatrixserverlib.Join,
 		DisplayName: profile.DisplayName,
 		AvatarURL:   profile.AvatarURL,
 	}
@@ -177,19 +205,19 @@ func createRoom(
 	var joinRules, historyVisibility string
 	switch r.Preset {
 	case presetPrivateChat:
-		joinRules = joinRuleInvite
+		joinRules = gomatrixserverlib.Invite
 		historyVisibility = historyVisibilityShared
 	case presetTrustedPrivateChat:
-		joinRules = joinRuleInvite
+		joinRules = gomatrixserverlib.Invite
 		historyVisibility = historyVisibilityShared
 		// TODO If trusted_private_chat, all invitees are given the same power level as the room creator.
 	case presetPublicChat:
-		joinRules = joinRulePublic
+		joinRules = gomatrixserverlib.Public
 		historyVisibility = historyVisibilityShared
 	default:
 		// Default room rules, r.Preset was previously checked for valid values so
 		// only a request with no preset should end up here.
-		joinRules = joinRuleInvite
+		joinRules = gomatrixserverlib.Invite
 		historyVisibility = historyVisibilityShared
 	}
 
@@ -214,7 +242,7 @@ func createRoom(
 	// harder to reason about, hence sticking to a strict static ordering.
 	// TODO: Synapse has txn/token ID on each event. Do we need to do this here?
 	eventsToMake := []fledglingEvent{
-		{"m.room.create", "", common.CreateContent{Creator: userID}},
+		{"m.room.create", "", r.CreationContent},
 		{"m.room.member", userID, membershipContent},
 		{"m.room.power_levels", "", common.InitialPowerLevelsContent(userID)},
 		// TODO: m.room.canonical_alias

clientapi/routing/device.go

@@ -106,13 +106,6 @@ func UpdateDeviceByID(
 	req *http.Request, deviceDB *devices.Database, device *authtypes.Device,
 	deviceID string,
 ) util.JSONResponse {
-	if req.Method != http.MethodPut {
-		return util.JSONResponse{
-			Code: http.StatusMethodNotAllowed,
-			JSON: jsonerror.NotFound("Bad Method"),
-		}
-	}
-
 	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
 	if err != nil {
 		return httputil.LogThenError(req, err)

clientapi/routing/directory.go

@@ -117,12 +117,16 @@ func SetLocalAlias(
 	// 1. The new method for checking for things matching an AS's namespace
 	// 2. Using an overall Regex object for all AS's just like we did for usernames
 	for _, appservice := range cfg.Derived.ApplicationServices {
-		if aliasNamespaces, ok := appservice.NamespaceMap["aliases"]; ok {
+		// Don't prevent AS from creating aliases in its own namespace
-			for _, namespace := range aliasNamespaces {
+		// Note that Dendrite uses SenderLocalpart as UserID for AS users
-				if namespace.Exclusive && namespace.RegexpObject.MatchString(alias) {
+		if device.UserID != appservice.SenderLocalpart {
-					return util.JSONResponse{
+			if aliasNamespaces, ok := appservice.NamespaceMap["aliases"]; ok {
-						Code: http.StatusBadRequest,
+				for _, namespace := range aliasNamespaces {
-						JSON: jsonerror.ASExclusive("Alias is reserved by an application service"),
+					if namespace.Exclusive && namespace.RegexpObject.MatchString(alias) {
+						return util.JSONResponse{
+							Code: http.StatusBadRequest,
+							JSON: jsonerror.ASExclusive("Alias is reserved by an application service"),
+						}
 					}
 				}
 			}
@@ -160,13 +164,36 @@ func SetLocalAlias(
 }
 
 // RemoveLocalAlias implements DELETE /directory/room/{roomAlias}
-// TODO: Check if the user has the power level to remove an alias
 func RemoveLocalAlias(
 	req *http.Request,
 	device *authtypes.Device,
 	alias string,
 	aliasAPI roomserverAPI.RoomserverAliasAPI,
 ) util.JSONResponse {
+
+	creatorQueryReq := roomserverAPI.GetCreatorIDForAliasRequest{
+		Alias: alias,
+	}
+	var creatorQueryRes roomserverAPI.GetCreatorIDForAliasResponse
+	if err := aliasAPI.GetCreatorIDForAlias(req.Context(), &creatorQueryReq, &creatorQueryRes); err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	if creatorQueryRes.UserID == "" {
+		return util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: jsonerror.NotFound("Alias does not exist"),
+		}
+	}
+
+	if creatorQueryRes.UserID != device.UserID {
+		// TODO: Still allow deletion if user is admin
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("You do not have permission to delete this alias"),
+		}
+	}
+
 	queryReq := roomserverAPI.RemoveRoomAliasRequest{
 		Alias:  alias,
 		UserID: device.UserID,

clientapi/routing/filter.go

@@ -17,13 +17,10 @@ package routing
 import (
 	"net/http"
 
-	"encoding/json"
-
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/clientapi/jsonerror"
-	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
@@ -32,12 +29,6 @@ import (
 func GetFilter(
 	req *http.Request, device *authtypes.Device, accountDB *accounts.Database, userID string, filterID string,
 ) util.JSONResponse {
-	if req.Method != http.MethodGet {
-		return util.JSONResponse{
-			Code: http.StatusMethodNotAllowed,
-			JSON: jsonerror.NotFound("Bad method"),
-		}
-	}
 	if userID != device.UserID {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
@@ -49,7 +40,7 @@ func GetFilter(
 		return httputil.LogThenError(req, err)
 	}
 
-	res, err := accountDB.GetFilter(req.Context(), localpart, filterID)
+	filter, err := accountDB.GetFilter(req.Context(), localpart, filterID)
 	if err != nil {
 		//TODO better error handling. This error message is *probably* right,
 		// but if there are obscure db errors, this will also be returned,
@@ -59,11 +50,6 @@ func GetFilter(
 			JSON: jsonerror.NotFound("No such filter"),
 		}
 	}
-	filter := gomatrix.Filter{}
-	err = json.Unmarshal(res, &filter)
-	if err != nil {
-		httputil.LogThenError(req, err)
-	}
 
 	return util.JSONResponse{
 		Code: http.StatusOK,
@@ -79,12 +65,6 @@ type filterResponse struct {
 func PutFilter(
 	req *http.Request, device *authtypes.Device, accountDB *accounts.Database, userID string,
 ) util.JSONResponse {
-	if req.Method != http.MethodPost {
-		return util.JSONResponse{
-			Code: http.StatusMethodNotAllowed,
-			JSON: jsonerror.NotFound("Bad method"),
-		}
-	}
 	if userID != device.UserID {
 		return util.JSONResponse{
 			Code: http.StatusForbidden,
@@ -97,21 +77,21 @@ func PutFilter(
 		return httputil.LogThenError(req, err)
 	}
 
-	var filter gomatrix.Filter
+	var filter gomatrixserverlib.Filter
 
 	if reqErr := httputil.UnmarshalJSONRequest(req, &filter); reqErr != nil {
 		return *reqErr
 	}
 
-	filterArray, err := json.Marshal(filter)
+	// Validate generates a user-friendly error
-	if err != nil {
+	if err = filter.Validate(); err != nil {
 		return util.JSONResponse{
 			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON("Filter is malformed"),
+			JSON: jsonerror.BadJSON("Invalid filter: " + err.Error()),
 		}
 	}
 
-	filterID, err := accountDB.PutFilter(req.Context(), localpart, filterArray)
+	filterID, err := accountDB.PutFilter(req.Context(), localpart, &filter)
 	if err != nil {
 		return httputil.LogThenError(req, err)
 	}

clientapi/routing/getevent.go

@@ -0,0 +1,127 @@
+// Copyright 2019 Alex Chen
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"net/http"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/common/config"
+	"github.com/matrix-org/dendrite/roomserver/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+)
+
+type getEventRequest struct {
+	req            *http.Request
+	device         *authtypes.Device
+	roomID         string
+	eventID        string
+	cfg            config.Dendrite
+	federation     *gomatrixserverlib.FederationClient
+	keyRing        gomatrixserverlib.KeyRing
+	requestedEvent gomatrixserverlib.Event
+}
+
+// GetEvent implements GET /_matrix/client/r0/rooms/{roomId}/event/{eventId}
+// https://matrix.org/docs/spec/client_server/r0.4.0.html#get-matrix-client-r0-rooms-roomid-event-eventid
+func GetEvent(
+	req *http.Request,
+	device *authtypes.Device,
+	roomID string,
+	eventID string,
+	cfg config.Dendrite,
+	queryAPI api.RoomserverQueryAPI,
+	federation *gomatrixserverlib.FederationClient,
+	keyRing gomatrixserverlib.KeyRing,
+) util.JSONResponse {
+	eventsReq := api.QueryEventsByIDRequest{
+		EventIDs: []string{eventID},
+	}
+	var eventsResp api.QueryEventsByIDResponse
+	err := queryAPI.QueryEventsByID(req.Context(), &eventsReq, &eventsResp)
+	if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	if len(eventsResp.Events) == 0 {
+		// Event not found locally
+		return util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: jsonerror.NotFound("The event was not found or you do not have permission to read this event"),
+		}
+	}
+
+	requestedEvent := eventsResp.Events[0]
+
+	r := getEventRequest{
+		req:            req,
+		device:         device,
+		roomID:         roomID,
+		eventID:        eventID,
+		cfg:            cfg,
+		federation:     federation,
+		keyRing:        keyRing,
+		requestedEvent: requestedEvent,
+	}
+
+	stateReq := api.QueryStateAfterEventsRequest{
+		RoomID:       r.requestedEvent.RoomID(),
+		PrevEventIDs: r.requestedEvent.PrevEventIDs(),
+		StateToFetch: []gomatrixserverlib.StateKeyTuple{{
+			EventType: gomatrixserverlib.MRoomMember,
+			StateKey:  device.UserID,
+		}},
+	}
+	var stateResp api.QueryStateAfterEventsResponse
+	if err := queryAPI.QueryStateAfterEvents(req.Context(), &stateReq, &stateResp); err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	if !stateResp.RoomExists {
+		util.GetLogger(req.Context()).Errorf("Expected to find room for event %s but failed", r.requestedEvent.EventID())
+		return jsonerror.InternalServerError()
+	}
+
+	if !stateResp.PrevEventsExist {
+		// Missing some events locally; stateResp.StateEvents unavailable.
+		return util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: jsonerror.NotFound("The event was not found or you do not have permission to read this event"),
+		}
+	}
+
+	for _, stateEvent := range stateResp.StateEvents {
+		if stateEvent.StateKeyEquals(r.device.UserID) {
+			membership, err := stateEvent.Membership()
+			if err != nil {
+				return httputil.LogThenError(req, err)
+			}
+			if membership == gomatrixserverlib.Join {
+				return util.JSONResponse{
+					Code: http.StatusOK,
+					JSON: gomatrixserverlib.ToClientEvent(r.requestedEvent, gomatrixserverlib.FormatAll),
+				}
+			}
+		}
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusNotFound,
+		JSON: jsonerror.NotFound("The event was not found or you do not have permission to read this event"),
+	}
+}

clientapi/routing/joinroom.go

@@ -70,7 +70,7 @@ func JoinRoomByIDOrAlias(
 		return httputil.LogThenError(req, err)
 	}
 
-	content["membership"] = "join"
+	content["membership"] = gomatrixserverlib.Join
 	content["displayname"] = profile.DisplayName
 	content["avatar_url"] = profile.AvatarURL
 
@@ -86,7 +86,10 @@ func JoinRoomByIDOrAlias(
 	}
 	return util.JSONResponse{
 		Code: http.StatusBadRequest,
-		JSON: jsonerror.BadJSON("Invalid first character for room ID or alias"),
+		JSON: jsonerror.BadJSON(
+			fmt.Sprintf("Invalid first character '%s' for room ID or alias",
+				string([]rune(roomIDOrAlias)[0])), // Wrapping with []rune makes this call UTF-8 safe
+		),
 	}
 }
 

clientapi/routing/login.go

@@ -18,7 +18,7 @@ import (
 	"net/http"
 
 	"context"
-	"database/sql"
+
 	"github.com/matrix-org/dendrite/clientapi/auth"
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
@@ -41,10 +41,12 @@ type flow struct {
 }
 
 type passwordRequest struct {
-	User               string  `json:"user"`
+	User     string `json:"user"`
-	Password           string  `json:"password"`
+	Password string `json:"password"`
+	// Both DeviceID and InitialDisplayName can be omitted, or empty strings ("")
+	// Thus a pointer is needed to differentiate between the two
 	InitialDisplayName *string `json:"initial_device_display_name"`
-	DeviceID           string  `json:"device_id"`
+	DeviceID           *string `json:"device_id"`
 }
 
 type loginResponse struct {
@@ -106,10 +108,10 @@ func Login(
 
 		token, err := auth.GenerateAccessToken()
 		if err != nil {
-			httputil.LogThenError(req, err)
+			return httputil.LogThenError(req, err)
 		}
 
-		dev, err := getDevice(req.Context(), r, deviceDB, acc, localpart, token)
+		dev, err := getDevice(req.Context(), r, deviceDB, acc, token)
 		if err != nil {
 			return util.JSONResponse{
 				Code: http.StatusInternalServerError,
@@ -133,20 +135,16 @@ func Login(
 	}
 }
 
-// check if device exists else create one
+// getDevice returns a new or existing device
 func getDevice(
 	ctx context.Context,
 	r passwordRequest,
 	deviceDB *devices.Database,
 	acc *authtypes.Account,
-	localpart, token string,
+	token string,
 ) (dev *authtypes.Device, err error) {
-	dev, err = deviceDB.GetDeviceByID(ctx, localpart, r.DeviceID)
+	dev, err = deviceDB.CreateDevice(
-	if err == sql.ErrNoRows {
+		ctx, acc.Localpart, r.DeviceID, token, r.InitialDisplayName,
-		// device doesn't exist, create one
+	)
-		dev, err = deviceDB.CreateDevice(
-			ctx, acc.Localpart, nil, token, r.InitialDisplayName,
-		)
-	}
 	return
 }

clientapi/routing/logout.go

@@ -20,7 +20,6 @@ import (
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/auth/storage/devices"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
-	"github.com/matrix-org/dendrite/clientapi/jsonerror"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/util"
 )
@@ -29,13 +28,6 @@ import (
 func Logout(
 	req *http.Request, deviceDB *devices.Database, device *authtypes.Device,
 ) util.JSONResponse {
-	if req.Method != http.MethodPost {
-		return util.JSONResponse{
-			Code: http.StatusMethodNotAllowed,
-			JSON: jsonerror.NotFound("Bad method"),
-		}
-	}
-
 	localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID)
 	if err != nil {
 		return httputil.LogThenError(req, err)

clientapi/routing/membership.go

@@ -58,27 +58,12 @@ func SendMembership(
 		}
 	}
 
-	inviteStored, err := threepid.CheckAndProcessInvite(
+	inviteStored, jsonErrResp := checkAndProcessThreepid(
-		req.Context(), device, &body, cfg, queryAPI, accountDB, producer,
+		req, device, &body, cfg, queryAPI, accountDB, producer,
 		membership, roomID, evTime,
 	)
-	if err == threepid.ErrMissingParameter {
+	if jsonErrResp != nil {
-		return util.JSONResponse{
+		return *jsonErrResp
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.BadJSON(err.Error()),
-		}
-	} else if err == threepid.ErrNotTrusted {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: jsonerror.NotTrusted(body.IDServer),
-		}
-	} else if err == common.ErrRoomNoExists {
-		return util.JSONResponse{
-			Code: http.StatusNotFound,
-			JSON: jsonerror.NotFound(err.Error()),
-		}
-	} else if err != nil {
-		return httputil.LogThenError(req, err)
 	}
 
 	// If an invite has been stored on an identity server, it means that a
@@ -114,9 +99,18 @@ func SendMembership(
 		return httputil.LogThenError(req, err)
 	}
 
+	var returnData interface{} = struct{}{}
+
+	// The join membership requires the room id to be sent in the response
+	if membership == gomatrixserverlib.Join {
+		returnData = struct {
+			RoomID string `json:"room_id"`
+		}{roomID}
+	}
+
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: struct{}{},
+		JSON: returnData,
 	}
 }
 
@@ -147,7 +141,7 @@ func buildMembershipEvent(
 
 	// "unban" or "kick" isn't a valid membership value, change it to "leave"
 	if membership == "unban" || membership == "kick" {
-		membership = "leave"
+		membership = gomatrixserverlib.Leave
 	}
 
 	content := common.MemberContent{
@@ -182,7 +176,7 @@ func loadProfile(
 
 	var profile *authtypes.Profile
 	if serverName == cfg.Matrix.ServerName {
-		profile, err = appserviceAPI.RetreiveUserProfile(ctx, userID, asAPI, accountDB)
+		profile, err = appserviceAPI.RetrieveUserProfile(ctx, userID, asAPI, accountDB)
 	} else {
 		profile = &authtypes.Profile{}
 	}
@@ -198,7 +192,7 @@ func loadProfile(
 func getMembershipStateKey(
 	body threepid.MembershipRequest, device *authtypes.Device, membership string,
 ) (stateKey string, reason string, err error) {
-	if membership == "ban" || membership == "unban" || membership == "kick" || membership == "invite" {
+	if membership == gomatrixserverlib.Ban || membership == "unban" || membership == "kick" || membership == gomatrixserverlib.Invite {
 		// If we're in this case, the state key is contained in the request body,
 		// possibly along with a reason (for "kick" and "ban") so we need to parse
 		// it
@@ -215,3 +209,41 @@ func getMembershipStateKey(
 
 	return
 }
+
+func checkAndProcessThreepid(
+	req *http.Request,
+	device *authtypes.Device,
+	body *threepid.MembershipRequest,
+	cfg config.Dendrite,
+	queryAPI roomserverAPI.RoomserverQueryAPI,
+	accountDB *accounts.Database,
+	producer *producers.RoomserverProducer,
+	membership, roomID string,
+	evTime time.Time,
+) (inviteStored bool, errRes *util.JSONResponse) {
+
+	inviteStored, err := threepid.CheckAndProcessInvite(
+		req.Context(), device, body, cfg, queryAPI, accountDB, producer,
+		membership, roomID, evTime,
+	)
+	if err == threepid.ErrMissingParameter {
+		return inviteStored, &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.BadJSON(err.Error()),
+		}
+	} else if err == threepid.ErrNotTrusted {
+		return inviteStored, &util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: jsonerror.NotTrusted(body.IDServer),
+		}
+	} else if err == common.ErrRoomNoExists {
+		return inviteStored, &util.JSONResponse{
+			Code: http.StatusNotFound,
+			JSON: jsonerror.NotFound(err.Error()),
+		}
+	} else if err != nil {
+		er := httputil.LogThenError(req, err)
+		return inviteStored, &er
+	}
+	return
+}

clientapi/routing/profile.go

@@ -30,49 +30,61 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/gomatrixserverlib"
 
+	"github.com/matrix-org/gomatrix"
 	"github.com/matrix-org/util"
 )
 
 // GetProfile implements GET /profile/{userID}
 func GetProfile(
-	req *http.Request, accountDB *accounts.Database, userID string, asAPI appserviceAPI.AppServiceQueryAPI,
+	req *http.Request, accountDB *accounts.Database, cfg *config.Dendrite,
+	userID string,
+	asAPI appserviceAPI.AppServiceQueryAPI,
+	federation *gomatrixserverlib.FederationClient,
 ) util.JSONResponse {
-	if req.Method != http.MethodGet {
+	profile, err := getProfile(req.Context(), accountDB, cfg, userID, asAPI, federation)
-		return util.JSONResponse{
-			Code: http.StatusMethodNotAllowed,
-			JSON: jsonerror.NotFound("Bad method"),
-		}
-	}
-	profile, err := appserviceAPI.RetreiveUserProfile(req.Context(), userID, asAPI, accountDB)
 	if err != nil {
+		if err == common.ErrProfileNoExists {
+			return util.JSONResponse{
+				Code: http.StatusNotFound,
+				JSON: jsonerror.NotFound("The user does not exist or does not have a profile"),
+			}
+		}
+
 		return httputil.LogThenError(req, err)
 	}
 
-	res := common.ProfileResponse{
-		AvatarURL:   profile.AvatarURL,
-		DisplayName: profile.DisplayName,
-	}
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: res,
+		JSON: common.ProfileResponse{
+			AvatarURL:   profile.AvatarURL,
+			DisplayName: profile.DisplayName,
+		},
 	}
 }
 
 // GetAvatarURL implements GET /profile/{userID}/avatar_url
 func GetAvatarURL(
-	req *http.Request, accountDB *accounts.Database, userID string, asAPI appserviceAPI.AppServiceQueryAPI,
+	req *http.Request, accountDB *accounts.Database, cfg *config.Dendrite,
+	userID string, asAPI appserviceAPI.AppServiceQueryAPI,
+	federation *gomatrixserverlib.FederationClient,
 ) util.JSONResponse {
-	profile, err := appserviceAPI.RetreiveUserProfile(req.Context(), userID, asAPI, accountDB)
+	profile, err := getProfile(req.Context(), accountDB, cfg, userID, asAPI, federation)
 	if err != nil {
+		if err == common.ErrProfileNoExists {
+			return util.JSONResponse{
+				Code: http.StatusNotFound,
+				JSON: jsonerror.NotFound("The user does not exist or does not have a profile"),
+			}
+		}
+
 		return httputil.LogThenError(req, err)
 	}
 
-	res := common.AvatarURL{
-		AvatarURL: profile.AvatarURL,
-	}
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: res,
+		JSON: common.AvatarURL{
+			AvatarURL: profile.AvatarURL,
+		},
 	}
 }
 
@@ -158,18 +170,27 @@ func SetAvatarURL(
 
 // GetDisplayName implements GET /profile/{userID}/displayname
 func GetDisplayName(
-	req *http.Request, accountDB *accounts.Database, userID string, asAPI appserviceAPI.AppServiceQueryAPI,
+	req *http.Request, accountDB *accounts.Database, cfg *config.Dendrite,
+	userID string, asAPI appserviceAPI.AppServiceQueryAPI,
+	federation *gomatrixserverlib.FederationClient,
 ) util.JSONResponse {
-	profile, err := appserviceAPI.RetreiveUserProfile(req.Context(), userID, asAPI, accountDB)
+	profile, err := getProfile(req.Context(), accountDB, cfg, userID, asAPI, federation)
 	if err != nil {
+		if err == common.ErrProfileNoExists {
+			return util.JSONResponse{
+				Code: http.StatusNotFound,
+				JSON: jsonerror.NotFound("The user does not exist or does not have a profile"),
+			}
+		}
+
 		return httputil.LogThenError(req, err)
 	}
-	res := common.DisplayName{
+
-		DisplayName: profile.DisplayName,
-	}
 	return util.JSONResponse{
 		Code: http.StatusOK,
-		JSON: res,
+		JSON: common.DisplayName{
+			DisplayName: profile.DisplayName,
+		},
 	}
 }
 
@@ -253,6 +274,48 @@ func SetDisplayName(
 	}
 }
 
+// getProfile gets the full profile of a user by querying the database or a
+// remote homeserver.
+// Returns an error when something goes wrong or specifically
+// common.ErrProfileNoExists when the profile doesn't exist.
+func getProfile(
+	ctx context.Context, accountDB *accounts.Database, cfg *config.Dendrite,
+	userID string,
+	asAPI appserviceAPI.AppServiceQueryAPI,
+	federation *gomatrixserverlib.FederationClient,
+) (*authtypes.Profile, error) {
+	localpart, domain, err := gomatrixserverlib.SplitID('@', userID)
+	if err != nil {
+		return nil, err
+	}
+
+	if domain != cfg.Matrix.ServerName {
+		profile, fedErr := federation.LookupProfile(ctx, domain, userID, "")
+		if fedErr != nil {
+			if x, ok := fedErr.(gomatrix.HTTPError); ok {
+				if x.Code == http.StatusNotFound {
+					return nil, common.ErrProfileNoExists
+				}
+			}
+
+			return nil, fedErr
+		}
+
+		return &authtypes.Profile{
+			Localpart:   localpart,
+			DisplayName: profile.DisplayName,
+			AvatarURL:   profile.AvatarURL,
+		}, nil
+	}
+
+	profile, err := appserviceAPI.RetrieveUserProfile(ctx, userID, asAPI, accountDB)
+	if err != nil {
+		return nil, err
+	}
+
+	return profile, nil
+}
+
 func buildMembershipEvents(
 	ctx context.Context,
 	memberships []authtypes.Membership,
@@ -270,7 +333,7 @@ func buildMembershipEvents(
 		}
 
 		content := common.MemberContent{
-			Membership: "join",
+			Membership: gomatrixserverlib.Join,
 		}
 
 		content.DisplayName = newProfile.DisplayName

clientapi/routing/register.go

@@ -85,6 +85,12 @@ func (d sessionsDict) GetCompletedStages(sessionID string) []authtypes.LoginType
 
 // AddCompletedStage records that a session has completed an auth stage.
 func (d *sessionsDict) AddCompletedStage(sessionID string, stage authtypes.LoginType) {
+	// Return if the stage is already present
+	for _, completedStage := range d.GetCompletedStages(sessionID) {
+		if completedStage == stage {
+			return
+		}
+	}
 	d.sessions[sessionID] = append(d.GetCompletedStages(sessionID), stage)
 }
 
@@ -115,7 +121,10 @@ type registerRequest struct {
 	// user-interactive auth params
 	Auth authDict `json:"auth"`
 
+	// Both DeviceID and InitialDisplayName can be omitted, or empty strings ("")
+	// Thus a pointer is needed to differentiate between the two
 	InitialDisplayName *string `json:"initial_device_display_name"`
+	DeviceID           *string `json:"device_id"`
 
 	// Prevent this user from logging in
 	InhibitLogin common.WeakBoolean `json:"inhibit_login"`
@@ -243,8 +252,8 @@ func validateRecaptcha(
 ) *util.JSONResponse {
 	if !cfg.Matrix.RecaptchaEnabled {
 		return &util.JSONResponse{
-			Code: http.StatusBadRequest,
+			Code: http.StatusConflict,
-			JSON: jsonerror.BadJSON("Captcha registration is disabled"),
+			JSON: jsonerror.Unknown("Captcha registration is disabled"),
 		}
 	}
 
@@ -279,8 +288,8 @@ func validateRecaptcha(
 	body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		return &util.JSONResponse{
-			Code: http.StatusInternalServerError,
+			Code: http.StatusGatewayTimeout,
-			JSON: jsonerror.BadJSON("Error in contacting captcha server" + err.Error()),
+			JSON: jsonerror.Unknown("Error in contacting captcha server" + err.Error()),
 		}
 	}
 	err = json.Unmarshal(body, &r)
@@ -620,7 +629,7 @@ func handleApplicationServiceRegistration(
 	// application service registration is entirely separate.
 	return completeRegistration(
 		req.Context(), accountDB, deviceDB, r.Username, "", appserviceID,
-		r.InhibitLogin, r.InitialDisplayName,
+		r.InhibitLogin, r.InitialDisplayName, r.DeviceID,
 	)
 }
 
@@ -640,7 +649,7 @@ func checkAndCompleteFlow(
 		// This flow was completed, registration can continue
 		return completeRegistration(
 			req.Context(), accountDB, deviceDB, r.Username, r.Password, "",
-			r.InhibitLogin, r.InitialDisplayName,
+			r.InhibitLogin, r.InitialDisplayName, r.DeviceID,
 		)
 	}
 
@@ -691,10 +700,10 @@ func LegacyRegister(
 			return util.MessageResponse(http.StatusForbidden, "HMAC incorrect")
 		}
 
-		return completeRegistration(req.Context(), accountDB, deviceDB, r.Username, r.Password, "", false, nil)
+		return completeRegistration(req.Context(), accountDB, deviceDB, r.Username, r.Password, "", false, nil, nil)
 	case authtypes.LoginTypeDummy:
 		// there is nothing to do
-		return completeRegistration(req.Context(), accountDB, deviceDB, r.Username, r.Password, "", false, nil)
+		return completeRegistration(req.Context(), accountDB, deviceDB, r.Username, r.Password, "", false, nil, nil)
 	default:
 		return util.JSONResponse{
 			Code: http.StatusNotImplemented,
@@ -732,13 +741,19 @@ func parseAndValidateLegacyLogin(req *http.Request, r *legacyRegisterRequest) *u
 	return nil
 }
 
+// completeRegistration runs some rudimentary checks against the submitted
+// input, then if successful creates an account and a newly associated device
+// We pass in each individual part of the request here instead of just passing a
+// registerRequest, as this function serves requests encoded as both
+// registerRequests and legacyRegisterRequests, which share some attributes but
+// not all
 func completeRegistration(
 	ctx context.Context,
 	accountDB *accounts.Database,
 	deviceDB *devices.Database,
 	username, password, appserviceID string,
 	inhibitLogin common.WeakBoolean,
-	displayName *string,
+	displayName, deviceID *string,
 ) util.JSONResponse {
 	if username == "" {
 		return util.JSONResponse{
@@ -767,6 +782,9 @@ func completeRegistration(
 		}
 	}
 
+	// Increment prometheus counter for created users
+	amtRegUsers.Inc()
+
 	// Check whether inhibit_login option is set. If so, don't create an access
 	// token or a device for this user
 	if inhibitLogin {
@@ -787,8 +805,7 @@ func completeRegistration(
 		}
 	}
 
-	// TODO: Use the device ID in the request.
+	dev, err := deviceDB.CreateDevice(ctx, username, deviceID, token, displayName)
-	dev, err := deviceDB.CreateDevice(ctx, username, nil, token, displayName)
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
@@ -796,9 +813,6 @@ func completeRegistration(
 		}
 	}
 
-	// Increment prometheus counter for created users
-	amtRegUsers.Inc()
-
 	return util.JSONResponse{
 		Code: http.StatusOK,
 		JSON: registerResponse{

clientapi/routing/room_tagging.go

@@ -0,0 +1,234 @@
+// Copyright 2019 Sumukha PK
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package routing
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"github.com/sirupsen/logrus"
+
+	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
+	"github.com/matrix-org/dendrite/clientapi/auth/storage/accounts"
+	"github.com/matrix-org/dendrite/clientapi/httputil"
+	"github.com/matrix-org/dendrite/clientapi/jsonerror"
+	"github.com/matrix-org/dendrite/clientapi/producers"
+	"github.com/matrix-org/gomatrix"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/util"
+)
+
+// newTag creates and returns a new gomatrix.TagContent
+func newTag() gomatrix.TagContent {
+	return gomatrix.TagContent{
+		Tags: make(map[string]gomatrix.TagProperties),
+	}
+}
+
+// GetTags implements GET /_matrix/client/r0/user/{userID}/rooms/{roomID}/tags
+func GetTags(
+	req *http.Request,
+	accountDB *accounts.Database,
+	device *authtypes.Device,
+	userID string,
+	roomID string,
+	syncProducer *producers.SyncAPIProducer,
+) util.JSONResponse {
+
+	if device.UserID != userID {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("Cannot retrieve another user's tags"),
+		}
+	}
+
+	_, data, err := obtainSavedTags(req, userID, roomID, accountDB)
+	if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	if len(data) == 0 {
+		return util.JSONResponse{
+			Code: http.StatusOK,
+			JSON: struct{}{},
+		}
+	}
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: data[0].Content,
+	}
+}
+
+// PutTag implements PUT /_matrix/client/r0/user/{userID}/rooms/{roomID}/tags/{tag}
+// Put functionality works by getting existing data from the DB (if any), adding
+// the tag to the "map" and saving the new "map" to the DB
+func PutTag(
+	req *http.Request,
+	accountDB *accounts.Database,
+	device *authtypes.Device,
+	userID string,
+	roomID string,
+	tag string,
+	syncProducer *producers.SyncAPIProducer,
+) util.JSONResponse {
+
+	if device.UserID != userID {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("Cannot modify another user's tags"),
+		}
+	}
+
+	var properties gomatrix.TagProperties
+	if reqErr := httputil.UnmarshalJSONRequest(req, &properties); reqErr != nil {
+		return *reqErr
+	}
+
+	localpart, data, err := obtainSavedTags(req, userID, roomID, accountDB)
+	if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	var tagContent gomatrix.TagContent
+	if len(data) > 0 {
+		if err = json.Unmarshal(data[0].Content, &tagContent); err != nil {
+			return httputil.LogThenError(req, err)
+		}
+	} else {
+		tagContent = newTag()
+	}
+	tagContent.Tags[tag] = properties
+	if err = saveTagData(req, localpart, roomID, accountDB, tagContent); err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	// Send data to syncProducer in order to inform clients of changes
+	// Run in a goroutine in order to prevent blocking the tag request response
+	go func() {
+		if err := syncProducer.SendData(userID, roomID, "m.tag"); err != nil {
+			logrus.WithError(err).Error("Failed to send m.tag account data update to syncapi")
+		}
+	}()
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}
+}
+
+// DeleteTag implements DELETE /_matrix/client/r0/user/{userID}/rooms/{roomID}/tags/{tag}
+// Delete functionality works by obtaining the saved tags, removing the intended tag from
+// the "map" and then saving the new "map" in the DB
+func DeleteTag(
+	req *http.Request,
+	accountDB *accounts.Database,
+	device *authtypes.Device,
+	userID string,
+	roomID string,
+	tag string,
+	syncProducer *producers.SyncAPIProducer,
+) util.JSONResponse {
+
+	if device.UserID != userID {
+		return util.JSONResponse{
+			Code: http.StatusForbidden,
+			JSON: jsonerror.Forbidden("Cannot modify another user's tags"),
+		}
+	}
+
+	localpart, data, err := obtainSavedTags(req, userID, roomID, accountDB)
+	if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	// If there are no tags in the database, exit
+	if len(data) == 0 {
+		// Spec only defines 200 responses for this endpoint so we don't return anything else.
+		return util.JSONResponse{
+			Code: http.StatusOK,
+			JSON: struct{}{},
+		}
+	}
+
+	var tagContent gomatrix.TagContent
+	err = json.Unmarshal(data[0].Content, &tagContent)
+	if err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	// Check whether the tag to be deleted exists
+	if _, ok := tagContent.Tags[tag]; ok {
+		delete(tagContent.Tags, tag)
+	} else {
+		// Spec only defines 200 responses for this endpoint so we don't return anything else.
+		return util.JSONResponse{
+			Code: http.StatusOK,
+			JSON: struct{}{},
+		}
+	}
+	if err = saveTagData(req, localpart, roomID, accountDB, tagContent); err != nil {
+		return httputil.LogThenError(req, err)
+	}
+
+	// Send data to syncProducer in order to inform clients of changes
+	// Run in a goroutine in order to prevent blocking the tag request response
+	go func() {
+		if err := syncProducer.SendData(userID, roomID, "m.tag"); err != nil {
+			logrus.WithError(err).Error("Failed to send m.tag account data update to syncapi")
+		}
+	}()
+
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: struct{}{},
+	}
+}
+
+// obtainSavedTags gets all tags scoped to a userID and roomID
+// from the database
+func obtainSavedTags(
+	req *http.Request,
+	userID string,
+	roomID string,
+	accountDB *accounts.Database,
+) (string, []gomatrixserverlib.ClientEvent, error) {
+	localpart, _, err := gomatrixserverlib.SplitID('@', userID)
+	if err != nil {
+		return "", nil, err
+	}
+
+	data, err := accountDB.GetAccountDataByType(
+		req.Context(), localpart, roomID, "m.tag",
+	)
+
+	return localpart, data, err
+}
+
+// saveTagData saves the provided tag data into the database
+func saveTagData(
+	req *http.Request,
+	localpart string,
+	roomID string,
+	accountDB *accounts.Database,
+	Tag gomatrix.TagContent,
+) error {
+	newTagData, err := json.Marshal(Tag)
+	if err != nil {
+		return err
+	}
+
+	return accountDB.SaveAccountData(req.Context(), localpart, roomID, "m.tag", string(newTagData))
+}

clientapi/routing/routing.go

@@ -41,6 +41,10 @@ const pathPrefixUnstable = "/_matrix/client/unstable"
 
 // Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client
 // to clients which need to make outbound HTTP requests.
+//
+// Due to Setup being used to call many other functions, a gocyclo nolint is
+// applied:
+// nolint: gocyclo
 func Setup(
 	apiMux *mux.Router, cfg config.Dendrite,
 	producer *producers.RoomserverProducer,
@@ -77,7 +81,11 @@ func Setup(
 	v1mux := apiMux.PathPrefix(pathPrefixV1).Subrouter()
 	unstableMux := apiMux.PathPrefix(pathPrefixUnstable).Subrouter()
 
-	authData := auth.Data{accountDB, deviceDB, cfg.Derived.ApplicationServices}
+	authData := auth.Data{
+		AccountDB:   accountDB,
+		DeviceDB:    deviceDB,
+		AppServices: cfg.Derived.ApplicationServices,
+	}
 
 	r0mux.Handle("/createRoom",
 		common.MakeAuthAPI("createRoom", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
@@ -85,8 +93,11 @@ func Setup(
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/join/{roomIDOrAlias}",
-		common.MakeAuthAPI("join", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+		common.MakeAuthAPI(gomatrixserverlib.Join, authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return JoinRoomByIDOrAlias(
 				req, device, vars["roomIDOrAlias"], cfg, federation, producer, queryAPI, aliasAPI, keyRing, accountDB,
 			)
@@ -94,27 +105,48 @@ func Setup(
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/{membership:(?:join|kick|ban|unban|leave|invite)}",
 		common.MakeAuthAPI("membership", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return SendMembership(req, accountDB, device, vars["roomID"], vars["membership"], cfg, queryAPI, asAPI, producer)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/send/{eventType}",
 		common.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, nil, cfg, queryAPI, producer, nil)
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/send/{eventType}/{txnID}",
 		common.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			txnID := vars["txnID"]
 			return SendEvent(req, device, vars["roomID"], vars["eventType"], &txnID,
 				nil, cfg, queryAPI, producer, transactionsCache)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
+	r0mux.Handle("/rooms/{roomID}/event/{eventID}",
+		common.MakeAuthAPI("rooms_get_event", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return GetEvent(req, device, vars["roomID"], vars["eventID"], cfg, queryAPI, federation, keyRing)
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/state/{eventType:[^/]+/?}",
 		common.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			emptyString := ""
 			eventType := vars["eventType"]
 			// If there's a trailing slash, remove it
@@ -126,7 +158,10 @@ func Setup(
 	).Methods(http.MethodPut, http.MethodOptions)
 	r0mux.Handle("/rooms/{roomID}/state/{eventType}/{stateKey}",
 		common.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			stateKey := vars["stateKey"]
 			return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, &stateKey, cfg, queryAPI, producer, nil)
 		}),
@@ -146,21 +181,30 @@ func Setup(
 
 	r0mux.Handle("/directory/room/{roomAlias}",
 		common.MakeExternalAPI("directory_room", func(req *http.Request) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return DirectoryRoom(req, vars["roomAlias"], federation, &cfg, aliasAPI)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/directory/room/{roomAlias}",
 		common.MakeAuthAPI("directory_room", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return SetLocalAlias(req, device, vars["roomAlias"], &cfg, aliasAPI)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/directory/room/{roomAlias}",
 		common.MakeAuthAPI("directory_room", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return RemoveLocalAlias(req, device, vars["roomAlias"], aliasAPI)
 		}),
 	).Methods(http.MethodDelete, http.MethodOptions)
@@ -179,7 +223,10 @@ func Setup(
 
 	r0mux.Handle("/rooms/{roomID}/typing/{userID}",
 		common.MakeAuthAPI("rooms_typing", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return SendTyping(req, device, vars["roomID"], vars["userID"], accountDB, typingProducer)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
@@ -219,14 +266,20 @@ func Setup(
 
 	r0mux.Handle("/user/{userId}/filter",
 		common.MakeAuthAPI("put_filter", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return PutFilter(req, device, accountDB, vars["userId"])
 		}),
 	).Methods(http.MethodPost, http.MethodOptions)
 
 	r0mux.Handle("/user/{userId}/filter/{filterId}",
 		common.MakeAuthAPI("get_filter", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return GetFilter(req, device, accountDB, vars["userId"], vars["filterId"])
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
@@ -235,21 +288,30 @@ func Setup(
 
 	r0mux.Handle("/profile/{userID}",
 		common.MakeExternalAPI("profile", func(req *http.Request) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
-			return GetProfile(req, accountDB, vars["userID"], asAPI)
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return GetProfile(req, accountDB, &cfg, vars["userID"], asAPI, federation)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/profile/{userID}/avatar_url",
 		common.MakeExternalAPI("profile_avatar_url", func(req *http.Request) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
-			return GetAvatarURL(req, accountDB, vars["userID"], asAPI)
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return GetAvatarURL(req, accountDB, &cfg, vars["userID"], asAPI, federation)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/profile/{userID}/avatar_url",
 		common.MakeAuthAPI("profile_avatar_url", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return SetAvatarURL(req, accountDB, device, vars["userID"], userUpdateProducer, &cfg, producer, queryAPI)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
@@ -258,14 +320,20 @@ func Setup(
 
 	r0mux.Handle("/profile/{userID}/displayname",
 		common.MakeExternalAPI("profile_displayname", func(req *http.Request) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
-			return GetDisplayName(req, accountDB, vars["userID"], asAPI)
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return GetDisplayName(req, accountDB, &cfg, vars["userID"], asAPI, federation)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/profile/{userID}/displayname",
 		common.MakeAuthAPI("profile_displayname", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return SetDisplayName(req, accountDB, device, vars["userID"], userUpdateProducer, &cfg, producer, queryAPI)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
@@ -335,28 +403,40 @@ func Setup(
 
 	r0mux.Handle("/user/{userID}/account_data/{type}",
 		common.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return SaveAccountData(req, accountDB, device, vars["userID"], "", vars["type"], syncProducer)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/user/{userID}/rooms/{roomID}/account_data/{type}",
 		common.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return SaveAccountData(req, accountDB, device, vars["userID"], vars["roomID"], vars["type"], syncProducer)
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
 
 	r0mux.Handle("/rooms/{roomID}/members",
 		common.MakeAuthAPI("rooms_members", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return GetMemberships(req, device, vars["roomID"], false, cfg, queryAPI)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/rooms/{roomID}/joined_members",
 		common.MakeAuthAPI("rooms_members", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return GetMemberships(req, device, vars["roomID"], true, cfg, queryAPI)
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
@@ -376,14 +456,20 @@ func Setup(
 
 	r0mux.Handle("/devices/{deviceID}",
 		common.MakeAuthAPI("get_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return GetDeviceByID(req, deviceDB, device, vars["deviceID"])
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
 
 	r0mux.Handle("/devices/{deviceID}",
 		common.MakeAuthAPI("device_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
-			vars := mux.Vars(req)
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
 			return UpdateDeviceByID(req, deviceDB, device, vars["deviceID"])
 		}),
 	).Methods(http.MethodPut, http.MethodOptions)
@@ -406,4 +492,34 @@ func Setup(
 			}}
 		}),
 	).Methods(http.MethodGet, http.MethodOptions)
+
+	r0mux.Handle("/user/{userId}/rooms/{roomId}/tags",
+		common.MakeAuthAPI("get_tags", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return GetTags(req, accountDB, device, vars["userId"], vars["roomId"], syncProducer)
+		}),
+	).Methods(http.MethodGet, http.MethodOptions)
+
+	r0mux.Handle("/user/{userId}/rooms/{roomId}/tags/{tag}",
+		common.MakeAuthAPI("put_tag", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return PutTag(req, accountDB, device, vars["userId"], vars["roomId"], vars["tag"], syncProducer)
+		}),
+	).Methods(http.MethodPut, http.MethodOptions)
+
+	r0mux.Handle("/user/{userId}/rooms/{roomId}/tags/{tag}",
+		common.MakeAuthAPI("delete_tag", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse {
+			vars, err := common.URLDecodeMapValues(mux.Vars(req))
+			if err != nil {
+				return util.ErrorResponse(err)
+			}
+			return DeleteTag(req, accountDB, device, vars["userId"], vars["roomId"], vars["tag"], syncProducer)
+		}),
+	).Methods(http.MethodDelete, http.MethodOptions)
 }

clientapi/routing/sendevent.go

@@ -50,7 +50,7 @@ func SendEvent(
 ) util.JSONResponse {
 	if txnID != nil {
 		// Try to fetch response from transactionsCache
-		if res, ok := txnCache.FetchTransaction(*txnID); ok {
+		if res, ok := txnCache.FetchTransaction(device.AccessToken, *txnID); ok {
 			return *res
 		}
 	}
@@ -83,7 +83,7 @@ func SendEvent(
 	}
 	// Add response to transactionsCache
 	if txnID != nil {
-		txnCache.AddTransaction(*txnID, &res)
+		txnCache.AddTransaction(device.AccessToken, *txnID, &res)
 	}
 
 	return res

clientapi/threepid/invites.go

@@ -91,7 +91,7 @@ func CheckAndProcessInvite(
 	producer *producers.RoomserverProducer, membership string, roomID string,
 	evTime time.Time,
 ) (inviteStoredOnIDServer bool, err error) {
-	if membership != "invite" || (body.Address == "" && body.IDServer == "" && body.Medium == "") {
+	if membership != gomatrixserverlib.Invite || (body.Address == "" && body.IDServer == "" && body.Medium == "") {
 		// If none of the 3PID-specific fields are supplied, it's a standard invite
 		// so return nil for it to be processed as such
 		return

cmd/client-api-proxy/main.go

@@ -17,13 +17,14 @@ package main
 import (
 	"flag"
 	"fmt"
-	log "github.com/sirupsen/logrus"
 	"net/http"
 	"net/http/httputil"
 	"net/url"
 	"os"
 	"strings"
 	"time"
+
+	log "github.com/sirupsen/logrus"
 )
 
 const usage = `Usage: %s

cmd/create-room-events/main.go

@@ -86,7 +86,7 @@ func main() {
 	// Build a m.room.member event.
 	b.Type = "m.room.member"
 	b.StateKey = userID
-	b.SetContent(map[string]string{"membership": "join"}) // nolint: errcheck
+	b.SetContent(map[string]string{"membership": gomatrixserverlib.Join}) // nolint: errcheck
 	b.AuthEvents = []gomatrixserverlib.EventReference{create}
 	member := buildAndOutput()
 
@@ -139,6 +139,6 @@ func writeEvent(event gomatrixserverlib.Event) {
 			panic(err)
 		}
 	} else {
-		panic(fmt.Errorf("Format %q is not valid, must be %q or %q", format, "InputRoomEvent", "Event"))
+		panic(fmt.Errorf("Format %q is not valid, must be %q or %q", *format, "InputRoomEvent", "Event"))
 	}
 }

cmd/federation-api-proxy/main.go

@@ -17,13 +17,14 @@ package main
 import (
 	"flag"
 	"fmt"
-	log "github.com/sirupsen/logrus"
 	"net/http"
 	"net/http/httputil"
 	"net/url"
 	"os"
 	"strings"
 	"time"
+
+	log "github.com/sirupsen/logrus"
 )
 
 const usage = `Usage: %s

cmd/kafka-producer/main.go

@@ -18,9 +18,10 @@ import (
 	"bufio"
 	"flag"
 	"fmt"
-	"github.com/Shopify/sarama"
 	"os"
 	"strings"
+
+	"github.com/Shopify/sarama"
 )
 
 const usage = `Usage: %s

cmd/mediaapi-integration-tests/main.go

@@ -41,7 +41,7 @@ var (
 	// Postgres docker container name (for running psql). If not set, psql must be in PATH.
 	postgresContainerName = os.Getenv("POSTGRES_CONTAINER")
 	// Test image to be uploaded/downloaded
-	testJPEG = test.Defaulting(os.Getenv("TEST_JPEG_PATH"), "src/github.com/matrix-org/dendrite/cmd/mediaapi-integration-tests/totem.jpg")
+	testJPEG = test.Defaulting(os.Getenv("TEST_JPEG_PATH"), "cmd/mediaapi-integration-tests/totem.jpg")
 	kafkaURI = test.Defaulting(os.Getenv("KAFKA_URIS"), "localhost:9092")
 )
 

common/basecomponent/base.go

@@ -71,7 +71,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string) *BaseDendrite {
 		componentName: componentName,
 		tracerCloser:  closer,
 		Cfg:           cfg,
-		APIMux:        mux.NewRouter(),
+		APIMux:        mux.NewRouter().UseEncodedPath(),
 		KafkaConsumer: kafkaConsumer,
 		KafkaProducer: kafkaProducer,
 	}