From ce189a737de1749a47296aec67c7403fc89e7504 Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Sat, 22 Jun 2019 21:40:55 +0800 Subject: [PATCH 01/17] Update dependency gomatrixserverlib to 178ed5e (#695) Update dependency gomatrixserverlib to 178ed5e --- go.mod | 28 ++++++++++++++-------------- go.sum | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index 4ab381206..574905727 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd github.com/crossdock/crossdock-go v0.0.0-20160816171116-049aabb0122b - github.com/davecgh/go-spew v1.1.0 + github.com/davecgh/go-spew v1.1.1 github.com/eapache/go-resiliency v0.0.0-20160104191539-b86b1ec0dd42 github.com/eapache/go-xerial-snappy v0.0.0-20160609142408-bb955e01b934 github.com/eapache/queue v1.1.0 @@ -22,10 +22,10 @@ require ( github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6 github.com/lib/pq v0.0.0-20170918175043-23da1db4f16d github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 - github.com/matrix-org/gomatrix v0.0.0-20171003113848-a7fc80c8060c - github.com/matrix-org/gomatrixserverlib v0.0.0-20181109104322-1c2cbc0872f0 + github.com/matrix-org/gomatrix v0.0.0-20190130130140-385f072fe9af + github.com/matrix-org/gomatrixserverlib v0.0.0-20190619132215-178ed5e3b8e2 github.com/matrix-org/naffka v0.0.0-20171115094957-662bfd0841d0 - github.com/matrix-org/util v0.0.0-20171013132526-8b1c8ab81986 + github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 github.com/matttproud/golang_protobuf_extensions v1.0.1 github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 github.com/nicksnyder/go-i18n v1.8.1 @@ -40,11 +40,11 @@ require ( github.com/prometheus/common v0.0.0-20170108231212-dd2f054febf4 github.com/prometheus/procfs v0.0.0-20170128160123-1878d9fbb537 github.com/rcrowley/go-metrics v0.0.0-20161128210544-1f30fe9094a5 - github.com/sirupsen/logrus v0.0.0-20170822132746-89742aefa4b2 - github.com/stretchr/testify v0.0.0-20170809224252-890a5c3458b4 - github.com/tidwall/gjson v1.0.2 - github.com/tidwall/match v0.0.0-20171002075945-1731857f09b1 - github.com/tidwall/sjson v1.0.0 + github.com/sirupsen/logrus v1.3.0 + github.com/stretchr/testify v1.2.2 + github.com/tidwall/gjson v1.1.5 + github.com/tidwall/match v1.0.1 + github.com/tidwall/sjson v1.0.3 github.com/uber-go/atomic v1.3.0 github.com/uber/jaeger-client-go v2.15.0+incompatible github.com/uber/jaeger-lib v1.5.0 @@ -52,14 +52,14 @@ require ( go.uber.org/atomic v1.3.0 go.uber.org/multierr v0.0.0-20170829224307-fb7d312c2c04 go.uber.org/zap v1.7.1 - golang.org/x/crypto v0.0.0-20181106171534-e4dc69e5b2fd - golang.org/x/net v0.0.0-20170927055102-0a9397675ba3 - golang.org/x/sys v0.0.0-20171012164349-43eea11bc926 + golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613 + golang.org/x/net v0.0.0-20190301231341-16b79f2e4e95 + golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 gopkg.in/Shopify/sarama.v1 v1.11.0 gopkg.in/airbrake/gobrake.v2 v2.0.9 gopkg.in/alecthomas/kingpin.v3-unstable v3.0.0-20170727041045-23bcc3c4eae3 gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 gopkg.in/h2non/bimg.v1 v1.0.18 - gopkg.in/macaroon.v2 v2.0.0 - gopkg.in/yaml.v2 v2.0.0-20171116090243-287cf08546ab + gopkg.in/macaroon.v2 v2.1.0 + gopkg.in/yaml.v2 v2.2.2 ) diff --git a/go.sum b/go.sum index 941c4f813..250b300b6 100644 --- a/go.sum +++ b/go.sum @@ -10,38 +10,54 @@ github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE github.com/crossdock/crossdock-go v0.0.0-20160816171116-049aabb0122b/go.mod h1:v9FBN7gdVTpiD/+LZ7Po0UKvROyT87uLVxTHVky/dlQ= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/eapache/go-resiliency v0.0.0-20160104191539-b86b1ec0dd42 h1:f8ERmXYuaC+kCSv2w+y3rBK/oVu6If4DEm3jywJJ0hc= github.com/eapache/go-resiliency v0.0.0-20160104191539-b86b1ec0dd42/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20160609142408-bb955e01b934 h1:oGLoaVIefp3tiOgi7+KInR/nNPvEpPM6GFo+El7fd14= github.com/eapache/go-xerial-snappy v0.0.0-20160609142408-bb955e01b934/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/frankban/quicktest v1.0.0/go.mod h1:R98jIehRai+d1/3Hv2//jOVCTJhW1VBavT6B6CuGq2k= github.com/golang/protobuf v0.0.0-20161117033126-8ee79997227b h1:fE/yi9pibxGEc0gSJuEShcsBXE2d5FW3OudsjE9tKzQ= github.com/golang/protobuf v0.0.0-20161117033126-8ee79997227b/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.0-20170119014723-7db9049039a0 h1:FMElzTwkd/2jQ2QzLEzt97JRgvFhYhnYiaQSwZ7tuyU= github.com/golang/snappy v0.0.0-20170119014723-7db9049039a0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/shlex v0.0.0-20150127133951-6f45313302b9/go.mod h1:RpwtwJQFrIEPstU94h88MWPXP2ektJZ8cZ0YntAmXiE= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.3.0 h1:HwSEKGN6U5T2aAQTfu5pW8fiwjSp3IgwdRbkICydk/c= github.com/gorilla/mux v1.3.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= github.com/jaegertracing/jaeger-client-go v0.0.0-20170921145708-3ad49a1d839b/go.mod h1:HWG7INeOG1ZE17I/S8eeb+svquXmBS/hf1Obi6hJUyQ= github.com/jaegertracing/jaeger-lib v0.0.0-20170920222118-21a3da6d66fe/go.mod h1:VqeqQrZmZr9G4WdLw4ei9tAHU54iJRkfoFHvTTQn4jQ= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6 h1:KAZ1BW2TCmT6PRihDPpocIy1QTtsAsrx6TneU/4+CMg= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v0.0.0-20170918175043-23da1db4f16d h1:Hdtccv31GWxWoCzWsIhZXy5NxEktzAkA8lywhTKu8O4= github.com/lib/pq v0.0.0-20170918175043-23da1db4f16d/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 h1:nMX2t7hbGF0NYDYySx0pCqEKGKAeZIiSqlWSspetlhY= github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg= github.com/matrix-org/gomatrix v0.0.0-20171003113848-a7fc80c8060c h1:aZap604NyBGhAUE0CyNHz6+Pryye5A5mHnYyO4KPPW8= github.com/matrix-org/gomatrix v0.0.0-20171003113848-a7fc80c8060c/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= +github.com/matrix-org/gomatrix v0.0.0-20190130130140-385f072fe9af h1:piaIBNQGIHnni27xRB7VKkEwoWCgAmeuYf8pxAyG0bI= +github.com/matrix-org/gomatrix v0.0.0-20190130130140-385f072fe9af/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrixserverlib v0.0.0-20181109104322-1c2cbc0872f0 h1:3UzhmERBbis4ZaB3imEbZwtDjGz/oVRC2cLLEajCzJA= github.com/matrix-org/gomatrixserverlib v0.0.0-20181109104322-1c2cbc0872f0/go.mod h1:YHyhIQUmuXyKtoVfDUMk/DyU93Taamlu6nPZkij/JtA= +github.com/matrix-org/gomatrixserverlib v0.0.0-20190619132215-178ed5e3b8e2 h1:pYajAEdi3sowj4iSunqctchhcMNW3rDjeeH0T4uDkMY= +github.com/matrix-org/gomatrixserverlib v0.0.0-20190619132215-178ed5e3b8e2/go.mod h1:sf0RcKOdiwJeTti7A313xsaejNUGYDq02MQZ4JD4w/E= github.com/matrix-org/naffka v0.0.0-20171115094957-662bfd0841d0 h1:p7WTwG+aXM86+yVrYAiCMW3ZHSmotVvuRbjtt3jC+4A= github.com/matrix-org/naffka v0.0.0-20171115094957-662bfd0841d0/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/util v0.0.0-20171013132526-8b1c8ab81986 h1:TiWl4hLvezAhRPM8tPcPDFTysZ7k4T/1J4GPp/iqlZo= github.com/matrix-org/util v0.0.0-20171013132526-8b1c8ab81986/go.mod h1:lePuOiXLNDott7NZfnQvJk0lAZ5HgvIuWGhel6J+RLA= +github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 h1:W7l5CP4V7wPyPb4tYE11dbmeAOwtFQBTW0rf4OonOS8= +github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5/go.mod h1:lePuOiXLNDott7NZfnQvJk0lAZ5HgvIuWGhel6J+RLA= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.1.4/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 h1:BvoENQQU+fZ9uukda/RzCAL/191HHwJA5b13R6diVlY= github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= github.com/nicksnyder/go-i18n v1.8.1/go.mod h1:HrK7VCrbOvQoUAQ7Vpy7i87N7JZZZ7R2xBGjv0j365Q= @@ -67,13 +83,23 @@ github.com/rcrowley/go-metrics v0.0.0-20161128210544-1f30fe9094a5 h1:gwcdIpH6NU2 github.com/rcrowley/go-metrics v0.0.0-20161128210544-1f30fe9094a5/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/sirupsen/logrus v0.0.0-20170822132746-89742aefa4b2 h1:+8J/sCAVv2Y9Ct1BKszDFJEVWv6Aynr2O4FYGUg6+Mc= github.com/sirupsen/logrus v0.0.0-20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= +github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME= +github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v0.0.0-20170809224252-890a5c3458b4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/tidwall/gjson v1.0.2 h1:5BsM7kyEAHAUGEGDkEKO9Mdyiuw6QQ6TSDdarP0Nnmk= github.com/tidwall/gjson v1.0.2/go.mod h1:c/nTNbUr0E0OrXEhq1pwa8iEgc2DOt4ZZqAt1HtCkPA= +github.com/tidwall/gjson v1.1.5 h1:QysILxBeUEY3GTLA0fQVgkQG1zme8NxGvhh2SSqWNwI= +github.com/tidwall/gjson v1.1.5/go.mod h1:c/nTNbUr0E0OrXEhq1pwa8iEgc2DOt4ZZqAt1HtCkPA= github.com/tidwall/match v0.0.0-20171002075945-1731857f09b1 h1:pWIN9LOlFRCJFqWIOEbHLvY0WWJddsjH2FQ6N0HKZdU= github.com/tidwall/match v0.0.0-20171002075945-1731857f09b1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= +github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= +github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= github.com/tidwall/sjson v1.0.0 h1:hOrzQPtGKlKAudQVmU43GkxEgG8TOgKyiKUyb7sE0rs= github.com/tidwall/sjson v1.0.0/go.mod h1:bURseu1nuBkFpIES5cz6zBtjmYeOQmEESshn7VpF15Y= +github.com/tidwall/sjson v1.0.3 h1:DeF+0LZqvIt4fKYw41aPB29ZGlvwVkHKktoXJ1YW9Y8= +github.com/tidwall/sjson v1.0.3/go.mod h1:bURseu1nuBkFpIES5cz6zBtjmYeOQmEESshn7VpF15Y= github.com/uber-go/atomic v1.3.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= github.com/uber/jaeger-client-go v2.15.0+incompatible h1:NP3qsSqNxh8VYr956ur1N/1C1PjvOJnJykCzcD5QHbk= github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= @@ -83,18 +109,29 @@ github.com/uber/tchannel-go v0.0.0-20170927010734-b3e26487e291/go.mod h1:Rrgz1eL go.uber.org/atomic v1.3.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v0.0.0-20170829224307-fb7d312c2c04/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.7.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181106171534-e4dc69e5b2fd h1:VtIkGDhk0ph3t+THbvXHfMZ8QHgsBO39Nh52+74pq7w= golang.org/x/crypto v0.0.0-20181106171534-e4dc69e5b2fd/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613 h1:MQ/ZZiDsUapFFiMS+vzwXkCTeEKaum+Do5rINYJDmxc= +golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/net v0.0.0-20170927055102-0a9397675ba3 h1:tTDpczhDVjW6WN3DinzKcw5juwkDTVn22I7MNlfxSXM= golang.org/x/net v0.0.0-20170927055102-0a9397675ba3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190301231341-16b79f2e4e95/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/sys v0.0.0-20171012164349-43eea11bc926 h1:PY6OU86NqbyZiOzaPnDw6oOjAGtYQqIua16z6y9QkwE= golang.org/x/sys v0.0.0-20171012164349-43eea11bc926/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 h1:I6FyU15t786LL7oL/hn43zqTuEGr4PN7F4XJ1p4E3Y8= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= gopkg.in/Shopify/sarama.v1 v1.11.0 h1:/3kaCyeYaPbr59IBjeqhIcUOB1vXlIVqXAYa5g5C5F0= gopkg.in/Shopify/sarama.v1 v1.11.0/go.mod h1:AxnvoaevB2nBjNK17cG61A3LleFcWFwVBHBt+cot4Oc= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v3-unstable v3.0.0-20170727041045-23bcc3c4eae3/go.mod h1:3HH7i1SgMqlzxCcBmUHW657sD4Kvv9sC3HpL3YukzwA= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/h2non/bimg.v1 v1.0.18/go.mod h1:PgsZL7dLwUbsGm1NYps320GxGgvQNTnecMCZqxV11So= +gopkg.in/h2non/gock.v1 v1.0.14/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE= gopkg.in/macaroon.v2 v2.0.0/go.mod h1:+I6LnTMkm/uV5ew/0nsulNjL16SK4+C8yDmRUzHR17I= +gopkg.in/macaroon.v2 v2.1.0/go.mod h1:OUb+TQP/OP0WOerC2Jp/3CwhIKyIa9kQjuc7H24e6/o= gopkg.in/yaml.v2 v2.0.0-20171116090243-287cf08546ab h1:yZ6iByf7GKeJ3gsd1Dr/xaj1DyJ//wxKX1Cdh8LhoAw= gopkg.in/yaml.v2 v2.0.0-20171116090243-287cf08546ab/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From eaf38ae87e89fc51190fb1c1a0c3e74bcb0884d4 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Mon, 24 Jun 2019 13:35:29 +0100 Subject: [PATCH 02/17] Not all systems have bash (#720) As not all systems have bash. Helpful for not making debugging alpine docker images a pain :> --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index eed4d52f5..9a8050f3c 100755 --- a/build.sh +++ b/build.sh @@ -1,3 +1,3 @@ -#!/bin/bash +#!/bin/sh GOBIN=$PWD/`dirname $0`/bin go install -v ./cmd/... From b88112b05db9da292d39b165f91b9af293c88500 Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Tue, 25 Jun 2019 18:32:15 +0800 Subject: [PATCH 03/17] Make federation state request 404 when event not in the room - fixes #625 (#716) Signed-off-by: Alex Chen --- federationapi/routing/state.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/federationapi/routing/state.go b/federationapi/routing/state.go index 58398bde9..86cf1cf54 100644 --- a/federationapi/routing/state.go +++ b/federationapi/routing/state.go @@ -103,6 +103,10 @@ func getState( return nil, resErr } + if event.RoomID() != roomID { + return nil, &util.JSONResponse{Code: http.StatusNotFound, JSON: nil} + } + prevEventIDs := getIDsFromEventRef(event.PrevEvents()) authEventIDs := getIDsFromEventRef(event.AuthEvents()) From 7792f12e6f6c0327d4a38cb1f6b879c3acf38f57 Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Tue, 25 Jun 2019 18:50:19 +0800 Subject: [PATCH 04/17] Fix testfile path hardcoded in show-expected-fail-tests.sh (#719) Signed-off-by: Alex Chen --- docs/sytest.md | 2 +- show-expected-fail-tests.sh | 19 +++++++++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/docs/sytest.md b/docs/sytest.md index 99987fea5..e936dc493 100644 --- a/docs/sytest.md +++ b/docs/sytest.md @@ -59,7 +59,7 @@ Once the tests are complete, run the helper script to see if you need to add any newly passing test names to `testfile` in the project's root directory: ```sh -../dendrite/show-expected-fail-tests.sh results.tap +../dendrite/show-expected-fail-tests.sh results.tap ../dendrite/testfile ``` If the script prints nothing/exits with 0, then you're good to go. diff --git a/show-expected-fail-tests.sh b/show-expected-fail-tests.sh index 277a2c8b9..f58416b2d 100755 --- a/show-expected-fail-tests.sh +++ b/show-expected-fail-tests.sh @@ -1,13 +1,28 @@ #! /bin/bash results_file=$1 +testfile=$2 + +fail_build=0 + +if [ ! -f "$results_file" ]; then + echo "ERROR: Specified results file ${results_file} doesn't exist." + fail_build=1 +fi + +if [ ! -f "$testfile" ]; then + echo "ERROR: Specified testfile ${testfile} doesn't exist." + fail_build=1 +fi + +[ "$fail_build" = 0 ] || exit 1 + passed_but_expected_fail=$(grep ' # TODO passed but expected fail' ${results_file} | sed -E 's/^ok [0-9]+ (\(expected fail\) )?//' | sed -E 's/( \([0-9]+ subtests\))? # TODO passed but expected fail$//') tests_to_add="" already_in_testfile="" -fail_build=0 while read -r test_id; do - grep "${test_id}" testfile > /dev/null 2>&1 + grep "${test_id}" "${testfile}" > /dev/null 2>&1 if [ "$?" != "0" ]; then tests_to_add="${tests_to_add}${test_id}\n" fail_build=1 From a0dec456c173de650bef03d3da97f0f721288fd1 Mon Sep 17 00:00:00 2001 From: Serra Allgood Date: Tue, 25 Jun 2019 05:43:18 -0700 Subject: [PATCH 05/17] rommserver/alias: Do not call appserviceAPI in GetRoomIDForAlias if local alias found #631 (#702) A conditional is added to wrap the call to appserviceAPI if a local alias is not found in the database. Fixes #631 Signed-off-by: Serra Allgood --- roomserver/alias/alias.go | 21 +++- roomserver/alias/alias_test.go | 196 +++++++++++++++++++++++++++++++++ 2 files changed, 211 insertions(+), 6 deletions(-) create mode 100644 roomserver/alias/alias_test.go diff --git a/roomserver/alias/alias.go b/roomserver/alias/alias.go index 27279aad8..6a34aacdd 100644 --- a/roomserver/alias/alias.go +++ b/roomserver/alias/alias.go @@ -96,12 +96,21 @@ func (r *RoomserverAliasAPI) GetRoomIDForAlias( return err } - // No rooms found locally, try our application services by making a call to - // the appservice component - aliasReq := appserviceAPI.RoomAliasExistsRequest{Alias: request.Alias} - var aliasResp appserviceAPI.RoomAliasExistsResponse - if err = r.AppserviceAPI.RoomAliasExists(ctx, &aliasReq, &aliasResp); err != nil { - return err + if roomID == "" { + // No room found locally, try our application services by making a call to + // the appservice component + aliasReq := appserviceAPI.RoomAliasExistsRequest{Alias: request.Alias} + var aliasResp appserviceAPI.RoomAliasExistsResponse + if err = r.AppserviceAPI.RoomAliasExists(ctx, &aliasReq, &aliasResp); err != nil { + return err + } + + if aliasResp.AliasExists { + roomID, err = r.DB.GetRoomIDForAlias(ctx, request.Alias) + if err != nil { + return err + } + } } response.RoomID = roomID diff --git a/roomserver/alias/alias_test.go b/roomserver/alias/alias_test.go new file mode 100644 index 000000000..4b9ca022d --- /dev/null +++ b/roomserver/alias/alias_test.go @@ -0,0 +1,196 @@ +// Copyright 2019 Serra Allgood +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package alias + +import ( + "context" + "fmt" + "strings" + "testing" + + appserviceAPI "github.com/matrix-org/dendrite/appservice/api" + roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" +) + +type MockRoomserverAliasAPIDatabase struct { + mode string + attempts int +} + +// These methods can be essentially noop +func (db MockRoomserverAliasAPIDatabase) SetRoomAlias(ctx context.Context, alias string, roomID string) error { + return nil +} + +func (db MockRoomserverAliasAPIDatabase) GetAliasesForRoomID(ctx context.Context, roomID string) ([]string, error) { + aliases := make([]string, 0) + return aliases, nil +} + +func (db MockRoomserverAliasAPIDatabase) RemoveRoomAlias(ctx context.Context, alias string) error { + return nil +} + +// This method needs to change depending on test case +func (db *MockRoomserverAliasAPIDatabase) GetRoomIDForAlias( + ctx context.Context, + alias string, +) (string, error) { + switch db.mode { + case "empty": + return "", nil + case "error": + return "", fmt.Errorf("found an error from GetRoomIDForAlias") + case "found": + return "123", nil + case "emptyFound": + switch db.attempts { + case 0: + db.attempts = 1 + return "", nil + case 1: + db.attempts = 0 + return "123", nil + default: + return "", nil + } + default: + return "", fmt.Errorf("unknown option used") + } +} + +type MockAppServiceQueryAPI struct { + mode string +} + +// This method can be noop +func (q MockAppServiceQueryAPI) UserIDExists( + ctx context.Context, + req *appserviceAPI.UserIDExistsRequest, + resp *appserviceAPI.UserIDExistsResponse, +) error { + return nil +} + +func (q MockAppServiceQueryAPI) RoomAliasExists( + ctx context.Context, + req *appserviceAPI.RoomAliasExistsRequest, + resp *appserviceAPI.RoomAliasExistsResponse, +) error { + switch q.mode { + case "error": + return fmt.Errorf("found an error from RoomAliasExists") + case "found": + resp.AliasExists = true + return nil + case "empty": + resp.AliasExists = false + return nil + default: + return fmt.Errorf("Unknown option used") + } +} + +func TestGetRoomIDForAlias(t *testing.T) { + type arguments struct { + ctx context.Context + request *roomserverAPI.GetRoomIDForAliasRequest + response *roomserverAPI.GetRoomIDForAliasResponse + } + args := arguments{ + context.Background(), + &roomserverAPI.GetRoomIDForAliasRequest{}, + &roomserverAPI.GetRoomIDForAliasResponse{}, + } + type testCase struct { + name string + dbMode string + queryMode string + wantError bool + errorMsg string + want string + } + tt := []testCase{ + { + "found local alias", + "found", + "error", + false, + "", + "123", + }, + { + "found appservice alias", + "emptyFound", + "found", + false, + "", + "123", + }, + { + "error returned from DB", + "error", + "", + true, + "GetRoomIDForAlias", + "", + }, + { + "error returned from appserviceAPI", + "empty", + "error", + true, + "RoomAliasExists", + "", + }, + { + "no errors but no alias", + "empty", + "empty", + false, + "", + "", + }, + } + + setup := func(dbMode, queryMode string) *RoomserverAliasAPI { + mockAliasAPIDB := &MockRoomserverAliasAPIDatabase{dbMode, 0} + mockAppServiceQueryAPI := MockAppServiceQueryAPI{queryMode} + + return &RoomserverAliasAPI{ + DB: mockAliasAPIDB, + AppserviceAPI: mockAppServiceQueryAPI, + } + } + + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + aliasAPI := setup(tc.dbMode, tc.queryMode) + + err := aliasAPI.GetRoomIDForAlias(args.ctx, args.request, args.response) + if tc.wantError { + if err == nil { + t.Fatalf("Got no error; wanted error from %s", tc.errorMsg) + } else if !strings.Contains(err.Error(), tc.errorMsg) { + t.Fatalf("Got %s; wanted error from %s", err, tc.errorMsg) + } + } else if err != nil { + t.Fatalf("Got %s; wanted no error", err) + } else if args.response.RoomID != tc.want { + t.Errorf("Got '%s'; wanted '%s'", args.response.RoomID, tc.want) + } + }) + } +} From 1eb77b8161cdf2e3b107c606d4f3f88209042ed8 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Mon, 1 Jul 2019 16:04:49 +0100 Subject: [PATCH 06/17] Don't print Sending EDU if there is noone to send to (#721) The logs had a lot of: ``` Sending EDU event destinations="[]" edu_type=m.typing ``` Which is useless if it isn't actually sending the event anywhere (destinations is empty). --- federationsender/queue/queue.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/federationsender/queue/queue.go b/federationsender/queue/queue.go index 4a38dc086..6a05c5f07 100644 --- a/federationsender/queue/queue.go +++ b/federationsender/queue/queue.go @@ -96,9 +96,11 @@ func (oqs *OutgoingQueues) SendEDU( // Remove our own server from the list of destinations. destinations = filterDestinations(oqs.origin, destinations) - log.WithFields(log.Fields{ - "destinations": destinations, "edu_type": e.Type, - }).Info("Sending EDU event") + if len(destinations) > 0 { + log.WithFields(log.Fields{ + "destinations": destinations, "edu_type": e.Type, + }).Info("Sending EDU event") + } oqs.queuesMutex.Lock() defer oqs.queuesMutex.Unlock() From 33a13925417612b032c1cbbb8ee62eb053faa5f8 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Wed, 3 Jul 2019 16:38:50 +0100 Subject: [PATCH 07/17] Encode URLs properly (#728) We were escaping the URL before performing any pattern matching on it. This meant that if you sent data that URLdecoded to a "/", it would count as a "/" in the URL, potentially causing a 404. This was causing some flaky tests with some randomly-generated query parameters. Now, we keep URLs encoded while doing the pattern matching, and only afterwards do we URL decode each query parameter individually before passing them to their respective handler functions. github.com/gorilla/mux was also updated to v1.7.3 to fix a bug with URL encoding and subrouters. --- appservice/routing/routing.go | 4 + clientapi/routing/joinroom.go | 5 +- clientapi/routing/routing.go | 119 ++++++++++++++++++++++++------ common/basecomponent/base.go | 2 +- common/routing.go | 35 +++++++++ federationapi/routing/routing.go | 69 +++++++++++++---- go.mod | 2 +- go.sum | 6 ++ mediaapi/routing/routing.go | 6 +- publicroomsapi/routing/routing.go | 14 +++- syncapi/routing/routing.go | 19 ++++- 11 files changed, 236 insertions(+), 45 deletions(-) create mode 100644 common/routing.go diff --git a/appservice/routing/routing.go b/appservice/routing/routing.go index f0b8461dc..3c19c8401 100644 --- a/appservice/routing/routing.go +++ b/appservice/routing/routing.go @@ -31,6 +31,10 @@ const pathPrefixApp = "/_matrix/app/r0" // Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client // to clients which need to make outbound HTTP requests. +// +// Due to Setup being used to call many other functions, a gocyclo nolint is +// applied: +// nolint: gocyclo func Setup( apiMux *mux.Router, cfg config.Dendrite, // nolint: unparam queryAPI api.RoomserverQueryAPI, aliasAPI api.RoomserverAliasAPI, // nolint: unparam diff --git a/clientapi/routing/joinroom.go b/clientapi/routing/joinroom.go index c98688de0..9c02a93ca 100644 --- a/clientapi/routing/joinroom.go +++ b/clientapi/routing/joinroom.go @@ -86,7 +86,10 @@ func JoinRoomByIDOrAlias( } return util.JSONResponse{ Code: http.StatusBadRequest, - JSON: jsonerror.BadJSON("Invalid first character for room ID or alias"), + JSON: jsonerror.BadJSON( + fmt.Sprintf("Invalid first character '%s' for room ID or alias", + string([]rune(roomIDOrAlias)[0])), // Wrapping with []rune makes this call UTF-8 safe + ), } } diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index b0ced79e7..8135e49af 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -41,6 +41,10 @@ const pathPrefixUnstable = "/_matrix/client/unstable" // Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client // to clients which need to make outbound HTTP requests. +// +// Due to Setup being used to call many other functions, a gocyclo nolint is +// applied: +// nolint: gocyclo func Setup( apiMux *mux.Router, cfg config.Dendrite, producer *producers.RoomserverProducer, @@ -90,7 +94,10 @@ func Setup( ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/join/{roomIDOrAlias}", common.MakeAuthAPI("join", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return JoinRoomByIDOrAlias( req, device, vars["roomIDOrAlias"], cfg, federation, producer, queryAPI, aliasAPI, keyRing, accountDB, ) @@ -98,19 +105,28 @@ func Setup( ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/{membership:(?:join|kick|ban|unban|leave|invite)}", common.MakeAuthAPI("membership", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return SendMembership(req, accountDB, device, vars["roomID"], vars["membership"], cfg, queryAPI, asAPI, producer) }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/send/{eventType}", common.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, nil, cfg, queryAPI, producer, nil) }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/send/{eventType}/{txnID}", common.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } txnID := vars["txnID"] return SendEvent(req, device, vars["roomID"], vars["eventType"], &txnID, nil, cfg, queryAPI, producer, transactionsCache) @@ -118,7 +134,10 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/state/{eventType:[^/]+/?}", common.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } emptyString := "" eventType := vars["eventType"] // If there's a trailing slash, remove it @@ -130,7 +149,10 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/state/{eventType}/{stateKey}", common.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } stateKey := vars["stateKey"] return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, &stateKey, cfg, queryAPI, producer, nil) }), @@ -150,21 +172,30 @@ func Setup( r0mux.Handle("/directory/room/{roomAlias}", common.MakeExternalAPI("directory_room", func(req *http.Request) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return DirectoryRoom(req, vars["roomAlias"], federation, &cfg, aliasAPI) }), ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/directory/room/{roomAlias}", common.MakeAuthAPI("directory_room", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return SetLocalAlias(req, device, vars["roomAlias"], &cfg, aliasAPI) }), ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/directory/room/{roomAlias}", common.MakeAuthAPI("directory_room", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return RemoveLocalAlias(req, device, vars["roomAlias"], aliasAPI) }), ).Methods(http.MethodDelete, http.MethodOptions) @@ -183,7 +214,10 @@ func Setup( r0mux.Handle("/rooms/{roomID}/typing/{userID}", common.MakeAuthAPI("rooms_typing", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return SendTyping(req, device, vars["roomID"], vars["userID"], accountDB, typingProducer) }), ).Methods(http.MethodPut, http.MethodOptions) @@ -223,14 +257,20 @@ func Setup( r0mux.Handle("/user/{userId}/filter", common.MakeAuthAPI("put_filter", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return PutFilter(req, device, accountDB, vars["userId"]) }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/user/{userId}/filter/{filterId}", common.MakeAuthAPI("get_filter", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return GetFilter(req, device, accountDB, vars["userId"], vars["filterId"]) }), ).Methods(http.MethodGet, http.MethodOptions) @@ -239,21 +279,30 @@ func Setup( r0mux.Handle("/profile/{userID}", common.MakeExternalAPI("profile", func(req *http.Request) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return GetProfile(req, accountDB, vars["userID"], asAPI) }), ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/profile/{userID}/avatar_url", common.MakeExternalAPI("profile_avatar_url", func(req *http.Request) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return GetAvatarURL(req, accountDB, vars["userID"], asAPI) }), ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/profile/{userID}/avatar_url", common.MakeAuthAPI("profile_avatar_url", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return SetAvatarURL(req, accountDB, device, vars["userID"], userUpdateProducer, &cfg, producer, queryAPI) }), ).Methods(http.MethodPut, http.MethodOptions) @@ -262,14 +311,20 @@ func Setup( r0mux.Handle("/profile/{userID}/displayname", common.MakeExternalAPI("profile_displayname", func(req *http.Request) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return GetDisplayName(req, accountDB, vars["userID"], asAPI) }), ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/profile/{userID}/displayname", common.MakeAuthAPI("profile_displayname", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return SetDisplayName(req, accountDB, device, vars["userID"], userUpdateProducer, &cfg, producer, queryAPI) }), ).Methods(http.MethodPut, http.MethodOptions) @@ -339,28 +394,40 @@ func Setup( r0mux.Handle("/user/{userID}/account_data/{type}", common.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return SaveAccountData(req, accountDB, device, vars["userID"], "", vars["type"], syncProducer) }), ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/user/{userID}/rooms/{roomID}/account_data/{type}", common.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return SaveAccountData(req, accountDB, device, vars["userID"], vars["roomID"], vars["type"], syncProducer) }), ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/members", common.MakeAuthAPI("rooms_members", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return GetMemberships(req, device, vars["roomID"], false, cfg, queryAPI) }), ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/joined_members", common.MakeAuthAPI("rooms_members", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return GetMemberships(req, device, vars["roomID"], true, cfg, queryAPI) }), ).Methods(http.MethodGet, http.MethodOptions) @@ -380,14 +447,20 @@ func Setup( r0mux.Handle("/devices/{deviceID}", common.MakeAuthAPI("get_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return GetDeviceByID(req, deviceDB, device, vars["deviceID"]) }), ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/devices/{deviceID}", common.MakeAuthAPI("device_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return UpdateDeviceByID(req, deviceDB, device, vars["deviceID"]) }), ).Methods(http.MethodPut, http.MethodOptions) diff --git a/common/basecomponent/base.go b/common/basecomponent/base.go index d1f507544..6a20aca3b 100644 --- a/common/basecomponent/base.go +++ b/common/basecomponent/base.go @@ -71,7 +71,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string) *BaseDendrite { componentName: componentName, tracerCloser: closer, Cfg: cfg, - APIMux: mux.NewRouter(), + APIMux: mux.NewRouter().UseEncodedPath(), KafkaConsumer: kafkaConsumer, KafkaProducer: kafkaProducer, } diff --git a/common/routing.go b/common/routing.go new file mode 100644 index 000000000..330912cde --- /dev/null +++ b/common/routing.go @@ -0,0 +1,35 @@ +// Copyright 2019 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + "net/url" +) + +// URLDecodeMapValues is a function that iterates through each of the items in a +// map, URL decodes the value, and returns a new map with the decoded values +// under the same key names +func URLDecodeMapValues(vmap map[string]string) (map[string]string, error) { + decoded := make(map[string]string, len(vmap)) + for key, value := range vmap { + decodedVal, err := url.QueryUnescape(value) + if err != nil { + return make(map[string]string), err + } + decoded[key] = decodedVal + } + + return decoded, nil +} diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index 035d54aa1..16704e0b2 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -35,6 +35,10 @@ const ( ) // Setup registers HTTP handlers with the given ServeMux. +// +// Due to Setup being used to call many other functions, a gocyclo nolint is +// applied: +// nolint: gocyclo func Setup( apiMux *mux.Router, cfg config.Dendrite, @@ -64,7 +68,10 @@ func Setup( v1fedmux.Handle("/send/{txnID}/", common.MakeFedAPI( "federation_send", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars := mux.Vars(httpReq) + vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) + if err != nil { + return util.ErrorResponse(err) + } return Send( httpReq, request, gomatrixserverlib.TransactionID(vars["txnID"]), cfg, query, producer, keys, federation, @@ -75,7 +82,10 @@ func Setup( v1fedmux.Handle("/invite/{roomID}/{eventID}", common.MakeFedAPI( "federation_invite", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars := mux.Vars(httpReq) + vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) + if err != nil { + return util.ErrorResponse(err) + } return Invite( httpReq, request, vars["roomID"], vars["eventID"], cfg, producer, keys, @@ -92,7 +102,10 @@ func Setup( v1fedmux.Handle("/exchange_third_party_invite/{roomID}", common.MakeFedAPI( "exchange_third_party_invite", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars := mux.Vars(httpReq) + vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) + if err != nil { + return util.ErrorResponse(err) + } return ExchangeThirdPartyInvite( httpReq, request, vars["roomID"], query, cfg, federation, producer, ) @@ -102,7 +115,10 @@ func Setup( v1fedmux.Handle("/event/{eventID}", common.MakeFedAPI( "federation_get_event", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars := mux.Vars(httpReq) + vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) + if err != nil { + return util.ErrorResponse(err) + } return GetEvent( httpReq.Context(), request, query, vars["eventID"], ) @@ -112,7 +128,10 @@ func Setup( v1fedmux.Handle("/state/{roomID}", common.MakeFedAPI( "federation_get_event_auth", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars := mux.Vars(httpReq) + vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) + if err != nil { + return util.ErrorResponse(err) + } return GetState( httpReq.Context(), request, query, vars["roomID"], ) @@ -122,7 +141,10 @@ func Setup( v1fedmux.Handle("/state_ids/{roomID}", common.MakeFedAPI( "federation_get_event_auth", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars := mux.Vars(httpReq) + vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) + if err != nil { + return util.ErrorResponse(err) + } return GetStateIDs( httpReq.Context(), request, query, vars["roomID"], ) @@ -150,7 +172,10 @@ func Setup( v1fedmux.Handle("/user/devices/{userID}", common.MakeFedAPI( "federation_user_devices", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars := mux.Vars(httpReq) + vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) + if err != nil { + return util.ErrorResponse(err) + } return GetUserDevices( httpReq, deviceDB, vars["userID"], ) @@ -160,7 +185,10 @@ func Setup( v1fedmux.Handle("/make_join/{roomID}/{userID}", common.MakeFedAPI( "federation_make_join", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars := mux.Vars(httpReq) + vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) + if err != nil { + return util.ErrorResponse(err) + } roomID := vars["roomID"] userID := vars["userID"] return MakeJoin( @@ -172,7 +200,10 @@ func Setup( v1fedmux.Handle("/send_join/{roomID}/{userID}", common.MakeFedAPI( "federation_send_join", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars := mux.Vars(httpReq) + vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) + if err != nil { + return util.ErrorResponse(err) + } roomID := vars["roomID"] userID := vars["userID"] return SendJoin( @@ -184,7 +215,10 @@ func Setup( v1fedmux.Handle("/make_leave/{roomID}/{userID}", common.MakeFedAPI( "federation_make_leave", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars := mux.Vars(httpReq) + vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) + if err != nil { + return util.ErrorResponse(err) + } roomID := vars["roomID"] userID := vars["userID"] return MakeLeave( @@ -196,7 +230,10 @@ func Setup( v1fedmux.Handle("/send_leave/{roomID}/{userID}", common.MakeFedAPI( "federation_send_leave", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars := mux.Vars(httpReq) + vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) + if err != nil { + return util.ErrorResponse(err) + } roomID := vars["roomID"] userID := vars["userID"] return SendLeave( @@ -215,7 +252,10 @@ func Setup( v1fedmux.Handle("/get_missing_events/{roomID}", common.MakeFedAPI( "federation_get_missing_events", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars := mux.Vars(httpReq) + vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) + if err != nil { + return util.ErrorResponse(err) + } return GetMissingEvents(httpReq, request, query, vars["roomID"]) }, )).Methods(http.MethodPost) @@ -223,7 +263,10 @@ func Setup( v1fedmux.Handle("/backfill/{roomID}/", common.MakeFedAPI( "federation_backfill", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars := mux.Vars(httpReq) + vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) + if err != nil { + return util.ErrorResponse(err) + } return Backfill(httpReq, request, query, vars["roomID"], cfg) }, )).Methods(http.MethodGet) diff --git a/go.mod b/go.mod index 574905727..072d9ef30 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/golang/snappy v0.0.0-20170119014723-7db9049039a0 github.com/google/shlex v0.0.0-20150127133951-6f45313302b9 github.com/gorilla/context v1.1.1 - github.com/gorilla/mux v1.3.0 + github.com/gorilla/mux v1.7.3 github.com/jaegertracing/jaeger-client-go v0.0.0-20170921145708-3ad49a1d839b github.com/jaegertracing/jaeger-lib v0.0.0-20170920222118-21a3da6d66fe github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6 diff --git a/go.sum b/go.sum index 250b300b6..ce3c07dd7 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,7 @@ github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE github.com/crossdock/crossdock-go v0.0.0-20160816171116-049aabb0122b/go.mod h1:v9FBN7gdVTpiD/+LZ7Po0UKvROyT87uLVxTHVky/dlQ= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/eapache/go-resiliency v0.0.0-20160104191539-b86b1ec0dd42 h1:f8ERmXYuaC+kCSv2w+y3rBK/oVu6If4DEm3jywJJ0hc= github.com/eapache/go-resiliency v0.0.0-20160104191539-b86b1ec0dd42/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= @@ -27,6 +28,8 @@ github.com/google/shlex v0.0.0-20150127133951-6f45313302b9/go.mod h1:RpwtwJQFrIE github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.3.0 h1:HwSEKGN6U5T2aAQTfu5pW8fiwjSp3IgwdRbkICydk/c= github.com/gorilla/mux v1.3.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= +github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= github.com/jaegertracing/jaeger-client-go v0.0.0-20170921145708-3ad49a1d839b/go.mod h1:HWG7INeOG1ZE17I/S8eeb+svquXmBS/hf1Obi6hJUyQ= github.com/jaegertracing/jaeger-lib v0.0.0-20170920222118-21a3da6d66fe/go.mod h1:VqeqQrZmZr9G4WdLw4ei9tAHU54iJRkfoFHvTTQn4jQ= @@ -117,6 +120,7 @@ golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613 h1:MQ/ZZiDsUapFFiMS+vzwXk golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/net v0.0.0-20170927055102-0a9397675ba3 h1:tTDpczhDVjW6WN3DinzKcw5juwkDTVn22I7MNlfxSXM= golang.org/x/net v0.0.0-20170927055102-0a9397675ba3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190301231341-16b79f2e4e95 h1:fY7Dsw114eJN4boqzVSbpVHO6rTdhq6/GnXeu+PKnzU= golang.org/x/net v0.0.0-20190301231341-16b79f2e4e95/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/sys v0.0.0-20171012164349-43eea11bc926 h1:PY6OU86NqbyZiOzaPnDw6oOjAGtYQqIua16z6y9QkwE= golang.org/x/sys v0.0.0-20171012164349-43eea11bc926/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -134,4 +138,6 @@ gopkg.in/macaroon.v2 v2.0.0/go.mod h1:+I6LnTMkm/uV5ew/0nsulNjL16SK4+C8yDmRUzHR17 gopkg.in/macaroon.v2 v2.1.0/go.mod h1:OUb+TQP/OP0WOerC2Jp/3CwhIKyIa9kQjuc7H24e6/o= gopkg.in/yaml.v2 v2.0.0-20171116090243-287cf08546ab h1:yZ6iByf7GKeJ3gsd1Dr/xaj1DyJ//wxKX1Cdh8LhoAw= gopkg.in/yaml.v2 v2.0.0-20171116090243-287cf08546ab/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= + diff --git a/mediaapi/routing/routing.go b/mediaapi/routing/routing.go index fb983ccc2..5bcce1772 100644 --- a/mediaapi/routing/routing.go +++ b/mediaapi/routing/routing.go @@ -34,6 +34,10 @@ import ( const pathPrefixR0 = "/_matrix/media/r0" // Setup registers the media API HTTP handlers +// +// Due to Setup being used to call many other functions, a gocyclo nolint is +// applied: +// nolint: gocyclo func Setup( apiMux *mux.Router, cfg *config.Dendrite, @@ -87,7 +91,7 @@ func makeDownloadAPI( // Content-Type will be overridden in case of returning file data, else we respond with JSON-formatted errors w.Header().Set("Content-Type", "application/json") - vars := mux.Vars(req) + vars, _ := common.URLDecodeMapValues(mux.Vars(req)) Download( w, req, diff --git a/publicroomsapi/routing/routing.go b/publicroomsapi/routing/routing.go index 6a4b79b7e..3a1c9eb58 100644 --- a/publicroomsapi/routing/routing.go +++ b/publicroomsapi/routing/routing.go @@ -30,6 +30,10 @@ import ( const pathPrefixR0 = "/_matrix/client/r0" // Setup configures the given mux with publicroomsapi server listeners +// +// Due to Setup being used to call many other functions, a gocyclo nolint is +// applied: +// nolint: gocyclo func Setup(apiMux *mux.Router, deviceDB *devices.Database, publicRoomsDB *storage.PublicRoomsServerDatabase) { r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter() @@ -41,14 +45,20 @@ func Setup(apiMux *mux.Router, deviceDB *devices.Database, publicRoomsDB *storag r0mux.Handle("/directory/list/room/{roomID}", common.MakeExternalAPI("directory_list", func(req *http.Request) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return directory.GetVisibility(req, publicRoomsDB, vars["roomID"]) }), ).Methods(http.MethodGet, http.MethodOptions) // TODO: Add AS support r0mux.Handle("/directory/list/room/{roomID}", common.MakeAuthAPI("directory_list", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return directory.SetVisibility(req, publicRoomsDB, vars["roomID"]) }), ).Methods(http.MethodPut, http.MethodOptions) diff --git a/syncapi/routing/routing.go b/syncapi/routing/routing.go index 93d939c30..cbdcfb6bb 100644 --- a/syncapi/routing/routing.go +++ b/syncapi/routing/routing.go @@ -30,6 +30,10 @@ import ( const pathPrefixR0 = "/_matrix/client/r0" // Setup configures the given mux with sync-server listeners +// +// Due to Setup being used to call many other functions, a gocyclo nolint is +// applied: +// nolint: gocyclo func Setup(apiMux *mux.Router, srp *sync.RequestPool, syncDB *storage.SyncServerDatabase, deviceDB *devices.Database) { r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter() @@ -45,17 +49,26 @@ func Setup(apiMux *mux.Router, srp *sync.RequestPool, syncDB *storage.SyncServer })).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/state", common.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return OnIncomingStateRequest(req, syncDB, vars["roomID"]) })).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/state/{type}", common.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return OnIncomingStateTypeRequest(req, syncDB, vars["roomID"], vars["type"], "") })).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", common.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars := mux.Vars(req) + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } return OnIncomingStateTypeRequest(req, syncDB, vars["roomID"], vars["type"], vars["stateKey"]) })).Methods(http.MethodGet, http.MethodOptions) } From da0dd06475a47fb28cf8381bda5f941caae6caad Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Fri, 5 Jul 2019 19:42:23 +0800 Subject: [PATCH 08/17] Update testfile and fix empty warning in show-expected-fail-tests.sh (#729) This PR adds the test that matrix-org/sytest#642 will allow to pass to testfile and also contains a minor fix for show-expected-fail-tests.sh so it doesn't show an empty warning. Signed-off-by: Alex Chen minecnly@gmail.com --- show-expected-fail-tests.sh | 1 + testfile | 1 + 2 files changed, 2 insertions(+) diff --git a/show-expected-fail-tests.sh b/show-expected-fail-tests.sh index f58416b2d..80b842ab1 100755 --- a/show-expected-fail-tests.sh +++ b/show-expected-fail-tests.sh @@ -22,6 +22,7 @@ tests_to_add="" already_in_testfile="" while read -r test_id; do + [ "${test_id}" = "" ] && continue grep "${test_id}" "${testfile}" > /dev/null 2>&1 if [ "$?" != "0" ]; then tests_to_add="${tests_to_add}${test_id}\n" diff --git a/testfile b/testfile index ea6fc9172..362df4513 100644 --- a/testfile +++ b/testfile @@ -142,3 +142,4 @@ Trying to get push rules with unknown rule_id fails with 404 Events come down the correct room local user can join room with version 5 User can invite local user to room with version 5 +Inbound federation can receive room-join requests From 69f8d5a77e9392d5285fcc1c8d68659eac88c67a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Behouba=20Manass=C3=A9?= Date: Mon, 8 Jul 2019 16:06:17 +0300 Subject: [PATCH 09/17] Content-Disposition HTTP header in mediaapi's responses added (#685) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This should fix #628 I also Noticed that GET /_matrix/media/r0/download/{serverName}/{mediaId}/{fileName} is not yet implemented, but it should work for both. Signed-off-by: Kouamé Behouba Manassé behouba@gmail.com --- mediaapi/routing/download.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mediaapi/routing/download.go b/mediaapi/routing/download.go index 9c8f43c44..38c436367 100644 --- a/mediaapi/routing/download.go +++ b/mediaapi/routing/download.go @@ -305,6 +305,10 @@ func (r *downloadRequest) respondFromLocalFile( }).Info("Responding with file") responseFile = file responseMetadata = r.MediaMetadata + + if len(responseMetadata.UploadName) > 0 { + w.Header().Set("Content-Disposition", fmt.Sprintf(`inline; filename*=utf-8"%s"`, responseMetadata.UploadName)) + } } w.Header().Set("Content-Type", string(responseMetadata.ContentType)) From c72517687878267a021641a689fdf73fe6d1c127 Mon Sep 17 00:00:00 2001 From: Anant Prakash Date: Tue, 9 Jul 2019 18:51:33 +0530 Subject: [PATCH 10/17] Fix http responses in validateRecaptcha (#431) fixes #421 --- clientapi/routing/register.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index b1522e82b..243f9dd23 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -243,8 +243,8 @@ func validateRecaptcha( ) *util.JSONResponse { if !cfg.Matrix.RecaptchaEnabled { return &util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.BadJSON("Captcha registration is disabled"), + Code: http.StatusConflict, + JSON: jsonerror.Unknown("Captcha registration is disabled"), } } @@ -279,8 +279,8 @@ func validateRecaptcha( body, err := ioutil.ReadAll(resp.Body) if err != nil { return &util.JSONResponse{ - Code: http.StatusInternalServerError, - JSON: jsonerror.BadJSON("Error in contacting captcha server" + err.Error()), + Code: http.StatusGatewayTimeout, + JSON: jsonerror.Unknown("Error in contacting captcha server" + err.Error()), } } err = json.Unmarshal(body, &r) From 6106ec1399b4dd2dc90ca577cfcb1ec019f05507 Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Wed, 10 Jul 2019 00:13:43 +0800 Subject: [PATCH 11/17] Fix getAliasesForRoomID has no HTTP handler in aliasAPI (#705) This PR adds back the HTTP handler for internal API GetAliasesForRoomID in roomserver, which seemed to be missing. --- roomserver/alias/alias.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/roomserver/alias/alias.go b/roomserver/alias/alias.go index 6a34aacdd..f699e3362 100644 --- a/roomserver/alias/alias.go +++ b/roomserver/alias/alias.go @@ -277,6 +277,20 @@ func (r *RoomserverAliasAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) + servMux.Handle( + roomserverAPI.RoomserverGetAliasesForRoomIDPath, + common.MakeInternalAPI("getAliasesForRoomID", func(req *http.Request) util.JSONResponse { + var request roomserverAPI.GetAliasesForRoomIDRequest + var response roomserverAPI.GetAliasesForRoomIDResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.GetAliasesForRoomID(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) servMux.Handle( roomserverAPI.RoomserverRemoveRoomAliasPath, common.MakeInternalAPI("removeRoomAlias", func(req *http.Request) util.JSONResponse { From 86e65bb22d49ef2e6b5d5724e6e1fd0327f200c6 Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Wed, 10 Jul 2019 00:33:52 +0800 Subject: [PATCH 12/17] Add back missing returns for httputil.LogThenError calls (#730) Signed-off-by: Alex Chen --- clientapi/routing/filter.go | 2 +- clientapi/routing/login.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/clientapi/routing/filter.go b/clientapi/routing/filter.go index 109c55da1..1ed91cd2f 100644 --- a/clientapi/routing/filter.go +++ b/clientapi/routing/filter.go @@ -62,7 +62,7 @@ func GetFilter( filter := gomatrix.Filter{} err = json.Unmarshal(res, &filter) if err != nil { - httputil.LogThenError(req, err) + return httputil.LogThenError(req, err) } return util.JSONResponse{ diff --git a/clientapi/routing/login.go b/clientapi/routing/login.go index abcf7f569..2e2d409f6 100644 --- a/clientapi/routing/login.go +++ b/clientapi/routing/login.go @@ -107,7 +107,7 @@ func Login( token, err := auth.GenerateAccessToken() if err != nil { - httputil.LogThenError(req, err) + return httputil.LogThenError(req, err) } dev, err := getDevice(req.Context(), r, deviceDB, acc, localpart, token) From d4918b83c6f0fe0c6578d53a1c22d02a37c0d8b9 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Fri, 12 Jul 2019 14:23:27 +0100 Subject: [PATCH 13/17] Backup and restore go.mod & go.sum during linting (#735) Every time before sending a PR I like to run ./scripts/build-test-lint.sh to make sure the CI won't complain about anything. The problem is that this script attempts to install golangci-lint, which causes modifications to go.mod/go.sum. This PR backs up and restores those files before and after linting. Ideally instead of this hacky backing up/restoring we'd use go gets -mod=readonly option, but that still modifies go.sum. This will be fixed in go 1.13 apparently. golang/go#30667 --- scripts/find-lint.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/find-lint.sh b/scripts/find-lint.sh index 6511272b2..25b311f94 100755 --- a/scripts/find-lint.sh +++ b/scripts/find-lint.sh @@ -22,7 +22,15 @@ then args="--fast" fi echo "Installing golangci-lint..." + +# Make a backup of go.{mod,sum} first +# TODO: Once go 1.13 is out, use go get's -mod=readonly option +# https://github.com/golang/go/issues/30667 +cp go.mod go.mod.bak && cp go.sum go.sum.bak go get github.com/golangci/golangci-lint/cmd/golangci-lint echo "Looking for lint..." golangci-lint run $args + +# Restore go.{mod,sum} +mv go.mod.bak go.mod && mv go.sum.bak go.sum From 7edf197eccc89cb7368ed31df47c9d15f24d0923 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Fri, 12 Jul 2019 14:29:30 +0100 Subject: [PATCH 14/17] Fix response to /rooms/{roomId}/join v2 (#734) Continuation of #684 but merged-forward. Also did a little code cleanup and added a new, passing test to the testfile. --- clientapi/routing/membership.go | 72 ++++++++++++++++++++++++--------- testfile | 1 + 2 files changed, 53 insertions(+), 20 deletions(-) diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index b308de79a..22e66f452 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -58,27 +58,12 @@ func SendMembership( } } - inviteStored, err := threepid.CheckAndProcessInvite( - req.Context(), device, &body, cfg, queryAPI, accountDB, producer, + inviteStored, jsonErrResp := checkAndProcessThreepid( + req, device, &body, cfg, queryAPI, accountDB, producer, membership, roomID, evTime, ) - if err == threepid.ErrMissingParameter { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.BadJSON(err.Error()), - } - } else if err == threepid.ErrNotTrusted { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.NotTrusted(body.IDServer), - } - } else if err == common.ErrRoomNoExists { - return util.JSONResponse{ - Code: http.StatusNotFound, - JSON: jsonerror.NotFound(err.Error()), - } - } else if err != nil { - return httputil.LogThenError(req, err) + if jsonErrResp != nil { + return *jsonErrResp } // If an invite has been stored on an identity server, it means that a @@ -114,9 +99,18 @@ func SendMembership( return httputil.LogThenError(req, err) } + var returnData interface{} = struct{}{} + + // The join membership requires the room id to be sent in the response + if membership == "join" { + returnData = struct { + RoomID string `json:"room_id"` + }{roomID} + } + return util.JSONResponse{ Code: http.StatusOK, - JSON: struct{}{}, + JSON: returnData, } } @@ -215,3 +209,41 @@ func getMembershipStateKey( return } + +func checkAndProcessThreepid( + req *http.Request, + device *authtypes.Device, + body *threepid.MembershipRequest, + cfg config.Dendrite, + queryAPI roomserverAPI.RoomserverQueryAPI, + accountDB *accounts.Database, + producer *producers.RoomserverProducer, + membership, roomID string, + evTime time.Time, +) (inviteStored bool, errRes *util.JSONResponse) { + + inviteStored, err := threepid.CheckAndProcessInvite( + req.Context(), device, body, cfg, queryAPI, accountDB, producer, + membership, roomID, evTime, + ) + if err == threepid.ErrMissingParameter { + return inviteStored, &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.BadJSON(err.Error()), + } + } else if err == threepid.ErrNotTrusted { + return inviteStored, &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.NotTrusted(body.IDServer), + } + } else if err == common.ErrRoomNoExists { + return inviteStored, &util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound(err.Error()), + } + } else if err != nil { + er := httputil.LogThenError(req, err) + return inviteStored, &er + } + return +} diff --git a/testfile b/testfile index 362df4513..e869be4f0 100644 --- a/testfile +++ b/testfile @@ -42,6 +42,7 @@ POST /join/:room_alias can join a room POST /join/:room_id can join a room POST /join/:room_id can join a room with custom content POST /join/:room_alias can join a room with custom content +POST /rooms/:room_id/join can join a room POST /rooms/:room_id/leave can leave a room POST /rooms/:room_id/invite can send an invite POST /rooms/:room_id/ban can ban a user From f8463063ac4f45e61dfd0b27647bf00a8d05daa1 Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Fri, 12 Jul 2019 21:36:17 +0800 Subject: [PATCH 15/17] Fix #661 appservice can't set aliases in its own namespace (#731) Fixes #661. --- clientapi/routing/directory.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/clientapi/routing/directory.go b/clientapi/routing/directory.go index b23dfbfb6..ab85e86a9 100644 --- a/clientapi/routing/directory.go +++ b/clientapi/routing/directory.go @@ -117,12 +117,16 @@ func SetLocalAlias( // 1. The new method for checking for things matching an AS's namespace // 2. Using an overall Regex object for all AS's just like we did for usernames for _, appservice := range cfg.Derived.ApplicationServices { - if aliasNamespaces, ok := appservice.NamespaceMap["aliases"]; ok { - for _, namespace := range aliasNamespaces { - if namespace.Exclusive && namespace.RegexpObject.MatchString(alias) { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.ASExclusive("Alias is reserved by an application service"), + // Don't prevent AS from creating aliases in its own namespace + // Note that Dendrite uses SenderLocalpart as UserID for AS users + if device.UserID != appservice.SenderLocalpart { + if aliasNamespaces, ok := appservice.NamespaceMap["aliases"]; ok { + for _, namespace := range aliasNamespaces { + if namespace.Exclusive && namespace.RegexpObject.MatchString(alias) { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.ASExclusive("Alias is reserved by an application service"), + } } } } From 29841bed6b1a88787211368e6052a87a658c5714 Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Fri, 12 Jul 2019 22:59:53 +0800 Subject: [PATCH 16/17] Add typing notifications to /sync responses - fixes #635 (#718) This PR adds a new consumer for typing notifications in syncapi. It also brings changes to syncserver.go and some related files so EDUs can better fit in /sync responses. Fixes #635. Fixes #574. --- syncapi/consumers/clientapi.go | 9 +- syncapi/consumers/roomserver.go | 12 +- syncapi/consumers/typingserver.go | 96 ++++++ syncapi/routing/routing.go | 2 +- syncapi/routing/state.go | 4 +- syncapi/storage/account_data_table.go | 4 +- syncapi/storage/output_room_events_table.go | 11 +- syncapi/storage/syncserver.go | 342 ++++++++++++++------ syncapi/sync/notifier.go | 68 ++-- syncapi/sync/notifier_test.go | 130 +++++--- syncapi/sync/request.go | 44 ++- syncapi/sync/requestpool.go | 20 +- syncapi/sync/userstream.go | 18 +- syncapi/syncapi.go | 16 +- syncapi/types/types.go | 43 ++- testfile | 3 + typingserver/api/output.go | 7 +- typingserver/cache/cache.go | 123 +++++-- typingserver/input/input.go | 12 +- 19 files changed, 712 insertions(+), 252 deletions(-) create mode 100644 syncapi/consumers/typingserver.go diff --git a/syncapi/consumers/clientapi.go b/syncapi/consumers/clientapi.go index d05a76920..f0db56427 100644 --- a/syncapi/consumers/clientapi.go +++ b/syncapi/consumers/clientapi.go @@ -22,6 +22,7 @@ import ( "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/sync" + "github.com/matrix-org/dendrite/syncapi/types" log "github.com/sirupsen/logrus" sarama "gopkg.in/Shopify/sarama.v1" ) @@ -29,7 +30,7 @@ import ( // OutputClientDataConsumer consumes events that originated in the client API server. type OutputClientDataConsumer struct { clientAPIConsumer *common.ContinualConsumer - db *storage.SyncServerDatabase + db *storage.SyncServerDatasource notifier *sync.Notifier } @@ -38,7 +39,7 @@ func NewOutputClientDataConsumer( cfg *config.Dendrite, kafkaConsumer sarama.Consumer, n *sync.Notifier, - store *storage.SyncServerDatabase, + store *storage.SyncServerDatasource, ) *OutputClientDataConsumer { consumer := common.ContinualConsumer{ @@ -78,7 +79,7 @@ func (s *OutputClientDataConsumer) onMessage(msg *sarama.ConsumerMessage) error "room_id": output.RoomID, }).Info("received data from client API server") - syncStreamPos, err := s.db.UpsertAccountData( + pduPos, err := s.db.UpsertAccountData( context.TODO(), string(msg.Key), output.RoomID, output.Type, ) if err != nil { @@ -89,7 +90,7 @@ func (s *OutputClientDataConsumer) onMessage(msg *sarama.ConsumerMessage) error }).Panicf("could not save account data") } - s.notifier.OnNewEvent(nil, string(msg.Key), syncStreamPos) + s.notifier.OnNewEvent(nil, "", []string{string(msg.Key)}, types.SyncPosition{PDUPosition: pduPos}) return nil } diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index 1866a9667..e4f1ab460 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -33,7 +33,7 @@ import ( // OutputRoomEventConsumer consumes events that originated in the room server. type OutputRoomEventConsumer struct { roomServerConsumer *common.ContinualConsumer - db *storage.SyncServerDatabase + db *storage.SyncServerDatasource notifier *sync.Notifier query api.RoomserverQueryAPI } @@ -43,7 +43,7 @@ func NewOutputRoomEventConsumer( cfg *config.Dendrite, kafkaConsumer sarama.Consumer, n *sync.Notifier, - store *storage.SyncServerDatabase, + store *storage.SyncServerDatasource, queryAPI api.RoomserverQueryAPI, ) *OutputRoomEventConsumer { @@ -126,7 +126,7 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent( } } - syncStreamPos, err := s.db.WriteEvent( + pduPos, err := s.db.WriteEvent( ctx, &ev, addsStateEvents, @@ -144,7 +144,7 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent( }).Panicf("roomserver output log: write event failure") return nil } - s.notifier.OnNewEvent(&ev, "", types.StreamPosition(syncStreamPos)) + s.notifier.OnNewEvent(&ev, "", nil, types.SyncPosition{PDUPosition: pduPos}) return nil } @@ -152,7 +152,7 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent( func (s *OutputRoomEventConsumer) onNewInviteEvent( ctx context.Context, msg api.OutputNewInviteEvent, ) error { - syncStreamPos, err := s.db.AddInviteEvent(ctx, msg.Event) + pduPos, err := s.db.AddInviteEvent(ctx, msg.Event) if err != nil { // panic rather than continue with an inconsistent database log.WithFields(log.Fields{ @@ -161,7 +161,7 @@ func (s *OutputRoomEventConsumer) onNewInviteEvent( }).Panicf("roomserver output log: write invite failure") return nil } - s.notifier.OnNewEvent(&msg.Event, "", syncStreamPos) + s.notifier.OnNewEvent(&msg.Event, "", nil, types.SyncPosition{PDUPosition: pduPos}) return nil } diff --git a/syncapi/consumers/typingserver.go b/syncapi/consumers/typingserver.go new file mode 100644 index 000000000..5d998a18a --- /dev/null +++ b/syncapi/consumers/typingserver.go @@ -0,0 +1,96 @@ +// Copyright 2019 Alex Chen +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package consumers + +import ( + "encoding/json" + + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/dendrite/syncapi/sync" + "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/dendrite/typingserver/api" + log "github.com/sirupsen/logrus" + sarama "gopkg.in/Shopify/sarama.v1" +) + +// OutputTypingEventConsumer consumes events that originated in the typing server. +type OutputTypingEventConsumer struct { + typingConsumer *common.ContinualConsumer + db *storage.SyncServerDatasource + notifier *sync.Notifier +} + +// NewOutputTypingEventConsumer creates a new OutputTypingEventConsumer. +// Call Start() to begin consuming from the typing server. +func NewOutputTypingEventConsumer( + cfg *config.Dendrite, + kafkaConsumer sarama.Consumer, + n *sync.Notifier, + store *storage.SyncServerDatasource, +) *OutputTypingEventConsumer { + + consumer := common.ContinualConsumer{ + Topic: string(cfg.Kafka.Topics.OutputTypingEvent), + Consumer: kafkaConsumer, + PartitionStore: store, + } + + s := &OutputTypingEventConsumer{ + typingConsumer: &consumer, + db: store, + notifier: n, + } + + consumer.ProcessMessage = s.onMessage + + return s +} + +// Start consuming from typing api +func (s *OutputTypingEventConsumer) Start() error { + s.db.SetTypingTimeoutCallback(func(userID, roomID string, latestSyncPosition int64) { + s.notifier.OnNewEvent(nil, roomID, nil, types.SyncPosition{TypingPosition: latestSyncPosition}) + }) + + return s.typingConsumer.Start() +} + +func (s *OutputTypingEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { + var output api.OutputTypingEvent + if err := json.Unmarshal(msg.Value, &output); err != nil { + // If the message was invalid, log it and move on to the next message in the stream + log.WithError(err).Errorf("typing server output log: message parse failure") + return nil + } + + log.WithFields(log.Fields{ + "room_id": output.Event.RoomID, + "user_id": output.Event.UserID, + "typing": output.Event.Typing, + }).Debug("received data from typing server") + + var typingPos int64 + typingEvent := output.Event + if typingEvent.Typing { + typingPos = s.db.AddTypingUser(typingEvent.UserID, typingEvent.RoomID, output.ExpireTime) + } else { + typingPos = s.db.RemoveTypingUser(typingEvent.UserID, typingEvent.RoomID) + } + + s.notifier.OnNewEvent(nil, output.Event.RoomID, nil, types.SyncPosition{TypingPosition: typingPos}) + return nil +} diff --git a/syncapi/routing/routing.go b/syncapi/routing/routing.go index cbdcfb6bb..0f5019fc3 100644 --- a/syncapi/routing/routing.go +++ b/syncapi/routing/routing.go @@ -34,7 +34,7 @@ const pathPrefixR0 = "/_matrix/client/r0" // Due to Setup being used to call many other functions, a gocyclo nolint is // applied: // nolint: gocyclo -func Setup(apiMux *mux.Router, srp *sync.RequestPool, syncDB *storage.SyncServerDatabase, deviceDB *devices.Database) { +func Setup(apiMux *mux.Router, srp *sync.RequestPool, syncDB *storage.SyncServerDatasource, deviceDB *devices.Database) { r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter() authData := auth.Data{ diff --git a/syncapi/routing/state.go b/syncapi/routing/state.go index 6b98a0b7b..5571a0525 100644 --- a/syncapi/routing/state.go +++ b/syncapi/routing/state.go @@ -40,7 +40,7 @@ type stateEventInStateResp struct { // TODO: Check if the user is in the room. If not, check if the room's history // is publicly visible. Current behaviour is returning an empty array if the // user cannot see the room's history. -func OnIncomingStateRequest(req *http.Request, db *storage.SyncServerDatabase, roomID string) util.JSONResponse { +func OnIncomingStateRequest(req *http.Request, db *storage.SyncServerDatasource, roomID string) util.JSONResponse { // TODO(#287): Auth request and handle the case where the user has left (where // we should return the state at the poin they left) @@ -84,7 +84,7 @@ func OnIncomingStateRequest(req *http.Request, db *storage.SyncServerDatabase, r // /rooms/{roomID}/state/{type}/{statekey} request. It will look in current // state to see if there is an event with that type and state key, if there // is then (by default) we return the content, otherwise a 404. -func OnIncomingStateTypeRequest(req *http.Request, db *storage.SyncServerDatabase, roomID string, evType, stateKey string) util.JSONResponse { +func OnIncomingStateTypeRequest(req *http.Request, db *storage.SyncServerDatasource, roomID string, evType, stateKey string) util.JSONResponse { // TODO(#287): Auth request and handle the case where the user has left (where // we should return the state at the poin they left) diff --git a/syncapi/storage/account_data_table.go b/syncapi/storage/account_data_table.go index d4d74d158..9b73ce7d6 100644 --- a/syncapi/storage/account_data_table.go +++ b/syncapi/storage/account_data_table.go @@ -19,8 +19,6 @@ import ( "database/sql" "github.com/matrix-org/dendrite/common" - - "github.com/matrix-org/dendrite/syncapi/types" ) const accountDataSchema = ` @@ -94,7 +92,7 @@ func (s *accountDataStatements) insertAccountData( func (s *accountDataStatements) selectAccountDataInRange( ctx context.Context, userID string, - oldPos, newPos types.StreamPosition, + oldPos, newPos int64, ) (data map[string][]string, err error) { data = make(map[string][]string) diff --git a/syncapi/storage/output_room_events_table.go b/syncapi/storage/output_room_events_table.go index 035db9882..06df017cb 100644 --- a/syncapi/storage/output_room_events_table.go +++ b/syncapi/storage/output_room_events_table.go @@ -23,7 +23,6 @@ import ( "github.com/lib/pq" "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" log "github.com/sirupsen/logrus" ) @@ -109,11 +108,11 @@ func (s *outputRoomEventsStatements) prepare(db *sql.DB) (err error) { return } -// selectStateInRange returns the state events between the two given stream positions, exclusive of oldPos, inclusive of newPos. +// selectStateInRange returns the state events between the two given PDU stream positions, exclusive of oldPos, inclusive of newPos. // Results are bucketed based on the room ID. If the same state is overwritten multiple times between the // two positions, only the most recent state is returned. func (s *outputRoomEventsStatements) selectStateInRange( - ctx context.Context, txn *sql.Tx, oldPos, newPos types.StreamPosition, + ctx context.Context, txn *sql.Tx, oldPos, newPos int64, ) (map[string]map[string]bool, map[string]streamEvent, error) { stmt := common.TxStmt(txn, s.selectStateInRangeStmt) @@ -171,7 +170,7 @@ func (s *outputRoomEventsStatements) selectStateInRange( eventIDToEvent[ev.EventID()] = streamEvent{ Event: ev, - streamPosition: types.StreamPosition(streamPos), + streamPosition: streamPos, } } @@ -223,7 +222,7 @@ func (s *outputRoomEventsStatements) insertEvent( // RecentEventsInRoom returns the most recent events in the given room, up to a maximum of 'limit'. func (s *outputRoomEventsStatements) selectRecentEvents( ctx context.Context, txn *sql.Tx, - roomID string, fromPos, toPos types.StreamPosition, limit int, + roomID string, fromPos, toPos int64, limit int, ) ([]streamEvent, error) { stmt := common.TxStmt(txn, s.selectRecentEventsStmt) rows, err := stmt.QueryContext(ctx, roomID, fromPos, toPos, limit) @@ -286,7 +285,7 @@ func rowsToStreamEvents(rows *sql.Rows) ([]streamEvent, error) { result = append(result, streamEvent{ Event: ev, - streamPosition: types.StreamPosition(streamPos), + streamPosition: streamPos, transactionID: transactionID, }) } diff --git a/syncapi/storage/syncserver.go b/syncapi/storage/syncserver.go index b0655a0a8..b4d7ccbd2 100644 --- a/syncapi/storage/syncserver.go +++ b/syncapi/storage/syncserver.go @@ -17,7 +17,10 @@ package storage import ( "context" "database/sql" + "encoding/json" "fmt" + "strconv" + "time" "github.com/sirupsen/logrus" @@ -28,6 +31,7 @@ import ( _ "github.com/lib/pq" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/dendrite/typingserver/cache" "github.com/matrix-org/gomatrixserverlib" ) @@ -35,33 +39,35 @@ type stateDelta struct { roomID string stateEvents []gomatrixserverlib.Event membership string - // The stream position of the latest membership event for this user, if applicable. + // The PDU stream position of the latest membership event for this user, if applicable. // Can be 0 if there is no membership event in this delta. - membershipPos types.StreamPosition + membershipPos int64 } -// Same as gomatrixserverlib.Event but also has the stream position for this event. +// Same as gomatrixserverlib.Event but also has the PDU stream position for this event. type streamEvent struct { gomatrixserverlib.Event - streamPosition types.StreamPosition + streamPosition int64 transactionID *api.TransactionID } -// SyncServerDatabase represents a sync server database -type SyncServerDatabase struct { +// SyncServerDatabase represents a sync server datasource which manages +// both the database for PDUs and caches for EDUs. +type SyncServerDatasource struct { db *sql.DB common.PartitionOffsetStatements accountData accountDataStatements events outputRoomEventsStatements roomstate currentRoomStateStatements invites inviteEventsStatements + typingCache *cache.TypingCache } // NewSyncServerDatabase creates a new sync server database -func NewSyncServerDatabase(dataSourceName string) (*SyncServerDatabase, error) { - var d SyncServerDatabase +func NewSyncServerDatasource(dbDataSourceName string) (*SyncServerDatasource, error) { + var d SyncServerDatasource var err error - if d.db, err = sql.Open("postgres", dataSourceName); err != nil { + if d.db, err = sql.Open("postgres", dbDataSourceName); err != nil { return nil, err } if err = d.PartitionOffsetStatements.Prepare(d.db, "syncapi"); err != nil { @@ -79,11 +85,12 @@ func NewSyncServerDatabase(dataSourceName string) (*SyncServerDatabase, error) { if err := d.invites.prepare(d.db); err != nil { return nil, err } + d.typingCache = cache.NewTypingCache() return &d, nil } // AllJoinedUsersInRooms returns a map of room ID to a list of all joined user IDs. -func (d *SyncServerDatabase) AllJoinedUsersInRooms(ctx context.Context) (map[string][]string, error) { +func (d *SyncServerDatasource) AllJoinedUsersInRooms(ctx context.Context) (map[string][]string, error) { return d.roomstate.selectJoinedUsers(ctx) } @@ -92,7 +99,7 @@ func (d *SyncServerDatabase) AllJoinedUsersInRooms(ctx context.Context) (map[str // If an event is not found in the database then it will be omitted from the list. // Returns an error if there was a problem talking with the database. // Does not include any transaction IDs in the returned events. -func (d *SyncServerDatabase) Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.Event, error) { +func (d *SyncServerDatasource) Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.Event, error) { streamEvents, err := d.events.selectEvents(ctx, nil, eventIDs) if err != nil { return nil, err @@ -104,38 +111,38 @@ func (d *SyncServerDatabase) Events(ctx context.Context, eventIDs []string) ([]g } // WriteEvent into the database. It is not safe to call this function from multiple goroutines, as it would create races -// when generating the stream position for this event. Returns the sync stream position for the inserted event. +// when generating the sync stream position for this event. Returns the sync stream position for the inserted event. // Returns an error if there was a problem inserting this event. -func (d *SyncServerDatabase) WriteEvent( +func (d *SyncServerDatasource) WriteEvent( ctx context.Context, ev *gomatrixserverlib.Event, addStateEvents []gomatrixserverlib.Event, addStateEventIDs, removeStateEventIDs []string, transactionID *api.TransactionID, -) (streamPos types.StreamPosition, returnErr error) { +) (pduPosition int64, returnErr error) { returnErr = common.WithTransaction(d.db, func(txn *sql.Tx) error { var err error pos, err := d.events.insertEvent(ctx, txn, ev, addStateEventIDs, removeStateEventIDs, transactionID) if err != nil { return err } - streamPos = types.StreamPosition(pos) + pduPosition = pos if len(addStateEvents) == 0 && len(removeStateEventIDs) == 0 { // Nothing to do, the event may have just been a message event. return nil } - return d.updateRoomState(ctx, txn, removeStateEventIDs, addStateEvents, streamPos) + return d.updateRoomState(ctx, txn, removeStateEventIDs, addStateEvents, pduPosition) }) return } -func (d *SyncServerDatabase) updateRoomState( +func (d *SyncServerDatasource) updateRoomState( ctx context.Context, txn *sql.Tx, removedEventIDs []string, addedEvents []gomatrixserverlib.Event, - streamPos types.StreamPosition, + pduPosition int64, ) error { // remove first, then add, as we do not ever delete state, but do replace state which is a remove followed by an add. for _, eventID := range removedEventIDs { @@ -157,7 +164,7 @@ func (d *SyncServerDatabase) updateRoomState( } membership = &value } - if err := d.roomstate.upsertRoomState(ctx, txn, event, membership, int64(streamPos)); err != nil { + if err := d.roomstate.upsertRoomState(ctx, txn, event, membership, pduPosition); err != nil { return err } } @@ -168,7 +175,7 @@ func (d *SyncServerDatabase) updateRoomState( // GetStateEvent returns the Matrix state event of a given type for a given room with a given state key // If no event could be found, returns nil // If there was an issue during the retrieval, returns an error -func (d *SyncServerDatabase) GetStateEvent( +func (d *SyncServerDatasource) GetStateEvent( ctx context.Context, roomID, evType, stateKey string, ) (*gomatrixserverlib.Event, error) { return d.roomstate.selectStateEvent(ctx, roomID, evType, stateKey) @@ -177,7 +184,7 @@ func (d *SyncServerDatabase) GetStateEvent( // GetStateEventsForRoom fetches the state events for a given room. // Returns an empty slice if no state events could be found for this room. // Returns an error if there was an issue with the retrieval. -func (d *SyncServerDatabase) GetStateEventsForRoom( +func (d *SyncServerDatasource) GetStateEventsForRoom( ctx context.Context, roomID string, ) (stateEvents []gomatrixserverlib.Event, err error) { err = common.WithTransaction(d.db, func(txn *sql.Tx) error { @@ -187,46 +194,49 @@ func (d *SyncServerDatabase) GetStateEventsForRoom( return } -// SyncStreamPosition returns the latest position in the sync stream. Returns 0 if there are no events yet. -func (d *SyncServerDatabase) SyncStreamPosition(ctx context.Context) (types.StreamPosition, error) { - return d.syncStreamPositionTx(ctx, nil) +// SyncPosition returns the latest positions for syncing. +func (d *SyncServerDatasource) SyncPosition(ctx context.Context) (types.SyncPosition, error) { + return d.syncPositionTx(ctx, nil) } -func (d *SyncServerDatabase) syncStreamPositionTx( +func (d *SyncServerDatasource) syncPositionTx( ctx context.Context, txn *sql.Tx, -) (types.StreamPosition, error) { - maxID, err := d.events.selectMaxEventID(ctx, txn) +) (sp types.SyncPosition, err error) { + + maxEventID, err := d.events.selectMaxEventID(ctx, txn) if err != nil { - return 0, err + return sp, err } maxAccountDataID, err := d.accountData.selectMaxAccountDataID(ctx, txn) if err != nil { - return 0, err + return sp, err } - if maxAccountDataID > maxID { - maxID = maxAccountDataID + if maxAccountDataID > maxEventID { + maxEventID = maxAccountDataID } maxInviteID, err := d.invites.selectMaxInviteID(ctx, txn) if err != nil { - return 0, err + return sp, err } - if maxInviteID > maxID { - maxID = maxInviteID + if maxInviteID > maxEventID { + maxEventID = maxInviteID } - return types.StreamPosition(maxID), nil + sp.PDUPosition = maxEventID + + sp.TypingPosition = d.typingCache.GetLatestSyncPosition() + + return } -// IncrementalSync returns all the data needed in order to create an incremental -// sync response for the given user. Events returned will include any client -// transaction IDs associated with the given device. These transaction IDs come -// from when the device sent the event via an API that included a transaction -// ID. -func (d *SyncServerDatabase) IncrementalSync( +// addPDUDeltaToResponse adds all PDU deltas to a sync response. +// IDs of all rooms the user joined are returned so EDU deltas can be added for them. +func (d *SyncServerDatasource) addPDUDeltaToResponse( ctx context.Context, device authtypes.Device, - fromPos, toPos types.StreamPosition, + fromPos, toPos int64, numRecentEventsPerRoom int, -) (*types.Response, error) { + res *types.Response, +) ([]string, error) { txn, err := d.db.BeginTx(ctx, &txReadOnlySnapshot) if err != nil { return nil, err @@ -235,7 +245,7 @@ func (d *SyncServerDatabase) IncrementalSync( defer common.EndTransaction(txn, &succeeded) // Work out which rooms to return in the response. This is done by getting not only the currently - // joined rooms, but also which rooms have membership transitions for this user between the 2 stream positions. + // joined rooms, but also which rooms have membership transitions for this user between the 2 PDU stream positions. // This works out what the 'state' key should be for each room as well as which membership block // to put the room into. deltas, err := d.getStateDeltas(ctx, &device, txn, fromPos, toPos, device.UserID) @@ -243,8 +253,9 @@ func (d *SyncServerDatabase) IncrementalSync( return nil, err } - res := types.NewResponse(toPos) + joinedRoomIDs := make([]string, 0, len(deltas)) for _, delta := range deltas { + joinedRoomIDs = append(joinedRoomIDs, delta.roomID) err = d.addRoomDeltaToResponse(ctx, &device, txn, fromPos, toPos, delta, numRecentEventsPerRoom, res) if err != nil { return nil, err @@ -257,52 +268,151 @@ func (d *SyncServerDatabase) IncrementalSync( } succeeded = true + return joinedRoomIDs, nil +} + +// addTypingDeltaToResponse adds all typing notifications to a sync response +// since the specified position. +func (d *SyncServerDatasource) addTypingDeltaToResponse( + since int64, + joinedRoomIDs []string, + res *types.Response, +) error { + var jr types.JoinResponse + var ok bool + var err error + for _, roomID := range joinedRoomIDs { + if typingUsers, updated := d.typingCache.GetTypingUsersIfUpdatedAfter( + roomID, since, + ); updated { + ev := gomatrixserverlib.ClientEvent{ + Type: gomatrixserverlib.MTyping, + } + ev.Content, err = json.Marshal(map[string]interface{}{ + "user_ids": typingUsers, + }) + if err != nil { + return err + } + + if jr, ok = res.Rooms.Join[roomID]; !ok { + jr = *types.NewJoinResponse() + } + jr.Ephemeral.Events = append(jr.Ephemeral.Events, ev) + res.Rooms.Join[roomID] = jr + } + } + return nil +} + +// addEDUDeltaToResponse adds updates for EDUs of each type since fromPos if +// the positions of that type are not equal in fromPos and toPos. +func (d *SyncServerDatasource) addEDUDeltaToResponse( + fromPos, toPos types.SyncPosition, + joinedRoomIDs []string, + res *types.Response, +) (err error) { + + if fromPos.TypingPosition != toPos.TypingPosition { + err = d.addTypingDeltaToResponse( + fromPos.TypingPosition, joinedRoomIDs, res, + ) + } + + return +} + +// IncrementalSync returns all the data needed in order to create an incremental +// sync response for the given user. Events returned will include any client +// transaction IDs associated with the given device. These transaction IDs come +// from when the device sent the event via an API that included a transaction +// ID. +func (d *SyncServerDatasource) IncrementalSync( + ctx context.Context, + device authtypes.Device, + fromPos, toPos types.SyncPosition, + numRecentEventsPerRoom int, +) (*types.Response, error) { + nextBatchPos := fromPos.WithUpdates(toPos) + res := types.NewResponse(nextBatchPos) + + var joinedRoomIDs []string + var err error + if fromPos.PDUPosition != toPos.PDUPosition { + joinedRoomIDs, err = d.addPDUDeltaToResponse( + ctx, device, fromPos.PDUPosition, toPos.PDUPosition, numRecentEventsPerRoom, res, + ) + } else { + joinedRoomIDs, err = d.roomstate.selectRoomIDsWithMembership( + ctx, nil, device.UserID, "join", + ) + } + if err != nil { + return nil, err + } + + err = d.addEDUDeltaToResponse( + fromPos, toPos, joinedRoomIDs, res, + ) + if err != nil { + return nil, err + } + return res, nil } -// CompleteSync a complete /sync API response for the given user. -func (d *SyncServerDatabase) CompleteSync( - ctx context.Context, userID string, numRecentEventsPerRoom int, -) (*types.Response, error) { +// getResponseWithPDUsForCompleteSync creates a response and adds all PDUs needed +// to it. It returns toPos and joinedRoomIDs for use of adding EDUs. +func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( + ctx context.Context, + userID string, + numRecentEventsPerRoom int, +) ( + res *types.Response, + toPos types.SyncPosition, + joinedRoomIDs []string, + err error, +) { // This needs to be all done in a transaction as we need to do multiple SELECTs, and we need to have - // a consistent view of the database throughout. This includes extracting the sync stream position. + // a consistent view of the database throughout. This includes extracting the sync position. // This does have the unfortunate side-effect that all the matrixy logic resides in this function, // but it's better to not hide the fact that this is being done in a transaction. txn, err := d.db.BeginTx(ctx, &txReadOnlySnapshot) if err != nil { - return nil, err + return } var succeeded bool defer common.EndTransaction(txn, &succeeded) - // Get the current stream position which we will base the sync response on. - pos, err := d.syncStreamPositionTx(ctx, txn) + // Get the current sync position which we will base the sync response on. + toPos, err = d.syncPositionTx(ctx, txn) if err != nil { - return nil, err + return } + res = types.NewResponse(toPos) + // Extract room state and recent events for all rooms the user is joined to. - roomIDs, err := d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, "join") + joinedRoomIDs, err = d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, "join") if err != nil { - return nil, err + return } // Build up a /sync response. Add joined rooms. - res := types.NewResponse(pos) - for _, roomID := range roomIDs { + for _, roomID := range joinedRoomIDs { var stateEvents []gomatrixserverlib.Event stateEvents, err = d.roomstate.selectCurrentState(ctx, txn, roomID) if err != nil { - return nil, err + return } // TODO: When filters are added, we may need to call this multiple times to get enough events. // See: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L316 var recentStreamEvents []streamEvent recentStreamEvents, err = d.events.selectRecentEvents( - ctx, txn, roomID, types.StreamPosition(0), pos, numRecentEventsPerRoom, + ctx, txn, roomID, 0, toPos.PDUPosition, numRecentEventsPerRoom, ) if err != nil { - return nil, err + return } // We don't include a device here as we don't need to send down @@ -311,10 +421,12 @@ func (d *SyncServerDatabase) CompleteSync( stateEvents = removeDuplicates(stateEvents, recentEvents) jr := types.NewJoinResponse() - if prevBatch := recentStreamEvents[0].streamPosition - 1; prevBatch > 0 { - jr.Timeline.PrevBatch = types.StreamPosition(prevBatch).String() + if prevPDUPos := recentStreamEvents[0].streamPosition - 1; prevPDUPos > 0 { + // Use the short form of batch token for prev_batch + jr.Timeline.PrevBatch = strconv.FormatInt(prevPDUPos, 10) } else { - jr.Timeline.PrevBatch = types.StreamPosition(1).String() + // Use the short form of batch token for prev_batch + jr.Timeline.PrevBatch = "1" } jr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = true @@ -322,12 +434,34 @@ func (d *SyncServerDatabase) CompleteSync( res.Rooms.Join[roomID] = *jr } - if err = d.addInvitesToResponse(ctx, txn, userID, 0, pos, res); err != nil { - return nil, err + if err = d.addInvitesToResponse(ctx, txn, userID, 0, toPos.PDUPosition, res); err != nil { + return } succeeded = true - return res, err + return res, toPos, joinedRoomIDs, err +} + +// CompleteSync returns a complete /sync API response for the given user. +func (d *SyncServerDatasource) CompleteSync( + ctx context.Context, userID string, numRecentEventsPerRoom int, +) (*types.Response, error) { + res, toPos, joinedRoomIDs, err := d.getResponseWithPDUsForCompleteSync( + ctx, userID, numRecentEventsPerRoom, + ) + if err != nil { + return nil, err + } + + // Use a zero value SyncPosition for fromPos so all EDU states are added. + err = d.addEDUDeltaToResponse( + types.SyncPosition{}, toPos, joinedRoomIDs, res, + ) + if err != nil { + return nil, err + } + + return res, nil } var txReadOnlySnapshot = sql.TxOptions{ @@ -345,8 +479,8 @@ var txReadOnlySnapshot = sql.TxOptions{ // Returns a map following the format data[roomID] = []dataTypes // If no data is retrieved, returns an empty map // If there was an issue with the retrieval, returns an error -func (d *SyncServerDatabase) GetAccountDataInRange( - ctx context.Context, userID string, oldPos, newPos types.StreamPosition, +func (d *SyncServerDatasource) GetAccountDataInRange( + ctx context.Context, userID string, oldPos, newPos int64, ) (map[string][]string, error) { return d.accountData.selectAccountDataInRange(ctx, userID, oldPos, newPos) } @@ -357,26 +491,24 @@ func (d *SyncServerDatabase) GetAccountDataInRange( // If no data with the given type, user ID and room ID exists in the database, // creates a new row, else update the existing one // Returns an error if there was an issue with the upsert -func (d *SyncServerDatabase) UpsertAccountData( +func (d *SyncServerDatasource) UpsertAccountData( ctx context.Context, userID, roomID, dataType string, -) (types.StreamPosition, error) { - pos, err := d.accountData.insertAccountData(ctx, userID, roomID, dataType) - return types.StreamPosition(pos), err +) (int64, error) { + return d.accountData.insertAccountData(ctx, userID, roomID, dataType) } // AddInviteEvent stores a new invite event for a user. // If the invite was successfully stored this returns the stream ID it was stored at. // Returns an error if there was a problem communicating with the database. -func (d *SyncServerDatabase) AddInviteEvent( +func (d *SyncServerDatasource) AddInviteEvent( ctx context.Context, inviteEvent gomatrixserverlib.Event, -) (types.StreamPosition, error) { - pos, err := d.invites.insertInviteEvent(ctx, inviteEvent) - return types.StreamPosition(pos), err +) (int64, error) { + return d.invites.insertInviteEvent(ctx, inviteEvent) } // RetireInviteEvent removes an old invite event from the database. // Returns an error if there was a problem communicating with the database. -func (d *SyncServerDatabase) RetireInviteEvent( +func (d *SyncServerDatasource) RetireInviteEvent( ctx context.Context, inviteEventID string, ) error { // TODO: Record that invite has been retired in a stream so that we can @@ -385,10 +517,30 @@ func (d *SyncServerDatabase) RetireInviteEvent( return err } -func (d *SyncServerDatabase) addInvitesToResponse( +func (d *SyncServerDatasource) SetTypingTimeoutCallback(fn cache.TimeoutCallbackFn) { + d.typingCache.SetTimeoutCallback(fn) +} + +// AddTypingUser adds a typing user to the typing cache. +// Returns the newly calculated sync position for typing notifications. +func (d *SyncServerDatasource) AddTypingUser( + userID, roomID string, expireTime *time.Time, +) int64 { + return d.typingCache.AddTypingUser(userID, roomID, expireTime) +} + +// RemoveTypingUser removes a typing user from the typing cache. +// Returns the newly calculated sync position for typing notifications. +func (d *SyncServerDatasource) RemoveTypingUser( + userID, roomID string, +) int64 { + return d.typingCache.RemoveUser(userID, roomID) +} + +func (d *SyncServerDatasource) addInvitesToResponse( ctx context.Context, txn *sql.Tx, userID string, - fromPos, toPos types.StreamPosition, + fromPos, toPos int64, res *types.Response, ) error { invites, err := d.invites.selectInviteEventsInRange( @@ -409,11 +561,11 @@ func (d *SyncServerDatabase) addInvitesToResponse( } // addRoomDeltaToResponse adds a room state delta to a sync response -func (d *SyncServerDatabase) addRoomDeltaToResponse( +func (d *SyncServerDatasource) addRoomDeltaToResponse( ctx context.Context, device *authtypes.Device, txn *sql.Tx, - fromPos, toPos types.StreamPosition, + fromPos, toPos int64, delta stateDelta, numRecentEventsPerRoom int, res *types.Response, @@ -445,10 +597,12 @@ func (d *SyncServerDatabase) addRoomDeltaToResponse( switch delta.membership { case "join": jr := types.NewJoinResponse() - if prevBatch := recentStreamEvents[0].streamPosition - 1; prevBatch > 0 { - jr.Timeline.PrevBatch = types.StreamPosition(prevBatch).String() + if prevPDUPos := recentStreamEvents[0].streamPosition - 1; prevPDUPos > 0 { + // Use the short form of batch token for prev_batch + jr.Timeline.PrevBatch = strconv.FormatInt(prevPDUPos, 10) } else { - jr.Timeline.PrevBatch = types.StreamPosition(1).String() + // Use the short form of batch token for prev_batch + jr.Timeline.PrevBatch = "1" } jr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true @@ -460,10 +614,12 @@ func (d *SyncServerDatabase) addRoomDeltaToResponse( // TODO: recentEvents may contain events that this user is not allowed to see because they are // no longer in the room. lr := types.NewLeaveResponse() - if prevBatch := recentStreamEvents[0].streamPosition - 1; prevBatch > 0 { - lr.Timeline.PrevBatch = types.StreamPosition(prevBatch).String() + if prevPDUPos := recentStreamEvents[0].streamPosition - 1; prevPDUPos > 0 { + // Use the short form of batch token for prev_batch + lr.Timeline.PrevBatch = strconv.FormatInt(prevPDUPos, 10) } else { - lr.Timeline.PrevBatch = types.StreamPosition(1).String() + // Use the short form of batch token for prev_batch + lr.Timeline.PrevBatch = "1" } lr.Timeline.Events = gomatrixserverlib.ToClientEvents(recentEvents, gomatrixserverlib.FormatSync) lr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true @@ -476,7 +632,7 @@ func (d *SyncServerDatabase) addRoomDeltaToResponse( // fetchStateEvents converts the set of event IDs into a set of events. It will fetch any which are missing from the database. // Returns a map of room ID to list of events. -func (d *SyncServerDatabase) fetchStateEvents( +func (d *SyncServerDatasource) fetchStateEvents( ctx context.Context, txn *sql.Tx, roomIDToEventIDSet map[string]map[string]bool, eventIDToEvent map[string]streamEvent, @@ -521,7 +677,7 @@ func (d *SyncServerDatabase) fetchStateEvents( return stateBetween, nil } -func (d *SyncServerDatabase) fetchMissingStateEvents( +func (d *SyncServerDatasource) fetchMissingStateEvents( ctx context.Context, txn *sql.Tx, eventIDs []string, ) ([]streamEvent, error) { // Fetch from the events table first so we pick up the stream ID for the @@ -560,9 +716,9 @@ func (d *SyncServerDatabase) fetchMissingStateEvents( return events, nil } -func (d *SyncServerDatabase) getStateDeltas( +func (d *SyncServerDatasource) getStateDeltas( ctx context.Context, device *authtypes.Device, txn *sql.Tx, - fromPos, toPos types.StreamPosition, userID string, + fromPos, toPos int64, userID string, ) ([]stateDelta, error) { // Implement membership change algorithm: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L821 // - Get membership list changes for this user in this sync response @@ -601,7 +757,7 @@ func (d *SyncServerDatabase) getStateDeltas( } s := make([]streamEvent, len(allState)) for i := 0; i < len(s); i++ { - s[i] = streamEvent{Event: allState[i], streamPosition: types.StreamPosition(0)} + s[i] = streamEvent{Event: allState[i], streamPosition: 0} } state[roomID] = s continue // we'll add this room in when we do joined rooms diff --git a/syncapi/sync/notifier.go b/syncapi/sync/notifier.go index 5ed701d8e..30ac3a2e5 100644 --- a/syncapi/sync/notifier.go +++ b/syncapi/sync/notifier.go @@ -26,7 +26,7 @@ import ( ) // Notifier will wake up sleeping requests when there is some new data. -// It does not tell requests what that data is, only the stream position which +// It does not tell requests what that data is, only the sync position which // they can use to get at it. This is done to prevent races whereby we tell the caller // the event, but the token has already advanced by the time they fetch it, resulting // in missed events. @@ -35,18 +35,18 @@ type Notifier struct { roomIDToJoinedUsers map[string]userIDSet // Protects currPos and userStreams. streamLock *sync.Mutex - // The latest sync stream position - currPos types.StreamPosition + // The latest sync position + currPos types.SyncPosition // A map of user_id => UserStream which can be used to wake a given user's /sync request. userStreams map[string]*UserStream // The last time we cleaned out stale entries from the userStreams map lastCleanUpTime time.Time } -// NewNotifier creates a new notifier set to the given stream position. +// NewNotifier creates a new notifier set to the given sync position. // In order for this to be of any use, the Notifier needs to be told all rooms and // the joined users within each of them by calling Notifier.Load(*storage.SyncServerDatabase). -func NewNotifier(pos types.StreamPosition) *Notifier { +func NewNotifier(pos types.SyncPosition) *Notifier { return &Notifier{ currPos: pos, roomIDToJoinedUsers: make(map[string]userIDSet), @@ -58,20 +58,30 @@ func NewNotifier(pos types.StreamPosition) *Notifier { // OnNewEvent is called when a new event is received from the room server. Must only be // called from a single goroutine, to avoid races between updates which could set the -// current position in the stream incorrectly. -// Can be called either with a *gomatrixserverlib.Event, or with an user ID -func (n *Notifier) OnNewEvent(ev *gomatrixserverlib.Event, userID string, pos types.StreamPosition) { +// current sync position incorrectly. +// Chooses which user sync streams to update by a provided *gomatrixserverlib.Event +// (based on the users in the event's room), +// a roomID directly, or a list of user IDs, prioritised by parameter ordering. +// posUpdate contains the latest position(s) for one or more types of events. +// If a position in posUpdate is 0, it means no updates are available of that type. +// Typically a consumer supplies a posUpdate with the latest sync position for the +// event type it handles, leaving other fields as 0. +func (n *Notifier) OnNewEvent( + ev *gomatrixserverlib.Event, roomID string, userIDs []string, + posUpdate types.SyncPosition, +) { // update the current position then notify relevant /sync streams. // This needs to be done PRIOR to waking up users as they will read this value. n.streamLock.Lock() defer n.streamLock.Unlock() - n.currPos = pos + latestPos := n.currPos.WithUpdates(posUpdate) + n.currPos = latestPos n.removeEmptyUserStreams() if ev != nil { // Map this event's room_id to a list of joined users, and wake them up. - userIDs := n.joinedUsers(ev.RoomID()) + usersToNotify := n.joinedUsers(ev.RoomID()) // If this is an invite, also add in the invitee to this list. if ev.Type() == "m.room.member" && ev.StateKey() != nil { targetUserID := *ev.StateKey() @@ -84,11 +94,11 @@ func (n *Notifier) OnNewEvent(ev *gomatrixserverlib.Event, userID string, pos ty // Keep the joined user map up-to-date switch membership { case "invite": - userIDs = append(userIDs, targetUserID) + usersToNotify = append(usersToNotify, targetUserID) case "join": // Manually append the new user's ID so they get notified // along all members in the room - userIDs = append(userIDs, targetUserID) + usersToNotify = append(usersToNotify, targetUserID) n.addJoinedUser(ev.RoomID(), targetUserID) case "leave": fallthrough @@ -98,11 +108,15 @@ func (n *Notifier) OnNewEvent(ev *gomatrixserverlib.Event, userID string, pos ty } } - for _, toNotifyUserID := range userIDs { - n.wakeupUser(toNotifyUserID, pos) - } - } else if len(userID) > 0 { - n.wakeupUser(userID, pos) + n.wakeupUsers(usersToNotify, latestPos) + } else if roomID != "" { + n.wakeupUsers(n.joinedUsers(roomID), latestPos) + } else if len(userIDs) > 0 { + n.wakeupUsers(userIDs, latestPos) + } else { + log.WithFields(log.Fields{ + "posUpdate": posUpdate.String, + }).Warn("Notifier.OnNewEvent called but caller supplied no user to wake up") } } @@ -127,7 +141,7 @@ func (n *Notifier) GetListener(req syncRequest) UserStreamListener { } // Load the membership states required to notify users correctly. -func (n *Notifier) Load(ctx context.Context, db *storage.SyncServerDatabase) error { +func (n *Notifier) Load(ctx context.Context, db *storage.SyncServerDatasource) error { roomToUsers, err := db.AllJoinedUsersInRooms(ctx) if err != nil { return err @@ -136,8 +150,11 @@ func (n *Notifier) Load(ctx context.Context, db *storage.SyncServerDatabase) err return nil } -// CurrentPosition returns the current stream position -func (n *Notifier) CurrentPosition() types.StreamPosition { +// CurrentPosition returns the current sync position +func (n *Notifier) CurrentPosition() types.SyncPosition { + n.streamLock.Lock() + defer n.streamLock.Unlock() + return n.currPos } @@ -156,12 +173,13 @@ func (n *Notifier) setUsersJoinedToRooms(roomIDToUserIDs map[string][]string) { } } -func (n *Notifier) wakeupUser(userID string, newPos types.StreamPosition) { - stream := n.fetchUserStream(userID, false) - if stream == nil { - return +func (n *Notifier) wakeupUsers(userIDs []string, newPos types.SyncPosition) { + for _, userID := range userIDs { + stream := n.fetchUserStream(userID, false) + if stream != nil { + stream.Broadcast(newPos) // wake up all goroutines Wait()ing on this stream + } } - stream.Broadcast(newPos) // wakeup all goroutines Wait()ing on this stream } // fetchUserStream retrieves a stream unique to the given user. If makeIfNotExists is true, diff --git a/syncapi/sync/notifier_test.go b/syncapi/sync/notifier_test.go index 4fa543936..904315e9f 100644 --- a/syncapi/sync/notifier_test.go +++ b/syncapi/sync/notifier_test.go @@ -32,19 +32,40 @@ var ( randomMessageEvent gomatrixserverlib.Event aliceInviteBobEvent gomatrixserverlib.Event bobLeaveEvent gomatrixserverlib.Event + syncPositionVeryOld types.SyncPosition + syncPositionBefore types.SyncPosition + syncPositionAfter types.SyncPosition + syncPositionNewEDU types.SyncPosition + syncPositionAfter2 types.SyncPosition ) var ( - streamPositionVeryOld = types.StreamPosition(5) - streamPositionBefore = types.StreamPosition(11) - streamPositionAfter = types.StreamPosition(12) - streamPositionAfter2 = types.StreamPosition(13) - roomID = "!test:localhost" - alice = "@alice:localhost" - bob = "@bob:localhost" + roomID = "!test:localhost" + alice = "@alice:localhost" + bob = "@bob:localhost" ) func init() { + baseSyncPos := types.SyncPosition{ + PDUPosition: 0, + TypingPosition: 0, + } + + syncPositionVeryOld = baseSyncPos + syncPositionVeryOld.PDUPosition = 5 + + syncPositionBefore = baseSyncPos + syncPositionBefore.PDUPosition = 11 + + syncPositionAfter = baseSyncPos + syncPositionAfter.PDUPosition = 12 + + syncPositionNewEDU = syncPositionAfter + syncPositionNewEDU.TypingPosition = 1 + + syncPositionAfter2 = baseSyncPos + syncPositionAfter2.PDUPosition = 13 + var err error randomMessageEvent, err = gomatrixserverlib.NewEventFromTrustedJSON([]byte(`{ "type": "m.room.message", @@ -92,19 +113,19 @@ func init() { // Test that the current position is returned if a request is already behind. func TestImmediateNotification(t *testing.T) { - n := NewNotifier(streamPositionBefore) - pos, err := waitForEvents(n, newTestSyncRequest(alice, streamPositionVeryOld)) + n := NewNotifier(syncPositionBefore) + pos, err := waitForEvents(n, newTestSyncRequest(alice, syncPositionVeryOld)) if err != nil { t.Fatalf("TestImmediateNotification error: %s", err) } - if pos != streamPositionBefore { - t.Fatalf("TestImmediateNotification want %d, got %d", streamPositionBefore, pos) + if pos != syncPositionBefore { + t.Fatalf("TestImmediateNotification want %d, got %d", syncPositionBefore, pos) } } // Test that new events to a joined room unblocks the request. func TestNewEventAndJoinedToRoom(t *testing.T) { - n := NewNotifier(streamPositionBefore) + n := NewNotifier(syncPositionBefore) n.setUsersJoinedToRooms(map[string][]string{ roomID: {alice, bob}, }) @@ -112,12 +133,12 @@ func TestNewEventAndJoinedToRoom(t *testing.T) { var wg sync.WaitGroup wg.Add(1) go func() { - pos, err := waitForEvents(n, newTestSyncRequest(bob, streamPositionBefore)) + pos, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionBefore)) if err != nil { t.Errorf("TestNewEventAndJoinedToRoom error: %s", err) } - if pos != streamPositionAfter { - t.Errorf("TestNewEventAndJoinedToRoom want %d, got %d", streamPositionAfter, pos) + if pos != syncPositionAfter { + t.Errorf("TestNewEventAndJoinedToRoom want %d, got %d", syncPositionAfter, pos) } wg.Done() }() @@ -125,14 +146,14 @@ func TestNewEventAndJoinedToRoom(t *testing.T) { stream := n.fetchUserStream(bob, true) waitForBlocking(stream, 1) - n.OnNewEvent(&randomMessageEvent, "", streamPositionAfter) + n.OnNewEvent(&randomMessageEvent, "", nil, syncPositionAfter) wg.Wait() } // Test that an invite unblocks the request func TestNewInviteEventForUser(t *testing.T) { - n := NewNotifier(streamPositionBefore) + n := NewNotifier(syncPositionBefore) n.setUsersJoinedToRooms(map[string][]string{ roomID: {alice, bob}, }) @@ -140,12 +161,12 @@ func TestNewInviteEventForUser(t *testing.T) { var wg sync.WaitGroup wg.Add(1) go func() { - pos, err := waitForEvents(n, newTestSyncRequest(bob, streamPositionBefore)) + pos, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionBefore)) if err != nil { t.Errorf("TestNewInviteEventForUser error: %s", err) } - if pos != streamPositionAfter { - t.Errorf("TestNewInviteEventForUser want %d, got %d", streamPositionAfter, pos) + if pos != syncPositionAfter { + t.Errorf("TestNewInviteEventForUser want %d, got %d", syncPositionAfter, pos) } wg.Done() }() @@ -153,14 +174,42 @@ func TestNewInviteEventForUser(t *testing.T) { stream := n.fetchUserStream(bob, true) waitForBlocking(stream, 1) - n.OnNewEvent(&aliceInviteBobEvent, "", streamPositionAfter) + n.OnNewEvent(&aliceInviteBobEvent, "", nil, syncPositionAfter) + + wg.Wait() +} + +// Test an EDU-only update wakes up the request. +func TestEDUWakeup(t *testing.T) { + n := NewNotifier(syncPositionAfter) + n.setUsersJoinedToRooms(map[string][]string{ + roomID: {alice, bob}, + }) + + var wg sync.WaitGroup + wg.Add(1) + go func() { + pos, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionAfter)) + if err != nil { + t.Errorf("TestNewInviteEventForUser error: %s", err) + } + if pos != syncPositionNewEDU { + t.Errorf("TestNewInviteEventForUser want %d, got %d", syncPositionNewEDU, pos) + } + wg.Done() + }() + + stream := n.fetchUserStream(bob, true) + waitForBlocking(stream, 1) + + n.OnNewEvent(&aliceInviteBobEvent, "", nil, syncPositionNewEDU) wg.Wait() } // Test that all blocked requests get woken up on a new event. func TestMultipleRequestWakeup(t *testing.T) { - n := NewNotifier(streamPositionBefore) + n := NewNotifier(syncPositionBefore) n.setUsersJoinedToRooms(map[string][]string{ roomID: {alice, bob}, }) @@ -168,12 +217,12 @@ func TestMultipleRequestWakeup(t *testing.T) { var wg sync.WaitGroup wg.Add(3) poll := func() { - pos, err := waitForEvents(n, newTestSyncRequest(bob, streamPositionBefore)) + pos, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionBefore)) if err != nil { t.Errorf("TestMultipleRequestWakeup error: %s", err) } - if pos != streamPositionAfter { - t.Errorf("TestMultipleRequestWakeup want %d, got %d", streamPositionAfter, pos) + if pos != syncPositionAfter { + t.Errorf("TestMultipleRequestWakeup want %d, got %d", syncPositionAfter, pos) } wg.Done() } @@ -184,7 +233,7 @@ func TestMultipleRequestWakeup(t *testing.T) { stream := n.fetchUserStream(bob, true) waitForBlocking(stream, 3) - n.OnNewEvent(&randomMessageEvent, "", streamPositionAfter) + n.OnNewEvent(&randomMessageEvent, "", nil, syncPositionAfter) wg.Wait() @@ -198,7 +247,7 @@ func TestMultipleRequestWakeup(t *testing.T) { func TestNewEventAndWasPreviouslyJoinedToRoom(t *testing.T) { // listen as bob. Make bob leave room. Make alice send event to room. // Make sure alice gets woken up only and not bob as well. - n := NewNotifier(streamPositionBefore) + n := NewNotifier(syncPositionBefore) n.setUsersJoinedToRooms(map[string][]string{ roomID: {alice, bob}, }) @@ -208,18 +257,18 @@ func TestNewEventAndWasPreviouslyJoinedToRoom(t *testing.T) { // Make bob leave the room leaveWG.Add(1) go func() { - pos, err := waitForEvents(n, newTestSyncRequest(bob, streamPositionBefore)) + pos, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionBefore)) if err != nil { t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom error: %s", err) } - if pos != streamPositionAfter { - t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom want %d, got %d", streamPositionAfter, pos) + if pos != syncPositionAfter { + t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom want %d, got %d", syncPositionAfter, pos) } leaveWG.Done() }() bobStream := n.fetchUserStream(bob, true) waitForBlocking(bobStream, 1) - n.OnNewEvent(&bobLeaveEvent, "", streamPositionAfter) + n.OnNewEvent(&bobLeaveEvent, "", nil, syncPositionAfter) leaveWG.Wait() // send an event into the room. Make sure alice gets it. Bob should not. @@ -227,19 +276,19 @@ func TestNewEventAndWasPreviouslyJoinedToRoom(t *testing.T) { aliceStream := n.fetchUserStream(alice, true) aliceWG.Add(1) go func() { - pos, err := waitForEvents(n, newTestSyncRequest(alice, streamPositionAfter)) + pos, err := waitForEvents(n, newTestSyncRequest(alice, syncPositionAfter)) if err != nil { t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom error: %s", err) } - if pos != streamPositionAfter2 { - t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom want %d, got %d", streamPositionAfter2, pos) + if pos != syncPositionAfter2 { + t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom want %d, got %d", syncPositionAfter2, pos) } aliceWG.Done() }() go func() { // this should timeout with an error (but the main goroutine won't wait for the timeout explicitly) - _, err := waitForEvents(n, newTestSyncRequest(bob, streamPositionAfter)) + _, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionAfter)) if err == nil { t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom expect error but got nil") } @@ -248,7 +297,7 @@ func TestNewEventAndWasPreviouslyJoinedToRoom(t *testing.T) { waitForBlocking(aliceStream, 1) waitForBlocking(bobStream, 1) - n.OnNewEvent(&randomMessageEvent, "", streamPositionAfter2) + n.OnNewEvent(&randomMessageEvent, "", nil, syncPositionAfter2) aliceWG.Wait() // it's possible that at this point alice has been informed and bob is about to be informed, so wait @@ -256,18 +305,17 @@ func TestNewEventAndWasPreviouslyJoinedToRoom(t *testing.T) { time.Sleep(1 * time.Millisecond) } -// same as Notifier.WaitForEvents but with a timeout. -func waitForEvents(n *Notifier, req syncRequest) (types.StreamPosition, error) { +func waitForEvents(n *Notifier, req syncRequest) (types.SyncPosition, error) { listener := n.GetListener(req) defer listener.Close() select { case <-time.After(5 * time.Second): - return types.StreamPosition(0), fmt.Errorf( + return types.SyncPosition{}, fmt.Errorf( "waitForEvents timed out waiting for %s (pos=%d)", req.device.UserID, req.since, ) case <-listener.GetNotifyChannel(*req.since): - p := listener.GetStreamPosition() + p := listener.GetSyncPosition() return p, nil } } @@ -280,7 +328,7 @@ func waitForBlocking(s *UserStream, numBlocking uint) { } } -func newTestSyncRequest(userID string, since types.StreamPosition) syncRequest { +func newTestSyncRequest(userID string, since types.SyncPosition) syncRequest { return syncRequest{ device: authtypes.Device{UserID: userID}, timeout: 1 * time.Minute, diff --git a/syncapi/sync/request.go b/syncapi/sync/request.go index 35a15f6f9..a5d2f60f4 100644 --- a/syncapi/sync/request.go +++ b/syncapi/sync/request.go @@ -16,8 +16,10 @@ package sync import ( "context" + "errors" "net/http" "strconv" + "strings" "time" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" @@ -36,7 +38,7 @@ type syncRequest struct { device authtypes.Device limit int timeout time.Duration - since *types.StreamPosition // nil means that no since token was supplied + since *types.SyncPosition // nil means that no since token was supplied wantFullState bool log *log.Entry } @@ -73,15 +75,41 @@ func getTimeout(timeoutMS string) time.Duration { } // getSyncStreamPosition tries to parse a 'since' token taken from the API to a -// stream position. If the string is empty then (nil, nil) is returned. -func getSyncStreamPosition(since string) (*types.StreamPosition, error) { +// types.SyncPosition. If the string is empty then (nil, nil) is returned. +// There are two forms of tokens: The full length form containing all PDU and EDU +// positions separated by "_", and the short form containing only the PDU +// position. Short form can be used for, e.g., `prev_batch` tokens. +func getSyncStreamPosition(since string) (*types.SyncPosition, error) { if since == "" { return nil, nil } - i, err := strconv.Atoi(since) - if err != nil { - return nil, err + + posStrings := strings.Split(since, "_") + if len(posStrings) != 2 && len(posStrings) != 1 { + // A token can either be full length or short (PDU-only). + return nil, errors.New("malformed batch token") + } + + positions := make([]int64, len(posStrings)) + for i, posString := range posStrings { + pos, err := strconv.ParseInt(posString, 10, 64) + if err != nil { + return nil, err + } + positions[i] = pos + } + + if len(positions) == 2 { + // Full length token; construct SyncPosition with every entry in + // `positions`. These entries must have the same order with the fields + // in struct SyncPosition, so we disable the govet check below. + return &types.SyncPosition{ //nolint:govet + positions[0], positions[1], + }, nil + } else { + // Token with PDU position only + return &types.SyncPosition{ + PDUPosition: positions[0], + }, nil } - token := types.StreamPosition(i) - return &token, nil } diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index 89137eb59..a6ec6bd92 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -31,13 +31,13 @@ import ( // RequestPool manages HTTP long-poll connections for /sync type RequestPool struct { - db *storage.SyncServerDatabase + db *storage.SyncServerDatasource accountDB *accounts.Database notifier *Notifier } // NewRequestPool makes a new RequestPool -func NewRequestPool(db *storage.SyncServerDatabase, n *Notifier, adb *accounts.Database) *RequestPool { +func NewRequestPool(db *storage.SyncServerDatasource, n *Notifier, adb *accounts.Database) *RequestPool { return &RequestPool{db, adb, n} } @@ -92,11 +92,13 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *authtype // respond with, so we skip the return an go back to waiting for content to // be sent down or the request timing out. var hasTimedOut bool + sincePos := *syncReq.since for { select { // Wait for notifier to wake us up - case <-userStreamListener.GetNotifyChannel(currPos): - currPos = userStreamListener.GetStreamPosition() + case <-userStreamListener.GetNotifyChannel(sincePos): + currPos = userStreamListener.GetSyncPosition() + sincePos = currPos // Or for timeout to expire case <-timer.C: // We just need to ensure we get out of the select after reaching the @@ -128,24 +130,24 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *authtype } } -func (rp *RequestPool) currentSyncForUser(req syncRequest, currentPos types.StreamPosition) (res *types.Response, err error) { +func (rp *RequestPool) currentSyncForUser(req syncRequest, latestPos types.SyncPosition) (res *types.Response, err error) { // TODO: handle ignored users if req.since == nil { res, err = rp.db.CompleteSync(req.ctx, req.device.UserID, req.limit) } else { - res, err = rp.db.IncrementalSync(req.ctx, req.device, *req.since, currentPos, req.limit) + res, err = rp.db.IncrementalSync(req.ctx, req.device, *req.since, latestPos, req.limit) } if err != nil { return } - res, err = rp.appendAccountData(res, req.device.UserID, req, currentPos) + res, err = rp.appendAccountData(res, req.device.UserID, req, latestPos.PDUPosition) return } func (rp *RequestPool) appendAccountData( - data *types.Response, userID string, req syncRequest, currentPos types.StreamPosition, + data *types.Response, userID string, req syncRequest, currentPos int64, ) (*types.Response, error) { // TODO: Account data doesn't have a sync position of its own, meaning that // account data might be sent multiple time to the client if multiple account @@ -179,7 +181,7 @@ func (rp *RequestPool) appendAccountData( } // Sync is not initial, get all account data since the latest sync - dataTypes, err := rp.db.GetAccountDataInRange(req.ctx, userID, *req.since, currentPos) + dataTypes, err := rp.db.GetAccountDataInRange(req.ctx, userID, req.since.PDUPosition, currentPos) if err != nil { return nil, err } diff --git a/syncapi/sync/userstream.go b/syncapi/sync/userstream.go index 77d09c202..beb10e487 100644 --- a/syncapi/sync/userstream.go +++ b/syncapi/sync/userstream.go @@ -34,8 +34,8 @@ type UserStream struct { lock sync.Mutex // Closed when there is an update. signalChannel chan struct{} - // The last stream position that there may have been an update for the suser - pos types.StreamPosition + // The last sync position that there may have been an update for the user + pos types.SyncPosition // The last time when we had some listeners waiting timeOfLastChannel time.Time // The number of listeners waiting @@ -51,7 +51,7 @@ type UserStreamListener struct { } // NewUserStream creates a new user stream -func NewUserStream(userID string, currPos types.StreamPosition) *UserStream { +func NewUserStream(userID string, currPos types.SyncPosition) *UserStream { return &UserStream{ UserID: userID, timeOfLastChannel: time.Now(), @@ -84,8 +84,8 @@ func (s *UserStream) GetListener(ctx context.Context) UserStreamListener { return listener } -// Broadcast a new stream position for this user. -func (s *UserStream) Broadcast(pos types.StreamPosition) { +// Broadcast a new sync position for this user. +func (s *UserStream) Broadcast(pos types.SyncPosition) { s.lock.Lock() defer s.lock.Unlock() @@ -118,9 +118,9 @@ func (s *UserStream) TimeOfLastNonEmpty() time.Time { return s.timeOfLastChannel } -// GetStreamPosition returns last stream position which the UserStream was +// GetStreamPosition returns last sync position which the UserStream was // notified about -func (s *UserStreamListener) GetStreamPosition() types.StreamPosition { +func (s *UserStreamListener) GetSyncPosition() types.SyncPosition { s.userStream.lock.Lock() defer s.userStream.lock.Unlock() @@ -132,11 +132,11 @@ func (s *UserStreamListener) GetStreamPosition() types.StreamPosition { // sincePos specifies from which point we want to be notified about. If there // has already been an update after sincePos we'll return a closed channel // immediately. -func (s *UserStreamListener) GetNotifyChannel(sincePos types.StreamPosition) <-chan struct{} { +func (s *UserStreamListener) GetNotifyChannel(sincePos types.SyncPosition) <-chan struct{} { s.userStream.lock.Lock() defer s.userStream.lock.Unlock() - if sincePos < s.userStream.pos { + if s.userStream.pos.IsAfter(sincePos) { // If the listener is behind, i.e. missed a potential update, then we // want them to wake up immediately. We do this by returning a new // closed stream, which returns immediately when selected. diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go index 2db54c3ce..4738feea2 100644 --- a/syncapi/syncapi.go +++ b/syncapi/syncapi.go @@ -28,7 +28,6 @@ import ( "github.com/matrix-org/dendrite/syncapi/routing" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/sync" - "github.com/matrix-org/dendrite/syncapi/types" ) // SetupSyncAPIComponent sets up and registers HTTP handlers for the SyncAPI @@ -39,17 +38,17 @@ func SetupSyncAPIComponent( accountsDB *accounts.Database, queryAPI api.RoomserverQueryAPI, ) { - syncDB, err := storage.NewSyncServerDatabase(string(base.Cfg.Database.SyncAPI)) + syncDB, err := storage.NewSyncServerDatasource(string(base.Cfg.Database.SyncAPI)) if err != nil { logrus.WithError(err).Panicf("failed to connect to sync db") } - pos, err := syncDB.SyncStreamPosition(context.Background()) + pos, err := syncDB.SyncPosition(context.Background()) if err != nil { - logrus.WithError(err).Panicf("failed to get stream position") + logrus.WithError(err).Panicf("failed to get sync position") } - notifier := sync.NewNotifier(types.StreamPosition(pos)) + notifier := sync.NewNotifier(pos) err = notifier.Load(context.Background(), syncDB) if err != nil { logrus.WithError(err).Panicf("failed to start notifier") @@ -71,5 +70,12 @@ func SetupSyncAPIComponent( logrus.WithError(err).Panicf("failed to start client data consumer") } + typingConsumer := consumers.NewOutputTypingEventConsumer( + base.Cfg, base.KafkaConsumer, notifier, syncDB, + ) + if err = typingConsumer.Start(); err != nil { + logrus.WithError(err).Panicf("failed to start typing server consumer") + } + routing.Setup(base.APIMux, requestPool, syncDB, deviceDB) } diff --git a/syncapi/types/types.go b/syncapi/types/types.go index d0b1c38ab..af7ec865f 100644 --- a/syncapi/types/types.go +++ b/syncapi/types/types.go @@ -21,12 +21,38 @@ import ( "github.com/matrix-org/gomatrixserverlib" ) -// StreamPosition represents the offset in the sync stream a client is at. -type StreamPosition int64 +// SyncPosition contains the PDU and EDU stream sync positions for a client. +type SyncPosition struct { + // PDUPosition is the stream position for PDUs the client is at. + PDUPosition int64 + // TypingPosition is the client's position for typing notifications. + TypingPosition int64 +} // String implements the Stringer interface. -func (sp StreamPosition) String() string { - return strconv.FormatInt(int64(sp), 10) +func (sp SyncPosition) String() string { + return strconv.FormatInt(sp.PDUPosition, 10) + "_" + + strconv.FormatInt(sp.TypingPosition, 10) +} + +// IsAfter returns whether one SyncPosition refers to states newer than another SyncPosition. +func (sp SyncPosition) IsAfter(other SyncPosition) bool { + return sp.PDUPosition > other.PDUPosition || + sp.TypingPosition > other.TypingPosition +} + +// WithUpdates returns a copy of the SyncPosition with updates applied from another SyncPosition. +// If the latter SyncPosition contains a field that is not 0, it is considered an update, +// and its value will replace the corresponding value in the SyncPosition on which WithUpdates is called. +func (sp SyncPosition) WithUpdates(other SyncPosition) SyncPosition { + ret := sp + if other.PDUPosition != 0 { + ret.PDUPosition = other.PDUPosition + } + if other.TypingPosition != 0 { + ret.TypingPosition = other.TypingPosition + } + return ret } // PrevEventRef represents a reference to a previous event in a state event upgrade @@ -53,11 +79,10 @@ type Response struct { } // NewResponse creates an empty response with initialised maps. -func NewResponse(pos StreamPosition) *Response { - res := Response{} - // Make sure we send the next_batch as a string. We don't want to confuse clients by sending this - // as an integer even though (at the moment) it is. - res.NextBatch = pos.String() +func NewResponse(pos SyncPosition) *Response { + res := Response{ + NextBatch: pos.String(), + } // Pre-initialise the maps. Synapse will return {} even if there are no rooms under a specific section, // so let's do the same thing. Bonus: this means we can't get dreaded 'assignment to entry in nil map' errors. res.Rooms.Join = make(map[string]JoinResponse) diff --git a/testfile b/testfile index e869be4f0..bdd421e2d 100644 --- a/testfile +++ b/testfile @@ -144,3 +144,6 @@ Events come down the correct room local user can join room with version 5 User can invite local user to room with version 5 Inbound federation can receive room-join requests +Typing events appear in initial sync +Typing events appear in incremental sync +Typing events appear in gapped sync diff --git a/typingserver/api/output.go b/typingserver/api/output.go index 813b9b7c7..8696acf49 100644 --- a/typingserver/api/output.go +++ b/typingserver/api/output.go @@ -12,14 +12,17 @@ package api +import "time" + // OutputTypingEvent is an entry in typing server output kafka log. // This contains the event with extra fields used to create 'm.typing' event // in clientapi & federation. type OutputTypingEvent struct { // The Event for the typing edu event. Event TypingEvent `json:"event"` - // Users typing in the room when the event was generated. - TypingUsers []string `json:"typing_users"` + // ExpireTime is the interval after which the user should no longer be + // considered typing. Only available if Event.Typing is true. + ExpireTime *time.Time } // TypingEvent represents a matrix edu event of type 'm.typing'. diff --git a/typingserver/cache/cache.go b/typingserver/cache/cache.go index 85d74cd19..8d84f856c 100644 --- a/typingserver/cache/cache.go +++ b/typingserver/cache/cache.go @@ -22,25 +22,66 @@ const defaultTypingTimeout = 10 * time.Second // userSet is a map of user IDs to a timer, timer fires at expiry. type userSet map[string]*time.Timer +// TimeoutCallbackFn is a function called right after the removal of a user +// from the typing user list due to timeout. +// latestSyncPosition is the typing sync position after the removal. +type TimeoutCallbackFn func(userID, roomID string, latestSyncPosition int64) + +type roomData struct { + syncPosition int64 + userSet userSet +} + // TypingCache maintains a list of users typing in each room. type TypingCache struct { sync.RWMutex - data map[string]userSet + latestSyncPosition int64 + data map[string]*roomData + timeoutCallback TimeoutCallbackFn +} + +// Create a roomData with its sync position set to the latest sync position. +// Must only be called after locking the cache. +func (t *TypingCache) newRoomData() *roomData { + return &roomData{ + syncPosition: t.latestSyncPosition, + userSet: make(userSet), + } } // NewTypingCache returns a new TypingCache initialised for use. func NewTypingCache() *TypingCache { - return &TypingCache{data: make(map[string]userSet)} + return &TypingCache{data: make(map[string]*roomData)} +} + +// SetTimeoutCallback sets a callback function that is called right after +// a user is removed from the typing user list due to timeout. +func (t *TypingCache) SetTimeoutCallback(fn TimeoutCallbackFn) { + t.timeoutCallback = fn } // GetTypingUsers returns the list of users typing in a room. -func (t *TypingCache) GetTypingUsers(roomID string) (users []string) { +func (t *TypingCache) GetTypingUsers(roomID string) []string { + users, _ := t.GetTypingUsersIfUpdatedAfter(roomID, 0) + // 0 should work above because the first position used will be 1. + return users +} + +// GetTypingUsersIfUpdatedAfter returns all users typing in this room with +// updated == true if the typing sync position of the room is after the given +// position. Otherwise, returns an empty slice with updated == false. +func (t *TypingCache) GetTypingUsersIfUpdatedAfter( + roomID string, position int64, +) (users []string, updated bool) { t.RLock() - usersMap, ok := t.data[roomID] - t.RUnlock() - if ok { - users = make([]string, 0, len(usersMap)) - for userID := range usersMap { + defer t.RUnlock() + + roomData, ok := t.data[roomID] + if ok && roomData.syncPosition > position { + updated = true + userSet := roomData.userSet + users = make([]string, 0, len(userSet)) + for userID := range userSet { users = append(users, userID) } } @@ -51,25 +92,41 @@ func (t *TypingCache) GetTypingUsers(roomID string) (users []string) { // AddTypingUser sets an user as typing in a room. // expire is the time when the user typing should time out. // if expire is nil, defaultTypingTimeout is assumed. -func (t *TypingCache) AddTypingUser(userID, roomID string, expire *time.Time) { +// Returns the latest sync position for typing after update. +func (t *TypingCache) AddTypingUser( + userID, roomID string, expire *time.Time, +) int64 { expireTime := getExpireTime(expire) if until := time.Until(expireTime); until > 0 { - timer := time.AfterFunc(until, t.timeoutCallback(userID, roomID)) - t.addUser(userID, roomID, timer) + timer := time.AfterFunc(until, func() { + latestSyncPosition := t.RemoveUser(userID, roomID) + if t.timeoutCallback != nil { + t.timeoutCallback(userID, roomID, latestSyncPosition) + } + }) + return t.addUser(userID, roomID, timer) } + return t.GetLatestSyncPosition() } // addUser with mutex lock & replace the previous timer. -func (t *TypingCache) addUser(userID, roomID string, expiryTimer *time.Timer) { +// Returns the latest typing sync position after update. +func (t *TypingCache) addUser( + userID, roomID string, expiryTimer *time.Timer, +) int64 { t.Lock() defer t.Unlock() + t.latestSyncPosition++ + if t.data[roomID] == nil { - t.data[roomID] = make(userSet) + t.data[roomID] = t.newRoomData() + } else { + t.data[roomID].syncPosition = t.latestSyncPosition } // Stop the timer to cancel the call to timeoutCallback - if timer, ok := t.data[roomID][userID]; ok { + if timer, ok := t.data[roomID].userSet[userID]; ok { // It may happen that at this stage timer fires but now we have a lock on t. // Hence the execution of timeoutCallback will happen after we unlock. // So we may lose a typing state, though this event is highly unlikely. @@ -78,26 +135,40 @@ func (t *TypingCache) addUser(userID, roomID string, expiryTimer *time.Timer) { timer.Stop() } - t.data[roomID][userID] = expiryTimer -} + t.data[roomID].userSet[userID] = expiryTimer -// Returns a function which is called after timeout happens. -// This removes the user. -func (t *TypingCache) timeoutCallback(userID, roomID string) func() { - return func() { - t.RemoveUser(userID, roomID) - } + return t.latestSyncPosition } // RemoveUser with mutex lock & stop the timer. -func (t *TypingCache) RemoveUser(userID, roomID string) { +// Returns the latest sync position for typing after update. +func (t *TypingCache) RemoveUser(userID, roomID string) int64 { t.Lock() defer t.Unlock() - if timer, ok := t.data[roomID][userID]; ok { - timer.Stop() - delete(t.data[roomID], userID) + roomData, ok := t.data[roomID] + if !ok { + return t.latestSyncPosition } + + timer, ok := roomData.userSet[userID] + if !ok { + return t.latestSyncPosition + } + + timer.Stop() + delete(roomData.userSet, userID) + + t.latestSyncPosition++ + t.data[roomID].syncPosition = t.latestSyncPosition + + return t.latestSyncPosition +} + +func (t *TypingCache) GetLatestSyncPosition() int64 { + t.Lock() + defer t.Unlock() + return t.latestSyncPosition } func getExpireTime(expire *time.Time) time.Time { diff --git a/typingserver/input/input.go b/typingserver/input/input.go index b9968ce4c..0e2fbe51f 100644 --- a/typingserver/input/input.go +++ b/typingserver/input/input.go @@ -57,15 +57,21 @@ func (t *TypingServerInputAPI) InputTypingEvent( } func (t *TypingServerInputAPI) sendEvent(ite *api.InputTypingEvent) error { - userIDs := t.Cache.GetTypingUsers(ite.RoomID) ev := &api.TypingEvent{ Type: gomatrixserverlib.MTyping, RoomID: ite.RoomID, UserID: ite.UserID, + Typing: ite.Typing, } ote := &api.OutputTypingEvent{ - Event: *ev, - TypingUsers: userIDs, + Event: *ev, + } + + if ev.Typing { + expireTime := ite.OriginServerTS.Time().Add( + time.Duration(ite.Timeout) * time.Millisecond, + ) + ote.ExpireTime = &expireTime } eventJSON, err := json.Marshal(ote) From e2251199a49ab0bb846c02ba37e1cd437a7f725b Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Fri, 12 Jul 2019 16:43:01 +0100 Subject: [PATCH 17/17] Lots of small typo fixes (#737) --- appservice/api/query.go | 4 ++-- clientapi/routing/createroom.go | 2 +- clientapi/routing/membership.go | 2 +- clientapi/routing/profile.go | 6 +++--- federationapi/routing/profile.go | 2 +- federationapi/routing/threepid.go | 2 +- go.sum | 1 - syncapi/storage/output_room_events_table.go | 2 +- typingserver/cache/cache.go | 11 ++++++----- 9 files changed, 16 insertions(+), 16 deletions(-) diff --git a/appservice/api/query.go b/appservice/api/query.go index 9ec214486..8ce3b4e04 100644 --- a/appservice/api/query.go +++ b/appservice/api/query.go @@ -134,9 +134,9 @@ func (h *httpAppServiceQueryAPI) UserIDExists( return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } -// RetreiveUserProfile is a wrapper that queries both the local database and +// RetrieveUserProfile is a wrapper that queries both the local database and // application services for a given user's profile -func RetreiveUserProfile( +func RetrieveUserProfile( ctx context.Context, userID string, asAPI AppServiceQueryAPI, diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index a7187c495..220ba6ae8 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -163,7 +163,7 @@ func createRoom( "roomID": roomID, }).Info("Creating new room") - profile, err := appserviceAPI.RetreiveUserProfile(req.Context(), userID, asAPI, accountDB) + profile, err := appserviceAPI.RetrieveUserProfile(req.Context(), userID, asAPI, accountDB) if err != nil { return httputil.LogThenError(req, err) } diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index 22e66f452..61898fecd 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -176,7 +176,7 @@ func loadProfile( var profile *authtypes.Profile if serverName == cfg.Matrix.ServerName { - profile, err = appserviceAPI.RetreiveUserProfile(ctx, userID, asAPI, accountDB) + profile, err = appserviceAPI.RetrieveUserProfile(ctx, userID, asAPI, accountDB) } else { profile = &authtypes.Profile{} } diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index e57d16fbf..eb1acab74 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -43,7 +43,7 @@ func GetProfile( JSON: jsonerror.NotFound("Bad method"), } } - profile, err := appserviceAPI.RetreiveUserProfile(req.Context(), userID, asAPI, accountDB) + profile, err := appserviceAPI.RetrieveUserProfile(req.Context(), userID, asAPI, accountDB) if err != nil { return httputil.LogThenError(req, err) } @@ -62,7 +62,7 @@ func GetProfile( func GetAvatarURL( req *http.Request, accountDB *accounts.Database, userID string, asAPI appserviceAPI.AppServiceQueryAPI, ) util.JSONResponse { - profile, err := appserviceAPI.RetreiveUserProfile(req.Context(), userID, asAPI, accountDB) + profile, err := appserviceAPI.RetrieveUserProfile(req.Context(), userID, asAPI, accountDB) if err != nil { return httputil.LogThenError(req, err) } @@ -160,7 +160,7 @@ func SetAvatarURL( func GetDisplayName( req *http.Request, accountDB *accounts.Database, userID string, asAPI appserviceAPI.AppServiceQueryAPI, ) util.JSONResponse { - profile, err := appserviceAPI.RetreiveUserProfile(req.Context(), userID, asAPI, accountDB) + profile, err := appserviceAPI.RetrieveUserProfile(req.Context(), userID, asAPI, accountDB) if err != nil { return httputil.LogThenError(req, err) } diff --git a/federationapi/routing/profile.go b/federationapi/routing/profile.go index aa4fcdc42..2b478cfbf 100644 --- a/federationapi/routing/profile.go +++ b/federationapi/routing/profile.go @@ -53,7 +53,7 @@ func GetProfile( return httputil.LogThenError(httpReq, err) } - profile, err := appserviceAPI.RetreiveUserProfile(httpReq.Context(), userID, asAPI, accountDB) + profile, err := appserviceAPI.RetrieveUserProfile(httpReq.Context(), userID, asAPI, accountDB) if err != nil { return httputil.LogThenError(httpReq, err) } diff --git a/federationapi/routing/threepid.go b/federationapi/routing/threepid.go index 27796067b..05ca8892e 100644 --- a/federationapi/routing/threepid.go +++ b/federationapi/routing/threepid.go @@ -194,7 +194,7 @@ func createInviteFrom3PIDInvite( StateKey: &inv.MXID, } - profile, err := appserviceAPI.RetreiveUserProfile(ctx, inv.MXID, asAPI, accountDB) + profile, err := appserviceAPI.RetrieveUserProfile(ctx, inv.MXID, asAPI, accountDB) if err != nil { return nil, err } diff --git a/go.sum b/go.sum index ce3c07dd7..ef6e94e10 100644 --- a/go.sum +++ b/go.sum @@ -140,4 +140,3 @@ gopkg.in/yaml.v2 v2.0.0-20171116090243-287cf08546ab h1:yZ6iByf7GKeJ3gsd1Dr/xaj1D gopkg.in/yaml.v2 v2.0.0-20171116090243-287cf08546ab/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= - diff --git a/syncapi/storage/output_room_events_table.go b/syncapi/storage/output_room_events_table.go index 06df017cb..34632aedf 100644 --- a/syncapi/storage/output_room_events_table.go +++ b/syncapi/storage/output_room_events_table.go @@ -235,7 +235,7 @@ func (s *outputRoomEventsStatements) selectRecentEvents( return nil, err } // The events need to be returned from oldest to latest, which isn't - // necessary the way the SQL query returns them, so a sort is necessary to + // necessarily the way the SQL query returns them, so a sort is necessary to // ensure the events are in the right order in the slice. sort.SliceStable(events, func(i int, j int) bool { return events[i].streamPosition < events[j].streamPosition diff --git a/typingserver/cache/cache.go b/typingserver/cache/cache.go index 8d84f856c..3f05c938e 100644 --- a/typingserver/cache/cache.go +++ b/typingserver/cache/cache.go @@ -127,11 +127,12 @@ func (t *TypingCache) addUser( // Stop the timer to cancel the call to timeoutCallback if timer, ok := t.data[roomID].userSet[userID]; ok { - // It may happen that at this stage timer fires but now we have a lock on t. - // Hence the execution of timeoutCallback will happen after we unlock. - // So we may lose a typing state, though this event is highly unlikely. - // This can be mitigated by keeping another time.Time in the map and check against it - // before removing. This however is not required in most practical scenario. + // It may happen that at this stage the timer fires, but we now have a lock on + // it. Hence the execution of timeoutCallback will happen after we unlock. So + // we may lose a typing state, though this is highly unlikely. This can be + // mitigated by keeping another time.Time in the map and checking against it + // before removing, but its occurrence is so infrequent it does not seem + // worthwhile. timer.Stop() }