From 8fb2c9c33c6bd7b1a58685668eedcbb57607f8c5 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 17 Dec 2019 16:47:45 +0000 Subject: [PATCH 01/14] Replace deprecated prometheus.InstrumentHandler and unsafe time.Ticker --- common/httpapi.go | 12 +++++- common/transactions/transactions.go | 2 +- go.mod | 26 ++++++----- go.sum | 67 +++++++++++++++++++++++++++++ mediaapi/routing/routing.go | 51 +++++++++++++--------- 5 files changed, 124 insertions(+), 34 deletions(-) diff --git a/common/httpapi.go b/common/httpapi.go index bf634ff4a..59b303b6d 100644 --- a/common/httpapi.go +++ b/common/httpapi.go @@ -11,6 +11,7 @@ import ( opentracing "github.com/opentracing/opentracing-go" "github.com/opentracing/opentracing-go/ext" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promhttp" ) @@ -59,7 +60,16 @@ func MakeHTMLAPI(metricsName string, f func(http.ResponseWriter, *http.Request) } } - return prometheus.InstrumentHandler(metricsName, http.HandlerFunc(withSpan)) + return promhttp.InstrumentHandlerCounter( + promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: metricsName, + Help: "Total number of http requests for HTML resources", + }, + []string{"code"}, + ), + http.HandlerFunc(withSpan), + ) } // MakeInternalAPI turns a util.JSONRequestHandler function into an http.Handler. diff --git a/common/transactions/transactions.go b/common/transactions/transactions.go index 80b403a98..d2eb0f27f 100644 --- a/common/transactions/transactions.go +++ b/common/transactions/transactions.go @@ -85,7 +85,7 @@ func (t *Cache) AddTransaction(accessToken, txnID string, res *util.JSONResponse // It guarantees that an entry will be present in cache for at least cleanupPeriod & at most 2 * cleanupPeriod. // This cycles the txnMaps forward, i.e. back map is assigned the front and front is assigned an empty map. func cacheCleanService(t *Cache) { - ticker := time.Tick(t.cleanupPeriod) + ticker := time.NewTicker(t.cleanupPeriod).C for range ticker { t.Lock() t.txnsMaps[1] = t.txnsMaps[0] diff --git a/go.mod b/go.mod index d51f0a33e..d2cb80bb8 100644 --- a/go.mod +++ b/go.mod @@ -3,16 +3,16 @@ module github.com/matrix-org/dendrite require ( github.com/Shopify/sarama v0.0.0-20170127151855-574d3147eee3 github.com/alecthomas/gometalinter v2.0.2+incompatible - github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf + github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 github.com/apache/thrift v0.0.0-20161221203622-b2a4d4ae21c7 - github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a + github.com/beorn7/perks v1.0.1 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd github.com/crossdock/crossdock-go v0.0.0-20160816171116-049aabb0122b github.com/davecgh/go-spew v1.1.1 github.com/eapache/go-resiliency v0.0.0-20160104191539-b86b1ec0dd42 github.com/eapache/go-xerial-snappy v0.0.0-20160609142408-bb955e01b934 github.com/eapache/queue v1.1.0 - github.com/golang/protobuf v0.0.0-20161117033126-8ee79997227b + github.com/golang/protobuf v1.3.2 github.com/golang/snappy v0.0.0-20170119014723-7db9049039a0 github.com/google/shlex v0.0.0-20150127133951-6f45313302b9 github.com/gorilla/context v1.1.1 @@ -28,18 +28,20 @@ require ( github.com/matrix-org/naffka v0.0.0-20171115094957-662bfd0841d0 github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 github.com/matttproud/golang_protobuf_extensions v1.0.1 + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 github.com/nicksnyder/go-i18n v1.8.1 github.com/opentracing/opentracing-go v0.0.0-20170806192116-8ebe5d4e236e github.com/pelletier/go-toml v0.0.0-20170904195809-1d6b12b7cb29 github.com/pierrec/lz4 v0.0.0-20161206202305-5c9560bfa9ac github.com/pierrec/xxHash v0.0.0-20160112165351-5a004441f897 - github.com/pkg/errors v0.0.0-20170505043639-c605e284fe17 + github.com/pkg/errors v0.8.1 github.com/pmezard/go-difflib v1.0.0 - github.com/prometheus/client_golang v0.0.0-20180519192340-c51dc758d4bb - github.com/prometheus/client_model v0.0.0-20150212101744-fa8ad6fec335 - github.com/prometheus/common v0.0.0-20170108231212-dd2f054febf4 - github.com/prometheus/procfs v0.0.0-20170128160123-1878d9fbb537 + github.com/prometheus/client_golang v1.2.1 + github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 + github.com/prometheus/common v0.7.0 + github.com/prometheus/procfs v0.0.5 github.com/rcrowley/go-metrics v0.0.0-20161128210544-1f30fe9094a5 github.com/sirupsen/logrus v1.4.2 github.com/stretchr/objx v0.2.0 // indirect @@ -54,9 +56,9 @@ require ( go.uber.org/atomic v1.3.0 go.uber.org/multierr v0.0.0-20170829224307-fb7d312c2c04 go.uber.org/zap v1.7.1 - golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613 - golang.org/x/net v0.0.0-20190301231341-16b79f2e4e95 - golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 + golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 + golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 + golang.org/x/sys v0.0.0-20191010194322-b09406accb47 gopkg.in/Shopify/sarama.v1 v1.11.0 gopkg.in/airbrake/gobrake.v2 v2.0.9 gopkg.in/alecthomas/kingpin.v3-unstable v3.0.0-20170727041045-23bcc3c4eae3 @@ -65,3 +67,5 @@ require ( gopkg.in/macaroon.v2 v2.1.0 gopkg.in/yaml.v2 v2.2.2 ) + +go 1.13 diff --git a/go.sum b/go.sum index 56781c9a6..962d0f5bb 100644 --- a/go.sum +++ b/go.sum @@ -1,10 +1,19 @@ github.com/Shopify/sarama v0.0.0-20170127151855-574d3147eee3 h1:j6BAEHYn1kUyW2j7kY0mOJ/R8A0qWwXpvUAEHGemm/g= github.com/Shopify/sarama v0.0.0-20170127151855-574d3147eee3/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/alecthomas/gometalinter v2.0.2+incompatible/go.mod h1:qfIpQGGz3d+NmgyPBqv+LSh50emm1pt72EtcX2vKYQk= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/apache/thrift v0.0.0-20161221203622-b2a4d4ae21c7/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a h1:BtpsbiV638WQZwhA98cEZw2BsbnQJrbd0BI7tsy0W1c= github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.1.0 h1:yTUvW7Vhb89inJ+8irsUqiWjh8iT6sQPZiQzI6ReGkA= +github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/crossdock/crossdock-go v0.0.0-20160816171116-049aabb0122b/go.mod h1:v9FBN7gdVTpiD/+LZ7Po0UKvROyT87uLVxTHVky/dlQ= @@ -19,24 +28,41 @@ github.com/eapache/go-xerial-snappy v0.0.0-20160609142408-bb955e01b934/go.mod h1 github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/frankban/quicktest v1.0.0/go.mod h1:R98jIehRai+d1/3Hv2//jOVCTJhW1VBavT6B6CuGq2k= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/protobuf v0.0.0-20161117033126-8ee79997227b h1:fE/yi9pibxGEc0gSJuEShcsBXE2d5FW3OudsjE9tKzQ= github.com/golang/protobuf v0.0.0-20161117033126-8ee79997227b/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.0-20170119014723-7db9049039a0 h1:FMElzTwkd/2jQ2QzLEzt97JRgvFhYhnYiaQSwZ7tuyU= github.com/golang/snappy v0.0.0-20170119014723-7db9049039a0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/shlex v0.0.0-20150127133951-6f45313302b9/go.mod h1:RpwtwJQFrIEPstU94h88MWPXP2ektJZ8cZ0YntAmXiE= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.3.0 h1:HwSEKGN6U5T2aAQTfu5pW8fiwjSp3IgwdRbkICydk/c= github.com/gorilla/mux v1.3.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= github.com/jaegertracing/jaeger-client-go v0.0.0-20170921145708-3ad49a1d839b/go.mod h1:HWG7INeOG1ZE17I/S8eeb+svquXmBS/hf1Obi6hJUyQ= github.com/jaegertracing/jaeger-lib v0.0.0-20170920222118-21a3da6d66fe/go.mod h1:VqeqQrZmZr9G4WdLw4ei9tAHU54iJRkfoFHvTTQn4jQ= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6 h1:KAZ1BW2TCmT6PRihDPpocIy1QTtsAsrx6TneU/4+CMg= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -68,7 +94,13 @@ github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 h1:W7l5CP4V7wPyPb4 github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5/go.mod h1:lePuOiXLNDott7NZfnQvJk0lAZ5HgvIuWGhel6J+RLA= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.1.4 h1:rCMZsU2ScVSYcAsOXgmC6+AKOK+6pmQTOcw03nfwYV0= github.com/miekg/dns v1.1.4/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 h1:BvoENQQU+fZ9uukda/RzCAL/191HHwJA5b13R6diVlY= github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= @@ -82,19 +114,38 @@ github.com/pierrec/xxHash v0.0.0-20160112165351-5a004441f897 h1:jp3jc/PyyTrTKjJJ github.com/pierrec/xxHash v0.0.0-20160112165351-5a004441f897/go.mod h1:w2waW5Zoa/Wc4Yqe0wgrIYAGKqRMf7czn2HNKXmuL+I= github.com/pkg/errors v0.0.0-20170505043639-c605e284fe17 h1:chPfVn+gpAM5CTpTyVU9j8J+xgRGwmoDlNDLjKnJiYo= github.com/pkg/errors v0.0.0-20170505043639-c605e284fe17/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.0.0-20180519192340-c51dc758d4bb h1:ghXIh3jvLRo/h3y2O7wBgcmH1th5NXQ4XHwK5CgI//I= github.com/prometheus/client_golang v0.0.0-20180519192340-c51dc758d4bb/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.2.1 h1:JnMpQc6ppsNgw9QPAGF6Dod479itz7lvlsMzzNayLOI= +github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U= github.com/prometheus/client_model v0.0.0-20150212101744-fa8ad6fec335 h1:0E/5GnGmzoDCtmzTycjGDWW33H0UBmAhR0h+FC8hWLs= github.com/prometheus/client_model v0.0.0-20150212101744-fa8ad6fec335/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20170108231212-dd2f054febf4 h1:bZG2YNnM/Fjd3kiqaVt13Apkhzz24wBKlxQ+URiggXk= github.com/prometheus/common v0.0.0-20170108231212-dd2f054febf4/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY= +github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/procfs v0.0.0-20170128160123-1878d9fbb537 h1:Lq69k27tHOmljEqDOHDy3b6kQyEie2yWeAiF0OKFMJ8= github.com/prometheus/procfs v0.0.0-20170128160123-1878d9fbb537/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8= +github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/rcrowley/go-metrics v0.0.0-20161128210544-1f30fe9094a5 h1:gwcdIpH6NU2iF8CmcqD+CP6+1CkRBOhHaPR+iu6raBY= github.com/rcrowley/go-metrics v0.0.0-20161128210544-1f30fe9094a5/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/sirupsen/logrus v0.0.0-20170822132746-89742aefa4b2 h1:+8J/sCAVv2Y9Ct1BKszDFJEVWv6Aynr2O4FYGUg6+Mc= github.com/sirupsen/logrus v0.0.0-20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= @@ -104,6 +155,7 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v0.0.0-20170809224252-890a5c3458b4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/tidwall/gjson v1.0.2 h1:5BsM7kyEAHAUGEGDkEKO9Mdyiuw6QQ6TSDdarP0Nnmk= github.com/tidwall/gjson v1.0.2/go.mod h1:c/nTNbUr0E0OrXEhq1pwa8iEgc2DOt4ZZqAt1HtCkPA= @@ -132,28 +184,43 @@ golang.org/x/crypto v0.0.0-20181106171534-e4dc69e5b2fd h1:VtIkGDhk0ph3t+THbvXHfM golang.org/x/crypto v0.0.0-20181106171534-e4dc69e5b2fd/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613 h1:MQ/ZZiDsUapFFiMS+vzwXkCTeEKaum+Do5rINYJDmxc= golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/net v0.0.0-20170927055102-0a9397675ba3 h1:tTDpczhDVjW6WN3DinzKcw5juwkDTVn22I7MNlfxSXM= golang.org/x/net v0.0.0-20170927055102-0a9397675ba3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190301231341-16b79f2e4e95 h1:fY7Dsw114eJN4boqzVSbpVHO6rTdhq6/GnXeu+PKnzU= golang.org/x/net v0.0.0-20190301231341-16b79f2e4e95/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20171012164349-43eea11bc926 h1:PY6OU86NqbyZiOzaPnDw6oOjAGtYQqIua16z6y9QkwE= golang.org/x/sys v0.0.0-20171012164349-43eea11bc926/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 h1:I6FyU15t786LL7oL/hn43zqTuEGr4PN7F4XJ1p4E3Y8= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 h1:LepdCS8Gf/MVejFIt8lsiexZATdoGVyp5bcyS+rYoUI= golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY= +golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= gopkg.in/Shopify/sarama.v1 v1.11.0 h1:/3kaCyeYaPbr59IBjeqhIcUOB1vXlIVqXAYa5g5C5F0= gopkg.in/Shopify/sarama.v1 v1.11.0/go.mod h1:AxnvoaevB2nBjNK17cG61A3LleFcWFwVBHBt+cot4Oc= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/alecthomas/kingpin.v3-unstable v3.0.0-20170727041045-23bcc3c4eae3/go.mod h1:3HH7i1SgMqlzxCcBmUHW657sD4Kvv9sC3HpL3YukzwA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/h2non/bimg.v1 v1.0.18/go.mod h1:PgsZL7dLwUbsGm1NYps320GxGgvQNTnecMCZqxV11So= +gopkg.in/h2non/gock.v1 v1.0.14 h1:fTeu9fcUvSnLNacYvYI54h+1/XEteDyHvrVCZEEEYNM= gopkg.in/h2non/gock.v1 v1.0.14/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE= gopkg.in/macaroon.v2 v2.0.0/go.mod h1:+I6LnTMkm/uV5ew/0nsulNjL16SK4+C8yDmRUzHR17I= gopkg.in/macaroon.v2 v2.1.0/go.mod h1:OUb+TQP/OP0WOerC2Jp/3CwhIKyIa9kQjuc7H24e6/o= gopkg.in/yaml.v2 v2.0.0-20171116090243-287cf08546ab h1:yZ6iByf7GKeJ3gsd1Dr/xaj1DyJ//wxKX1Cdh8LhoAw= gopkg.in/yaml.v2 v2.0.0-20171116090243-287cf08546ab/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/mediaapi/routing/routing.go b/mediaapi/routing/routing.go index 5bcce1772..232a57fd1 100644 --- a/mediaapi/routing/routing.go +++ b/mediaapi/routing/routing.go @@ -29,6 +29,8 @@ import ( "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/prometheus/client_golang/prometheus/promhttp" ) const pathPrefixR0 = "/_matrix/media/r0" @@ -83,26 +85,33 @@ func makeDownloadAPI( activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration, ) http.HandlerFunc { - return prometheus.InstrumentHandler(name, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - req = util.RequestWithLogging(req) + return promhttp.InstrumentHandlerCounter( + promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: name, + Help: "Total number of media_api requests for either thumbnails or full downloads", + }, + []string{"code"}, + ), http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + req = util.RequestWithLogging(req) - // Set common headers returned regardless of the outcome of the request - util.SetCORSHeaders(w) - // Content-Type will be overridden in case of returning file data, else we respond with JSON-formatted errors - w.Header().Set("Content-Type", "application/json") - - vars, _ := common.URLDecodeMapValues(mux.Vars(req)) - Download( - w, - req, - gomatrixserverlib.ServerName(vars["serverName"]), - types.MediaID(vars["mediaId"]), - cfg, - db, - client, - activeRemoteRequests, - activeThumbnailGeneration, - name == "thumbnail", - ) - })) + // Set common headers returned regardless of the outcome of the request + util.SetCORSHeaders(w) + // Content-Type will be overridden in case of returning file data, else we respond with JSON-formatted errors + w.Header().Set("Content-Type", "application/json") + vars := mux.Vars(req) + Download( + w, + req, + gomatrixserverlib.ServerName(vars["serverName"]), + types.MediaID(vars["mediaId"]), + cfg, + db, + client, + activeRemoteRequests, + activeThumbnailGeneration, + name == "thumbnail", + ) + }, + )) } From de5d463dd2b5f8955b714d440d02777269255ed0 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 17 Dec 2019 16:54:21 +0000 Subject: [PATCH 02/14] goimports --- go.mod | 2 -- mediaapi/routing/routing.go | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d2cb80bb8..723bf99d3 100644 --- a/go.mod +++ b/go.mod @@ -28,8 +28,6 @@ require ( github.com/matrix-org/naffka v0.0.0-20171115094957-662bfd0841d0 github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 github.com/matttproud/golang_protobuf_extensions v1.0.1 - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.1 // indirect github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 github.com/nicksnyder/go-i18n v1.8.1 github.com/opentracing/opentracing-go v0.0.0-20170806192116-8ebe5d4e236e diff --git a/mediaapi/routing/routing.go b/mediaapi/routing/routing.go index 232a57fd1..bc919bacd 100644 --- a/mediaapi/routing/routing.go +++ b/mediaapi/routing/routing.go @@ -113,5 +113,5 @@ func makeDownloadAPI( name == "thumbnail", ) }, - )) + )) } From 2baf5baa75e1a3e1834bd2dde2fa3601ab834f0d Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 17 Dec 2019 17:05:39 +0000 Subject: [PATCH 03/14] re-add temporarily missing deps? --- go.mod | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go.mod b/go.mod index 723bf99d3..d2cb80bb8 100644 --- a/go.mod +++ b/go.mod @@ -28,6 +28,8 @@ require ( github.com/matrix-org/naffka v0.0.0-20171115094957-662bfd0841d0 github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 github.com/matttproud/golang_protobuf_extensions v1.0.1 + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 github.com/nicksnyder/go-i18n v1.8.1 github.com/opentracing/opentracing-go v0.0.0-20170806192116-8ebe5d4e236e From 9b5d6c9745bf8cb1c5a2057ee6218640c82e5250 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 18 Dec 2019 15:10:53 +0000 Subject: [PATCH 04/14] Refactor InstrumentHandlerCounter definition --- mediaapi/routing/routing.go | 56 ++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/mediaapi/routing/routing.go b/mediaapi/routing/routing.go index bc919bacd..3a8cb9ca6 100644 --- a/mediaapi/routing/routing.go +++ b/mediaapi/routing/routing.go @@ -85,33 +85,33 @@ func makeDownloadAPI( activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration, ) http.HandlerFunc { - return promhttp.InstrumentHandlerCounter( - promauto.NewCounterVec( - prometheus.CounterOpts{ - Name: name, - Help: "Total number of media_api requests for either thumbnails or full downloads", - }, - []string{"code"}, - ), http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - req = util.RequestWithLogging(req) - - // Set common headers returned regardless of the outcome of the request - util.SetCORSHeaders(w) - // Content-Type will be overridden in case of returning file data, else we respond with JSON-formatted errors - w.Header().Set("Content-Type", "application/json") - vars := mux.Vars(req) - Download( - w, - req, - gomatrixserverlib.ServerName(vars["serverName"]), - types.MediaID(vars["mediaId"]), - cfg, - db, - client, - activeRemoteRequests, - activeThumbnailGeneration, - name == "thumbnail", - ) + counterVec := promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: name, + Help: "Total number of media_api requests for either thumbnails or full downloads", }, - )) + []string{"code"}, + ) + httpHandler := func(w http.ResponseWriter, req *http.Request) { + req = util.RequestWithLogging(req) + + // Set common headers returned regardless of the outcome of the request + util.SetCORSHeaders(w) + // Content-Type will be overridden in case of returning file data, else we respond with JSON-formatted errors + w.Header().Set("Content-Type", "application/json") + vars := mux.Vars(req) + Download( + w, + req, + gomatrixserverlib.ServerName(vars["serverName"]), + types.MediaID(vars["mediaId"]), + cfg, + db, + client, + activeRemoteRequests, + activeThumbnailGeneration, + name == "thumbnail", + ) + } + return promhttp.InstrumentHandlerCounter(counterVec, http.HandlerFunc(httpHandler)) } From f392ce8ed744654c25f9ea58ed609db8edb25b68 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 18 Dec 2019 15:29:27 +0000 Subject: [PATCH 05/14] URL decode args --- mediaapi/routing/routing.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediaapi/routing/routing.go b/mediaapi/routing/routing.go index 3a8cb9ca6..0f2266640 100644 --- a/mediaapi/routing/routing.go +++ b/mediaapi/routing/routing.go @@ -99,7 +99,7 @@ func makeDownloadAPI( util.SetCORSHeaders(w) // Content-Type will be overridden in case of returning file data, else we respond with JSON-formatted errors w.Header().Set("Content-Type", "application/json") - vars := mux.Vars(req) + vars, _ := common.URLDecodeMapValues(mux.Vars(req)) Download( w, req, From b4d638cd04d64bf574d8af75fc22efb72b4a0798 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 20 Dec 2019 13:24:57 +0000 Subject: [PATCH 06/14] Return server names (#833) * Remove unnecessary map->array processing * Return server names in room federation directory query * Knock off a TODO --- cmd/dendrite-federation-api-server/main.go | 3 ++- cmd/dendrite-monolith-server/main.go | 2 +- federationapi/federationapi.go | 5 ++++- federationapi/routing/query.go | 11 +++++++++-- federationapi/routing/routing.go | 4 +++- federationsender/query/query.go | 9 +++------ 6 files changed, 22 insertions(+), 12 deletions(-) diff --git a/cmd/dendrite-federation-api-server/main.go b/cmd/dendrite-federation-api-server/main.go index c83845d25..367f5dc0c 100644 --- a/cmd/dendrite-federation-api-server/main.go +++ b/cmd/dendrite-federation-api-server/main.go @@ -29,6 +29,7 @@ func main() { deviceDB := base.CreateDeviceDB() keyDB := base.CreateKeyDB() federation := base.CreateFederationClient() + federationSender := base.CreateHTTPFederationSenderAPIs() keyRing := keydb.CreateKeyRing(federation.Client, keyDB) alias, input, query := base.CreateHTTPRoomserverAPIs() @@ -36,7 +37,7 @@ func main() { federationapi.SetupFederationAPIComponent( base, accountDB, deviceDB, federation, &keyRing, - alias, input, query, asQuery, + alias, input, query, asQuery, federationSender, ) base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationAPI), string(base.Cfg.Listen.FederationAPI)) diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 0a320616e..5ea6b154a 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -67,7 +67,7 @@ func main() { federation, &keyRing, alias, input, query, typingInputAPI, asQuery, transactions.New(), fedSenderAPI, ) - federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery) + federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fedSenderAPI) mediaapi.SetupMediaAPIComponent(base, deviceDB) publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB) syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query) diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go index 87402d976..c2aef06c8 100644 --- a/federationapi/federationapi.go +++ b/federationapi/federationapi.go @@ -19,6 +19,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/common/basecomponent" + federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" // TODO: Are we really wanting to pull in the producer from clientapi @@ -39,11 +40,13 @@ func SetupFederationAPIComponent( inputAPI roomserverAPI.RoomserverInputAPI, queryAPI roomserverAPI.RoomserverQueryAPI, asAPI appserviceAPI.AppServiceQueryAPI, + federationSenderAPI federationSenderAPI.FederationSenderQueryAPI, ) { roomserverProducer := producers.NewRoomserverProducer(inputAPI) routing.Setup( base.APIMux, *base.Cfg, queryAPI, aliasAPI, asAPI, - roomserverProducer, *keyRing, federation, accountsDB, deviceDB, + roomserverProducer, federationSenderAPI, *keyRing, federation, accountsDB, + deviceDB, ) } diff --git a/federationapi/routing/query.go b/federationapi/routing/query.go index 781b8ea46..ed2d8b741 100644 --- a/federationapi/routing/query.go +++ b/federationapi/routing/query.go @@ -21,6 +21,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/common/config" + federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" @@ -33,6 +34,7 @@ func RoomAliasToID( federation *gomatrixserverlib.FederationClient, cfg config.Dendrite, aliasAPI roomserverAPI.RoomserverAliasAPI, + senderAPI federationSenderAPI.FederationSenderQueryAPI, ) util.JSONResponse { roomAlias := httpReq.FormValue("room_alias") if roomAlias == "" { @@ -59,10 +61,15 @@ func RoomAliasToID( } if queryRes.RoomID != "" { - // TODO: List servers that are aware of this room alias + serverQueryReq := federationSenderAPI.QueryJoinedHostServerNamesInRoomRequest{RoomID: queryRes.RoomID} + var serverQueryRes federationSenderAPI.QueryJoinedHostServerNamesInRoomResponse + if err = senderAPI.QueryJoinedHostServerNamesInRoom(httpReq.Context(), &serverQueryReq, &serverQueryRes); err != nil { + return httputil.LogThenError(httpReq, err) + } + resp = gomatrixserverlib.RespDirectory{ RoomID: queryRes.RoomID, - Servers: []gomatrixserverlib.ServerName{}, + Servers: serverQueryRes.ServerNames, } } else { // If no alias was found, return an error diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index 9f576790b..eae1fabdb 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -24,6 +24,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/config" + federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -46,6 +47,7 @@ func Setup( aliasAPI roomserverAPI.RoomserverAliasAPI, asAPI appserviceAPI.AppServiceQueryAPI, producer *producers.RoomserverProducer, + federationSenderAPI federationSenderAPI.FederationSenderQueryAPI, keys gomatrixserverlib.KeyRing, federation *gomatrixserverlib.FederationClient, accountDB *accounts.Database, @@ -156,7 +158,7 @@ func Setup( "federation_query_room_alias", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { return RoomAliasToID( - httpReq, federation, cfg, aliasAPI, + httpReq, federation, cfg, aliasAPI, federationSenderAPI, ) }, )).Methods(http.MethodGet) diff --git a/federationsender/query/query.go b/federationsender/query/query.go index 088244826..8c35bb29e 100644 --- a/federationsender/query/query.go +++ b/federationsender/query/query.go @@ -45,15 +45,12 @@ func (f *FederationSenderQueryAPI) QueryJoinedHostServerNamesInRoom( return } - serverNamesSet := make(map[gomatrixserverlib.ServerName]bool, len(joinedHosts)) + response.ServerNames = make([]gomatrixserverlib.ServerName, 0, len(joinedHosts)) for _, host := range joinedHosts { - serverNamesSet[host.ServerName] = true + response.ServerNames = append(response.ServerNames, host.ServerName) } - response.ServerNames = make([]gomatrixserverlib.ServerName, 0, len(serverNamesSet)) - for name := range serverNamesSet { - response.ServerNames = append(response.ServerNames, name) - } + // TODO: remove duplicates? return } From af9568ba4468eb106ec305df496fbe1319fd74eb Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Fri, 20 Dec 2019 14:41:32 +0000 Subject: [PATCH 07/14] Fix /send_join and /send_leave (#821) Fix the /send_join and /send_leave endpoints, so that they use the v2 endpoints as mandated by MSC1802. Also comment out the SyTest tests that are failing because of lack of support for the v1 endpoints. --- federationapi/routing/routing.go | 6 ++++-- testfile | 25 ++++++++++++++++++++----- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index eae1fabdb..302f7ed0b 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -33,6 +33,7 @@ import ( const ( pathPrefixV2Keys = "/_matrix/key/v2" pathPrefixV1Federation = "/_matrix/federation/v1" + pathPrefixV2Federation = "/_matrix/federation/v2" ) // Setup registers HTTP handlers with the given ServeMux. @@ -55,6 +56,7 @@ func Setup( ) { v2keysmux := apiMux.PathPrefix(pathPrefixV2Keys).Subrouter() v1fedmux := apiMux.PathPrefix(pathPrefixV1Federation).Subrouter() + v2fedmux := apiMux.PathPrefix(pathPrefixV2Federation).Subrouter() localKeys := common.MakeExternalAPI("localkeys", func(req *http.Request) util.JSONResponse { return LocalKeys(cfg) @@ -200,7 +202,7 @@ func Setup( }, )).Methods(http.MethodGet) - v1fedmux.Handle("/send_join/{roomID}/{userID}", common.MakeFedAPI( + v2fedmux.Handle("/send_join/{roomID}/{userID}", common.MakeFedAPI( "federation_send_join", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) @@ -230,7 +232,7 @@ func Setup( }, )).Methods(http.MethodGet) - v1fedmux.Handle("/send_leave/{roomID}/{userID}", common.MakeFedAPI( + v2fedmux.Handle("/send_leave/{roomID}/{userID}", common.MakeFedAPI( "federation_send_leave", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) diff --git a/testfile b/testfile index 5ad3426e9..8a4e9de12 100644 --- a/testfile +++ b/testfile @@ -126,8 +126,14 @@ Checking local federation server Inbound federation can query profile data Outbound federation can send room-join requests Outbound federation can send events -Inbound federation can backfill events -Backfill checks the events requested belong to the room +# SyTest currently only implements the v1 endpoints for /send_join and /send_leave, +# whereas Dendrite only supports the v2 endpoints for those, so let's ignore this +# test for now. +#Inbound federation can backfill events +# SyTest currently only implements the v1 endpoints for /send_join and /send_leave, +# whereas Dendrite only supports the v2 endpoints for those, so let's ignore this +# test for now. +#Backfill checks the events requested belong to the room Can upload without a file name Can download without a file name locally Can upload with ASCII file name @@ -143,7 +149,10 @@ Trying to get push rules with unknown rule_id fails with 404 Events come down the correct room local user can join room with version 5 User can invite local user to room with version 5 -Inbound federation can receive v1 room-join requests +# SyTest currently only implements the v1 endpoints for /send_join and /send_leave, +# whereas Dendrite only supports the v2 endpoints for those, so let's ignore this +# test for now. +#Inbound federation can receive v1 room-join requests Typing events appear in initial sync Typing events appear in incremental sync Typing events appear in gapped sync @@ -156,8 +165,14 @@ User can create and send/receive messages in a room with version 1 POST /createRoom ignores attempts to set the room version via creation_content Inbound federation rejects remote attempts to join local users to rooms Inbound federation rejects remote attempts to kick local users to rooms -An event which redacts itself should be ignored -A pair of events which redact each other should be ignored +# SyTest currently only implements the v1 endpoints for /send_join and /send_leave, +# whereas Dendrite only supports the v2 endpoints for those, so let's ignore this +# test for now. +#An event which redacts itself should be ignored +# SyTest currently only implements the v1 endpoints for /send_join and /send_leave, +# whereas Dendrite only supports the v2 endpoints for those, so let's ignore this +# test for now. +#A pair of events which redact each other should be ignored Full state sync includes joined rooms A message sent after an initial sync appears in the timeline of an incremental sync. Can add tag From e2d73855ebdb236aaa4388685872792ba3907925 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar <31231064+abhishekkumar2718@users.noreply.github.com> Date: Fri, 20 Dec 2019 20:12:57 +0530 Subject: [PATCH 08/14] Refuse /send_join without m.room.create (#824) Signed-off-by: Abhishek Kumar --- federationapi/routing/join.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go index 6f6574dd7..e2885dd99 100644 --- a/federationapi/routing/join.go +++ b/federationapi/routing/join.go @@ -154,16 +154,23 @@ func SendJoin( // Fetch the state and auth chain. We do this before we send the events // on, in case this fails. - var stateAndAuthChainRepsonse api.QueryStateAndAuthChainResponse + var stateAndAuthChainResponse api.QueryStateAndAuthChainResponse err = query.QueryStateAndAuthChain(httpReq.Context(), &api.QueryStateAndAuthChainRequest{ PrevEventIDs: event.PrevEventIDs(), AuthEventIDs: event.AuthEventIDs(), RoomID: roomID, - }, &stateAndAuthChainRepsonse) + }, &stateAndAuthChainResponse) if err != nil { return httputil.LogThenError(httpReq, err) } + if !stateAndAuthChainResponse.RoomExists { + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound("Room does not exist"), + } + } + // Send the events to the room server. // We are responsible for notifying other servers that the user has joined // the room, so set SendAsServer to cfg.Matrix.ServerName @@ -177,8 +184,8 @@ func SendJoin( return util.JSONResponse{ Code: http.StatusOK, JSON: map[string]interface{}{ - "state": stateAndAuthChainRepsonse.StateEvents, - "auth_chain": stateAndAuthChainRepsonse.AuthChainEvents, + "state": stateAndAuthChainResponse.StateEvents, + "auth_chain": stateAndAuthChainResponse.AuthChainEvents, }, } } From f1e229e9d4a6c7f6ad7eda146ec0e3a8a6603af1 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Fri, 20 Dec 2019 14:44:34 +0000 Subject: [PATCH 09/14] AS should use the v1 endpoint, rather than r0 (#827) --- appservice/routing/routing.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appservice/routing/routing.go b/appservice/routing/routing.go index 3c19c8401..0e4bd6bab 100644 --- a/appservice/routing/routing.go +++ b/appservice/routing/routing.go @@ -27,7 +27,7 @@ import ( "github.com/matrix-org/util" ) -const pathPrefixApp = "/_matrix/app/r0" +const pathPrefixApp = "/_matrix/app/v1" // Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client // to clients which need to make outbound HTTP requests. From 52dfa40471e1ba220960ed8f14033d7e03a96385 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 20 Dec 2019 14:46:59 +0000 Subject: [PATCH 10/14] docker: Passthrough parameters to dendrite-monolith-server --- docker/services/monolith.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/services/monolith.sh b/docker/services/monolith.sh index a038728b4..2287555cd 100644 --- a/docker/services/monolith.sh +++ b/docker/services/monolith.sh @@ -2,4 +2,4 @@ bash ./docker/build.sh -./bin/dendrite-monolith-server --tls-cert=server.crt --tls-key=server.key +./bin/dendrite-monolith-server --tls-cert=server.crt --tls-key=server.key $@ From 4f75e4febe5ec3ed8666fc723e0f2350f372bf2d Mon Sep 17 00:00:00 2001 From: S7evinK Date: Fri, 20 Dec 2019 16:00:25 +0100 Subject: [PATCH 11/14] Fix copy & paste error (#812) --- roomserver/api/query.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roomserver/api/query.go b/roomserver/api/query.go index a544f8aa2..e52c74ac3 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -332,7 +332,7 @@ const RoomserverQueryMissingEventsPath = "/api/roomserver/queryMissingEvents" // RoomserverQueryStateAndAuthChainPath is the HTTP path for the QueryStateAndAuthChain API const RoomserverQueryStateAndAuthChainPath = "/api/roomserver/queryStateAndAuthChain" -// RoomserverQueryBackfillPath is the HTTP path for the QueryMissingEvents API +// RoomserverQueryBackfillPath is the HTTP path for the QueryBackfill API const RoomserverQueryBackfillPath = "/api/roomserver/QueryBackfill" // NewRoomserverQueryAPIHTTP creates a RoomserverQueryAPI implemented by talking to a HTTP POST API. @@ -475,6 +475,6 @@ func (h *httpRoomserverQueryAPI) QueryBackfill( span, ctx := opentracing.StartSpanFromContext(ctx, "QueryBackfill") defer span.Finish() - apiURL := h.roomserverURL + RoomserverQueryMissingEventsPath + apiURL := h.roomserverURL + RoomserverQueryBackfillPath return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } From b34fce0d85be4626fcc9d7ffaa110aca148cdb4b Mon Sep 17 00:00:00 2001 From: S7evinK Date: Fri, 20 Dec 2019 16:02:09 +0100 Subject: [PATCH 12/14] Use gomatrixserverlib.Transaction instead of local type (#590) (#811) --- federationapi/routing/backfill.go | 10 ++++--- federationapi/types/types.go | 43 ------------------------------- 2 files changed, 6 insertions(+), 47 deletions(-) delete mode 100644 federationapi/types/types.go diff --git a/federationapi/routing/backfill.go b/federationapi/routing/backfill.go index d996db6a3..5c6b0087f 100644 --- a/federationapi/routing/backfill.go +++ b/federationapi/routing/backfill.go @@ -17,11 +17,11 @@ package routing import ( "net/http" "strconv" + "time" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/common/config" - "github.com/matrix-org/dendrite/federationapi/types" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -90,9 +90,11 @@ func Backfill( } } - txn := types.NewTransaction() - txn.Origin = cfg.Matrix.ServerName - txn.PDUs = evs + txn := gomatrixserverlib.Transaction{ + Origin: cfg.Matrix.ServerName, + PDUs: evs, + OriginServerTS: gomatrixserverlib.AsTimestamp(time.Now()), + } // Send the events to the client. return util.JSONResponse{ diff --git a/federationapi/types/types.go b/federationapi/types/types.go deleted file mode 100644 index 24838d547..000000000 --- a/federationapi/types/types.go +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2018 New Vector Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package types - -import ( - "time" - - "github.com/matrix-org/gomatrixserverlib" -) - -// Transaction is the representation of a transaction from the federation API -// See https://matrix.org/docs/spec/server_server/unstable.html for more info. -type Transaction struct { - // The server_name of the homeserver sending this transaction. - Origin gomatrixserverlib.ServerName `json:"origin"` - // POSIX timestamp in milliseconds on originating homeserver when this - // transaction started. - OriginServerTS int64 `json:"origin_server_ts"` - // List of persistent updates to rooms. - PDUs []gomatrixserverlib.Event `json:"pdus"` -} - -// NewTransaction sets the timestamp of a new transaction instance and then -// returns the said instance. -func NewTransaction() Transaction { - // Retrieve the current timestamp in nanoseconds and make it a milliseconds - // one. - ts := time.Now().UnixNano() / int64(time.Millisecond) - - return Transaction{OriginServerTS: ts} -} From dc25fb6837f4e301d572b71d2f480f0c052a399c Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Wed, 25 Dec 2019 18:14:58 +0000 Subject: [PATCH 13/14] Move files back if linting fails (#810) --- scripts/find-lint.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/find-lint.sh b/scripts/find-lint.sh index ca991b62c..c9663e4e8 100755 --- a/scripts/find-lint.sh +++ b/scripts/find-lint.sh @@ -29,8 +29,15 @@ echo "Installing golangci-lint..." cp go.mod go.mod.bak && cp go.sum go.sum.bak go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.19.1 +# Run linting echo "Looking for lint..." -golangci-lint run $args + +# Capture exit code to ensure go.{mod,sum} is restored before exiting +exit_code=0 + +golangci-lint run $args || exit_code=1 # Restore go.{mod,sum} mv go.mod.bak go.mod && mv go.sum.bak go.sum + +exit $exit_code From 5741235833489d2769c72c24f1b5d93abe9d21ed Mon Sep 17 00:00:00 2001 From: Arshpreet <30545756+arsh-7@users.noreply.github.com> Date: Wed, 25 Dec 2019 23:58:10 +0530 Subject: [PATCH 14/14] replaced gometalinter description with golangci-lint (#837) --- CODE_STYLE.md | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/CODE_STYLE.md b/CODE_STYLE.md index e8208172c..1f40c76fc 100644 --- a/CODE_STYLE.md +++ b/CODE_STYLE.md @@ -1,21 +1,23 @@ # Code Style -We follow the standard Go style using gofmt, but with a few extra +We follow the standard Go style using goimports, but with a few extra considerations. ## Linters -We use `gometalinter` to run a number of linters, the exact list can be found -in [linter.json](linter.json). Some of these are slow and expensive to run, but -a subset can be found in [linter-fast.json](linter-fast.json) that run quickly -enough to be run as part of an IDE. +We use `golangci-lint` to run a number of linters, the exact list can be found +under linters in [.golangci.yml](.golangci.yml). +[Installation](https://github.com/golangci/golangci-lint#install) and [Editor +Integration](https://github.com/golangci/golangci-lint#editor-integration) for +it can be found in the readme of golangci-lint. For rare cases where a linter is giving a spurious warning, it can be disabled -for that line or statement using a [comment directive](https://github.com/alecthomas/gometalinter#comment-directives), e.g. -`// nolint: gocyclo`. This should be used sparingly and only when its clear -that the lint warning is spurious. +for that line or statement using a [comment +directive](https://github.com/golangci/golangci-lint#nolint), e.g. `var +bad_name int //nolint:golint,unused`. This should be used sparingly and only +when its clear that the lint warning is spurious. -The linters are vendored, and can be run using [scripts/find-lint.sh](scripts/find-lint.sh) +The linters can be run using [scripts/find-lint.sh](scripts/find-lint.sh) (see file for docs) or as part of a build/test/lint cycle using [scripts/build-test-lint.sh](scripts/build-test-lint.sh). @@ -82,9 +84,9 @@ sets up linting correctly: ```json { - "go.gopath": "${workspaceRoot}:${workspaceRoot}/vendor", - "go.lintOnSave": "workspace", - "go.lintTool": "gometalinter", - "go.lintFlags": ["--config=linter-fast.json", "--concurrency=5"] + "go.lintTool":"golangci-lint", + "go.lintFlags": [ + "--fast" + ] } ```