From 423c656146579daac1e6739ef05b4b79f4c5b611 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Behouba=20Manass=C3=A9?= Date: Wed, 22 Apr 2020 16:13:11 +0300 Subject: [PATCH 001/200] Response from /send_join now use gomatrixserverlib.RespSendJoin (#796) * response from /send_join now use gomatrixserverlib.RespSendJoin * Update Dendrite gomatrixserverlib version * Fix spelling Co-authored-by: Andrew Morgan Co-authored-by: Cnly Co-authored-by: Neil Alexander --- clientapi/routing/joinroom.go | 2 +- federationapi/routing/join.go | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/clientapi/routing/joinroom.go b/clientapi/routing/joinroom.go index f72bb9162..d0dee7c2a 100644 --- a/clientapi/routing/joinroom.go +++ b/clientapi/routing/joinroom.go @@ -386,7 +386,7 @@ func (r joinRoomReq) joinRoomUsingServer(roomID string, server gomatrixserverlib if err = r.producer.SendEventWithState( r.req.Context(), - gomatrixserverlib.RespState(respSendJoin.RespState), + respSendJoin.ToRespState(), event.Headered(respMakeJoin.RoomVersion), ); err != nil { util.GetLogger(r.req.Context()).WithError(err).Error("r.producer.SendEventWithState") diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go index e06785954..0c899ab9f 100644 --- a/federationapi/routing/join.go +++ b/federationapi/routing/join.go @@ -252,9 +252,12 @@ func SendJoin( return util.JSONResponse{ Code: http.StatusOK, - JSON: map[string]interface{}{ - "state": gomatrixserverlib.UnwrapEventHeaders(stateAndAuthChainResponse.StateEvents), - "auth_chain": gomatrixserverlib.UnwrapEventHeaders(stateAndAuthChainResponse.AuthChainEvents), + JSON: gomatrixserverlib.RespSendJoin{ + RespState: gomatrixserverlib.RespState{ + StateEvents: gomatrixserverlib.UnwrapEventHeaders(stateAndAuthChainResponse.StateEvents), + AuthEvents: gomatrixserverlib.UnwrapEventHeaders(stateAndAuthChainResponse.AuthChainEvents), + }, + Origin: cfg.Matrix.ServerName, }, } } From c30b12b5a1deea21e54d4718ca11a3b6366c443d Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 22 Apr 2020 15:26:56 +0100 Subject: [PATCH 002/200] Fix sarama import URLs (#856) * Fix sarama import URLs * Update gomatrixserverlib * Update naffka * Update naffka * Update in kafka-producer --- appservice/consumers/roomserver.go | 2 +- clientapi/consumers/roomserver.go | 2 +- clientapi/producers/syncapi.go | 2 +- clientapi/producers/userupdate.go | 2 +- cmd/kafka-producer/main.go | 2 +- common/basecomponent/base.go | 2 +- common/consumers.go | 2 +- eduserver/input/input.go | 2 +- federationsender/consumers/eduserver.go | 2 +- federationsender/consumers/roomserver.go | 2 +- go.mod | 20 +++----- go.sum | 64 +++++++++++------------- publicroomsapi/consumers/roomserver.go | 2 +- roomserver/input/input.go | 2 +- syncapi/consumers/clientapi.go | 2 +- syncapi/consumers/eduserver.go | 2 +- syncapi/consumers/roomserver.go | 2 +- 17 files changed, 51 insertions(+), 63 deletions(-) diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go index 6ae58e85c..3bd364c58 100644 --- a/appservice/consumers/roomserver.go +++ b/appservice/consumers/roomserver.go @@ -26,8 +26,8 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" + "github.com/Shopify/sarama" log "github.com/sirupsen/logrus" - sarama "gopkg.in/Shopify/sarama.v1" ) // OutputRoomEventConsumer consumes events that originated in the room server. diff --git a/clientapi/consumers/roomserver.go b/clientapi/consumers/roomserver.go index 6d5bb09a6..3c7905721 100644 --- a/clientapi/consumers/roomserver.go +++ b/clientapi/consumers/roomserver.go @@ -24,8 +24,8 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" + "github.com/Shopify/sarama" log "github.com/sirupsen/logrus" - sarama "gopkg.in/Shopify/sarama.v1" ) // OutputRoomEventConsumer consumes events that originated in the room server. diff --git a/clientapi/producers/syncapi.go b/clientapi/producers/syncapi.go index 6bfcd51aa..0a446e2f5 100644 --- a/clientapi/producers/syncapi.go +++ b/clientapi/producers/syncapi.go @@ -19,7 +19,7 @@ import ( "github.com/matrix-org/dendrite/common" - sarama "gopkg.in/Shopify/sarama.v1" + "github.com/Shopify/sarama" ) // SyncAPIProducer produces events for the sync API server to consume diff --git a/clientapi/producers/userupdate.go b/clientapi/producers/userupdate.go index 2a5dfc70a..426b6d509 100644 --- a/clientapi/producers/userupdate.go +++ b/clientapi/producers/userupdate.go @@ -17,7 +17,7 @@ package producers import ( "encoding/json" - sarama "gopkg.in/Shopify/sarama.v1" + "github.com/Shopify/sarama" ) // UserUpdateProducer produces events related to user updates. diff --git a/cmd/kafka-producer/main.go b/cmd/kafka-producer/main.go index f5f243e4e..18ee3cdf2 100644 --- a/cmd/kafka-producer/main.go +++ b/cmd/kafka-producer/main.go @@ -21,7 +21,7 @@ import ( "os" "strings" - sarama "gopkg.in/Shopify/sarama.v1" + sarama "github.com/Shopify/sarama" ) const usage = `Usage: %s diff --git a/common/basecomponent/base.go b/common/basecomponent/base.go index 68a77cf99..5e2d659bf 100644 --- a/common/basecomponent/base.go +++ b/common/basecomponent/base.go @@ -33,8 +33,8 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/common" + "github.com/Shopify/sarama" "github.com/gorilla/mux" - sarama "gopkg.in/Shopify/sarama.v1" appserviceAPI "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/common/config" diff --git a/common/consumers.go b/common/consumers.go index f33993494..bc759d994 100644 --- a/common/consumers.go +++ b/common/consumers.go @@ -18,7 +18,7 @@ import ( "context" "fmt" - sarama "gopkg.in/Shopify/sarama.v1" + "github.com/Shopify/sarama" ) // A PartitionOffset is the offset into a partition of the input log. diff --git a/eduserver/input/input.go b/eduserver/input/input.go index 845909452..6a4a4bb4e 100644 --- a/eduserver/input/input.go +++ b/eduserver/input/input.go @@ -18,12 +18,12 @@ import ( "net/http" "time" + "github.com/Shopify/sarama" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" - "gopkg.in/Shopify/sarama.v1" ) // EDUServerInputAPI implements api.EDUServerInputAPI diff --git a/federationsender/consumers/eduserver.go b/federationsender/consumers/eduserver.go index 4d2445f3c..269701d77 100644 --- a/federationsender/consumers/eduserver.go +++ b/federationsender/consumers/eduserver.go @@ -16,6 +16,7 @@ import ( "context" "encoding/json" + "github.com/Shopify/sarama" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/eduserver/api" @@ -23,7 +24,6 @@ import ( "github.com/matrix-org/dendrite/federationsender/storage" "github.com/matrix-org/gomatrixserverlib" log "github.com/sirupsen/logrus" - "gopkg.in/Shopify/sarama.v1" ) // OutputTypingEventConsumer consumes events that originate in EDU server. diff --git a/federationsender/consumers/roomserver.go b/federationsender/consumers/roomserver.go index f59405af0..a36fb3792 100644 --- a/federationsender/consumers/roomserver.go +++ b/federationsender/consumers/roomserver.go @@ -19,6 +19,7 @@ import ( "encoding/json" "fmt" + "github.com/Shopify/sarama" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/federationsender/queue" @@ -27,7 +28,6 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" log "github.com/sirupsen/logrus" - sarama "gopkg.in/Shopify/sarama.v1" ) // OutputRoomEventConsumer consumes events that originated in the room server. diff --git a/go.mod b/go.mod index 8d91902d1..daf232781 100644 --- a/go.mod +++ b/go.mod @@ -1,9 +1,9 @@ module github.com/matrix-org/dendrite require ( + github.com/Shopify/sarama v1.26.1 github.com/gorilla/mux v1.7.3 github.com/hashicorp/golang-lru v0.5.4 - github.com/kr/pretty v0.2.0 // indirect github.com/lib/pq v1.2.0 github.com/libp2p/go-libp2p v0.6.0 github.com/libp2p/go-libp2p-circuit v0.1.4 @@ -17,26 +17,20 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f github.com/matrix-org/go-sqlite3-js v0.0.0-20200325174927-327088cdef10 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200421090225-4ea81b29f5f7 - github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 + github.com/matrix-org/gomatrixserverlib v0.0.0-20200422082552-d7b4202c47f3 + github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 - github.com/mattn/go-sqlite3 v2.0.3+incompatible + github.com/mattn/go-sqlite3 v2.0.2+incompatible github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 github.com/ngrok/sqlmw v0.0.0-20200129213757-d5c93a81bec6 github.com/opentracing/opentracing-go v1.1.0 - github.com/pierrec/lz4 v2.5.0+incompatible // indirect github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.4.1 - github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect github.com/sirupsen/logrus v1.4.2 - github.com/tidwall/gjson v1.6.0 // indirect - github.com/tidwall/pretty v1.0.1 // indirect - github.com/uber/jaeger-client-go v2.22.1+incompatible - github.com/uber/jaeger-lib v2.2.0+incompatible - go.uber.org/atomic v1.6.0 + github.com/uber/jaeger-client-go v2.15.0+incompatible + github.com/uber/jaeger-lib v1.5.0 + go.uber.org/atomic v1.4.0 golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d - golang.org/x/tools v0.0.0-20200402223321-bcf690261a44 // indirect - gopkg.in/Shopify/sarama.v1 v1.20.1 gopkg.in/h2non/bimg.v1 v1.0.18 gopkg.in/yaml.v2 v2.2.8 ) diff --git a/go.sum b/go.sum index e76f6d007..c8f7e43fe 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,12 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/zstd v1.4.4 h1:+IawcoXhCBylN7ccwdwf8LOH2jKq7NavGpEPanrlTzE= github.com/DataDog/zstd v1.4.4/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= +github.com/Shopify/sarama v1.26.1 h1:3jnfWKD7gVwbB1KSy/lE0szA9duPuSFLViK0o/d3DgA= +github.com/Shopify/sarama v1.26.1/go.mod h1:NbSGBSSndYaIhRcBtY9V0U7AyH+x71bG668AuWys/yU= github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= @@ -61,19 +62,18 @@ github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUn github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q= -github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q= -github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/frankban/quicktest v1.0.0/go.mod h1:R98jIehRai+d1/3Hv2//jOVCTJhW1VBavT6B6CuGq2k= github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3BTYk= github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJYCFOLkIBwI7xFExG03bbkOkCvUPI= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= @@ -119,6 +119,8 @@ github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/U github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= +github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= @@ -140,7 +142,6 @@ github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67Fexh github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-datastore v0.1.0/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= -github.com/ipfs/go-datastore v0.3.1 h1:SS1t869a6cctoSYmZXUk8eL6AzVXgASmKIWFNQkQ1jU= github.com/ipfs/go-datastore v0.3.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= github.com/ipfs/go-datastore v0.4.0/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= github.com/ipfs/go-datastore v0.4.1/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= @@ -158,7 +159,6 @@ github.com/ipfs/go-ds-leveldb v0.4.1/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1 github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-util v0.0.1 h1:Wz9bL2wB2YBJqggkA4dD7oSmqB4cAnpNbGrlHJulv50= github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= -github.com/ipfs/go-log v0.0.1 h1:9XTUN/rW64BCG1YhPK9Hoy3q8nr4gOmHHBpgFdfw6Lc= github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= github.com/ipfs/go-log v1.0.2 h1:s19ZwJxH8rPWzypjcDpqPLIyV7BnbLqvpli3iZoqYK0= github.com/ipfs/go-log v1.0.2/go.mod h1:1MNjMxe0u6xvJZgeqbJ8vdo2TKaGwZ1a0Bpza+sr2Sk= @@ -178,6 +178,8 @@ github.com/jbenet/go-temp-err-catcher v0.0.0-20150120210811-aac704a3f4f2/go.mod github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY= github.com/jbenet/goprocess v0.1.3 h1:YKyIEECS/XvcfHtBzxtjBBbWK+MbvA6dG8ASiqwvr10= github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= +github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8= +github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= @@ -188,9 +190,10 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= -github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/klauspost/compress v1.9.8 h1:VMAMUUOh+gaxKTMk+zqbjsSjsIcUcL/LF4o63i82QyA= +github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= @@ -364,10 +367,12 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:kmRjpmFOenVpOaV/DRlo9p6z/IbOKlUC+hhKsAAh8Qg= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200421090225-4ea81b29f5f7 h1:4vE84tE3r7BitCt2HQvT231JrhMjDfjDVDqVoiVPv0w= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200421090225-4ea81b29f5f7/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200422082552-d7b4202c47f3 h1:xis1ojN99vjygwqudzB9VQq3cM2SJ7aCAMlXj/YN+88= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200422082552-d7b4202c47f3/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= +github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y= +github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f/go.mod h1:y0oDTjZDv5SM9a2rp3bl+CU+bvTRINQsdb7YlDql5Go= github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 h1:W7l5CP4V7wPyPb4tYE11dbmeAOwtFQBTW0rf4OonOS8= github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5/go.mod h1:lePuOiXLNDott7NZfnQvJk0lAZ5HgvIuWGhel6J+RLA= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= @@ -380,8 +385,6 @@ github.com/mattn/go-isatty v0.0.5 h1:tHXDdz1cpzGaovsTB+TVB8q90WEokoVmfMqoVcrLUgw github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-sqlite3 v2.0.2+incompatible h1:qzw9c2GNT8UFrgWNDhCTqRqYUSmu/Dav/9Z58LGpk7U= github.com/mattn/go-sqlite3 v2.0.2+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U= -github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= @@ -475,8 +478,6 @@ github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFSt github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pierrec/lz4 v2.4.1+incompatible h1:mFe7ttWaflA46Mhqh+jUfjp2qTbPYxLB2/OyBppH9dg= github.com/pierrec/lz4 v2.4.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pierrec/lz4 v2.5.0+incompatible h1:MbdIZ43A//duwOjQqK3nP+up+65yraNFyX3Vp6Rwues= -github.com/pierrec/lz4 v2.5.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -505,10 +506,6 @@ github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLk github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ= -github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ= -github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME= @@ -550,17 +547,12 @@ github.com/uber-go/atomic v1.3.0 h1:ylWoWcs+jXihgo3Us1Sdsatf2R6+OlBGm8fexR3oFG4= github.com/uber-go/atomic v1.3.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= github.com/uber/jaeger-client-go v2.15.0+incompatible h1:NP3qsSqNxh8VYr956ur1N/1C1PjvOJnJykCzcD5QHbk= github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= -github.com/uber/jaeger-client-go v2.22.1+incompatible h1:NHcubEkVbahf9t3p75TOCR83gdUHXjRJvjoBh1yACsM= -github.com/uber/jaeger-client-go v2.22.1+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v1.5.0 h1:OHbgr8l656Ub3Fw5k9SWnBfIEwvoHQ+W2y+Aa9D1Uyo= github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= -github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw= -github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= -github.com/whyrusleeping/go-logging v0.0.1 h1:fwpzlmT0kRC/Fmd0MdmGgJG/CXIZ6gFq46FQZjprUcc= github.com/whyrusleeping/go-logging v0.0.1/go.mod h1:lDPYj54zutzG1XYfHAhcc7oNXEburHQBn+Iqd4yS4vE= github.com/whyrusleeping/mafmt v1.2.8 h1:TCghSl5kkwEE0j+sU/gudyhVMRlpBin8fMBBHg59EbA= github.com/whyrusleeping/mafmt v1.2.8/go.mod h1:faQJFPbLSxzD9xpA02ttW/tS9vZykNvXwGvqIpk20FA= @@ -571,8 +563,9 @@ github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7/go. github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee h1:lYbXeSvJi5zk5GLKVuid9TVjS9a0OmLIDKTfoZBL6Ow= github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee/go.mod h1:m2aV4LZI4Aez7dP5PMyVKEHhUyEJ/RjmPEDOpDvudHg= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= +github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= +github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -580,9 +573,8 @@ go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.0 h1:vs7fgriifsPbGdK3bNuMWapNn3qnZhCRXc19NRdq010= go.uber.org/atomic v1.3.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= -go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= @@ -605,16 +597,13 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d h1:2+ZP7EfsZV7Vvmx3TIqSlSzATMkTAKqM14YGFPoSKjI= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d h1:1ZiEyfaQIg3Qh0EoqpwAakHVhecoE5wlSg5GjnafJGw= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -629,6 +618,7 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -673,11 +663,6 @@ golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c h1:IGkKhmfzcztjm6gYkykvu/NiS8kaqbCWAEWWAyf8J5U= -golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200402223321-bcf690261a44 h1:bMm0eoDiGkM5VfIyKjxDvoflW5GLp7+VCo+60n8F+TE= -golang.org/x/tools v0.0.0-20200402223321-bcf690261a44/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= @@ -692,7 +677,6 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= gopkg.in/Shopify/sarama.v1 v1.20.1 h1:Gi09A3fJXm0Jgt8kuKZ8YK+r60GfYn7MQuEmI3oq6hE= gopkg.in/Shopify/sarama.v1 v1.20.1/go.mod h1:AxnvoaevB2nBjNK17cG61A3LleFcWFwVBHBt+cot4Oc= -gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -707,6 +691,16 @@ gopkg.in/h2non/bimg.v1 v1.0.18 h1:qn6/RpBHt+7WQqoBcK+aF2puc6nC78eZj5LexxoalT4= gopkg.in/h2non/bimg.v1 v1.0.18/go.mod h1:PgsZL7dLwUbsGm1NYps320GxGgvQNTnecMCZqxV11So= gopkg.in/h2non/gock.v1 v1.0.14 h1:fTeu9fcUvSnLNacYvYI54h+1/XEteDyHvrVCZEEEYNM= gopkg.in/h2non/gock.v1 v1.0.14/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE= +gopkg.in/jcmturner/aescts.v1 v1.0.1 h1:cVVZBK2b1zY26haWB4vbBiZrfFQnfbTVrE3xZq6hrEw= +gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= +gopkg.in/jcmturner/dnsutils.v1 v1.0.1 h1:cIuC1OLRGZrld+16ZJvvZxVJeKPsvd5eUIvxfoN5hSM= +gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= +gopkg.in/jcmturner/goidentity.v3 v3.0.0 h1:1duIyWiTaYvVx3YX2CYtpJbUFd7/UuPYCfgXtQ3VTbI= +gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4= +gopkg.in/jcmturner/gokrb5.v7 v7.5.0 h1:a9tsXlIDD9SKxotJMK3niV7rPZAJeX2aD/0yg3qlIrg= +gopkg.in/jcmturner/gokrb5.v7 v7.5.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= +gopkg.in/jcmturner/rpc.v1 v1.1.0 h1:QHIUxTX1ISuAv9dD2wJ9HWQVuWDX/Zc0PfeC2tjc4rU= +gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= gopkg.in/macaroon.v2 v2.1.0 h1:HZcsjBCzq9t0eBPMKqTN/uSN6JOm78ZJ2INbqcBQOUI= gopkg.in/macaroon.v2 v2.1.0/go.mod h1:OUb+TQP/OP0WOerC2Jp/3CwhIKyIa9kQjuc7H24e6/o= gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= diff --git a/publicroomsapi/consumers/roomserver.go b/publicroomsapi/consumers/roomserver.go index 2bbd92b72..853761c36 100644 --- a/publicroomsapi/consumers/roomserver.go +++ b/publicroomsapi/consumers/roomserver.go @@ -18,13 +18,13 @@ import ( "context" "encoding/json" + "github.com/Shopify/sarama" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" log "github.com/sirupsen/logrus" - sarama "gopkg.in/Shopify/sarama.v1" ) // OutputRoomEventConsumer consumes events that originated in the room server. diff --git a/roomserver/input/input.go b/roomserver/input/input.go index bd029d8df..3b519ecba 100644 --- a/roomserver/input/input.go +++ b/roomserver/input/input.go @@ -21,10 +21,10 @@ import ( "net/http" "sync" + "github.com/Shopify/sarama" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/util" - sarama "gopkg.in/Shopify/sarama.v1" ) // RoomserverInputAPI implements api.RoomserverInputAPI diff --git a/syncapi/consumers/clientapi.go b/syncapi/consumers/clientapi.go index 17f2c522c..f5b8c43ec 100644 --- a/syncapi/consumers/clientapi.go +++ b/syncapi/consumers/clientapi.go @@ -18,13 +18,13 @@ import ( "context" "encoding/json" + "github.com/Shopify/sarama" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/sync" "github.com/matrix-org/dendrite/syncapi/types" log "github.com/sirupsen/logrus" - sarama "gopkg.in/Shopify/sarama.v1" ) // OutputClientDataConsumer consumes events that originated in the client API server. diff --git a/syncapi/consumers/eduserver.go b/syncapi/consumers/eduserver.go index 5491c1e9f..249452af5 100644 --- a/syncapi/consumers/eduserver.go +++ b/syncapi/consumers/eduserver.go @@ -17,6 +17,7 @@ package consumers import ( "encoding/json" + "github.com/Shopify/sarama" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/eduserver/api" @@ -24,7 +25,6 @@ import ( "github.com/matrix-org/dendrite/syncapi/sync" "github.com/matrix-org/dendrite/syncapi/types" log "github.com/sirupsen/logrus" - sarama "gopkg.in/Shopify/sarama.v1" ) // OutputTypingEventConsumer consumes events that originated in the EDU server. diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index f1e68c262..1d512972d 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -19,6 +19,7 @@ import ( "encoding/json" "fmt" + "github.com/Shopify/sarama" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/roomserver/api" @@ -27,7 +28,6 @@ import ( "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" log "github.com/sirupsen/logrus" - sarama "gopkg.in/Shopify/sarama.v1" ) // OutputRoomEventConsumer consumes events that originated in the room server. From a202d88fe527c83179d2ebf57ac0d146c3173f9d Mon Sep 17 00:00:00 2001 From: Kegsay Date: Fri, 24 Apr 2020 10:38:58 +0100 Subject: [PATCH 003/200] Use a single storage.Database interface (#978) --- roomserver/input/authevents.go | 5 +- roomserver/input/events.go | 59 ++------------------- roomserver/input/input.go | 3 +- roomserver/input/latest_events.go | 5 +- roomserver/input/membership.go | 3 +- roomserver/query/query.go | 75 +++------------------------ roomserver/query/query_test.go | 4 +- roomserver/state/database/database.go | 67 ------------------------ roomserver/state/shared/shared.go | 1 - roomserver/state/state.go | 6 +-- roomserver/storage/interface.go | 41 ++++++++++++++- 11 files changed, 65 insertions(+), 204 deletions(-) delete mode 100644 roomserver/state/database/database.go delete mode 100644 roomserver/state/shared/shared.go diff --git a/roomserver/input/authevents.go b/roomserver/input/authevents.go index 456a01c79..2c2e14b3a 100644 --- a/roomserver/input/authevents.go +++ b/roomserver/input/authevents.go @@ -18,6 +18,7 @@ import ( "context" "sort" + "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -26,7 +27,7 @@ import ( // Returns the numeric IDs for the auth events. func checkAuthEvents( ctx context.Context, - db RoomEventDatabase, + db storage.Database, event gomatrixserverlib.HeaderedEvent, authEventIDs []string, ) ([]types.EventNID, error) { @@ -127,7 +128,7 @@ func (ae *authEvents) lookupEvent(typeNID types.EventTypeNID, stateKey string) * // loadAuthEvents loads the events needed for authentication from the supplied room state. func loadAuthEvents( ctx context.Context, - db RoomEventDatabase, + db storage.Database, needed gomatrixserverlib.StateNeeded, state []types.StateEntry, ) (result authEvents, err error) { diff --git a/roomserver/input/events.go b/roomserver/input/events.go index 2bb0d0a05..393c1f419 100644 --- a/roomserver/input/events.go +++ b/roomserver/input/events.go @@ -23,63 +23,12 @@ import ( "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/state" - "github.com/matrix-org/dendrite/roomserver/state/database" + "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" log "github.com/sirupsen/logrus" ) -// A RoomEventDatabase has the storage APIs needed to store a room event. -type RoomEventDatabase interface { - database.RoomStateDatabase - // Stores a matrix room event in the database - StoreEvent( - ctx context.Context, - event gomatrixserverlib.Event, - txnAndSessionID *api.TransactionID, - authEventNIDs []types.EventNID, - ) (types.RoomNID, types.StateAtEvent, error) - // Look up the state entries for a list of string event IDs - // Returns an error if the there is an error talking to the database - // Returns a types.MissingEventError if the event IDs aren't in the database. - StateEntriesForEventIDs( - ctx context.Context, eventIDs []string, - ) ([]types.StateEntry, error) - // Set the state at an event. - SetState( - ctx context.Context, - eventNID types.EventNID, - stateNID types.StateSnapshotNID, - ) error - // Look up the latest events in a room in preparation for an update. - // The RoomRecentEventsUpdater must have Commit or Rollback called on it if this doesn't return an error. - // Returns the latest events in the room and the last eventID sent to the log along with an updater. - // If this returns an error then no further action is required. - GetLatestEventsForUpdate( - ctx context.Context, roomNID types.RoomNID, - ) (updater types.RoomRecentEventsUpdater, err error) - // Look up the string event IDs for a list of numeric event IDs - EventIDs( - ctx context.Context, eventNIDs []types.EventNID, - ) (map[types.EventNID]string, error) - // Build a membership updater for the target user in a room. - MembershipUpdater( - ctx context.Context, roomID, targerUserID string, - roomVersion gomatrixserverlib.RoomVersion, - ) (types.MembershipUpdater, error) - // Look up event ID by transaction's info. - // This is used to determine if the room event is processed/processing already. - // Returns an empty string if no such event exists. - GetTransactionEventID( - ctx context.Context, transactionID string, - sessionID int64, userID string, - ) (string, error) - // Look up the room version for a given room. - GetRoomVersionForRoom( - ctx context.Context, roomID string, - ) (gomatrixserverlib.RoomVersion, error) -} - // OutputRoomEventWriter has the APIs needed to write an event to the output logs. type OutputRoomEventWriter interface { // Write a list of events for a room @@ -93,7 +42,7 @@ type OutputRoomEventWriter interface { // state deltas when sending to kafka streams func processRoomEvent( ctx context.Context, - db RoomEventDatabase, + db storage.Database, ow OutputRoomEventWriter, input api.InputRoomEvent, ) (eventID string, err error) { @@ -153,7 +102,7 @@ func processRoomEvent( func calculateAndSetState( ctx context.Context, - db RoomEventDatabase, + db storage.Database, input api.InputRoomEvent, roomNID types.RoomNID, stateAtEvent *types.StateAtEvent, @@ -184,7 +133,7 @@ func calculateAndSetState( func processInviteEvent( ctx context.Context, - db RoomEventDatabase, + db storage.Database, ow OutputRoomEventWriter, input api.InputInviteEvent, ) (err error) { diff --git a/roomserver/input/input.go b/roomserver/input/input.go index 3b519ecba..cb588380a 100644 --- a/roomserver/input/input.go +++ b/roomserver/input/input.go @@ -24,12 +24,13 @@ import ( "github.com/Shopify/sarama" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/util" ) // RoomserverInputAPI implements api.RoomserverInputAPI type RoomserverInputAPI struct { - DB RoomEventDatabase + DB storage.Database Producer sarama.SyncProducer // The kafkaesque topic to output new room events to. // This is the name used in kafka to identify the stream to write events to. diff --git a/roomserver/input/latest_events.go b/roomserver/input/latest_events.go index 525a6f518..cac3968d9 100644 --- a/roomserver/input/latest_events.go +++ b/roomserver/input/latest_events.go @@ -23,6 +23,7 @@ import ( "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/state" + "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -47,7 +48,7 @@ import ( // Can only be called once at a time func updateLatestEvents( ctx context.Context, - db RoomEventDatabase, + db storage.Database, ow OutputRoomEventWriter, roomNID types.RoomNID, stateAtEvent types.StateAtEvent, @@ -86,7 +87,7 @@ func updateLatestEvents( // when there are so many variables to pass around. type latestEventsUpdater struct { ctx context.Context - db RoomEventDatabase + db storage.Database updater types.RoomRecentEventsUpdater ow OutputRoomEventWriter roomNID types.RoomNID diff --git a/roomserver/input/membership.go b/roomserver/input/membership.go index ee39ff5eb..8629cb238 100644 --- a/roomserver/input/membership.go +++ b/roomserver/input/membership.go @@ -19,6 +19,7 @@ import ( "fmt" "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -29,7 +30,7 @@ import ( // consumers about the invites added or retired by the change in current state. func updateMemberships( ctx context.Context, - db RoomEventDatabase, + db storage.Database, updater types.RoomRecentEventsUpdater, removed, added []types.StateEntry, ) ([]api.OutputEvent, error) { diff --git a/roomserver/query/query.go b/roomserver/query/query.go index 224d9fa22..e9286b4e6 100644 --- a/roomserver/query/query.go +++ b/roomserver/query/query.go @@ -26,79 +26,16 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/auth" "github.com/matrix-org/dendrite/roomserver/state" - "github.com/matrix-org/dendrite/roomserver/state/database" + "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/roomserver/version" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) -// RoomserverQueryAPIEventDB has a convenience API to fetch events directly by -// EventIDs. -type RoomserverQueryAPIEventDB interface { - // Look up the Events for a list of event IDs. Does not error if event was - // not found. - // Returns an error if the retrieval went wrong. - EventsFromIDs(ctx context.Context, eventIDs []string) ([]types.Event, error) -} - -// RoomserverQueryAPIDatabase has the storage APIs needed to implement the query API. -type RoomserverQueryAPIDatabase interface { - database.RoomStateDatabase - RoomserverQueryAPIEventDB - // Look up the numeric ID for the room. - // Returns 0 if the room doesn't exists. - // Returns an error if there was a problem talking to the database. - RoomNID(ctx context.Context, roomID string) (types.RoomNID, error) - // Look up event references for the latest events in the room and the current state snapshot. - // Returns the latest events, the current state and the maximum depth of the latest events plus 1. - // Returns an error if there was a problem talking to the database. - LatestEventIDs( - ctx context.Context, roomNID types.RoomNID, - ) ([]gomatrixserverlib.EventReference, types.StateSnapshotNID, int64, error) - // Look up the numeric IDs for a list of events. - // Returns an error if there was a problem talking to the database. - EventNIDs(ctx context.Context, eventIDs []string) (map[string]types.EventNID, error) - // Lookup the event IDs for a batch of event numeric IDs. - // Returns an error if the retrieval went wrong. - EventIDs(ctx context.Context, eventNIDs []types.EventNID) (map[types.EventNID]string, error) - // Lookup the membership of a given user in a given room. - // Returns the numeric ID of the latest membership event sent from this user - // in this room, along a boolean set to true if the user is still in this room, - // false if not. - // Returns an error if there was a problem talking to the database. - GetMembership( - ctx context.Context, roomNID types.RoomNID, requestSenderUserID string, - ) (membershipEventNID types.EventNID, stillInRoom bool, err error) - // Lookup the membership event numeric IDs for all user that are or have - // been members of a given room. Only lookup events of "join" membership if - // joinOnly is set to true. - // Returns an error if there was a problem talking to the database. - GetMembershipEventNIDsForRoom( - ctx context.Context, roomNID types.RoomNID, joinOnly bool, - ) ([]types.EventNID, error) - // Look up the active invites targeting a user in a room and return the - // numeric state key IDs for the user IDs who sent them. - // Returns an error if there was a problem talking to the database. - GetInvitesForUser( - ctx context.Context, - roomNID types.RoomNID, - targetUserNID types.EventStateKeyNID, - ) (senderUserNIDs []types.EventStateKeyNID, err error) - // Look up the string event state keys for a list of numeric event state keys - // Returns an error if there was a problem talking to the database. - EventStateKeys( - context.Context, []types.EventStateKeyNID, - ) (map[types.EventStateKeyNID]string, error) - // Look up the room version for a given room. - GetRoomVersionForRoom( - ctx context.Context, roomID string, - ) (gomatrixserverlib.RoomVersion, error) -} - // RoomserverQueryAPI is an implementation of api.RoomserverQueryAPI type RoomserverQueryAPI struct { - DB RoomserverQueryAPIDatabase + DB storage.Database ImmutableCache caching.ImmutableCache } @@ -741,7 +678,7 @@ func (r *RoomserverQueryAPI) QueryStateAndAuthChain( } authEventIDs = util.UniqueStrings(authEventIDs) // de-dupe - authEvents, err := getAuthChain(ctx, r.DB, authEventIDs) + authEvents, err := getAuthChain(ctx, r.DB.EventsFromIDs, authEventIDs) if err != nil { return err } @@ -788,12 +725,14 @@ func (r *RoomserverQueryAPI) loadStateAtEventIDs(ctx context.Context, eventIDs [ return r.loadStateEvents(ctx, stateEntries) } +type eventsFromIDs func(context.Context, []string) ([]types.Event, error) + // getAuthChain fetches the auth chain for the given auth events. An auth chain // is the list of all events that are referenced in the auth_events section, and // all their auth_events, recursively. The returned set of events contain the // given events. Will *not* error if we don't have all auth events. func getAuthChain( - ctx context.Context, dB RoomserverQueryAPIEventDB, authEventIDs []string, + ctx context.Context, fn eventsFromIDs, authEventIDs []string, ) ([]gomatrixserverlib.Event, error) { // List of event IDs to fetch. On each pass, these events will be requested // from the database and the `eventsToFetch` will be updated with any new @@ -804,7 +743,7 @@ func getAuthChain( for len(eventsToFetch) > 0 { // Try to retrieve the events from the database. - events, err := dB.EventsFromIDs(ctx, eventsToFetch) + events, err := fn(ctx, eventsToFetch) if err != nil { return nil, err } diff --git a/roomserver/query/query_test.go b/roomserver/query/query_test.go index 7e040c6fb..8fb6a082e 100644 --- a/roomserver/query/query_test.go +++ b/roomserver/query/query_test.go @@ -106,7 +106,7 @@ func TestGetAuthChainSingle(t *testing.T) { t.Fatalf("Failed to add events to db: %v", err) } - result, err := getAuthChain(context.TODO(), db, []string{"e"}) + result, err := getAuthChain(context.TODO(), db.EventsFromIDs, []string{"e"}) if err != nil { t.Fatalf("getAuthChain failed: %v", err) } @@ -139,7 +139,7 @@ func TestGetAuthChainMultiple(t *testing.T) { t.Fatalf("Failed to add events to db: %v", err) } - result, err := getAuthChain(context.TODO(), db, []string{"e", "f"}) + result, err := getAuthChain(context.TODO(), db.EventsFromIDs, []string{"e", "f"}) if err != nil { t.Fatalf("getAuthChain failed: %v", err) } diff --git a/roomserver/state/database/database.go b/roomserver/state/database/database.go deleted file mode 100644 index 80f1b14f4..000000000 --- a/roomserver/state/database/database.go +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2017 Vector Creations Ltd -// Copyright 2018 New Vector Ltd -// Copyright 2019-2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package database - -import ( - "context" - - "github.com/matrix-org/dendrite/roomserver/types" - "github.com/matrix-org/gomatrixserverlib" -) - -// A RoomStateDatabase has the storage APIs needed to load state from the database -type RoomStateDatabase interface { - // Store the room state at an event in the database - AddState( - ctx context.Context, - roomNID types.RoomNID, - stateBlockNIDs []types.StateBlockNID, - state []types.StateEntry, - ) (types.StateSnapshotNID, error) - // Look up the state of a room at each event for a list of string event IDs. - // Returns an error if there is an error talking to the database - // Returns a types.MissingEventError if the room state for the event IDs aren't in the database - StateAtEventIDs(ctx context.Context, eventIDs []string) ([]types.StateAtEvent, error) - // Look up the numeric IDs for a list of string event types. - // Returns a map from string event type to numeric ID for the event type. - EventTypeNIDs(ctx context.Context, eventTypes []string) (map[string]types.EventTypeNID, error) - // Look up the numeric IDs for a list of string event state keys. - // Returns a map from string state key to numeric ID for the state key. - EventStateKeyNIDs(ctx context.Context, eventStateKeys []string) (map[string]types.EventStateKeyNID, error) - // Look up the numeric state data IDs for each numeric state snapshot ID - // The returned slice is sorted by numeric state snapshot ID. - StateBlockNIDs(ctx context.Context, stateNIDs []types.StateSnapshotNID) ([]types.StateBlockNIDList, error) - // Look up the state data for each numeric state data ID - // The returned slice is sorted by numeric state data ID. - StateEntries(ctx context.Context, stateBlockNIDs []types.StateBlockNID) ([]types.StateEntryList, error) - // Look up the state data for the state key tuples for each numeric state block ID - // This is used to fetch a subset of the room state at a snapshot. - // If a block doesn't contain any of the requested tuples then it can be discarded from the result. - // The returned slice is sorted by numeric state block ID. - StateEntriesForTuples( - ctx context.Context, - stateBlockNIDs []types.StateBlockNID, - stateKeyTuples []types.StateKeyTuple, - ) ([]types.StateEntryList, error) - // Look up the Events for a list of numeric event IDs. - // Returns a sorted list of events. - Events(ctx context.Context, eventNIDs []types.EventNID) ([]types.Event, error) - // Look up snapshot NID for an event ID string - SnapshotNIDFromEventID(ctx context.Context, eventID string) (types.StateSnapshotNID, error) - // Look up a room version from the room NID. - GetRoomVersionForRoomNID(ctx context.Context, roomNID types.RoomNID) (gomatrixserverlib.RoomVersion, error) -} diff --git a/roomserver/state/shared/shared.go b/roomserver/state/shared/shared.go deleted file mode 100644 index a29b5e403..000000000 --- a/roomserver/state/shared/shared.go +++ /dev/null @@ -1 +0,0 @@ -package shared diff --git a/roomserver/state/state.go b/roomserver/state/state.go index 3f68e0747..389c94400 100644 --- a/roomserver/state/state.go +++ b/roomserver/state/state.go @@ -22,7 +22,7 @@ import ( "sort" "time" - "github.com/matrix-org/dendrite/roomserver/state/database" + "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/util" "github.com/prometheus/client_golang/prometheus" @@ -31,10 +31,10 @@ import ( ) type StateResolution struct { - db database.RoomStateDatabase + db storage.Database } -func NewStateResolution(db database.RoomStateDatabase) StateResolution { +func NewStateResolution(db storage.Database) StateResolution { return StateResolution{ db: db, } diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go index 50369d806..0235e51ed 100644 --- a/roomserver/storage/interface.go +++ b/roomserver/storage/interface.go @@ -18,13 +18,50 @@ import ( "context" "github.com/matrix-org/dendrite/roomserver/api" - statedb "github.com/matrix-org/dendrite/roomserver/state/database" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" ) type Database interface { - statedb.RoomStateDatabase + // Store the room state at an event in the database + AddState( + ctx context.Context, + roomNID types.RoomNID, + stateBlockNIDs []types.StateBlockNID, + state []types.StateEntry, + ) (types.StateSnapshotNID, error) + // Look up the state of a room at each event for a list of string event IDs. + // Returns an error if there is an error talking to the database + // Returns a types.MissingEventError if the room state for the event IDs aren't in the database + StateAtEventIDs(ctx context.Context, eventIDs []string) ([]types.StateAtEvent, error) + // Look up the numeric IDs for a list of string event types. + // Returns a map from string event type to numeric ID for the event type. + EventTypeNIDs(ctx context.Context, eventTypes []string) (map[string]types.EventTypeNID, error) + // Look up the numeric IDs for a list of string event state keys. + // Returns a map from string state key to numeric ID for the state key. + EventStateKeyNIDs(ctx context.Context, eventStateKeys []string) (map[string]types.EventStateKeyNID, error) + // Look up the numeric state data IDs for each numeric state snapshot ID + // The returned slice is sorted by numeric state snapshot ID. + StateBlockNIDs(ctx context.Context, stateNIDs []types.StateSnapshotNID) ([]types.StateBlockNIDList, error) + // Look up the state data for each numeric state data ID + // The returned slice is sorted by numeric state data ID. + StateEntries(ctx context.Context, stateBlockNIDs []types.StateBlockNID) ([]types.StateEntryList, error) + // Look up the state data for the state key tuples for each numeric state block ID + // This is used to fetch a subset of the room state at a snapshot. + // If a block doesn't contain any of the requested tuples then it can be discarded from the result. + // The returned slice is sorted by numeric state block ID. + StateEntriesForTuples( + ctx context.Context, + stateBlockNIDs []types.StateBlockNID, + stateKeyTuples []types.StateKeyTuple, + ) ([]types.StateEntryList, error) + // Look up the Events for a list of numeric event IDs. + // Returns a sorted list of events. + Events(ctx context.Context, eventNIDs []types.EventNID) ([]types.Event, error) + // Look up snapshot NID for an event ID string + SnapshotNIDFromEventID(ctx context.Context, eventID string) (types.StateSnapshotNID, error) + // Look up a room version from the room NID. + GetRoomVersionForRoomNID(ctx context.Context, roomNID types.RoomNID) (gomatrixserverlib.RoomVersion, error) StoreEvent(ctx context.Context, event gomatrixserverlib.Event, txnAndSessionID *api.TransactionID, authEventNIDs []types.EventNID) (types.RoomNID, types.StateAtEvent, error) StateEntriesForEventIDs(ctx context.Context, eventIDs []string) ([]types.StateEntry, error) EventStateKeys(ctx context.Context, eventStateKeyNIDs []types.EventStateKeyNID) (map[types.EventStateKeyNID]string, error) From be558f02aa956cfff51c5dfcf57575b46254b0db Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 24 Apr 2020 10:38:21 +0100 Subject: [PATCH 004/200] Add new sytests to list --- are-we-synapse-yet.list | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/are-we-synapse-yet.list b/are-we-synapse-yet.list index 5a900b3ef..71b05d1c4 100644 --- a/are-we-synapse-yet.list +++ b/are-we-synapse-yet.list @@ -28,6 +28,10 @@ log POST /login returns the same device_id as that in the request log POST /login can log in as a user with just the local part of the id log POST /login as non-existing user is rejected log POST /login wrong password is rejected +log Interactive authentication types include SSO +log Can perform interactive authentication with SSO +log The user must be consistent through an interactive authentication session with SSO +log The operation must be consistent through an interactive authentication session v1s GET /events initially v1s GET /initialSync initially csa Version responds 200 OK with valid structure @@ -44,6 +48,7 @@ dev DELETE /device/{deviceId} dev DELETE /device/{deviceId} requires UI auth user to match device owner dev DELETE /device/{deviceId} with no body gives a 401 dev The deleted device must be consistent through an interactive auth session +dev Users receive device_list updates for their own devices pre GET /presence/:user_id/status fetches initial status pre PUT /presence/:user_id/status updates my presence crm POST /createRoom makes a public room @@ -827,4 +832,4 @@ syn Multiple calls to /sync should not cause 500 errors gst Guest user can call /events on another world_readable room (SYN-606) gst Real user can call /events on another world_readable room (SYN-606) gst Events come down the correct room -pub Asking for a remote rooms list, but supplying the local server's name, returns the local rooms list \ No newline at end of file +pub Asking for a remote rooms list, but supplying the local server's name, returns the local rooms list From 3ab8ebf6b8fbc813bfb3e0e0735e76a69a8ed2dd Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 24 Apr 2020 16:30:25 +0100 Subject: [PATCH 005/200] More invite support (#979) * Update gomatixserverlib * Try to build invite stripped state if not given to us * SendInvite improvements * Transpose invite_room_state into invite_state.events for sync API * Remove syncapi debugging output * Use RespInviteV2 * Update gomatrixserverlib * Send the invite event as a normal roomserver event too, for incorporating into room (should this be done by the roomserver automatically for invite inputs?) * Federation sender use invite_room_state, room server try to insert membership state * Check supported room versions on the invite endpoint * Prevent roomserver query API from trying to handle requests for stub rooms * Adding a nolint * Replace IsRoomStub with RoomNIDExcludingStubs, fix query API to use that instead * Review comments --- clientapi/producers/roomserver.go | 3 ++ clientapi/routing/membership.go | 42 ++++++++++----- federationapi/routing/invite.go | 16 +++++- federationsender/consumers/roomserver.go | 48 +++-------------- go.mod | 3 +- go.sum | 4 +- roomserver/api/input.go | 2 + roomserver/input/events.go | 66 +++++++++++++++++++++++- roomserver/input/membership.go | 2 +- roomserver/query/query.go | 6 +-- roomserver/storage/interface.go | 4 ++ roomserver/storage/postgres/storage.go | 17 ++++++ roomserver/storage/sqlite3/storage.go | 17 ++++++ syncapi/storage/postgres/syncserver.go | 6 +-- syncapi/storage/sqlite3/syncserver.go | 6 +-- syncapi/types/types.go | 10 ++-- 16 files changed, 175 insertions(+), 77 deletions(-) diff --git a/clientapi/producers/roomserver.go b/clientapi/producers/roomserver.go index 391ea07bf..fac1e3c7c 100644 --- a/clientapi/producers/roomserver.go +++ b/clientapi/producers/roomserver.go @@ -106,12 +106,15 @@ func (c *RoomserverProducer) SendInputRoomEvents( func (c *RoomserverProducer) SendInvite( ctx context.Context, inviteEvent gomatrixserverlib.HeaderedEvent, inviteRoomState []gomatrixserverlib.InviteV2StrippedState, + sendAsServer gomatrixserverlib.ServerName, txnID *api.TransactionID, ) error { request := api.InputRoomEventsRequest{ InputInviteEvents: []api.InputInviteEvent{{ Event: inviteEvent, InviteRoomState: inviteRoomState, RoomVersion: inviteEvent.RoomVersion, + SendAsServer: string(sendAsServer), + TransactionID: txnID, }}, } var response api.InputRoomEventsResponse diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index 9f386b718..c597dd27d 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -40,6 +40,8 @@ var errMissingUserID = errors.New("'user_id' must be supplied") // SendMembership implements PUT /rooms/{roomID}/(join|kick|ban|unban|leave|invite) // by building a m.room.member event then sending it to the room server +// TODO: Can we improve the cyclo count here? Separate code paths for invites? +// nolint:gocyclo func SendMembership( req *http.Request, accountDB accounts.Database, device *authtypes.Device, roomID string, membership string, cfg *config.Dendrite, @@ -104,23 +106,39 @@ func SendMembership( return jsonerror.InternalServerError() } - if _, err := producer.SendEvents( - req.Context(), - []gomatrixserverlib.HeaderedEvent{(*event).Headered(verRes.RoomVersion)}, - cfg.Matrix.ServerName, - nil, - ); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("producer.SendEvents failed") - return jsonerror.InternalServerError() - } - var returnData interface{} = struct{}{} - // The join membership requires the room id to be sent in the response - if membership == gomatrixserverlib.Join { + switch membership { + case gomatrixserverlib.Invite: + // Invites need to be handled specially + err = producer.SendInvite( + req.Context(), + event.Headered(verRes.RoomVersion), + nil, // ask the roomserver to draw up invite room state for us + cfg.Matrix.ServerName, + nil, + ) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("producer.SendInvite failed") + return jsonerror.InternalServerError() + } + case gomatrixserverlib.Join: + // The join membership requires the room id to be sent in the response returnData = struct { RoomID string `json:"room_id"` }{roomID} + default: + } + + _, err = producer.SendEvents( + req.Context(), + []gomatrixserverlib.HeaderedEvent{event.Headered(verRes.RoomVersion)}, + cfg.Matrix.ServerName, + nil, + ) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("producer.SendEvents failed") + return jsonerror.InternalServerError() } return util.JSONResponse{ diff --git a/federationapi/routing/invite.go b/federationapi/routing/invite.go index 4b367e004..064abe7e9 100644 --- a/federationapi/routing/invite.go +++ b/federationapi/routing/invite.go @@ -16,11 +16,13 @@ package routing import ( "encoding/json" + "fmt" "net/http" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/common/config" + roomserverVersion "github.com/matrix-org/dendrite/roomserver/version" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -44,6 +46,16 @@ func Invite( } event := inviteReq.Event() + // Check that we can accept invites for this room version. + if _, err := roomserverVersion.SupportedRoomVersion(inviteReq.RoomVersion()); err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.UnsupportedRoomVersion( + fmt.Sprintf("Room version %q is not supported by this server.", inviteReq.RoomVersion()), + ), + } + } + // Check that the room ID is correct. if event.RoomID() != roomID { return util.JSONResponse{ @@ -90,6 +102,8 @@ func Invite( httpReq.Context(), signedEvent.Headered(inviteReq.RoomVersion()), inviteReq.InviteRoomState(), + event.Origin(), + nil, ); err != nil { util.GetLogger(httpReq.Context()).WithError(err).Error("producer.SendInvite failed") return jsonerror.InternalServerError() @@ -99,6 +113,6 @@ func Invite( // the other servers in the room that we have been invited. return util.JSONResponse{ Code: http.StatusOK, - JSON: gomatrixserverlib.RespInvite{Event: signedEvent}, + JSON: gomatrixserverlib.RespInviteV2{Event: signedEvent}, } } diff --git a/federationsender/consumers/roomserver.go b/federationsender/consumers/roomserver.go index a36fb3792..18c8324b4 100644 --- a/federationsender/consumers/roomserver.go +++ b/federationsender/consumers/roomserver.go @@ -28,6 +28,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" log "github.com/sirupsen/logrus" + "github.com/tidwall/gjson" ) // OutputRoomEventConsumer consumes events that originated in the room server. @@ -187,49 +188,12 @@ func (s *OutputRoomEventConsumer) processInvite(oie api.OutputNewInviteEvent) er return nil } - // When sending a v2 invite, the inviting server should try and include - // a "stripped down" version of the room state. This is pretty much just - // enough information for the remote side to show something useful to the - // user, like the room name, aliases etc. + // Try to extract the room invite state. The roomserver will have stashed + // this for us in invite_room_state if it didn't already exist. strippedState := []gomatrixserverlib.InviteV2StrippedState{} - stateWanted := []string{ - gomatrixserverlib.MRoomName, gomatrixserverlib.MRoomCanonicalAlias, - gomatrixserverlib.MRoomAliases, gomatrixserverlib.MRoomJoinRules, - } - - // For each of the state keys that we want to try and send, ask the - // roomserver if we have a state event for that room that matches the - // state key. - for _, wanted := range stateWanted { - queryReq := api.QueryLatestEventsAndStateRequest{ - RoomID: oie.Event.RoomID(), - StateToFetch: []gomatrixserverlib.StateKeyTuple{ - gomatrixserverlib.StateKeyTuple{ - EventType: wanted, - StateKey: "", - }, - }, - } - // If this fails then we just move onto the next event - we don't - // actually know at this point whether the room even has that type - // of state. - queryRes := api.QueryLatestEventsAndStateResponse{} - if err := s.query.QueryLatestEventsAndState(context.TODO(), &queryReq, &queryRes); err != nil { - log.WithFields(log.Fields{ - "room_id": queryReq.RoomID, - "event_type": wanted, - }).WithError(err).Info("couldn't find state to strip") - continue - } - // Append the stripped down copy of the state to our list. - for _, headeredEvent := range queryRes.StateEvents { - event := headeredEvent.Unwrap() - strippedState = append(strippedState, gomatrixserverlib.NewInviteV2StrippedState(&event)) - - log.WithFields(log.Fields{ - "room_id": queryReq.RoomID, - "event_type": event.Type(), - }).Info("adding stripped state") + if inviteRoomState := gjson.GetBytes(oie.Event.Unsigned(), "invite_room_state"); inviteRoomState.Exists() { + if err := json.Unmarshal([]byte(inviteRoomState.Raw), &strippedState); err != nil { + log.WithError(err).Warn("failed to extract invite_room_state from event unsigned") } } diff --git a/go.mod b/go.mod index daf232781..37a67fcdd 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f github.com/matrix-org/go-sqlite3-js v0.0.0-20200325174927-327088cdef10 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200422082552-d7b4202c47f3 + github.com/matrix-org/gomatrixserverlib v0.0.0-20200424101831-2f10e8068538 github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible @@ -27,6 +27,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.4.1 github.com/sirupsen/logrus v1.4.2 + github.com/tidwall/gjson v1.6.0 github.com/uber/jaeger-client-go v2.15.0+incompatible github.com/uber/jaeger-lib v1.5.0 go.uber.org/atomic v1.4.0 diff --git a/go.sum b/go.sum index c8f7e43fe..517a77b3d 100644 --- a/go.sum +++ b/go.sum @@ -367,8 +367,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:kmRjpmFOenVpOaV/DRlo9p6z/IbOKlUC+hhKsAAh8Qg= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200422082552-d7b4202c47f3 h1:xis1ojN99vjygwqudzB9VQq3cM2SJ7aCAMlXj/YN+88= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200422082552-d7b4202c47f3/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200424101831-2f10e8068538 h1:kj2LdNOdg2+vydS9HrPdbECEVeusRg9VTSOkYm61reA= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200424101831-2f10e8068538/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y= diff --git a/roomserver/api/input.go b/roomserver/api/input.go index 87e3983e3..bb4e040de 100644 --- a/roomserver/api/input.go +++ b/roomserver/api/input.go @@ -89,6 +89,8 @@ type InputInviteEvent struct { RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"` Event gomatrixserverlib.HeaderedEvent `json:"event"` InviteRoomState []gomatrixserverlib.InviteV2StrippedState `json:"invite_room_state"` + SendAsServer string `json:"send_as_server"` + TransactionID *TransactionID `json:"transaction_id"` } // InputRoomEventsRequest is a request to InputRoomEvents diff --git a/roomserver/input/events.go b/roomserver/input/events.go index 393c1f419..205035d98 100644 --- a/roomserver/input/events.go +++ b/roomserver/input/events.go @@ -196,8 +196,23 @@ func processInviteEvent( event := input.Event.Unwrap() - if err = event.SetUnsignedField("invite_room_state", input.InviteRoomState); err != nil { - return err + if len(input.InviteRoomState) > 0 { + // If we were supplied with some invite room state already (which is + // most likely to be if the event came in over federation) then use + // that. + if err = event.SetUnsignedField("invite_room_state", input.InviteRoomState); err != nil { + return err + } + } else { + // There's no invite room state, so let's have a go at building it + // up from local data (which is most likely to be if the event came + // from the CS API). If we know about the room then we can insert + // the invite room state, if we don't then we just fail quietly. + if irs, ierr := buildInviteStrippedState(ctx, db, input); ierr == nil { + if err = event.SetUnsignedField("invite_room_state", irs); err != nil { + return err + } + } } outputUpdates, err := updateToInviteMembership(updater, &event, nil, input.Event.RoomVersion) @@ -212,3 +227,50 @@ func processInviteEvent( succeeded = true return nil } + +func buildInviteStrippedState( + ctx context.Context, + db storage.Database, + input api.InputInviteEvent, +) ([]gomatrixserverlib.InviteV2StrippedState, error) { + roomNID, err := db.RoomNID(ctx, input.Event.RoomID()) + if err != nil || roomNID == 0 { + return nil, fmt.Errorf("room %q unknown", input.Event.RoomID()) + } + stateWanted := []gomatrixserverlib.StateKeyTuple{} + for _, t := range []string{ + gomatrixserverlib.MRoomName, gomatrixserverlib.MRoomCanonicalAlias, + gomatrixserverlib.MRoomAliases, gomatrixserverlib.MRoomJoinRules, + } { + stateWanted = append(stateWanted, gomatrixserverlib.StateKeyTuple{ + EventType: t, + StateKey: "", + }) + } + _, currentStateSnapshotNID, _, err := db.LatestEventIDs(ctx, roomNID) + if err != nil { + return nil, err + } + roomState := state.NewStateResolution(db) + stateEntries, err := roomState.LoadStateAtSnapshotForStringTuples( + ctx, currentStateSnapshotNID, stateWanted, + ) + if err != nil { + return nil, err + } + stateNIDs := []types.EventNID{} + for _, stateNID := range stateEntries { + stateNIDs = append(stateNIDs, stateNID.EventNID) + } + stateEvents, err := db.Events(ctx, stateNIDs) + if err != nil { + return nil, err + } + inviteState := []gomatrixserverlib.InviteV2StrippedState{ + gomatrixserverlib.NewInviteV2StrippedState(&input.Event.Event), + } + for _, event := range stateEvents { + inviteState = append(inviteState, gomatrixserverlib.NewInviteV2StrippedState(&event.Event)) + } + return inviteState, nil +} diff --git a/roomserver/input/membership.go b/roomserver/input/membership.go index 8629cb238..351e63d61 100644 --- a/roomserver/input/membership.go +++ b/roomserver/input/membership.go @@ -144,7 +144,7 @@ func updateToInviteMembership( // consider a single stream of events when determining whether a user // is invited, rather than having to combine multiple streams themselves. onie := api.OutputNewInviteEvent{ - Event: (*add).Headered(roomVersion), + Event: add.Headered(roomVersion), RoomVersion: roomVersion, } updates = append(updates, api.OutputEvent{ diff --git a/roomserver/query/query.go b/roomserver/query/query.go index e9286b4e6..7508d7902 100644 --- a/roomserver/query/query.go +++ b/roomserver/query/query.go @@ -54,7 +54,7 @@ func (r *RoomserverQueryAPI) QueryLatestEventsAndState( roomState := state.NewStateResolution(r.DB) response.QueryLatestEventsAndStateRequest = *request - roomNID, err := r.DB.RoomNID(ctx, request.RoomID) + roomNID, err := r.DB.RoomNIDExcludingStubs(ctx, request.RoomID) if err != nil { return err } @@ -114,7 +114,7 @@ func (r *RoomserverQueryAPI) QueryStateAfterEvents( roomState := state.NewStateResolution(r.DB) response.QueryStateAfterEventsRequest = *request - roomNID, err := r.DB.RoomNID(ctx, request.RoomID) + roomNID, err := r.DB.RoomNIDExcludingStubs(ctx, request.RoomID) if err != nil { return err } @@ -649,7 +649,7 @@ func (r *RoomserverQueryAPI) QueryStateAndAuthChain( response *api.QueryStateAndAuthChainResponse, ) error { response.QueryStateAndAuthChainRequest = *request - roomNID, err := r.DB.RoomNID(ctx, request.RoomID) + roomNID, err := r.DB.RoomNIDExcludingStubs(ctx, request.RoomID) if err != nil { return err } diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go index 0235e51ed..a13c44d6e 100644 --- a/roomserver/storage/interface.go +++ b/roomserver/storage/interface.go @@ -71,6 +71,10 @@ type Database interface { GetLatestEventsForUpdate(ctx context.Context, roomNID types.RoomNID) (types.RoomRecentEventsUpdater, error) GetTransactionEventID(ctx context.Context, transactionID string, sessionID int64, userID string) (string, error) RoomNID(ctx context.Context, roomID string) (types.RoomNID, error) + // RoomNIDExcludingStubs is a special variation of RoomNID that will return 0 as if the room + // does not exist if the room has no latest events. This can happen when we've received an + // invite over federation for a room that we don't know anything else about yet. + RoomNIDExcludingStubs(ctx context.Context, roomID string) (types.RoomNID, error) LatestEventIDs(ctx context.Context, roomNID types.RoomNID) ([]gomatrixserverlib.EventReference, types.StateSnapshotNID, int64, error) GetInvitesForUser(ctx context.Context, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID) (senderUserIDs []types.EventStateKeyNID, err error) SetRoomAlias(ctx context.Context, alias string, roomID string, creatorUserID string) error diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go index 6f2b96610..5b5c61b0f 100644 --- a/roomserver/storage/postgres/storage.go +++ b/roomserver/storage/postgres/storage.go @@ -471,6 +471,23 @@ func (d *Database) RoomNID(ctx context.Context, roomID string) (types.RoomNID, e return roomNID, err } +// RoomNIDExcludingStubs implements query.RoomserverQueryAPIDB +func (d *Database) RoomNIDExcludingStubs(ctx context.Context, roomID string) (roomNID types.RoomNID, err error) { + roomNID, err = d.RoomNID(ctx, roomID) + if err != nil { + return + } + latestEvents, _, err := d.statements.selectLatestEventNIDs(ctx, roomNID) + if err != nil { + return + } + if len(latestEvents) == 0 { + roomNID = 0 + return + } + return +} + // LatestEventIDs implements query.RoomserverQueryAPIDatabase func (d *Database) LatestEventIDs( ctx context.Context, roomNID types.RoomNID, diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index 444a8fdd5..5df9c4e08 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -590,6 +590,23 @@ func (d *Database) RoomNID(ctx context.Context, roomID string) (roomNID types.Ro return } +// RoomNIDExcludingStubs implements query.RoomserverQueryAPIDB +func (d *Database) RoomNIDExcludingStubs(ctx context.Context, roomID string) (roomNID types.RoomNID, err error) { + roomNID, err = d.RoomNID(ctx, roomID) + if err != nil { + return + } + latestEvents, _, err := d.statements.selectLatestEventNIDs(ctx, nil, roomNID) + if err != nil { + return + } + if len(latestEvents) == 0 { + roomNID = 0 + return + } + return +} + // LatestEventIDs implements query.RoomserverQueryAPIDatabase func (d *Database) LatestEventIDs( ctx context.Context, roomNID types.RoomNID, diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go index 7fd75f066..1e078ef46 100644 --- a/syncapi/storage/postgres/syncserver.go +++ b/syncapi/storage/postgres/syncserver.go @@ -752,11 +752,7 @@ func (d *SyncServerDatasource) addInvitesToResponse( return err } for roomID, inviteEvent := range invites { - ir := types.NewInviteResponse() - ir.InviteState.Events = gomatrixserverlib.ToClientEvents( - []gomatrixserverlib.Event{inviteEvent.Event}, gomatrixserverlib.FormatSync, - ) - // TODO: add the invite state from the invite event. + ir := types.NewInviteResponse(inviteEvent) res.Rooms.Invite[roomID] = *ir } return nil diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index 29051cd06..cdfd29b84 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -799,11 +799,7 @@ func (d *SyncServerDatasource) addInvitesToResponse( return err } for roomID, inviteEvent := range invites { - ir := types.NewInviteResponse() - ir.InviteState.Events = gomatrixserverlib.HeaderedToClientEvents( - []gomatrixserverlib.HeaderedEvent{inviteEvent}, gomatrixserverlib.FormatSync, - ) - // TODO: add the invite state from the invite event. + ir := types.NewInviteResponse(inviteEvent) res.Rooms.Invite[roomID] = *ir } return nil diff --git a/syncapi/types/types.go b/syncapi/types/types.go index 718906ecd..cfd49ff1f 100644 --- a/syncapi/types/types.go +++ b/syncapi/types/types.go @@ -23,6 +23,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" + "github.com/tidwall/gjson" ) var ( @@ -247,14 +248,17 @@ func NewJoinResponse() *JoinResponse { // InviteResponse represents a /sync response for a room which is under the 'invite' key. type InviteResponse struct { InviteState struct { - Events []gomatrixserverlib.ClientEvent `json:"events"` + Events json.RawMessage `json:"events"` } `json:"invite_state"` } // NewInviteResponse creates an empty response with initialised arrays. -func NewInviteResponse() *InviteResponse { +func NewInviteResponse(event gomatrixserverlib.HeaderedEvent) *InviteResponse { res := InviteResponse{} - res.InviteState.Events = make([]gomatrixserverlib.ClientEvent, 0) + res.InviteState.Events = json.RawMessage{'[', ']'} + if inviteRoomState := gjson.GetBytes(event.Unsigned(), "invite_room_state"); inviteRoomState.Exists() { + res.InviteState.Events = json.RawMessage(inviteRoomState.Raw) + } return &res } From 87f05721b07cc6d57bcb4ce1d9ad77a9e1847054 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 27 Apr 2020 15:47:36 +0100 Subject: [PATCH 006/200] Update gomatrixserverlib --- federationapi/routing/backfill.go | 5 ++++- go.mod | 2 +- go.sum | 4 ++-- syncapi/routing/messages.go | 5 ++++- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/federationapi/routing/backfill.go b/federationapi/routing/backfill.go index 62471b8a9..6f85ba4be 100644 --- a/federationapi/routing/backfill.go +++ b/federationapi/routing/backfill.go @@ -97,7 +97,10 @@ func Backfill( } var eventJSONs []json.RawMessage - for _, e := range gomatrixserverlib.ReverseTopologicalOrdering(evs) { + for _, e := range gomatrixserverlib.ReverseTopologicalOrdering( + evs, + gomatrixserverlib.TopologicalOrderByPrevEvents, + ) { eventJSONs = append(eventJSONs, e.JSON()) } diff --git a/go.mod b/go.mod index 37a67fcdd..fd1c2de81 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f github.com/matrix-org/go-sqlite3-js v0.0.0-20200325174927-327088cdef10 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200424101831-2f10e8068538 + github.com/matrix-org/gomatrixserverlib v0.0.0-20200427134702-21db6d1430e3 github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible diff --git a/go.sum b/go.sum index 517a77b3d..156f67252 100644 --- a/go.sum +++ b/go.sum @@ -367,8 +367,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:kmRjpmFOenVpOaV/DRlo9p6z/IbOKlUC+hhKsAAh8Qg= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200424101831-2f10e8068538 h1:kj2LdNOdg2+vydS9HrPdbECEVeusRg9VTSOkYm61reA= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200424101831-2f10e8068538/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200427134702-21db6d1430e3 h1:aJMAKjfXG5I8TqPxJQbQIkGSWM770oxkpgsPHE8C06E= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200427134702-21db6d1430e3/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y= diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index 873ee9366..53a9a963a 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -210,7 +210,10 @@ func (r *messagesReq) retrieveEvents() ( } // Sort the events to ensure we send them in the right order. - events = gomatrixserverlib.HeaderedReverseTopologicalOrdering(events) + events = gomatrixserverlib.HeaderedReverseTopologicalOrdering( + events, + gomatrixserverlib.TopologicalOrderByPrevEvents, + ) if r.backwardOrdering { // This reverses the array from old->new to new->old sort.SliceStable(events, func(i, j int) bool { From 3a858afca2368f588b2681de4f4816f26686f540 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 28 Apr 2020 10:53:07 +0100 Subject: [PATCH 007/200] Loopback event from invite response (#982) * Working invite v2 support * Fix copyright notice * Update gomatrixserverlib * Add fallthrough * Add missing continue * Update sytest-whitelist, gomatrixserverlib * Update gomatrixserverlib to test matrix-org/gomatrixserverlib#181 * Update gomatrixserverlib --- clientapi/routing/membership.go | 22 +++---- cmd/dendrite-demo-libp2p/main.go | 2 +- cmd/dendrite-federation-sender-server/main.go | 4 +- cmd/dendrite-monolith-server/main.go | 2 +- federationsender/federationsender.go | 6 +- federationsender/producers/roomserver.go | 66 +++++++++++++++++++ federationsender/queue/destinationqueue.go | 28 +++++++- federationsender/queue/queue.go | 22 +++++-- go.mod | 2 +- go.sum | 4 +- sytest-whitelist | 5 ++ 11 files changed, 135 insertions(+), 28 deletions(-) create mode 100644 federationsender/producers/roomserver.go diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index c597dd27d..dff194dd3 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -127,18 +127,18 @@ func SendMembership( returnData = struct { RoomID string `json:"room_id"` }{roomID} + fallthrough default: - } - - _, err = producer.SendEvents( - req.Context(), - []gomatrixserverlib.HeaderedEvent{event.Headered(verRes.RoomVersion)}, - cfg.Matrix.ServerName, - nil, - ) - if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("producer.SendEvents failed") - return jsonerror.InternalServerError() + _, err = producer.SendEvents( + req.Context(), + []gomatrixserverlib.HeaderedEvent{event.Headered(verRes.RoomVersion)}, + cfg.Matrix.ServerName, + nil, + ) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("producer.SendEvents failed") + return jsonerror.InternalServerError() + } } return util.JSONResponse{ diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index f280c7483..0365a6f27 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -153,7 +153,7 @@ func main() { asQuery := appservice.SetupAppServiceAPIComponent( &base.Base, accountDB, deviceDB, federation, alias, query, transactions.New(), ) - fedSenderAPI := federationsender.SetupFederationSenderComponent(&base.Base, federation, query) + fedSenderAPI := federationsender.SetupFederationSenderComponent(&base.Base, federation, query, input) clientapi.SetupClientAPIComponent( &base.Base, deviceDB, accountDB, diff --git a/cmd/dendrite-federation-sender-server/main.go b/cmd/dendrite-federation-sender-server/main.go index 71fc0b015..1593afaa5 100644 --- a/cmd/dendrite-federation-sender-server/main.go +++ b/cmd/dendrite-federation-sender-server/main.go @@ -26,10 +26,10 @@ func main() { federation := base.CreateFederationClient() - _, _, query := base.CreateHTTPRoomserverAPIs() + _, input, query := base.CreateHTTPRoomserverAPIs() federationsender.SetupFederationSenderComponent( - base, federation, query, + base, federation, query, input, ) base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationSender), string(base.Cfg.Listen.FederationSender)) diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 6b0d83ae1..70a59ed68 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -62,7 +62,7 @@ func main() { asQuery := appservice.SetupAppServiceAPIComponent( base, accountDB, deviceDB, federation, alias, query, transactions.New(), ) - fedSenderAPI := federationsender.SetupFederationSenderComponent(base, federation, query) + fedSenderAPI := federationsender.SetupFederationSenderComponent(base, federation, query, input) clientapi.SetupClientAPIComponent( base, deviceDB, accountDB, diff --git a/federationsender/federationsender.go b/federationsender/federationsender.go index a318d2099..a06caf402 100644 --- a/federationsender/federationsender.go +++ b/federationsender/federationsender.go @@ -20,6 +20,7 @@ import ( "github.com/matrix-org/dendrite/common/basecomponent" "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/federationsender/consumers" + "github.com/matrix-org/dendrite/federationsender/producers" "github.com/matrix-org/dendrite/federationsender/query" "github.com/matrix-org/dendrite/federationsender/queue" "github.com/matrix-org/dendrite/federationsender/storage" @@ -34,13 +35,16 @@ func SetupFederationSenderComponent( base *basecomponent.BaseDendrite, federation *gomatrixserverlib.FederationClient, rsQueryAPI roomserverAPI.RoomserverQueryAPI, + rsInputAPI roomserverAPI.RoomserverInputAPI, ) api.FederationSenderQueryAPI { federationSenderDB, err := storage.NewDatabase(string(base.Cfg.Database.FederationSender)) if err != nil { logrus.WithError(err).Panic("failed to connect to federation sender db") } - queues := queue.NewOutgoingQueues(base.Cfg.Matrix.ServerName, federation) + roomserverProducer := producers.NewRoomserverProducer(rsInputAPI, base.Cfg.Matrix.ServerName) + + queues := queue.NewOutgoingQueues(base.Cfg.Matrix.ServerName, federation, roomserverProducer) rsConsumer := consumers.NewOutputRoomEventConsumer( base.Cfg, base.KafkaConsumer, queues, diff --git a/federationsender/producers/roomserver.go b/federationsender/producers/roomserver.go new file mode 100644 index 000000000..0395f9628 --- /dev/null +++ b/federationsender/producers/roomserver.go @@ -0,0 +1,66 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package producers + +import ( + "context" + + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/gomatrixserverlib" +) + +// RoomserverProducer produces events for the roomserver to consume. +type RoomserverProducer struct { + InputAPI api.RoomserverInputAPI + serverName gomatrixserverlib.ServerName +} + +// NewRoomserverProducer creates a new RoomserverProducer +func NewRoomserverProducer( + inputAPI api.RoomserverInputAPI, serverName gomatrixserverlib.ServerName, +) *RoomserverProducer { + return &RoomserverProducer{ + InputAPI: inputAPI, + serverName: serverName, + } +} + +// SendInviteResponse drops an invite response back into the roomserver so that users +// already in the room will be notified of the new invite. The invite response is signed +// by the remote side. +func (c *RoomserverProducer) SendInviteResponse( + ctx context.Context, res gomatrixserverlib.RespInviteV2, roomVersion gomatrixserverlib.RoomVersion, +) (string, error) { + ev := res.Event.Headered(roomVersion) + ire := api.InputRoomEvent{ + Kind: api.KindNew, + Event: ev, + AuthEventIDs: ev.AuthEventIDs(), + SendAsServer: string(c.serverName), + TransactionID: nil, + } + return c.SendInputRoomEvents(ctx, []api.InputRoomEvent{ire}) +} + +// SendInputRoomEvents writes the given input room events to the roomserver input API. +func (c *RoomserverProducer) SendInputRoomEvents( + ctx context.Context, ires []api.InputRoomEvent, +) (eventID string, err error) { + request := api.InputRoomEventsRequest{InputRoomEvents: ires} + var response api.InputRoomEventsResponse + err = c.InputAPI.InputRoomEvents(ctx, &request, &response) + eventID = response.EventID + return +} diff --git a/federationsender/queue/destinationqueue.go b/federationsender/queue/destinationqueue.go index 7d4dc850b..89526fcfd 100644 --- a/federationsender/queue/destinationqueue.go +++ b/federationsender/queue/destinationqueue.go @@ -21,6 +21,7 @@ import ( "sync" "time" + "github.com/matrix-org/dendrite/federationsender/producers" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" log "github.com/sirupsen/logrus" @@ -32,6 +33,7 @@ import ( // ensures that only one request is in flight to a given destination // at a time. type destinationQueue struct { + rsProducer *producers.RoomserverProducer client *gomatrixserverlib.FederationClient origin gomatrixserverlib.ServerName destination gomatrixserverlib.ServerName @@ -165,18 +167,38 @@ func (oq *destinationQueue) nextInvites() bool { } for _, inviteReq := range oq.pendingInvites { - ev := inviteReq.Event() + ev, roomVersion := inviteReq.Event(), inviteReq.RoomVersion() - if _, err := oq.client.SendInviteV2( + log.WithFields(log.Fields{ + "event_id": ev.EventID(), + "room_version": roomVersion, + "destination": oq.destination, + }).Info("sending invite") + + inviteRes, err := oq.client.SendInviteV2( context.TODO(), oq.destination, *inviteReq, - ); err != nil { + ) + if err != nil { log.WithFields(log.Fields{ "event_id": ev.EventID(), "state_key": ev.StateKey(), "destination": oq.destination, }).WithError(err).Error("failed to send invite") + continue + } + + if _, err = oq.rsProducer.SendInviteResponse( + context.TODO(), + inviteRes, + roomVersion, + ); err != nil { + log.WithFields(log.Fields{ + "event_id": ev.EventID(), + "state_key": ev.StateKey(), + "destination": oq.destination, + }).WithError(err).Error("failed to return signed invite to roomserver") } } diff --git a/federationsender/queue/queue.go b/federationsender/queue/queue.go index 88d47f120..33abc8fdd 100644 --- a/federationsender/queue/queue.go +++ b/federationsender/queue/queue.go @@ -18,6 +18,7 @@ import ( "fmt" "sync" + "github.com/matrix-org/dendrite/federationsender/producers" "github.com/matrix-org/gomatrixserverlib" log "github.com/sirupsen/logrus" ) @@ -25,19 +26,25 @@ import ( // OutgoingQueues is a collection of queues for sending transactions to other // matrix servers type OutgoingQueues struct { - origin gomatrixserverlib.ServerName - client *gomatrixserverlib.FederationClient + rsProducer *producers.RoomserverProducer + origin gomatrixserverlib.ServerName + client *gomatrixserverlib.FederationClient // The queuesMutex protects queues queuesMutex sync.Mutex queues map[gomatrixserverlib.ServerName]*destinationQueue } // NewOutgoingQueues makes a new OutgoingQueues -func NewOutgoingQueues(origin gomatrixserverlib.ServerName, client *gomatrixserverlib.FederationClient) *OutgoingQueues { +func NewOutgoingQueues( + origin gomatrixserverlib.ServerName, + client *gomatrixserverlib.FederationClient, + rsProducer *producers.RoomserverProducer, +) *OutgoingQueues { return &OutgoingQueues{ - origin: origin, - client: client, - queues: map[gomatrixserverlib.ServerName]*destinationQueue{}, + rsProducer: rsProducer, + origin: origin, + client: client, + queues: map[gomatrixserverlib.ServerName]*destinationQueue{}, } } @@ -67,6 +74,7 @@ func (oqs *OutgoingQueues) SendEvent( oq := oqs.queues[destination] if oq == nil { oq = &destinationQueue{ + rsProducer: oqs.rsProducer, origin: oqs.origin, destination: destination, client: oqs.client, @@ -111,6 +119,7 @@ func (oqs *OutgoingQueues) SendInvite( oq := oqs.queues[destination] if oq == nil { oq = &destinationQueue{ + rsProducer: oqs.rsProducer, origin: oqs.origin, destination: destination, client: oqs.client, @@ -151,6 +160,7 @@ func (oqs *OutgoingQueues) SendEDU( oq := oqs.queues[destination] if oq == nil { oq = &destinationQueue{ + rsProducer: oqs.rsProducer, origin: oqs.origin, destination: destination, client: oqs.client, diff --git a/go.mod b/go.mod index fd1c2de81..f566e6d9a 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f github.com/matrix-org/go-sqlite3-js v0.0.0-20200325174927-327088cdef10 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200427134702-21db6d1430e3 + github.com/matrix-org/gomatrixserverlib v0.0.0-20200428095012-a95e289995b1 github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible diff --git a/go.sum b/go.sum index 156f67252..535a999d3 100644 --- a/go.sum +++ b/go.sum @@ -367,8 +367,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:kmRjpmFOenVpOaV/DRlo9p6z/IbOKlUC+hhKsAAh8Qg= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200427134702-21db6d1430e3 h1:aJMAKjfXG5I8TqPxJQbQIkGSWM770oxkpgsPHE8C06E= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200427134702-21db6d1430e3/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200428095012-a95e289995b1 h1:TB4V69eOtvmHdFp0+BgLNrDCcCwq6QDUOTjmi8fjC/M= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200428095012-a95e289995b1/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y= diff --git a/sytest-whitelist b/sytest-whitelist index 7bd2a63c4..439c306c7 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -253,3 +253,8 @@ User can invite local user to room with version 3 User can invite local user to room with version 4 A pair of servers can establish a join in a v2 room Can logout all devices +State from remote users is included in the timeline in an incremental sync +User can invite remote user to room with version 1 +User can invite remote user to room with version 2 +User can invite remote user to room with version 3 +User can invite remote user to room with version 4 From 6d832ae544a6221eb01dc7bad170d3b25a534a1e Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 28 Apr 2020 11:46:47 +0100 Subject: [PATCH 008/200] Implement backfill in the roomserver (#983) * Initial cut for backfilling The syncserver now asks the roomserver via QueryBackfill (which already existed to *handle* backfill requests) which then makes federation requests via gomatrixserverlib.RequestBackfill. Currently, tests fail on subsequent /messages requests because we don't know which servers are in the room, because we are unable to get state snapshots from a backfilled event because that code doesn't exist yet. * WIP backfill, doesn't work * Make initial backfill pass checks * Persist backfilled events with state snapshots * Remove debug lines * Linting * Review comments --- cmd/dendrite-demo-libp2p/main.go | 2 +- cmd/dendrite-monolith-server/main.go | 2 +- cmd/dendrite-room-server/main.go | 6 +- federationapi/routing/backfill.go | 1 + go.mod | 1 + go.sum | 11 ++ roomserver/api/query.go | 46 +----- roomserver/input/events.go | 8 +- roomserver/query/backfill.go | 217 +++++++++++++++++++++++++++ roomserver/query/query.go | 147 +++++++++++------- roomserver/roomserver.go | 8 +- roomserver/storage/interface.go | 3 +- syncapi/routing/messages.go | 108 +++---------- 13 files changed, 367 insertions(+), 193 deletions(-) create mode 100644 roomserver/query/backfill.go diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index 0365a6f27..f1d18d0e0 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -148,7 +148,7 @@ func main() { federation := createFederationClient(base) keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) - alias, input, query := roomserver.SetupRoomServerComponent(&base.Base) + alias, input, query := roomserver.SetupRoomServerComponent(&base.Base, keyRing) eduInputAPI := eduserver.SetupEDUServerComponent(&base.Base, cache.New()) asQuery := appservice.SetupAppServiceAPIComponent( &base.Base, accountDB, deviceDB, federation, alias, query, transactions.New(), diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 70a59ed68..8cfb75665 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -57,7 +57,7 @@ func main() { federation := base.CreateFederationClient() keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) - alias, input, query := roomserver.SetupRoomServerComponent(base) + alias, input, query := roomserver.SetupRoomServerComponent(base, keyRing) eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New()) asQuery := appservice.SetupAppServiceAPIComponent( base, accountDB, deviceDB, federation, alias, query, transactions.New(), diff --git a/cmd/dendrite-room-server/main.go b/cmd/dendrite-room-server/main.go index 41b705755..f33a2b88f 100644 --- a/cmd/dendrite-room-server/main.go +++ b/cmd/dendrite-room-server/main.go @@ -18,6 +18,7 @@ import ( _ "net/http/pprof" "github.com/matrix-org/dendrite/common/basecomponent" + "github.com/matrix-org/dendrite/common/keydb" "github.com/matrix-org/dendrite/roomserver" ) @@ -25,8 +26,11 @@ func main() { cfg := basecomponent.ParseFlags() base := basecomponent.NewBaseDendrite(cfg, "RoomServerAPI") defer base.Close() // nolint: errcheck + keyDB := base.CreateKeyDB() + federation := base.CreateFederationClient() + keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) - roomserver.SetupRoomServerComponent(base) + roomserver.SetupRoomServerComponent(base, keyRing) base.SetupAndServeHTTP(string(base.Cfg.Bind.RoomServer), string(base.Cfg.Listen.RoomServer)) diff --git a/federationapi/routing/backfill.go b/federationapi/routing/backfill.go index 6f85ba4be..6f49b9a81 100644 --- a/federationapi/routing/backfill.go +++ b/federationapi/routing/backfill.go @@ -69,6 +69,7 @@ func Backfill( // Populate the request. req := api.QueryBackfillRequest{ + RoomID: roomID, EarliestEventsIDs: eIDs, ServerName: request.Origin(), } diff --git a/go.mod b/go.mod index f566e6d9a..73171d89c 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,7 @@ require ( github.com/opentracing/opentracing-go v1.1.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.4.1 + github.com/prometheus/common v0.9.1 github.com/sirupsen/logrus v1.4.2 github.com/tidwall/gjson v1.6.0 github.com/uber/jaeger-client-go v2.15.0+incompatible diff --git a/go.sum b/go.sum index 535a999d3..9c7213126 100644 --- a/go.sum +++ b/go.sum @@ -367,6 +367,16 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:kmRjpmFOenVpOaV/DRlo9p6z/IbOKlUC+hhKsAAh8Qg= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200422082552-d7b4202c47f3 h1:xis1ojN99vjygwqudzB9VQq3cM2SJ7aCAMlXj/YN+88= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200422082552-d7b4202c47f3/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200423090438-562549dbe799 h1:OsoUMTirIpeuZJdYkKKiYe6jm0E5viZR7aOS9K465QI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200423090438-562549dbe799/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200424154222-2827b39252bd h1:243fMfK0XqTQsdUY3IIqtxPX5g9MfPTaAP92CseqOek= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200424154222-2827b39252bd/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200427134702-21db6d1430e3 h1:aJMAKjfXG5I8TqPxJQbQIkGSWM770oxkpgsPHE8C06E= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200427134702-21db6d1430e3/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200427152419-6a0535cc473a h1:tlXCVU3eab9kksGYBRA3oyrmIRwD/aPujo5KJCdlCVQ= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200427152419-6a0535cc473a/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/gomatrixserverlib v0.0.0-20200428095012-a95e289995b1 h1:TB4V69eOtvmHdFp0+BgLNrDCcCwq6QDUOTjmi8fjC/M= github.com/matrix-org/gomatrixserverlib v0.0.0-20200428095012-a95e289995b1/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= @@ -677,6 +687,7 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= gopkg.in/Shopify/sarama.v1 v1.20.1 h1:Gi09A3fJXm0Jgt8kuKZ8YK+r60GfYn7MQuEmI3oq6hE= gopkg.in/Shopify/sarama.v1 v1.20.1/go.mod h1:AxnvoaevB2nBjNK17cG61A3LleFcWFwVBHBt+cot4Oc= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/roomserver/api/query.go b/roomserver/api/query.go index b272b1ebd..11fa5c9ca 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -229,6 +229,8 @@ type QueryStateAndAuthChainResponse struct { // QueryBackfillRequest is a request to QueryBackfill. type QueryBackfillRequest struct { + // The room to backfill + RoomID string `json:"room_id"` // Events to start paginating from. EarliestEventsIDs []string `json:"earliest_event_ids"` // The maximum number of events to retrieve. @@ -243,21 +245,7 @@ type QueryBackfillResponse struct { Events []gomatrixserverlib.HeaderedEvent `json:"events"` } -// QueryServersInRoomAtEventRequest is a request to QueryServersInRoomAtEvent -type QueryServersInRoomAtEventRequest struct { - // ID of the room to retrieve member servers for. - RoomID string `json:"room_id"` - // ID of the event for which to retrieve member servers. - EventID string `json:"event_id"` -} - -// QueryServersInRoomAtEventResponse is a response to QueryServersInRoomAtEvent -type QueryServersInRoomAtEventResponse struct { - // Servers present in the room for these events. - Servers []gomatrixserverlib.ServerName `json:"servers"` -} - -// QueryRoomVersionCapabilities asks for the default room version +// QueryRoomVersionCapabilitiesRequest asks for the default room version type QueryRoomVersionCapabilitiesRequest struct{} // QueryRoomVersionCapabilitiesResponse is a response to QueryRoomVersionCapabilitiesRequest @@ -266,12 +254,12 @@ type QueryRoomVersionCapabilitiesResponse struct { AvailableRoomVersions map[gomatrixserverlib.RoomVersion]string `json:"available"` } -// QueryRoomVersionForRoom asks for the room version for a given room. +// QueryRoomVersionForRoomRequest asks for the room version for a given room. type QueryRoomVersionForRoomRequest struct { RoomID string `json:"room_id"` } -// QueryRoomVersionCapabilitiesResponse is a response to QueryServersInRoomAtEventResponse +// QueryRoomVersionForRoomResponse is a response to QueryRoomVersionForRoomRequest type QueryRoomVersionForRoomResponse struct { RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"` } @@ -350,12 +338,6 @@ type RoomserverQueryAPI interface { response *QueryBackfillResponse, ) error - QueryServersInRoomAtEvent( - ctx context.Context, - request *QueryServersInRoomAtEventRequest, - response *QueryServersInRoomAtEventResponse, - ) error - // Asks for the default room version as preferred by the server. QueryRoomVersionCapabilities( ctx context.Context, @@ -401,13 +383,10 @@ const RoomserverQueryStateAndAuthChainPath = "/api/roomserver/queryStateAndAuthC // RoomserverQueryBackfillPath is the HTTP path for the QueryBackfillPath API const RoomserverQueryBackfillPath = "/api/roomserver/queryBackfill" -// RoomserverQueryServersInRoomAtEventPath is the HTTP path for the QueryServersInRoomAtEvent API -const RoomserverQueryServersInRoomAtEventPath = "/api/roomserver/queryServersInRoomAtEvents" - // RoomserverQueryRoomVersionCapabilitiesPath is the HTTP path for the QueryRoomVersionCapabilities API const RoomserverQueryRoomVersionCapabilitiesPath = "/api/roomserver/queryRoomVersionCapabilities" -// RoomserverQueryRoomVersionCapabilitiesPath is the HTTP path for the QueryRoomVersionCapabilities API +// RoomserverQueryRoomVersionForRoomPath is the HTTP path for the QueryRoomVersionForRoom API const RoomserverQueryRoomVersionForRoomPath = "/api/roomserver/queryRoomVersionForRoom" // NewRoomserverQueryAPIHTTP creates a RoomserverQueryAPI implemented by talking to a HTTP POST API. @@ -555,19 +534,6 @@ func (h *httpRoomserverQueryAPI) QueryBackfill( return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } -// QueryServersInRoomAtEvent implements RoomServerQueryAPI -func (h *httpRoomserverQueryAPI) QueryServersInRoomAtEvent( - ctx context.Context, - request *QueryServersInRoomAtEventRequest, - response *QueryServersInRoomAtEventResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "QueryServersInRoomAtEvent") - defer span.Finish() - - apiURL := h.roomserverURL + RoomserverQueryServersInRoomAtEventPath - return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} - // QueryRoomVersionCapabilities implements RoomServerQueryAPI func (h *httpRoomserverQueryAPI) QueryRoomVersionCapabilities( ctx context.Context, diff --git a/roomserver/input/events.go b/roomserver/input/events.go index 205035d98..69828d9f9 100644 --- a/roomserver/input/events.go +++ b/roomserver/input/events.go @@ -26,6 +26,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" + "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus" ) @@ -53,6 +54,7 @@ func processRoomEvent( // Check that the event passes authentication checks and work out the numeric IDs for the auth events. authEventNIDs, err := checkAuthEvents(ctx, db, headered, input.AuthEventIDs) if err != nil { + logrus.WithError(err).WithField("event_id", event.EventID()).Error("processRoomEvent.checkAuthEvents failed for event") return } @@ -77,6 +79,7 @@ func processRoomEvent( // For outliers we can stop after we've stored the event itself as it // doesn't have any associated state to store and we don't need to // notify anyone about it. + logrus.WithField("event_id", event.EventID()).WithField("type", event.Type()).WithField("room", event.RoomID()).Info("Stored outlier") return event.EventID(), nil } @@ -89,11 +92,6 @@ func processRoomEvent( } } - if input.Kind == api.KindBackfill { - // Backfill is not implemented. - panic("Not implemented") - } - // Update the extremities of the event graph for the room return event.EventID(), updateLatestEvents( ctx, db, ow, roomNID, stateAtEvent, event, input.SendAsServer, input.TransactionID, diff --git a/roomserver/query/backfill.go b/roomserver/query/backfill.go new file mode 100644 index 000000000..09a515e99 --- /dev/null +++ b/roomserver/query/backfill.go @@ -0,0 +1,217 @@ +package query + +import ( + "context" + + "github.com/matrix-org/dendrite/roomserver/storage" + "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/gomatrixserverlib" + "github.com/sirupsen/logrus" +) + +// backfillRequester implements gomatrixserverlib.BackfillRequester +type backfillRequester struct { + db storage.Database + fedClient *gomatrixserverlib.FederationClient + thisServer gomatrixserverlib.ServerName + + // per-request state + servers []gomatrixserverlib.ServerName + eventIDToBeforeStateIDs map[string][]string + eventIDMap map[string]gomatrixserverlib.Event +} + +func newBackfillRequester(db storage.Database, fedClient *gomatrixserverlib.FederationClient, thisServer gomatrixserverlib.ServerName) *backfillRequester { + return &backfillRequester{ + db: db, + fedClient: fedClient, + thisServer: thisServer, + eventIDToBeforeStateIDs: make(map[string][]string), + eventIDMap: make(map[string]gomatrixserverlib.Event), + } +} + +func (b *backfillRequester) StateIDsBeforeEvent(ctx context.Context, targetEvent gomatrixserverlib.HeaderedEvent) ([]string, error) { + b.eventIDMap[targetEvent.EventID()] = targetEvent.Unwrap() + if ids, ok := b.eventIDToBeforeStateIDs[targetEvent.EventID()]; ok { + return ids, nil + } + // if we have exactly 1 prev event and we know the state of the room at that prev event, then just roll forward the prev event. + // Else, we have to hit /state_ids because either we don't know the state at all at this event (new backwards extremity) or + // we don't know the result of state res to merge forks (2 or more prev_events) + if len(targetEvent.PrevEventIDs()) == 1 { + prevEventID := targetEvent.PrevEventIDs()[0] + prevEvent, ok := b.eventIDMap[prevEventID] + if !ok { + goto FederationHit + } + prevEventStateIDs, ok := b.eventIDToBeforeStateIDs[prevEventID] + if !ok { + goto FederationHit + } + newStateIDs := b.calculateNewStateIDs(targetEvent.Unwrap(), prevEvent, prevEventStateIDs) + if newStateIDs != nil { + b.eventIDToBeforeStateIDs[targetEvent.EventID()] = newStateIDs + return newStateIDs, nil + } + // else we failed to calculate the new state, so fallthrough + } + +FederationHit: + var lastErr error + logrus.WithField("event_id", targetEvent.EventID()).Info("Requesting /state_ids at event") + for _, srv := range b.servers { // hit any valid server + c := gomatrixserverlib.FederatedStateProvider{ + FedClient: b.fedClient, + AuthEventsOnly: false, + Server: srv, + } + res, err := c.StateIDsBeforeEvent(ctx, targetEvent) + if err != nil { + lastErr = err + continue + } + b.eventIDToBeforeStateIDs[targetEvent.EventID()] = res + return res, nil + } + return nil, lastErr +} + +func (b *backfillRequester) calculateNewStateIDs(targetEvent, prevEvent gomatrixserverlib.Event, prevEventStateIDs []string) []string { + newStateIDs := prevEventStateIDs[:] + if prevEvent.StateKey() == nil { + // state is the same as the previous event + b.eventIDToBeforeStateIDs[targetEvent.EventID()] = newStateIDs + return newStateIDs + } + + missingState := false // true if we are missing the info for a state event ID + foundEvent := false // true if we found a (type, state_key) match + // find which state ID to replace, if any + for i, id := range newStateIDs { + ev, ok := b.eventIDMap[id] + if !ok { + missingState = true + continue + } + // The state IDs BEFORE the target event are the state IDs BEFORE the prev_event PLUS the prev_event itself + if ev.Type() == prevEvent.Type() && ev.StateKey() != nil && *ev.StateKey() == *prevEvent.StateKey() { + newStateIDs[i] = prevEvent.EventID() + foundEvent = true + break + } + } + if !foundEvent && !missingState { + // we can be certain that this is new state + newStateIDs = append(newStateIDs, prevEvent.EventID()) + foundEvent = true + } + + if foundEvent { + b.eventIDToBeforeStateIDs[targetEvent.EventID()] = newStateIDs + return newStateIDs + } + return nil +} + +func (b *backfillRequester) StateBeforeEvent(ctx context.Context, roomVer gomatrixserverlib.RoomVersion, event gomatrixserverlib.HeaderedEvent, eventIDs []string) (map[string]*gomatrixserverlib.Event, error) { + // try to fetch the events from the database first + events, err := b.ProvideEvents(roomVer, eventIDs) + if err != nil { + // non-fatal, fallthrough + logrus.WithError(err).Info("Failed to fetch events") + } else { + logrus.Infof("Fetched %d/%d events from the database", len(events), len(eventIDs)) + if len(events) == len(eventIDs) { + result := make(map[string]*gomatrixserverlib.Event) + for i := range events { + result[events[i].EventID()] = &events[i] + b.eventIDMap[events[i].EventID()] = events[i] + } + return result, nil + } + } + + c := gomatrixserverlib.FederatedStateProvider{ + FedClient: b.fedClient, + AuthEventsOnly: false, + Server: b.servers[0], + } + result, err := c.StateBeforeEvent(ctx, roomVer, event, eventIDs) + if err != nil { + return nil, err + } + for eventID, ev := range result { + b.eventIDMap[eventID] = *ev + } + return result, nil +} + +// ServersAtEvent is called when trying to determine which server to request from. +// It returns a list of servers which can be queried for backfill requests. These servers +// will be servers that are in the room already. The entries at the beginning are preferred servers +// and will be tried first. An empty list will fail the request. +func (b *backfillRequester) ServersAtEvent(ctx context.Context, roomID, eventID string) (servers []gomatrixserverlib.ServerName) { + // getMembershipsBeforeEventNID requires a NID, so retrieving the NID for + // the event is necessary. + NIDs, err := b.db.EventNIDs(ctx, []string{eventID}) + if err != nil { + logrus.WithField("event_id", eventID).WithError(err).Error("ServersAtEvent: failed to get event NID for event") + return + } + + // Retrieve all "m.room.member" state events of "join" membership, which + // contains the list of users in the room before the event, therefore all + // the servers in it at that moment. + events, err := getMembershipsBeforeEventNID(ctx, b.db, NIDs[eventID], true) + if err != nil { + logrus.WithField("event_id", eventID).WithError(err).Error("ServersAtEvent: failed to get memberships before event") + return + } + + // Store the server names in a temporary map to avoid duplicates. + serverSet := make(map[gomatrixserverlib.ServerName]bool) + for _, event := range events { + serverSet[event.Origin()] = true + } + for server := range serverSet { + if server == b.thisServer { + continue + } + servers = append(servers, server) + } + b.servers = servers + return +} + +// Backfill performs a backfill request to the given server. +// https://matrix.org/docs/spec/server_server/latest#get-matrix-federation-v1-backfill-roomid +func (b *backfillRequester) Backfill(ctx context.Context, server gomatrixserverlib.ServerName, roomID string, fromEventIDs []string, limit int) (*gomatrixserverlib.Transaction, error) { + tx, err := b.fedClient.Backfill(ctx, server, roomID, limit, fromEventIDs) + return &tx, err +} + +func (b *backfillRequester) ProvideEvents(roomVer gomatrixserverlib.RoomVersion, eventIDs []string) ([]gomatrixserverlib.Event, error) { + ctx := context.Background() + nidMap, err := b.db.EventNIDs(ctx, eventIDs) + if err != nil { + logrus.WithError(err).WithField("event_ids", eventIDs).Error("Failed to find events") + return nil, err + } + eventNIDs := make([]types.EventNID, len(nidMap)) + i := 0 + for _, nid := range nidMap { + eventNIDs[i] = nid + i++ + } + eventsWithNids, err := b.db.Events(ctx, eventNIDs) + if err != nil { + logrus.WithError(err).WithField("event_nids", eventNIDs).Error("Failed to load events") + return nil, err + } + events := make([]gomatrixserverlib.Event, len(eventsWithNids)) + for i := range eventsWithNids { + events[i] = eventsWithNids[i].Event + } + return events, nil +} diff --git a/roomserver/query/query.go b/roomserver/query/query.go index 7508d7902..a54fa58d9 100644 --- a/roomserver/query/query.go +++ b/roomserver/query/query.go @@ -19,6 +19,7 @@ package query import ( "context" "encoding/json" + "fmt" "net/http" "github.com/matrix-org/dendrite/common" @@ -31,12 +32,16 @@ import ( "github.com/matrix-org/dendrite/roomserver/version" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" + "github.com/sirupsen/logrus" ) // RoomserverQueryAPI is an implementation of api.RoomserverQueryAPI type RoomserverQueryAPI struct { DB storage.Database ImmutableCache caching.ImmutableCache + ServerName gomatrixserverlib.ServerName + KeyRing gomatrixserverlib.JSONVerifier + FedClient *gomatrixserverlib.FederationClient } // QueryLatestEventsAndState implements api.RoomserverQueryAPI @@ -281,7 +286,7 @@ func (r *RoomserverQueryAPI) QueryMembershipsForRoom( events, err = r.DB.Events(ctx, eventNIDs) } else { - events, err = r.getMembershipsBeforeEventNID(ctx, membershipEventNID, request.JoinedOnly) + events, err = getMembershipsBeforeEventNID(ctx, r.DB, membershipEventNID, request.JoinedOnly) } if err != nil { @@ -300,19 +305,19 @@ func (r *RoomserverQueryAPI) QueryMembershipsForRoom( // of the event's room as it was when this event was fired, then filters the state events to // only keep the "m.room.member" events with a "join" membership. These events are returned. // Returns an error if there was an issue fetching the events. -func (r *RoomserverQueryAPI) getMembershipsBeforeEventNID( - ctx context.Context, eventNID types.EventNID, joinedOnly bool, +func getMembershipsBeforeEventNID( + ctx context.Context, db storage.Database, eventNID types.EventNID, joinedOnly bool, ) ([]types.Event, error) { - roomState := state.NewStateResolution(r.DB) + roomState := state.NewStateResolution(db) events := []types.Event{} // Lookup the event NID - eIDs, err := r.DB.EventIDs(ctx, []types.EventNID{eventNID}) + eIDs, err := db.EventIDs(ctx, []types.EventNID{eventNID}) if err != nil { return nil, err } eventIDs := []string{eIDs[eventNID]} - prevState, err := r.DB.StateAtEventIDs(ctx, eventIDs) + prevState, err := db.StateAtEventIDs(ctx, eventIDs) if err != nil { return nil, err } @@ -332,7 +337,7 @@ func (r *RoomserverQueryAPI) getMembershipsBeforeEventNID( } // Get all of the events in this state - stateEvents, err := r.DB.Events(ctx, eventNIDs) + stateEvents, err := db.Events(ctx, eventNIDs) if err != nil { return nil, err } @@ -484,6 +489,13 @@ func (r *RoomserverQueryAPI) QueryBackfill( request *api.QueryBackfillRequest, response *api.QueryBackfillResponse, ) error { + // if we are requesting the backfill then we need to do a federation hit + // TODO: we could be more sensible and fetch as many events we already have then request the rest + // which is what the syncapi does already. + if request.ServerName == r.ServerName { + return r.backfillViaFederation(ctx, request, response) + } + // someone else is requesting the backfill, try to service their request. var err error var front []string @@ -525,6 +537,55 @@ func (r *RoomserverQueryAPI) QueryBackfill( return err } +func (r *RoomserverQueryAPI) backfillViaFederation(ctx context.Context, req *api.QueryBackfillRequest, res *api.QueryBackfillResponse) error { + roomVer, err := r.DB.GetRoomVersionForRoom(ctx, req.RoomID) + if err != nil { + return fmt.Errorf("backfillViaFederation: unknown room version for room %s : %w", req.RoomID, err) + } + requester := newBackfillRequester(r.DB, r.FedClient, r.ServerName) + events, err := gomatrixserverlib.RequestBackfill( + ctx, requester, + r.KeyRing, req.RoomID, roomVer, req.EarliestEventsIDs, req.Limit) + if err != nil { + return err + } + logrus.WithField("room_id", req.RoomID).Infof("backfilled %d events", len(events)) + + // persist these new events - auth checks have already been done + roomNID, backfilledEventMap := persistEvents(ctx, r.DB, events) + if err != nil { + return err + } + + for _, ev := range backfilledEventMap { + // now add state for these events + stateIDs, ok := requester.eventIDToBeforeStateIDs[ev.EventID()] + if !ok { + // this should be impossible as all events returned must have pass Step 5 of the PDU checks + // which requires a list of state IDs. + logrus.WithError(err).WithField("event_id", ev.EventID()).Error("Failed to find state IDs for event which passed auth checks") + continue + } + var entries []types.StateEntry + if entries, err = r.DB.StateEntriesForEventIDs(ctx, stateIDs); err != nil { + return err + } + + var beforeStateSnapshotNID types.StateSnapshotNID + if beforeStateSnapshotNID, err = r.DB.AddState(ctx, roomNID, nil, entries); err != nil { + return err + } + if err = r.DB.SetState(ctx, ev.EventNID, beforeStateSnapshotNID); err != nil { + logrus.WithError(err).WithField("event_id", ev.EventID()).Error("Failed to set state before event") + } + } + + // TODO: update backwards extremities, as that should be moved from syncapi to roomserver at some point. + + res.Events = events + return nil +} + func (r *RoomserverQueryAPI) isServerCurrentlyInRoom(ctx context.Context, serverName gomatrixserverlib.ServerName, roomID string) (bool, error) { roomNID, err := r.DB.RoomNID(ctx, roomID) if err != nil { @@ -778,39 +839,33 @@ func getAuthChain( return authEvents, nil } -// QueryServersInRoomAtEvent implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryServersInRoomAtEvent( - ctx context.Context, - request *api.QueryServersInRoomAtEventRequest, - response *api.QueryServersInRoomAtEventResponse, -) error { - // getMembershipsBeforeEventNID requires a NID, so retrieving the NID for - // the event is necessary. - NIDs, err := r.DB.EventNIDs(ctx, []string{request.EventID}) - if err != nil { - return err +func persistEvents(ctx context.Context, db storage.Database, events []gomatrixserverlib.HeaderedEvent) (types.RoomNID, map[string]types.Event) { + var roomNID types.RoomNID + backfilledEventMap := make(map[string]types.Event) + for _, ev := range events { + nidMap, err := db.EventNIDs(ctx, ev.AuthEventIDs()) + if err != nil { // this shouldn't happen as RequestBackfill already found them + logrus.WithError(err).WithField("auth_events", ev.AuthEventIDs()).Error("Failed to find one or more auth events") + continue + } + authNids := make([]types.EventNID, len(nidMap)) + i := 0 + for _, nid := range nidMap { + authNids[i] = nid + i++ + } + var stateAtEvent types.StateAtEvent + roomNID, stateAtEvent, err = db.StoreEvent(ctx, ev.Unwrap(), nil, authNids) + if err != nil { + logrus.WithError(err).WithField("event_id", ev.EventID()).Error("Failed to store backfilled event") + continue + } + backfilledEventMap[ev.EventID()] = types.Event{ + EventNID: stateAtEvent.StateEntry.EventNID, + Event: ev.Unwrap(), + } } - - // Retrieve all "m.room.member" state events of "join" membership, which - // contains the list of users in the room before the event, therefore all - // the servers in it at that moment. - events, err := r.getMembershipsBeforeEventNID(ctx, NIDs[request.EventID], true) - if err != nil { - return err - } - - // Store the server names in a temporary map to avoid duplicates. - servers := make(map[gomatrixserverlib.ServerName]bool) - for _, event := range events { - servers[event.Origin()] = true - } - - // Populate the response. - for server := range servers { - response.Servers = append(response.Servers, server) - } - - return nil + return roomNID, backfilledEventMap } // QueryRoomVersionCapabilities implements api.RoomserverQueryAPI @@ -994,20 +1049,6 @@ func (r *RoomserverQueryAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle( - api.RoomserverQueryServersInRoomAtEventPath, - common.MakeInternalAPI("QueryServersInRoomAtEvent", func(req *http.Request) util.JSONResponse { - var request api.QueryServersInRoomAtEventRequest - var response api.QueryServersInRoomAtEventResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryServersInRoomAtEvent(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) servMux.Handle( api.RoomserverQueryRoomVersionCapabilitiesPath, common.MakeInternalAPI("QueryRoomVersionCapabilities", func(req *http.Request) util.JSONResponse { diff --git a/roomserver/roomserver.go b/roomserver/roomserver.go index fa4f20626..ea1c5c4c3 100644 --- a/roomserver/roomserver.go +++ b/roomserver/roomserver.go @@ -18,6 +18,7 @@ import ( "net/http" "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/gomatrixserverlib" asQuery "github.com/matrix-org/dendrite/appservice/query" "github.com/matrix-org/dendrite/common/basecomponent" @@ -33,7 +34,7 @@ import ( // allowing other components running in the same process to hit the query the // APIs directly instead of having to use HTTP. func SetupRoomServerComponent( - base *basecomponent.BaseDendrite, + base *basecomponent.BaseDendrite, keyRing gomatrixserverlib.JSONVerifier, ) (api.RoomserverAliasAPI, api.RoomserverInputAPI, api.RoomserverQueryAPI) { roomserverDB, err := storage.Open(string(base.Cfg.Database.RoomServer)) if err != nil { @@ -51,6 +52,11 @@ func SetupRoomServerComponent( queryAPI := query.RoomserverQueryAPI{ DB: roomserverDB, ImmutableCache: base.ImmutableCache, + ServerName: base.Cfg.Matrix.ServerName, + FedClient: base.CreateFederationClient(), + // TODO: We should have a key server so we don't keep adding components + // which talk to the same DB. + KeyRing: keyRing, } queryAPI.SetupHTTP(http.DefaultServeMux) diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go index a13c44d6e..fb39eca63 100644 --- a/roomserver/storage/interface.go +++ b/roomserver/storage/interface.go @@ -31,7 +31,8 @@ type Database interface { state []types.StateEntry, ) (types.StateSnapshotNID, error) // Look up the state of a room at each event for a list of string event IDs. - // Returns an error if there is an error talking to the database + // Returns an error if there is an error talking to the database. + // The length of []types.StateAtEvent is guaranteed to equal the length of eventIDs if no error is returned. // Returns a types.MissingEventError if the room state for the event IDs aren't in the database StateAtEventIDs(ctx context.Context, eventIDs []string) ([]types.StateAtEvent, error) // Look up the numeric IDs for a list of string event types. diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index 53a9a963a..c48414ab6 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -285,7 +285,7 @@ func (r *messagesReq) handleEmptyEventsSlice() ( // Check if we have backward extremities for this room. if len(backwardExtremities) > 0 { // If so, retrieve as much events as needed through backfilling. - events, err = r.backfill(backwardExtremities, r.limit) + events, err = r.backfill(r.roomID, backwardExtremities, r.limit) if err != nil { return } @@ -334,7 +334,7 @@ func (r *messagesReq) handleNonEmptyEventsSlice(streamEvents []types.StreamEvent if len(backwardExtremities) > 0 && !isSetLargeEnough && r.backwardOrdering { var pdus []gomatrixserverlib.HeaderedEvent // Only ask the remote server for enough events to reach the limit. - pdus, err = r.backfill(backwardExtremities, r.limit-len(streamEvents)) + pdus, err = r.backfill(r.roomID, backwardExtremities, r.limit-len(streamEvents)) if err != nil { return } @@ -358,45 +358,29 @@ func (r *messagesReq) handleNonEmptyEventsSlice(streamEvents []types.StreamEvent // event, or if there is no remote homeserver to contact. // Returns an error if there was an issue with retrieving the list of servers in // the room or sending the request. -func (r *messagesReq) backfill(fromEventIDs []string, limit int) ([]gomatrixserverlib.HeaderedEvent, error) { - verReq := api.QueryRoomVersionForRoomRequest{RoomID: r.roomID} - verRes := api.QueryRoomVersionForRoomResponse{} - if err := r.queryAPI.QueryRoomVersionForRoom(r.ctx, &verReq, &verRes); err != nil { - return nil, err - } - - srvToBackfillFrom, err := r.serverToBackfillFrom(fromEventIDs) +func (r *messagesReq) backfill(roomID string, fromEventIDs []string, limit int) ([]gomatrixserverlib.HeaderedEvent, error) { + var res api.QueryBackfillResponse + err := r.queryAPI.QueryBackfill(context.Background(), &api.QueryBackfillRequest{ + RoomID: roomID, + EarliestEventsIDs: fromEventIDs, + Limit: limit, + ServerName: r.cfg.Matrix.ServerName, + }, &res) if err != nil { - return nil, fmt.Errorf("Cannot find server to backfill from: %w", err) + return nil, fmt.Errorf("QueryBackfill failed: %w", err) } + util.GetLogger(r.ctx).WithField("new_events", len(res.Events)).Info("Storing new events from backfill") - headered := make([]gomatrixserverlib.HeaderedEvent, 0) - - // If the roomserver responded with at least one server that isn't us, - // send it a request for backfill. - util.GetLogger(r.ctx).WithField("server", srvToBackfillFrom).WithField("limit", limit).Info("Backfilling from server") - txn, err := r.federation.Backfill( - r.ctx, srvToBackfillFrom, r.roomID, limit, fromEventIDs, - ) - if err != nil { - return nil, err - } - - for _, p := range txn.PDUs { - event, e := gomatrixserverlib.NewEventFromUntrustedJSON(p, verRes.RoomVersion) - if e != nil { - continue - } - headered = append(headered, event.Headered(verRes.RoomVersion)) - } - util.GetLogger(r.ctx).WithField("server", srvToBackfillFrom).WithField("new_events", len(headered)).Info("Storing new events from backfill") + // TODO: we should only be inserting events into the database from the roomserver's kafka output stream. + // Currently, this can race with live events for the room and cause problems. It's also just a bit unclear + // when you have multiple entry points to write events. // Store the events in the database, while marking them as unfit to show // up in responses to sync requests. - for i := range headered { + for i := range res.Events { if _, err = r.db.WriteEvent( r.ctx, - &headered[i], + &res.Events[i], []gomatrixserverlib.HeaderedEvent{}, []string{}, []string{}, @@ -406,63 +390,7 @@ func (r *messagesReq) backfill(fromEventIDs []string, limit int) ([]gomatrixserv } } - return headered, nil -} - -func (r *messagesReq) serverToBackfillFrom(fromEventIDs []string) (gomatrixserverlib.ServerName, error) { - // Query the list of servers in the room when one of the backward extremities - // was sent. - var serversResponse api.QueryServersInRoomAtEventResponse - serversRequest := api.QueryServersInRoomAtEventRequest{ - RoomID: r.roomID, - EventID: fromEventIDs[0], - } - if err := r.queryAPI.QueryServersInRoomAtEvent(r.ctx, &serversRequest, &serversResponse); err != nil { - util.GetLogger(r.ctx).WithError(err).Warn("Failed to query servers in room at event, falling back to event sender") - // FIXME: We shouldn't be doing this but in situations where we have already backfilled once - // the query API doesn't work as backfilled events do not make it to the room server. - // This means QueryServersInRoomAtEvent returns an error as it doesn't have the event ID in question. - // We need to inject backfilled events into the room server and store them appropriately. - events, err := r.db.Events(r.ctx, fromEventIDs) - if err != nil { - return "", err - } - if len(events) == 0 { - // should be impossible as these event IDs are backwards extremities - return "", fmt.Errorf("backfill: missing backwards extremities, event IDs: %s", fromEventIDs) - } - // The rationale here is that the last event was unlikely to be sent by us, so poke the server who sent it. - // We shouldn't be doing this really, but as a heuristic it should work pretty well for now. - for _, e := range events { - _, srv, srverr := gomatrixserverlib.SplitID('@', e.Sender()) - if srverr != nil { - util.GetLogger(r.ctx).WithError(srverr).Warn("Failed to extract domain from event sender") - continue - } - if srv != r.cfg.Matrix.ServerName { - return srv, nil - } - } - // no valid events which have a remote server, fail. - return "", err - } - - // Use the first server from the response, except if that server is us. - // In that case, use the second one if the roomserver responded with - // enough servers. If not, use an empty string to prevent the backfill - // from happening as there's no server to direct the request towards. - // TODO: Be smarter at selecting the server to direct the request - // towards. - srvToBackfillFrom := serversResponse.Servers[0] - if srvToBackfillFrom == r.cfg.Matrix.ServerName { - if len(serversResponse.Servers) > 1 { - srvToBackfillFrom = serversResponse.Servers[1] - } else { - util.GetLogger(r.ctx).Info("Not enough servers to backfill from") - return "", nil - } - } - return srvToBackfillFrom, nil + return res.Events, nil } // setToDefault returns the default value for the "to" query parameter of a From 5071ecb8b3c51b3cf70794fd2dbcfd1a866058e3 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 28 Apr 2020 12:50:49 +0100 Subject: [PATCH 009/200] Fix URLDecodeMapValues (#984) * Update gomatrixserverlib * Fix URLDecodeMapValues * Update gomatrixserverlib --- common/routing.go | 2 +- go.mod | 3 +-- go.sum | 14 ++------------ 3 files changed, 4 insertions(+), 15 deletions(-) diff --git a/common/routing.go b/common/routing.go index 330912cde..cee0c162a 100644 --- a/common/routing.go +++ b/common/routing.go @@ -24,7 +24,7 @@ import ( func URLDecodeMapValues(vmap map[string]string) (map[string]string, error) { decoded := make(map[string]string, len(vmap)) for key, value := range vmap { - decodedVal, err := url.QueryUnescape(value) + decodedVal, err := url.PathUnescape(value) if err != nil { return make(map[string]string), err } diff --git a/go.mod b/go.mod index 73171d89c..56c58c6c8 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f github.com/matrix-org/go-sqlite3-js v0.0.0-20200325174927-327088cdef10 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200428095012-a95e289995b1 + github.com/matrix-org/gomatrixserverlib v0.0.0-20200428112024-9f47f9bfa4b2 github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible @@ -26,7 +26,6 @@ require ( github.com/opentracing/opentracing-go v1.1.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.4.1 - github.com/prometheus/common v0.9.1 github.com/sirupsen/logrus v1.4.2 github.com/tidwall/gjson v1.6.0 github.com/uber/jaeger-client-go v2.15.0+incompatible diff --git a/go.sum b/go.sum index 9c7213126..de844b2a1 100644 --- a/go.sum +++ b/go.sum @@ -367,18 +367,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:kmRjpmFOenVpOaV/DRlo9p6z/IbOKlUC+hhKsAAh8Qg= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200422082552-d7b4202c47f3 h1:xis1ojN99vjygwqudzB9VQq3cM2SJ7aCAMlXj/YN+88= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200422082552-d7b4202c47f3/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200423090438-562549dbe799 h1:OsoUMTirIpeuZJdYkKKiYe6jm0E5viZR7aOS9K465QI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200423090438-562549dbe799/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200424154222-2827b39252bd h1:243fMfK0XqTQsdUY3IIqtxPX5g9MfPTaAP92CseqOek= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200424154222-2827b39252bd/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200427134702-21db6d1430e3 h1:aJMAKjfXG5I8TqPxJQbQIkGSWM770oxkpgsPHE8C06E= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200427134702-21db6d1430e3/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200427152419-6a0535cc473a h1:tlXCVU3eab9kksGYBRA3oyrmIRwD/aPujo5KJCdlCVQ= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200427152419-6a0535cc473a/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200428095012-a95e289995b1 h1:TB4V69eOtvmHdFp0+BgLNrDCcCwq6QDUOTjmi8fjC/M= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200428095012-a95e289995b1/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200428112024-9f47f9bfa4b2 h1:sy2QOqJhb4WXzq8bJhsCntAUYb64Dl6txsFtXWtxxSg= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200428112024-9f47f9bfa4b2/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y= From 35b7cbd5d8673d8ea82b36d90d9ad5b79dc3c5b7 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 28 Apr 2020 15:50:24 +0100 Subject: [PATCH 010/200] sql/backwards_extremities: Shift to table format and share code (#985) * sql/backwards_extremities: Shift to table format and share code This is an initial cut to reduce boilerplate at the storage layer. It removes the need for 2x `_table.go` files, one for each DB engine, replacing it with a single struct which has an interface which implements the raw SQL statements. The actual impl sits alongside the interface declaration which is generally regarded as best practice (though no canonical sources). Especially in this case where the impl is tiny (functions returning strings) and relies heavily on the function signatures of the table struct (for parameters), having the context in the same file is useful. * Remove _table redundancy --- .../postgres/backward_extremities_table.go | 124 ------------- syncapi/storage/postgres/syncserver.go | 18 +- .../sqlite3/backward_extremities_table.go | 124 ------------- syncapi/storage/sqlite3/syncserver.go | 18 +- .../storage/tables/backward_extremities.go | 175 ++++++++++++++++++ 5 files changed, 195 insertions(+), 264 deletions(-) delete mode 100644 syncapi/storage/postgres/backward_extremities_table.go delete mode 100644 syncapi/storage/sqlite3/backward_extremities_table.go create mode 100644 syncapi/storage/tables/backward_extremities.go diff --git a/syncapi/storage/postgres/backward_extremities_table.go b/syncapi/storage/postgres/backward_extremities_table.go deleted file mode 100644 index cb3629644..000000000 --- a/syncapi/storage/postgres/backward_extremities_table.go +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright 2018 New Vector Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package postgres - -import ( - "context" - "database/sql" - - "github.com/matrix-org/dendrite/common" -) - -// The purpose of this table is to keep track of backwards extremities for a room. -// Backwards extremities are the earliest (DAG-wise) known events which we have -// the entire event JSON. These event IDs are used in federation requests to fetch -// even earlier events. -// -// We persist the previous event IDs as well, one per row, so when we do fetch even -// earlier events we can simply delete rows which referenced it. Consider the graph: -// A -// | Event C has 1 prev_event ID: A. -// B C -// |___| Event D has 2 prev_event IDs: B and C. -// | -// D -// The earliest known event we have is D, so this table has 2 rows. -// A backfill request gives us C but not B. We delete rows where prev_event=C. This -// still means that D is a backwards extremity as we do not have event B. However, event -// C is *also* a backwards extremity at this point as we do not have event A. Later, -// when we fetch event B, we delete rows where prev_event=B. This then removes D as -// a backwards extremity because there are no more rows with event_id=B. -const backwardExtremitiesSchema = ` --- Stores output room events received from the roomserver. -CREATE TABLE IF NOT EXISTS syncapi_backward_extremities ( - -- The 'room_id' key for the event. - room_id TEXT NOT NULL, - -- The event ID for the last known event. This is the backwards extremity. - event_id TEXT NOT NULL, - -- The prev_events for the last known event. This is used to update extremities. - prev_event_id TEXT NOT NULL, - - PRIMARY KEY(room_id, event_id, prev_event_id) -); -` - -const insertBackwardExtremitySQL = "" + - "INSERT INTO syncapi_backward_extremities (room_id, event_id, prev_event_id)" + - " VALUES ($1, $2, $3)" + - " ON CONFLICT DO NOTHING" - -const selectBackwardExtremitiesForRoomSQL = "" + - "SELECT DISTINCT event_id FROM syncapi_backward_extremities WHERE room_id = $1" - -const deleteBackwardExtremitySQL = "" + - "DELETE FROM syncapi_backward_extremities WHERE room_id = $1 AND prev_event_id = $2" - -type backwardExtremitiesStatements struct { - insertBackwardExtremityStmt *sql.Stmt - selectBackwardExtremitiesForRoomStmt *sql.Stmt - deleteBackwardExtremityStmt *sql.Stmt -} - -func (s *backwardExtremitiesStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(backwardExtremitiesSchema) - if err != nil { - return - } - if s.insertBackwardExtremityStmt, err = db.Prepare(insertBackwardExtremitySQL); err != nil { - return - } - if s.selectBackwardExtremitiesForRoomStmt, err = db.Prepare(selectBackwardExtremitiesForRoomSQL); err != nil { - return - } - if s.deleteBackwardExtremityStmt, err = db.Prepare(deleteBackwardExtremitySQL); err != nil { - return - } - return -} - -func (s *backwardExtremitiesStatements) insertsBackwardExtremity( - ctx context.Context, txn *sql.Tx, roomID, eventID string, prevEventID string, -) (err error) { - _, err = txn.Stmt(s.insertBackwardExtremityStmt).ExecContext(ctx, roomID, eventID, prevEventID) - return -} - -func (s *backwardExtremitiesStatements) selectBackwardExtremitiesForRoom( - ctx context.Context, roomID string, -) (eventIDs []string, err error) { - rows, err := s.selectBackwardExtremitiesForRoomStmt.QueryContext(ctx, roomID) - if err != nil { - return - } - defer common.CloseAndLogIfError(ctx, rows, "selectBackwardExtremitiesForRoom: rows.close() failed") - - for rows.Next() { - var eID string - if err = rows.Scan(&eID); err != nil { - return - } - - eventIDs = append(eventIDs, eID) - } - - return eventIDs, rows.Err() -} - -func (s *backwardExtremitiesStatements) deleteBackwardExtremity( - ctx context.Context, txn *sql.Tx, roomID, knownEventID string, -) (err error) { - _, err = txn.Stmt(s.deleteBackwardExtremityStmt).ExecContext(ctx, roomID, knownEventID) - return -} diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go index 1e078ef46..9d61ccfc3 100644 --- a/syncapi/storage/postgres/syncserver.go +++ b/syncapi/storage/postgres/syncserver.go @@ -32,6 +32,7 @@ import ( _ "github.com/lib/pq" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/eduserver/cache" + "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -56,7 +57,7 @@ type SyncServerDatasource struct { invites inviteEventsStatements eduCache *cache.EDUCache topology outputRoomEventsTopologyStatements - backwardExtremities backwardExtremitiesStatements + backwardExtremities tables.BackwardsExtremities } // NewSyncServerDatasource creates a new sync server database @@ -75,16 +76,17 @@ func NewSyncServerDatasource(dbDataSourceName string) (*SyncServerDatasource, er if err = d.events.prepare(d.db); err != nil { return nil, err } - if err := d.roomstate.prepare(d.db); err != nil { + if err = d.roomstate.prepare(d.db); err != nil { return nil, err } - if err := d.invites.prepare(d.db); err != nil { + if err = d.invites.prepare(d.db); err != nil { return nil, err } - if err := d.topology.prepare(d.db); err != nil { + if err = d.topology.prepare(d.db); err != nil { return nil, err } - if err := d.backwardExtremities.prepare(d.db); err != nil { + d.backwardExtremities, err = tables.NewBackwardsExtremities(d.db, &tables.PostgresBackwardsExtremitiesStatements{}) + if err != nil { return nil, err } d.eduCache = cache.New() @@ -116,7 +118,7 @@ func (d *SyncServerDatasource) Events(ctx context.Context, eventIDs []string) ([ // the events listed in the event's 'prev_events'. This function also updates the backwards extremities table // to account for the fact that the given event is no longer a backwards extremity, but may be marked as such. func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, txn *sql.Tx, ev *gomatrixserverlib.HeaderedEvent) error { - if err := d.backwardExtremities.deleteBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID()); err != nil { + if err := d.backwardExtremities.DeleteBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID()); err != nil { return err } @@ -137,7 +139,7 @@ func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, tx // If the event is missing, consider it a backward extremity. if !found { - if err = d.backwardExtremities.insertsBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID(), eID); err != nil { + if err = d.backwardExtremities.InsertsBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID(), eID); err != nil { return err } } @@ -314,7 +316,7 @@ func (d *SyncServerDatasource) SyncPosition(ctx context.Context) (types.Paginati func (d *SyncServerDatasource) BackwardExtremitiesForRoom( ctx context.Context, roomID string, ) (backwardExtremities []string, err error) { - return d.backwardExtremities.selectBackwardExtremitiesForRoom(ctx, roomID) + return d.backwardExtremities.SelectBackwardExtremitiesForRoom(ctx, roomID) } // MaxTopologicalPosition returns the highest topological position for a given diff --git a/syncapi/storage/sqlite3/backward_extremities_table.go b/syncapi/storage/sqlite3/backward_extremities_table.go deleted file mode 100644 index 3d8cb91fc..000000000 --- a/syncapi/storage/sqlite3/backward_extremities_table.go +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright 2018 New Vector Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sqlite3 - -import ( - "context" - "database/sql" - - "github.com/matrix-org/dendrite/common" -) - -// The purpose of this table is to keep track of backwards extremities for a room. -// Backwards extremities are the earliest (DAG-wise) known events which we have -// the entire event JSON. These event IDs are used in federation requests to fetch -// even earlier events. -// -// We persist the previous event IDs as well, one per row, so when we do fetch even -// earlier events we can simply delete rows which referenced it. Consider the graph: -// A -// | Event C has 1 prev_event ID: A. -// B C -// |___| Event D has 2 prev_event IDs: B and C. -// | -// D -// The earliest known event we have is D, so this table has 2 rows. -// A backfill request gives us C but not B. We delete rows where prev_event=C. This -// still means that D is a backwards extremity as we do not have event B. However, event -// C is *also* a backwards extremity at this point as we do not have event A. Later, -// when we fetch event B, we delete rows where prev_event=B. This then removes D as -// a backwards extremity because there are no more rows with event_id=B. -const backwardExtremitiesSchema = ` --- Stores output room events received from the roomserver. -CREATE TABLE IF NOT EXISTS syncapi_backward_extremities ( - -- The 'room_id' key for the event. - room_id TEXT NOT NULL, - -- The event ID for the last known event. This is the backwards extremity. - event_id TEXT NOT NULL, - -- The prev_events for the last known event. This is used to update extremities. - prev_event_id TEXT NOT NULL, - - PRIMARY KEY(room_id, event_id, prev_event_id) -); -` - -const insertBackwardExtremitySQL = "" + - "INSERT INTO syncapi_backward_extremities (room_id, event_id, prev_event_id)" + - " VALUES ($1, $2, $3)" + - " ON CONFLICT (room_id, event_id, prev_event_id) DO NOTHING" - -const selectBackwardExtremitiesForRoomSQL = "" + - "SELECT DISTINCT event_id FROM syncapi_backward_extremities WHERE room_id = $1" - -const deleteBackwardExtremitySQL = "" + - "DELETE FROM syncapi_backward_extremities WHERE room_id = $1 AND prev_event_id = $2" - -type backwardExtremitiesStatements struct { - insertBackwardExtremityStmt *sql.Stmt - selectBackwardExtremitiesForRoomStmt *sql.Stmt - deleteBackwardExtremityStmt *sql.Stmt -} - -func (s *backwardExtremitiesStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(backwardExtremitiesSchema) - if err != nil { - return - } - if s.insertBackwardExtremityStmt, err = db.Prepare(insertBackwardExtremitySQL); err != nil { - return - } - if s.selectBackwardExtremitiesForRoomStmt, err = db.Prepare(selectBackwardExtremitiesForRoomSQL); err != nil { - return - } - if s.deleteBackwardExtremityStmt, err = db.Prepare(deleteBackwardExtremitySQL); err != nil { - return - } - return -} - -func (s *backwardExtremitiesStatements) insertsBackwardExtremity( - ctx context.Context, txn *sql.Tx, roomID, eventID string, prevEventID string, -) (err error) { - _, err = txn.Stmt(s.insertBackwardExtremityStmt).ExecContext(ctx, roomID, eventID, prevEventID) - return -} - -func (s *backwardExtremitiesStatements) selectBackwardExtremitiesForRoom( - ctx context.Context, roomID string, -) (eventIDs []string, err error) { - rows, err := s.selectBackwardExtremitiesForRoomStmt.QueryContext(ctx, roomID) - if err != nil { - return - } - defer common.CloseAndLogIfError(ctx, rows, "selectBackwardExtremitiesForRoom: rows.close() failed") - - for rows.Next() { - var eID string - if err = rows.Scan(&eID); err != nil { - return - } - - eventIDs = append(eventIDs, eID) - } - - return eventIDs, rows.Err() -} - -func (s *backwardExtremitiesStatements) deleteBackwardExtremity( - ctx context.Context, txn *sql.Tx, roomID, knownEventID string, -) (err error) { - _, err = txn.Stmt(s.deleteBackwardExtremityStmt).ExecContext(ctx, roomID, knownEventID) - return -} diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index cdfd29b84..35774830e 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -35,6 +35,7 @@ import ( "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/eduserver/cache" + "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -60,7 +61,7 @@ type SyncServerDatasource struct { invites inviteEventsStatements eduCache *cache.EDUCache topology outputRoomEventsTopologyStatements - backwardExtremities backwardExtremitiesStatements + backwardExtremities tables.BackwardsExtremities } // NewSyncServerDatasource creates a new sync server database @@ -102,16 +103,17 @@ func (d *SyncServerDatasource) prepare() (err error) { if err = d.events.prepare(d.db, &d.streamID); err != nil { return err } - if err := d.roomstate.prepare(d.db, &d.streamID); err != nil { + if err = d.roomstate.prepare(d.db, &d.streamID); err != nil { return err } - if err := d.invites.prepare(d.db, &d.streamID); err != nil { + if err = d.invites.prepare(d.db, &d.streamID); err != nil { return err } - if err := d.topology.prepare(d.db); err != nil { + if err = d.topology.prepare(d.db); err != nil { return err } - if err := d.backwardExtremities.prepare(d.db); err != nil { + d.backwardExtremities, err = tables.NewBackwardsExtremities(d.db, &tables.SqliteBackwardsExtremitiesStatements{}) + if err != nil { return err } return nil @@ -142,7 +144,7 @@ func (d *SyncServerDatasource) Events(ctx context.Context, eventIDs []string) ([ // the events listed in the event's 'prev_events'. This function also updates the backwards extremities table // to account for the fact that the given event is no longer a backwards extremity, but may be marked as such. func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, txn *sql.Tx, ev *gomatrixserverlib.HeaderedEvent) error { - if err := d.backwardExtremities.deleteBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID()); err != nil { + if err := d.backwardExtremities.DeleteBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID()); err != nil { return err } @@ -163,7 +165,7 @@ func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, tx // If the event is missing, consider it a backward extremity. if !found { - if err = d.backwardExtremities.insertsBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID(), eID); err != nil { + if err = d.backwardExtremities.InsertsBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID(), eID); err != nil { return err } } @@ -344,7 +346,7 @@ func (d *SyncServerDatasource) SyncPosition(ctx context.Context) (tok types.Pagi func (d *SyncServerDatasource) BackwardExtremitiesForRoom( ctx context.Context, roomID string, ) (backwardExtremities []string, err error) { - return d.backwardExtremities.selectBackwardExtremitiesForRoom(ctx, roomID) + return d.backwardExtremities.SelectBackwardExtremitiesForRoom(ctx, roomID) } // MaxTopologicalPosition returns the highest topological position for a given diff --git a/syncapi/storage/tables/backward_extremities.go b/syncapi/storage/tables/backward_extremities.go new file mode 100644 index 000000000..babd9aaa1 --- /dev/null +++ b/syncapi/storage/tables/backward_extremities.go @@ -0,0 +1,175 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tables + +import ( + "context" + "database/sql" + + "github.com/matrix-org/dendrite/common" +) + +// BackwardsExtremitiesStatements contains the SQL statements to implement. +// See BackwardsExtremities to see the parameter and response types. +type BackwardsExtremitiesStatements interface { + Schema() string + InsertBackwardExtremity() string + SelectBackwardExtremitiesForRoom() string + DeleteBackwardExtremity() string +} + +type PostgresBackwardsExtremitiesStatements struct{} + +func (s *PostgresBackwardsExtremitiesStatements) Schema() string { + return `-- Stores output room events received from the roomserver. + CREATE TABLE IF NOT EXISTS syncapi_backward_extremities ( + -- The 'room_id' key for the event. + room_id TEXT NOT NULL, + -- The event ID for the last known event. This is the backwards extremity. + event_id TEXT NOT NULL, + -- The prev_events for the last known event. This is used to update extremities. + prev_event_id TEXT NOT NULL, + + PRIMARY KEY(room_id, event_id, prev_event_id) + ); + ` +} +func (s *PostgresBackwardsExtremitiesStatements) InsertBackwardExtremity() string { + return "" + + "INSERT INTO syncapi_backward_extremities (room_id, event_id, prev_event_id)" + + " VALUES ($1, $2, $3)" + + " ON CONFLICT DO NOTHING" +} +func (s *PostgresBackwardsExtremitiesStatements) SelectBackwardExtremitiesForRoom() string { + return "SELECT DISTINCT event_id FROM syncapi_backward_extremities WHERE room_id = $1" +} +func (s *PostgresBackwardsExtremitiesStatements) DeleteBackwardExtremity() string { + return "DELETE FROM syncapi_backward_extremities WHERE room_id = $1 AND prev_event_id = $2" +} + +type SqliteBackwardsExtremitiesStatements struct{} + +func (s *SqliteBackwardsExtremitiesStatements) Schema() string { + return `-- Stores output room events received from the roomserver. +CREATE TABLE IF NOT EXISTS syncapi_backward_extremities ( + -- The 'room_id' key for the event. + room_id TEXT NOT NULL, + -- The event ID for the last known event. This is the backwards extremity. + event_id TEXT NOT NULL, + -- The prev_events for the last known event. This is used to update extremities. + prev_event_id TEXT NOT NULL, + + PRIMARY KEY(room_id, event_id, prev_event_id) +); +` +} + +func (s *SqliteBackwardsExtremitiesStatements) InsertBackwardExtremity() string { + return "" + + "INSERT INTO syncapi_backward_extremities (room_id, event_id, prev_event_id)" + + " VALUES ($1, $2, $3)" + + " ON CONFLICT (room_id, event_id, prev_event_id) DO NOTHING" +} + +func (s *SqliteBackwardsExtremitiesStatements) SelectBackwardExtremitiesForRoom() string { + return "" + + "SELECT DISTINCT event_id FROM syncapi_backward_extremities WHERE room_id = $1" +} + +func (s *SqliteBackwardsExtremitiesStatements) DeleteBackwardExtremity() string { + return "" + + "DELETE FROM syncapi_backward_extremities WHERE room_id = $1 AND prev_event_id = $2" +} + +// BackwardsExtremities keeps track of backwards extremities for a room. +// Backwards extremities are the earliest (DAG-wise) known events which we have +// the entire event JSON. These event IDs are used in federation requests to fetch +// even earlier events. +// +// We persist the previous event IDs as well, one per row, so when we do fetch even +// earlier events we can simply delete rows which referenced it. Consider the graph: +// A +// | Event C has 1 prev_event ID: A. +// B C +// |___| Event D has 2 prev_event IDs: B and C. +// | +// D +// The earliest known event we have is D, so this table has 2 rows. +// A backfill request gives us C but not B. We delete rows where prev_event=C. This +// still means that D is a backwards extremity as we do not have event B. However, event +// C is *also* a backwards extremity at this point as we do not have event A. Later, +// when we fetch event B, we delete rows where prev_event=B. This then removes D as +// a backwards extremity because there are no more rows with event_id=B. +type BackwardsExtremities struct { + insertBackwardExtremityStmt *sql.Stmt + selectBackwardExtremitiesForRoomStmt *sql.Stmt + deleteBackwardExtremityStmt *sql.Stmt +} + +// NewBackwardsExtremities prepares the table +func NewBackwardsExtremities(db *sql.DB, stmts BackwardsExtremitiesStatements) (table BackwardsExtremities, err error) { + _, err = db.Exec(stmts.Schema()) + if err != nil { + return + } + if table.insertBackwardExtremityStmt, err = db.Prepare(stmts.InsertBackwardExtremity()); err != nil { + return + } + if table.selectBackwardExtremitiesForRoomStmt, err = db.Prepare(stmts.SelectBackwardExtremitiesForRoom()); err != nil { + return + } + if table.deleteBackwardExtremityStmt, err = db.Prepare(stmts.DeleteBackwardExtremity()); err != nil { + return + } + return +} + +// InsertsBackwardExtremity inserts a new backwards extremity. +func (s *BackwardsExtremities) InsertsBackwardExtremity( + ctx context.Context, txn *sql.Tx, roomID, eventID string, prevEventID string, +) (err error) { + _, err = txn.Stmt(s.insertBackwardExtremityStmt).ExecContext(ctx, roomID, eventID, prevEventID) + return +} + +// SelectBackwardExtremitiesForRoom retrieves all backwards extremities for the room. +func (s *BackwardsExtremities) SelectBackwardExtremitiesForRoom( + ctx context.Context, roomID string, +) (eventIDs []string, err error) { + rows, err := s.selectBackwardExtremitiesForRoomStmt.QueryContext(ctx, roomID) + if err != nil { + return + } + defer common.CloseAndLogIfError(ctx, rows, "selectBackwardExtremitiesForRoom: rows.close() failed") + + for rows.Next() { + var eID string + if err = rows.Scan(&eID); err != nil { + return + } + + eventIDs = append(eventIDs, eID) + } + + return eventIDs, rows.Err() +} + +// DeleteBackwardExtremity removes a backwards extremity for a room, if one existed. +func (s *BackwardsExtremities) DeleteBackwardExtremity( + ctx context.Context, txn *sql.Tx, roomID, knownEventID string, +) (err error) { + _, err = txn.Stmt(s.deleteBackwardExtremityStmt).ExecContext(ctx, roomID, knownEventID) + return +} From 0354836b57aa0f1e39c43961669ab1afcb757460 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 28 Apr 2020 16:22:00 +0100 Subject: [PATCH 011/200] Unbreak the wasm build (#986) --- build.sh | 4 +++- cmd/dendritejs/main.go | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/build.sh b/build.sh index 3ef148891..087f4ae72 100755 --- a/build.sh +++ b/build.sh @@ -3,4 +3,6 @@ # Put installed packages into ./bin export GOBIN=$PWD/`dirname $0`/bin -go install -v $PWD/`dirname $0`/cmd/... \ No newline at end of file +go install -v $PWD/`dirname $0`/cmd/... + +GOOS=js GOARCH=wasm go build -o main.wasm ./cmd/dendritejs diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 9bd8f2ee2..23d4327d7 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -123,12 +123,12 @@ func main() { } p2pPublicRoomProvider := NewLibP2PPublicRoomsProvider(node) - alias, input, query := roomserver.SetupRoomServerComponent(base) + alias, input, query := roomserver.SetupRoomServerComponent(base, keyRing) eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New()) asQuery := appservice.SetupAppServiceAPIComponent( base, accountDB, deviceDB, federation, alias, query, transactions.New(), ) - fedSenderAPI := federationsender.SetupFederationSenderComponent(base, federation, query) + fedSenderAPI := federationsender.SetupFederationSenderComponent(base, federation, query, input) clientapi.SetupClientAPIComponent( base, deviceDB, accountDB, From a4b9edb28e32b505cf3a67bcba1acacd4a882155 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 28 Apr 2020 16:51:16 +0100 Subject: [PATCH 012/200] Dependency inject the federation client so p2p binaries work as expected (#987) --- cmd/dendrite-demo-libp2p/main.go | 2 +- cmd/dendrite-monolith-server/main.go | 2 +- cmd/dendrite-room-server/main.go | 2 +- cmd/dendritejs/main.go | 2 +- roomserver/roomserver.go | 3 ++- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index f1d18d0e0..66d103eaf 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -148,7 +148,7 @@ func main() { federation := createFederationClient(base) keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) - alias, input, query := roomserver.SetupRoomServerComponent(&base.Base, keyRing) + alias, input, query := roomserver.SetupRoomServerComponent(&base.Base, keyRing, federation) eduInputAPI := eduserver.SetupEDUServerComponent(&base.Base, cache.New()) asQuery := appservice.SetupAppServiceAPIComponent( &base.Base, accountDB, deviceDB, federation, alias, query, transactions.New(), diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 8cfb75665..f3e720a88 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -57,7 +57,7 @@ func main() { federation := base.CreateFederationClient() keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) - alias, input, query := roomserver.SetupRoomServerComponent(base, keyRing) + alias, input, query := roomserver.SetupRoomServerComponent(base, keyRing, federation) eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New()) asQuery := appservice.SetupAppServiceAPIComponent( base, accountDB, deviceDB, federation, alias, query, transactions.New(), diff --git a/cmd/dendrite-room-server/main.go b/cmd/dendrite-room-server/main.go index f33a2b88f..98410452f 100644 --- a/cmd/dendrite-room-server/main.go +++ b/cmd/dendrite-room-server/main.go @@ -30,7 +30,7 @@ func main() { federation := base.CreateFederationClient() keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) - roomserver.SetupRoomServerComponent(base, keyRing) + roomserver.SetupRoomServerComponent(base, keyRing, federation) base.SetupAndServeHTTP(string(base.Cfg.Bind.RoomServer), string(base.Cfg.Listen.RoomServer)) diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 23d4327d7..6300d249f 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -123,7 +123,7 @@ func main() { } p2pPublicRoomProvider := NewLibP2PPublicRoomsProvider(node) - alias, input, query := roomserver.SetupRoomServerComponent(base, keyRing) + alias, input, query := roomserver.SetupRoomServerComponent(base, keyRing, federation) eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New()) asQuery := appservice.SetupAppServiceAPIComponent( base, accountDB, deviceDB, federation, alias, query, transactions.New(), diff --git a/roomserver/roomserver.go b/roomserver/roomserver.go index ea1c5c4c3..f6a373a4f 100644 --- a/roomserver/roomserver.go +++ b/roomserver/roomserver.go @@ -35,6 +35,7 @@ import ( // APIs directly instead of having to use HTTP. func SetupRoomServerComponent( base *basecomponent.BaseDendrite, keyRing gomatrixserverlib.JSONVerifier, + fedClient *gomatrixserverlib.FederationClient, ) (api.RoomserverAliasAPI, api.RoomserverInputAPI, api.RoomserverQueryAPI) { roomserverDB, err := storage.Open(string(base.Cfg.Database.RoomServer)) if err != nil { @@ -53,7 +54,7 @@ func SetupRoomServerComponent( DB: roomserverDB, ImmutableCache: base.ImmutableCache, ServerName: base.Cfg.Matrix.ServerName, - FedClient: base.CreateFederationClient(), + FedClient: fedClient, // TODO: We should have a key server so we don't keep adding components // which talk to the same DB. KeyRing: keyRing, From a308e61331f549ae0964f83dff88abc282033ed3 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 29 Apr 2020 11:34:31 +0100 Subject: [PATCH 013/200] Federation sender API remodel (#988) * Define an input API for the federationsender * Wiring for rooomserver input API and federation sender input API * Whoops, commit common too * Merge input API into query API * Rename FederationSenderQueryAPI to FederationSenderInternalAPI * Fix dendritejs * Rename Input to Perform * Fix a couple of inputs -> performs * Remove needless storage interface, add comments --- clientapi/clientapi.go | 4 +- clientapi/routing/directory.go | 2 +- clientapi/routing/routing.go | 2 +- cmd/dendrite-client-api-server/main.go | 4 +- cmd/dendrite-demo-libp2p/main.go | 6 +- cmd/dendrite-federation-api-server/main.go | 4 +- cmd/dendrite-monolith-server/main.go | 7 +- cmd/dendrite-room-server/main.go | 4 +- cmd/dendritejs/main.go | 1 + common/basecomponent/base.go | 8 +-- federationapi/federationapi.go | 2 +- federationapi/routing/query.go | 2 +- federationapi/routing/routing.go | 2 +- federationsender/api/api.go | 53 ++++++++++++++ federationsender/api/perform.go | 56 +++++++++++++++ federationsender/api/query.go | 80 ++++++---------------- federationsender/federationsender.go | 4 +- federationsender/query/api.go | 77 +++++++++++++++++++++ federationsender/query/perform.go | 25 +++++++ federationsender/query/query.go | 57 ++------------- roomserver/api/input.go | 16 ++++- roomserver/input/input.go | 12 ++++ roomserver/roomserver.go | 1 + 23 files changed, 293 insertions(+), 136 deletions(-) create mode 100644 federationsender/api/api.go create mode 100644 federationsender/api/perform.go create mode 100644 federationsender/query/api.go create mode 100644 federationsender/query/perform.go diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go index 1339f7c8c..6a857e52c 100644 --- a/clientapi/clientapi.go +++ b/clientapi/clientapi.go @@ -44,7 +44,7 @@ func SetupClientAPIComponent( eduInputAPI eduServerAPI.EDUServerInputAPI, asAPI appserviceAPI.AppServiceQueryAPI, transactionsCache *transactions.Cache, - fedSenderAPI federationSenderAPI.FederationSenderQueryAPI, + fsAPI federationSenderAPI.FederationSenderInternalAPI, ) { roomserverProducer := producers.NewRoomserverProducer(inputAPI, queryAPI) eduProducer := producers.NewEDUServerProducer(eduInputAPI) @@ -69,6 +69,6 @@ func SetupClientAPIComponent( routing.Setup( base.APIMux, base.Cfg, roomserverProducer, queryAPI, aliasAPI, asAPI, accountsDB, deviceDB, federation, *keyRing, userUpdateProducer, - syncProducer, eduProducer, transactionsCache, fedSenderAPI, + syncProducer, eduProducer, transactionsCache, fsAPI, ) } diff --git a/clientapi/routing/directory.go b/clientapi/routing/directory.go index 248696ab2..101ba11ff 100644 --- a/clientapi/routing/directory.go +++ b/clientapi/routing/directory.go @@ -47,7 +47,7 @@ func DirectoryRoom( federation *gomatrixserverlib.FederationClient, cfg *config.Dendrite, rsAPI roomserverAPI.RoomserverAliasAPI, - fedSenderAPI federationSenderAPI.FederationSenderQueryAPI, + fedSenderAPI federationSenderAPI.FederationSenderInternalAPI, ) util.JSONResponse { _, domain, err := gomatrixserverlib.SplitID('#', roomAlias) if err != nil { diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 5dc6d7db9..9ab22cbec 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -60,7 +60,7 @@ func Setup( syncProducer *producers.SyncAPIProducer, eduProducer *producers.EDUServerProducer, transactionsCache *transactions.Cache, - federationSender federationSenderAPI.FederationSenderQueryAPI, + federationSender federationSenderAPI.FederationSenderInternalAPI, ) { apiMux.Handle("/_matrix/client/versions", diff --git a/cmd/dendrite-client-api-server/main.go b/cmd/dendrite-client-api-server/main.go index 815a978a8..c8f629689 100644 --- a/cmd/dendrite-client-api-server/main.go +++ b/cmd/dendrite-client-api-server/main.go @@ -37,12 +37,12 @@ func main() { asQuery := base.CreateHTTPAppServiceAPIs() alias, input, query := base.CreateHTTPRoomserverAPIs() - fedSenderAPI := base.CreateHTTPFederationSenderAPIs() + fsAPI := base.CreateHTTPFederationSenderAPIs() eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New()) clientapi.SetupClientAPIComponent( base, deviceDB, accountDB, federation, &keyRing, - alias, input, query, eduInputAPI, asQuery, transactions.New(), fedSenderAPI, + alias, input, query, eduInputAPI, asQuery, transactions.New(), fsAPI, ) base.SetupAndServeHTTP(string(base.Cfg.Bind.ClientAPI), string(base.Cfg.Listen.ClientAPI)) diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index 66d103eaf..b9fbfc53e 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -153,15 +153,15 @@ func main() { asQuery := appservice.SetupAppServiceAPIComponent( &base.Base, accountDB, deviceDB, federation, alias, query, transactions.New(), ) - fedSenderAPI := federationsender.SetupFederationSenderComponent(&base.Base, federation, query, input) + fsAPI := federationsender.SetupFederationSenderComponent(&base.Base, federation, query, input) clientapi.SetupClientAPIComponent( &base.Base, deviceDB, accountDB, federation, &keyRing, alias, input, query, - eduInputAPI, asQuery, transactions.New(), fedSenderAPI, + eduInputAPI, asQuery, transactions.New(), fsAPI, ) eduProducer := producers.NewEDUServerProducer(eduInputAPI) - federationapi.SetupFederationAPIComponent(&base.Base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fedSenderAPI, eduProducer) + federationapi.SetupFederationAPIComponent(&base.Base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fsAPI, eduProducer) mediaapi.SetupMediaAPIComponent(&base.Base, deviceDB) publicRoomsDB, err := storage.NewPublicRoomsServerDatabaseWithPubSub(string(base.Base.Cfg.Database.PublicRoomsAPI), base.LibP2PPubsub) if err != nil { diff --git a/cmd/dendrite-federation-api-server/main.go b/cmd/dendrite-federation-api-server/main.go index dd06cd3f9..4267cf166 100644 --- a/cmd/dendrite-federation-api-server/main.go +++ b/cmd/dendrite-federation-api-server/main.go @@ -32,7 +32,7 @@ func main() { deviceDB := base.CreateDeviceDB() keyDB := base.CreateKeyDB() federation := base.CreateFederationClient() - federationSender := base.CreateHTTPFederationSenderAPIs() + fsAPI := base.CreateHTTPFederationSenderAPIs() keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) alias, input, query := base.CreateHTTPRoomserverAPIs() @@ -42,7 +42,7 @@ func main() { federationapi.SetupFederationAPIComponent( base, accountDB, deviceDB, federation, &keyRing, - alias, input, query, asQuery, federationSender, eduProducer, + alias, input, query, asQuery, fsAPI, eduProducer, ) base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationAPI), string(base.Cfg.Listen.FederationAPI)) diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index f3e720a88..e806f6f3f 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -62,15 +62,16 @@ func main() { asQuery := appservice.SetupAppServiceAPIComponent( base, accountDB, deviceDB, federation, alias, query, transactions.New(), ) - fedSenderAPI := federationsender.SetupFederationSenderComponent(base, federation, query, input) + fsAPI := federationsender.SetupFederationSenderComponent(base, federation, query, input) + input.SetFederationSenderAPI(fsAPI) clientapi.SetupClientAPIComponent( base, deviceDB, accountDB, federation, &keyRing, alias, input, query, - eduInputAPI, asQuery, transactions.New(), fedSenderAPI, + eduInputAPI, asQuery, transactions.New(), fsAPI, ) eduProducer := producers.NewEDUServerProducer(eduInputAPI) - federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fedSenderAPI, eduProducer) + federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fsAPI, eduProducer) mediaapi.SetupMediaAPIComponent(base, deviceDB) publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI)) if err != nil { diff --git a/cmd/dendrite-room-server/main.go b/cmd/dendrite-room-server/main.go index 98410452f..3f9913e24 100644 --- a/cmd/dendrite-room-server/main.go +++ b/cmd/dendrite-room-server/main.go @@ -30,7 +30,9 @@ func main() { federation := base.CreateFederationClient() keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) - roomserver.SetupRoomServerComponent(base, keyRing, federation) + fsAPI := base.CreateHTTPFederationSenderAPIs() + _, input, _ := roomserver.SetupRoomServerComponent(base, keyRing, federation) + input.SetFederationSenderAPI(fsAPI) base.SetupAndServeHTTP(string(base.Cfg.Bind.RoomServer), string(base.Cfg.Listen.RoomServer)) diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 6300d249f..7665138eb 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -129,6 +129,7 @@ func main() { base, accountDB, deviceDB, federation, alias, query, transactions.New(), ) fedSenderAPI := federationsender.SetupFederationSenderComponent(base, federation, query, input) + input.SetFederationSenderAPI(fedSenderAPI) clientapi.SetupClientAPIComponent( base, deviceDB, accountDB, diff --git a/common/basecomponent/base.go b/common/basecomponent/base.go index 5e2d659bf..f245dd50b 100644 --- a/common/basecomponent/base.go +++ b/common/basecomponent/base.go @@ -149,12 +149,12 @@ func (b *BaseDendrite) CreateHTTPEDUServerAPIs() eduServerAPI.EDUServerInputAPI return e } -// CreateHTTPFederationSenderAPIs returns FederationSenderQueryAPI for hitting +// CreateHTTPFederationSenderAPIs returns FederationSenderInternalAPI for hitting // the federation sender over HTTP -func (b *BaseDendrite) CreateHTTPFederationSenderAPIs() federationSenderAPI.FederationSenderQueryAPI { - f, err := federationSenderAPI.NewFederationSenderQueryAPIHTTP(b.Cfg.FederationSenderURL(), b.httpClient) +func (b *BaseDendrite) CreateHTTPFederationSenderAPIs() federationSenderAPI.FederationSenderInternalAPI { + f, err := federationSenderAPI.NewFederationSenderInternalAPIHTTP(b.Cfg.FederationSenderURL(), b.httpClient) if err != nil { - logrus.WithError(err).Panic("NewFederationSenderQueryAPIHTTP failed", b.httpClient) + logrus.WithError(err).Panic("NewFederationSenderInternalAPIHTTP failed", b.httpClient) } return f } diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go index ed96322b8..72e2b54ac 100644 --- a/federationapi/federationapi.go +++ b/federationapi/federationapi.go @@ -40,7 +40,7 @@ func SetupFederationAPIComponent( inputAPI roomserverAPI.RoomserverInputAPI, queryAPI roomserverAPI.RoomserverQueryAPI, asAPI appserviceAPI.AppServiceQueryAPI, - federationSenderAPI federationSenderAPI.FederationSenderQueryAPI, + federationSenderAPI federationSenderAPI.FederationSenderInternalAPI, eduProducer *producers.EDUServerProducer, ) { roomserverProducer := producers.NewRoomserverProducer(inputAPI, queryAPI) diff --git a/federationapi/routing/query.go b/federationapi/routing/query.go index 7cb50e525..13c92451d 100644 --- a/federationapi/routing/query.go +++ b/federationapi/routing/query.go @@ -33,7 +33,7 @@ func RoomAliasToID( federation *gomatrixserverlib.FederationClient, cfg *config.Dendrite, aliasAPI roomserverAPI.RoomserverAliasAPI, - senderAPI federationSenderAPI.FederationSenderQueryAPI, + senderAPI federationSenderAPI.FederationSenderInternalAPI, ) util.JSONResponse { roomAlias := httpReq.FormValue("room_alias") if roomAlias == "" { diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index 83bac5550..ebaeec6e6 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -49,7 +49,7 @@ func Setup( asAPI appserviceAPI.AppServiceQueryAPI, producer *producers.RoomserverProducer, eduProducer *producers.EDUServerProducer, - federationSenderAPI federationSenderAPI.FederationSenderQueryAPI, + federationSenderAPI federationSenderAPI.FederationSenderInternalAPI, keys gomatrixserverlib.KeyRing, federation *gomatrixserverlib.FederationClient, accountDB accounts.Database, diff --git a/federationsender/api/api.go b/federationsender/api/api.go new file mode 100644 index 000000000..1340179e1 --- /dev/null +++ b/federationsender/api/api.go @@ -0,0 +1,53 @@ +package api + +import ( + "context" + "errors" + "net/http" +) + +// FederationSenderInternalAPI is used to query information from the federation sender. +type FederationSenderInternalAPI interface { + // Query the joined hosts and the membership events accounting for their participation in a room. + // Note that if a server has multiple users in the room, it will have multiple entries in the returned slice. + // See `QueryJoinedHostServerNamesInRoom` for a de-duplicated version. + QueryJoinedHostsInRoom( + ctx context.Context, + request *QueryJoinedHostsInRoomRequest, + response *QueryJoinedHostsInRoomResponse, + ) error + // Query the server names of the joined hosts in a room. + // Unlike QueryJoinedHostsInRoom, this function returns a de-duplicated slice + // containing only the server names (without information for membership events). + QueryJoinedHostServerNamesInRoom( + ctx context.Context, + request *QueryJoinedHostServerNamesInRoomRequest, + response *QueryJoinedHostServerNamesInRoomResponse, + ) error + // Handle an instruction to make_join & send_join with a remote server. + PerformJoinRequest( + ctx context.Context, + request *PerformJoinRequest, + response *PerformJoinResponse, + ) error + // Handle an instruction to make_leave & send_leave with a remote server. + PerformLeaveRequest( + ctx context.Context, + request *PerformLeaveRequest, + response *PerformLeaveResponse, + ) error +} + +// NewFederationSenderInternalAPIHTTP creates a FederationSenderInternalAPI implemented by talking to a HTTP POST API. +// If httpClient is nil an error is returned +func NewFederationSenderInternalAPIHTTP(federationSenderURL string, httpClient *http.Client) (FederationSenderInternalAPI, error) { + if httpClient == nil { + return nil, errors.New("NewFederationSenderInternalAPIHTTP: httpClient is ") + } + return &httpFederationSenderInternalAPI{federationSenderURL, httpClient}, nil +} + +type httpFederationSenderInternalAPI struct { + federationSenderURL string + httpClient *http.Client +} diff --git a/federationsender/api/perform.go b/federationsender/api/perform.go new file mode 100644 index 000000000..8c30ecbef --- /dev/null +++ b/federationsender/api/perform.go @@ -0,0 +1,56 @@ +package api + +import ( + "context" + + commonHTTP "github.com/matrix-org/dendrite/common/http" + "github.com/opentracing/opentracing-go" +) + +const ( + // FederationSenderPerformJoinRequestPath is the HTTP path for the PerformJoinRequest API. + FederationSenderPerformJoinRequestPath = "/api/federationsender/performJoinRequest" + + // FederationSenderPerformLeaveRequestPath is the HTTP path for the PerformLeaveRequest API. + FederationSenderPerformLeaveRequestPath = "/api/federationsender/performLeaveRequest" +) + +type PerformJoinRequest struct { + RoomID string `json:"room_id"` +} + +type PerformJoinResponse struct { +} + +// Handle an instruction to make_join & send_join with a remote server. +func (h *httpFederationSenderInternalAPI) PerformJoinRequest( + ctx context.Context, + request *PerformJoinRequest, + response *PerformJoinResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "PerformJoinRequest") + defer span.Finish() + + apiURL := h.federationSenderURL + FederationSenderPerformJoinRequestPath + return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +type PerformLeaveRequest struct { + RoomID string `json:"room_id"` +} + +type PerformLeaveResponse struct { +} + +// Handle an instruction to make_leave & send_leave with a remote server. +func (h *httpFederationSenderInternalAPI) PerformLeaveRequest( + ctx context.Context, + request *PerformLeaveRequest, + response *PerformLeaveResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "PerformLeaveRequest") + defer span.Finish() + + apiURL := h.federationSenderURL + FederationSenderPerformLeaveRequestPath + return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} diff --git a/federationsender/api/query.go b/federationsender/api/query.go index 7c0ca7ff2..7a58cc863 100644 --- a/federationsender/api/query.go +++ b/federationsender/api/query.go @@ -2,16 +2,20 @@ package api import ( "context" - "errors" - "net/http" commonHTTP "github.com/matrix-org/dendrite/common/http" + "github.com/matrix-org/dendrite/federationsender/types" "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/dendrite/federationsender/types" "github.com/opentracing/opentracing-go" ) +// FederationSenderQueryJoinedHostsInRoomPath is the HTTP path for the QueryJoinedHostsInRoom API. +const FederationSenderQueryJoinedHostsInRoomPath = "/api/federationsender/queryJoinedHostsInRoom" + +// FederationSenderQueryJoinedHostServerNamesInRoomPath is the HTTP path for the QueryJoinedHostServerNamesInRoom API. +const FederationSenderQueryJoinedHostServerNamesInRoomPath = "/api/federationsender/queryJoinedHostServerNamesInRoom" + // QueryJoinedHostsInRoomRequest is a request to QueryJoinedHostsInRoom type QueryJoinedHostsInRoomRequest struct { RoomID string `json:"room_id"` @@ -22,6 +26,19 @@ type QueryJoinedHostsInRoomResponse struct { JoinedHosts []types.JoinedHost `json:"joined_hosts"` } +// QueryJoinedHostsInRoom implements FederationSenderInternalAPI +func (h *httpFederationSenderInternalAPI) QueryJoinedHostsInRoom( + ctx context.Context, + request *QueryJoinedHostsInRoomRequest, + response *QueryJoinedHostsInRoomResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "QueryJoinedHostsInRoom") + defer span.Finish() + + apiURL := h.federationSenderURL + FederationSenderQueryJoinedHostsInRoomPath + return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + // QueryJoinedHostServerNamesRequest is a request to QueryJoinedHostServerNames type QueryJoinedHostServerNamesInRoomRequest struct { RoomID string `json:"room_id"` @@ -32,61 +49,8 @@ type QueryJoinedHostServerNamesInRoomResponse struct { ServerNames []gomatrixserverlib.ServerName `json:"server_names"` } -// FederationSenderQueryAPI is used to query information from the federation sender. -type FederationSenderQueryAPI interface { - // Query the joined hosts and the membership events accounting for their participation in a room. - // Note that if a server has multiple users in the room, it will have multiple entries in the returned slice. - // See `QueryJoinedHostServerNamesInRoom` for a de-duplicated version. - QueryJoinedHostsInRoom( - ctx context.Context, - request *QueryJoinedHostsInRoomRequest, - response *QueryJoinedHostsInRoomResponse, - ) error - // Query the server names of the joined hosts in a room. - // Unlike QueryJoinedHostsInRoom, this function returns a de-duplicated slice - // containing only the server names (without information for membership events). - QueryJoinedHostServerNamesInRoom( - ctx context.Context, - request *QueryJoinedHostServerNamesInRoomRequest, - response *QueryJoinedHostServerNamesInRoomResponse, - ) error -} - -// FederationSenderQueryJoinedHostsInRoomPath is the HTTP path for the QueryJoinedHostsInRoom API. -const FederationSenderQueryJoinedHostsInRoomPath = "/api/federationsender/queryJoinedHostsInRoom" - -// FederationSenderQueryJoinedHostServerNamesInRoomPath is the HTTP path for the QueryJoinedHostServerNamesInRoom API. -const FederationSenderQueryJoinedHostServerNamesInRoomPath = "/api/federationsender/queryJoinedHostServerNamesInRoom" - -// NewFederationSenderQueryAPIHTTP creates a FederationSenderQueryAPI implemented by talking to a HTTP POST API. -// If httpClient is nil an error is returned -func NewFederationSenderQueryAPIHTTP(federationSenderURL string, httpClient *http.Client) (FederationSenderQueryAPI, error) { - if httpClient == nil { - return nil, errors.New("NewFederationSenderQueryAPIHTTP: httpClient is ") - } - return &httpFederationSenderQueryAPI{federationSenderURL, httpClient}, nil -} - -type httpFederationSenderQueryAPI struct { - federationSenderURL string - httpClient *http.Client -} - -// QueryJoinedHostsInRoom implements FederationSenderQueryAPI -func (h *httpFederationSenderQueryAPI) QueryJoinedHostsInRoom( - ctx context.Context, - request *QueryJoinedHostsInRoomRequest, - response *QueryJoinedHostsInRoomResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "QueryJoinedHostsInRoom") - defer span.Finish() - - apiURL := h.federationSenderURL + FederationSenderQueryJoinedHostsInRoomPath - return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} - -// QueryJoinedHostServerNamesInRoom implements FederationSenderQueryAPI -func (h *httpFederationSenderQueryAPI) QueryJoinedHostServerNamesInRoom( +// QueryJoinedHostServerNamesInRoom implements FederationSenderInternalAPI +func (h *httpFederationSenderInternalAPI) QueryJoinedHostServerNamesInRoom( ctx context.Context, request *QueryJoinedHostServerNamesInRoomRequest, response *QueryJoinedHostServerNamesInRoomResponse, diff --git a/federationsender/federationsender.go b/federationsender/federationsender.go index a06caf402..355775f8a 100644 --- a/federationsender/federationsender.go +++ b/federationsender/federationsender.go @@ -36,7 +36,7 @@ func SetupFederationSenderComponent( federation *gomatrixserverlib.FederationClient, rsQueryAPI roomserverAPI.RoomserverQueryAPI, rsInputAPI roomserverAPI.RoomserverInputAPI, -) api.FederationSenderQueryAPI { +) api.FederationSenderInternalAPI { federationSenderDB, err := storage.NewDatabase(string(base.Cfg.Database.FederationSender)) if err != nil { logrus.WithError(err).Panic("failed to connect to federation sender db") @@ -61,7 +61,7 @@ func SetupFederationSenderComponent( logrus.WithError(err).Panic("failed to start typing server consumer") } - queryAPI := query.FederationSenderQueryAPI{ + queryAPI := query.FederationSenderInternalAPI{ DB: federationSenderDB, } queryAPI.SetupHTTP(http.DefaultServeMux) diff --git a/federationsender/query/api.go b/federationsender/query/api.go new file mode 100644 index 000000000..e33bcc111 --- /dev/null +++ b/federationsender/query/api.go @@ -0,0 +1,77 @@ +package query + +import ( + "encoding/json" + "net/http" + + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/federationsender/storage" + rsAPI "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/util" +) + +// FederationSenderInternalAPI is an implementation of api.FederationSenderInternalAPI +type FederationSenderInternalAPI struct { + api.FederationSenderInternalAPI + DB storage.Database + RoomserverInputAPI rsAPI.RoomserverInputAPI +} + +// SetupHTTP adds the FederationSenderInternalAPI handlers to the http.ServeMux. +func (f *FederationSenderInternalAPI) SetupHTTP(servMux *http.ServeMux) { + servMux.Handle( + api.FederationSenderQueryJoinedHostsInRoomPath, + common.MakeInternalAPI("QueryJoinedHostsInRoom", func(req *http.Request) util.JSONResponse { + var request api.QueryJoinedHostsInRoomRequest + var response api.QueryJoinedHostsInRoomResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := f.QueryJoinedHostsInRoom(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.FederationSenderQueryJoinedHostServerNamesInRoomPath, + common.MakeInternalAPI("QueryJoinedHostServerNamesInRoom", func(req *http.Request) util.JSONResponse { + var request api.QueryJoinedHostServerNamesInRoomRequest + var response api.QueryJoinedHostServerNamesInRoomResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := f.QueryJoinedHostServerNamesInRoom(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle(api.FederationSenderPerformJoinRequestPath, + common.MakeInternalAPI("PerformJoinRequest", func(req *http.Request) util.JSONResponse { + var request api.PerformJoinRequest + var response api.PerformJoinResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := f.PerformJoinRequest(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle(api.FederationSenderPerformLeaveRequestPath, + common.MakeInternalAPI("PerformLeaveRequest", func(req *http.Request) util.JSONResponse { + var request api.PerformLeaveRequest + var response api.PerformLeaveResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := f.PerformLeaveRequest(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) +} diff --git a/federationsender/query/perform.go b/federationsender/query/perform.go new file mode 100644 index 000000000..2486873c2 --- /dev/null +++ b/federationsender/query/perform.go @@ -0,0 +1,25 @@ +package query + +import ( + "context" + + "github.com/matrix-org/dendrite/federationsender/api" +) + +// PerformJoinRequest implements api.FederationSenderInternalAPI +func (r *FederationSenderInternalAPI) PerformJoinRequest( + ctx context.Context, + request *api.PerformJoinRequest, + response *api.PerformJoinResponse, +) (err error) { + return nil +} + +// PerformLeaveRequest implements api.FederationSenderInternalAPI +func (r *FederationSenderInternalAPI) PerformLeaveRequest( + ctx context.Context, + request *api.PerformLeaveRequest, + response *api.PerformLeaveResponse, +) (err error) { + return nil +} diff --git a/federationsender/query/query.go b/federationsender/query/query.go index 8c35bb29e..ec668204d 100644 --- a/federationsender/query/query.go +++ b/federationsender/query/query.go @@ -2,30 +2,13 @@ package query import ( "context" - "encoding/json" - "net/http" - "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/federationsender/api" - "github.com/matrix-org/dendrite/federationsender/types" "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" ) -// FederationSenderQueryDatabase has the APIs needed to implement the query API. -type FederationSenderQueryDatabase interface { - GetJoinedHosts( - ctx context.Context, roomID string, - ) ([]types.JoinedHost, error) -} - -// FederationSenderQueryAPI is an implementation of api.FederationSenderQueryAPI -type FederationSenderQueryAPI struct { - DB FederationSenderQueryDatabase -} - -// QueryJoinedHostsInRoom implements api.FederationSenderQueryAPI -func (f *FederationSenderQueryAPI) QueryJoinedHostsInRoom( +// QueryJoinedHostsInRoom implements api.FederationSenderInternalAPI +func (f *FederationSenderInternalAPI) QueryJoinedHostsInRoom( ctx context.Context, request *api.QueryJoinedHostsInRoomRequest, response *api.QueryJoinedHostsInRoomResponse, @@ -34,8 +17,8 @@ func (f *FederationSenderQueryAPI) QueryJoinedHostsInRoom( return } -// QueryJoinedHostServerNamesInRoom implements api.FederationSenderQueryAPI -func (f *FederationSenderQueryAPI) QueryJoinedHostServerNamesInRoom( +// QueryJoinedHostServerNamesInRoom implements api.FederationSenderInternalAPI +func (f *FederationSenderInternalAPI) QueryJoinedHostServerNamesInRoom( ctx context.Context, request *api.QueryJoinedHostServerNamesInRoomRequest, response *api.QueryJoinedHostServerNamesInRoomResponse, @@ -54,35 +37,3 @@ func (f *FederationSenderQueryAPI) QueryJoinedHostServerNamesInRoom( return } - -// SetupHTTP adds the FederationSenderQueryAPI handlers to the http.ServeMux. -func (f *FederationSenderQueryAPI) SetupHTTP(servMux *http.ServeMux) { - servMux.Handle( - api.FederationSenderQueryJoinedHostsInRoomPath, - common.MakeInternalAPI("QueryJoinedHostsInRoom", func(req *http.Request) util.JSONResponse { - var request api.QueryJoinedHostsInRoomRequest - var response api.QueryJoinedHostsInRoomResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := f.QueryJoinedHostsInRoom(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.FederationSenderQueryJoinedHostServerNamesInRoomPath, - common.MakeInternalAPI("QueryJoinedHostServerNamesInRoom", func(req *http.Request) util.JSONResponse { - var request api.QueryJoinedHostServerNamesInRoomRequest - var response api.QueryJoinedHostServerNamesInRoomResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := f.QueryJoinedHostServerNamesInRoom(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) -} diff --git a/roomserver/api/input.go b/roomserver/api/input.go index bb4e040de..d9cffad27 100644 --- a/roomserver/api/input.go +++ b/roomserver/api/input.go @@ -21,6 +21,7 @@ import ( "net/http" commonHTTP "github.com/matrix-org/dendrite/common/http" + fsAPI "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/gomatrixserverlib" opentracing "github.com/opentracing/opentracing-go" ) @@ -106,6 +107,9 @@ type InputRoomEventsResponse struct { // RoomserverInputAPI is used to write events to the room server. type RoomserverInputAPI interface { + // needed to avoid chicken and egg scenario when setting up the + // interdependencies between the roomserver and the FS input API + SetFederationSenderAPI(fsInputAPI fsAPI.FederationSenderInternalAPI) InputRoomEvents( ctx context.Context, request *InputRoomEventsRequest, @@ -122,12 +126,22 @@ func NewRoomserverInputAPIHTTP(roomserverURL string, httpClient *http.Client) (R if httpClient == nil { return nil, errors.New("NewRoomserverInputAPIHTTP: httpClient is ") } - return &httpRoomserverInputAPI{roomserverURL, httpClient}, nil + return &httpRoomserverInputAPI{roomserverURL, httpClient, nil}, nil } type httpRoomserverInputAPI struct { roomserverURL string httpClient *http.Client + // The federation sender API allows us to send federation + // requests from the new perform input requests, still TODO. + fsInputAPI fsAPI.FederationSenderInternalAPI +} + +// SetFederationSenderInputAPI passes in a federation sender input API reference +// so that we can avoid the chicken-and-egg problem of both the roomserver input API +// and the federation sender input API being interdependent. +func (h *httpRoomserverInputAPI) SetFederationSenderAPI(fsInputAPI fsAPI.FederationSenderInternalAPI) { + h.fsInputAPI = fsInputAPI } // InputRoomEvents implements RoomserverInputAPI diff --git a/roomserver/input/input.go b/roomserver/input/input.go index cb588380a..20b6afc4b 100644 --- a/roomserver/input/input.go +++ b/roomserver/input/input.go @@ -26,6 +26,8 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/util" + + fsAPI "github.com/matrix-org/dendrite/federationsender/api" ) // RoomserverInputAPI implements api.RoomserverInputAPI @@ -37,6 +39,16 @@ type RoomserverInputAPI struct { OutputRoomEventTopic string // Protects calls to processRoomEvent mutex sync.Mutex + // The federation sender API allows us to send federation + // requests from the new perform input requests, still TODO. + fsAPI fsAPI.FederationSenderInternalAPI +} + +// SetFederationSenderInputAPI passes in a federation sender input API reference +// so that we can avoid the chicken-and-egg problem of both the roomserver input API +// and the federation sender input API being interdependent. +func (r *RoomserverInputAPI) SetFederationSenderAPI(fsAPI fsAPI.FederationSenderInternalAPI) { + r.fsAPI = fsAPI } // WriteOutputEvents implements OutputRoomEventWriter diff --git a/roomserver/roomserver.go b/roomserver/roomserver.go index f6a373a4f..6fb2caff9 100644 --- a/roomserver/roomserver.go +++ b/roomserver/roomserver.go @@ -21,6 +21,7 @@ import ( "github.com/matrix-org/gomatrixserverlib" asQuery "github.com/matrix-org/dendrite/appservice/query" + "github.com/matrix-org/dendrite/common/basecomponent" "github.com/matrix-org/dendrite/roomserver/alias" "github.com/matrix-org/dendrite/roomserver/input" From 64e94e9a6f0a138e7fe771f540b57988bc344b59 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 29 Apr 2020 15:29:39 +0100 Subject: [PATCH 014/200] Join room support in federation sender (#989) * Implement PerformJoinRequest * Rename perform functions * Check send join response * Temporary wiring to test federation sender room joins * Actually pass through the config * Make sure membership content shows join --- clientapi/routing/joinroom.go | 124 ++---------------- clientapi/routing/routing.go | 3 +- cmd/dendrite-demo-libp2p/main.go | 2 +- cmd/dendrite-federation-sender-server/main.go | 5 +- cmd/dendrite-monolith-server/main.go | 2 +- cmd/dendritejs/main.go | 2 +- federationsender/api/api.go | 4 +- federationsender/api/perform.go | 10 +- federationsender/federationsender.go | 9 +- federationsender/producers/roomserver.go | 36 +++++ federationsender/query/api.go | 30 ++++- federationsender/query/perform.go | 99 +++++++++++++- federationsender/query/perform/join.go | 70 ++++++++++ federationsender/query/query.go | 4 +- 14 files changed, 265 insertions(+), 135 deletions(-) create mode 100644 federationsender/query/perform/join.go diff --git a/clientapi/routing/joinroom.go b/clientapi/routing/joinroom.go index d0dee7c2a..f55d1b6ab 100644 --- a/clientapi/routing/joinroom.go +++ b/clientapi/routing/joinroom.go @@ -27,12 +27,11 @@ import ( "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/config" - "github.com/matrix-org/dendrite/roomserver/api" + federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" - "github.com/sirupsen/logrus" ) // JoinRoomByIDOrAlias implements the "/join/{roomIDOrAlias}" API. @@ -46,6 +45,7 @@ func JoinRoomByIDOrAlias( producer *producers.RoomserverProducer, queryAPI roomserverAPI.RoomserverQueryAPI, aliasAPI roomserverAPI.RoomserverAliasAPI, + fsAPI federationSenderAPI.FederationSenderInternalAPI, keyRing gomatrixserverlib.KeyRing, accountDB accounts.Database, ) util.JSONResponse { @@ -79,7 +79,8 @@ func JoinRoomByIDOrAlias( content["avatar_url"] = profile.AvatarURL r := joinRoomReq{ - req, evTime, content, device.UserID, cfg, federation, producer, queryAPI, aliasAPI, keyRing, + req, evTime, content, device.UserID, cfg, federation, producer, + queryAPI, aliasAPI, fsAPI, keyRing, } if strings.HasPrefix(roomIDOrAlias, "!") { @@ -107,6 +108,7 @@ type joinRoomReq struct { producer *producers.RoomserverProducer queryAPI roomserverAPI.RoomserverQueryAPI aliasAPI roomserverAPI.RoomserverAliasAPI + fsAPI federationSenderAPI.FederationSenderInternalAPI keyRing gomatrixserverlib.KeyRing } @@ -326,71 +328,15 @@ func (r joinRoomReq) joinRoomUsingServers( // server was invalid this returns an error. // Otherwise this returns a JSONResponse. func (r joinRoomReq) joinRoomUsingServer(roomID string, server gomatrixserverlib.ServerName) (*util.JSONResponse, error) { - // Ask the room server for information about room versions. - var request api.QueryRoomVersionCapabilitiesRequest - var response api.QueryRoomVersionCapabilitiesResponse - if err := r.queryAPI.QueryRoomVersionCapabilities(r.req.Context(), &request, &response); err != nil { + fedJoinReq := federationSenderAPI.PerformJoinRequest{ + RoomID: roomID, + UserID: r.userID, + ServerName: server, + } + fedJoinRes := federationSenderAPI.PerformJoinResponse{} + if err := r.fsAPI.PerformJoin(r.req.Context(), &fedJoinReq, &fedJoinRes); err != nil { return nil, err } - var supportedVersions []gomatrixserverlib.RoomVersion - for version := range response.AvailableRoomVersions { - supportedVersions = append(supportedVersions, version) - } - respMakeJoin, err := r.federation.MakeJoin(r.req.Context(), server, roomID, r.userID, supportedVersions) - if err != nil { - // TODO: Check if the user was not allowed to join the room. - return nil, fmt.Errorf("r.federation.MakeJoin: %w", err) - } - - // Set all the fields to be what they should be, this should be a no-op - // but it's possible that the remote server returned us something "odd" - err = r.writeToBuilder(&respMakeJoin.JoinEvent, roomID) - if err != nil { - return nil, fmt.Errorf("r.writeToBuilder: %w", err) - } - - if respMakeJoin.RoomVersion == "" { - respMakeJoin.RoomVersion = gomatrixserverlib.RoomVersionV1 - } - if _, err = respMakeJoin.RoomVersion.EventFormat(); err != nil { - return &util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.UnsupportedRoomVersion( - fmt.Sprintf("Room version '%s' is not supported", respMakeJoin.RoomVersion), - ), - }, nil - } - - event, err := respMakeJoin.JoinEvent.Build( - r.evTime, r.cfg.Matrix.ServerName, r.cfg.Matrix.KeyID, - r.cfg.Matrix.PrivateKey, respMakeJoin.RoomVersion, - ) - if err != nil { - return nil, fmt.Errorf("respMakeJoin.JoinEvent.Build: %w", err) - } - - respSendJoin, err := r.federation.SendJoin(r.req.Context(), server, event, respMakeJoin.RoomVersion) - if err != nil { - return nil, fmt.Errorf("r.federation.SendJoin: %w", err) - } - - if err = r.checkSendJoinResponse(event, server, respMakeJoin, respSendJoin); err != nil { - return nil, err - } - - util.GetLogger(r.req.Context()).WithFields(logrus.Fields{ - "room_id": roomID, - "num_auth_events": len(respSendJoin.AuthEvents), - "num_state_events": len(respSendJoin.StateEvents), - }).Info("Room join signature and auth verification passed") - - if err = r.producer.SendEventWithState( - r.req.Context(), - respSendJoin.ToRespState(), - event.Headered(respMakeJoin.RoomVersion), - ); err != nil { - util.GetLogger(r.req.Context()).WithError(err).Error("r.producer.SendEventWithState") - } return &util.JSONResponse{ Code: http.StatusOK, @@ -400,49 +346,3 @@ func (r joinRoomReq) joinRoomUsingServer(roomID string, server gomatrixserverlib }{roomID}, }, nil } - -// checkSendJoinResponse checks that all of the signatures are correct -// and that the join is allowed by the supplied state. -func (r joinRoomReq) checkSendJoinResponse( - event gomatrixserverlib.Event, - server gomatrixserverlib.ServerName, - respMakeJoin gomatrixserverlib.RespMakeJoin, - respSendJoin gomatrixserverlib.RespSendJoin, -) error { - // A list of events that we have retried, if they were not included in - // the auth events supplied in the send_join. - retries := map[string]bool{} - -retryCheck: - // TODO: Can we expand Check here to return a list of missing auth - // events rather than failing one at a time? - if err := respSendJoin.Check(r.req.Context(), r.keyRing, event); err != nil { - switch e := err.(type) { - case gomatrixserverlib.MissingAuthEventError: - // Check that we haven't already retried for this event, prevents - // us from ending up in endless loops - if !retries[e.AuthEventID] { - // Ask the server that we're talking to right now for the event - tx, txerr := r.federation.GetEvent(r.req.Context(), server, e.AuthEventID) - if txerr != nil { - return fmt.Errorf("r.federation.GetEvent: %w", txerr) - } - // For each event returned, add it to the auth events. - for _, pdu := range tx.PDUs { - ev, everr := gomatrixserverlib.NewEventFromUntrustedJSON(pdu, respMakeJoin.RoomVersion) - if everr != nil { - return fmt.Errorf("gomatrixserverlib.NewEventFromUntrustedJSON: %w", everr) - } - respSendJoin.AuthEvents = append(respSendJoin.AuthEvents, ev) - } - // Mark the event as retried and then give the check another go. - retries[e.AuthEventID] = true - goto retryCheck - } - return fmt.Errorf("respSendJoin (after retries): %w", e) - default: - return fmt.Errorf("respSendJoin: %w", err) - } - } - return nil -} diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 9ab22cbec..e62b51935 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -101,7 +101,8 @@ func Setup( return util.ErrorResponse(err) } return JoinRoomByIDOrAlias( - req, device, vars["roomIDOrAlias"], cfg, federation, producer, queryAPI, aliasAPI, keyRing, accountDB, + req, device, vars["roomIDOrAlias"], cfg, federation, producer, + queryAPI, aliasAPI, federationSender, keyRing, accountDB, ) }), ).Methods(http.MethodPost, http.MethodOptions) diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index b9fbfc53e..a2a4675b3 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -153,7 +153,7 @@ func main() { asQuery := appservice.SetupAppServiceAPIComponent( &base.Base, accountDB, deviceDB, federation, alias, query, transactions.New(), ) - fsAPI := federationsender.SetupFederationSenderComponent(&base.Base, federation, query, input) + fsAPI := federationsender.SetupFederationSenderComponent(&base.Base, federation, query, input, &keyRing) clientapi.SetupClientAPIComponent( &base.Base, deviceDB, accountDB, diff --git a/cmd/dendrite-federation-sender-server/main.go b/cmd/dendrite-federation-sender-server/main.go index 1593afaa5..f8d43b990 100644 --- a/cmd/dendrite-federation-sender-server/main.go +++ b/cmd/dendrite-federation-sender-server/main.go @@ -16,6 +16,7 @@ package main import ( "github.com/matrix-org/dendrite/common/basecomponent" + "github.com/matrix-org/dendrite/common/keydb" "github.com/matrix-org/dendrite/federationsender" ) @@ -25,11 +26,13 @@ func main() { defer base.Close() // nolint: errcheck federation := base.CreateFederationClient() + keyDB := base.CreateKeyDB() + keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) _, input, query := base.CreateHTTPRoomserverAPIs() federationsender.SetupFederationSenderComponent( - base, federation, query, input, + base, federation, query, input, &keyRing, ) base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationSender), string(base.Cfg.Listen.FederationSender)) diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index e806f6f3f..f43f8b04c 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -62,7 +62,7 @@ func main() { asQuery := appservice.SetupAppServiceAPIComponent( base, accountDB, deviceDB, federation, alias, query, transactions.New(), ) - fsAPI := federationsender.SetupFederationSenderComponent(base, federation, query, input) + fsAPI := federationsender.SetupFederationSenderComponent(base, federation, query, input, &keyRing) input.SetFederationSenderAPI(fsAPI) clientapi.SetupClientAPIComponent( diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 7665138eb..1f2f20fb4 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -128,7 +128,7 @@ func main() { asQuery := appservice.SetupAppServiceAPIComponent( base, accountDB, deviceDB, federation, alias, query, transactions.New(), ) - fedSenderAPI := federationsender.SetupFederationSenderComponent(base, federation, query, input) + fedSenderAPI := federationsender.SetupFederationSenderComponent(base, federation, query, input, &keyRing) input.SetFederationSenderAPI(fedSenderAPI) clientapi.SetupClientAPIComponent( diff --git a/federationsender/api/api.go b/federationsender/api/api.go index 1340179e1..10dc66da2 100644 --- a/federationsender/api/api.go +++ b/federationsender/api/api.go @@ -25,13 +25,13 @@ type FederationSenderInternalAPI interface { response *QueryJoinedHostServerNamesInRoomResponse, ) error // Handle an instruction to make_join & send_join with a remote server. - PerformJoinRequest( + PerformJoin( ctx context.Context, request *PerformJoinRequest, response *PerformJoinResponse, ) error // Handle an instruction to make_leave & send_leave with a remote server. - PerformLeaveRequest( + PerformLeave( ctx context.Context, request *PerformLeaveRequest, response *PerformLeaveResponse, diff --git a/federationsender/api/perform.go b/federationsender/api/perform.go index 8c30ecbef..87736f294 100644 --- a/federationsender/api/perform.go +++ b/federationsender/api/perform.go @@ -4,6 +4,7 @@ import ( "context" commonHTTP "github.com/matrix-org/dendrite/common/http" + "github.com/matrix-org/gomatrixserverlib" "github.com/opentracing/opentracing-go" ) @@ -16,14 +17,17 @@ const ( ) type PerformJoinRequest struct { - RoomID string `json:"room_id"` + RoomID string `json:"room_id"` + UserID string `json:"user_id"` + ServerName gomatrixserverlib.ServerName `json:"server_name"` + Content map[string]interface{} `json:"content"` } type PerformJoinResponse struct { } // Handle an instruction to make_join & send_join with a remote server. -func (h *httpFederationSenderInternalAPI) PerformJoinRequest( +func (h *httpFederationSenderInternalAPI) PerformJoin( ctx context.Context, request *PerformJoinRequest, response *PerformJoinResponse, @@ -43,7 +47,7 @@ type PerformLeaveResponse struct { } // Handle an instruction to make_leave & send_leave with a remote server. -func (h *httpFederationSenderInternalAPI) PerformLeaveRequest( +func (h *httpFederationSenderInternalAPI) PerformLeave( ctx context.Context, request *PerformLeaveRequest, response *PerformLeaveResponse, diff --git a/federationsender/federationsender.go b/federationsender/federationsender.go index 355775f8a..aa9a7bc9c 100644 --- a/federationsender/federationsender.go +++ b/federationsender/federationsender.go @@ -36,6 +36,7 @@ func SetupFederationSenderComponent( federation *gomatrixserverlib.FederationClient, rsQueryAPI roomserverAPI.RoomserverQueryAPI, rsInputAPI roomserverAPI.RoomserverInputAPI, + keyRing *gomatrixserverlib.KeyRing, ) api.FederationSenderInternalAPI { federationSenderDB, err := storage.NewDatabase(string(base.Cfg.Database.FederationSender)) if err != nil { @@ -61,10 +62,10 @@ func SetupFederationSenderComponent( logrus.WithError(err).Panic("failed to start typing server consumer") } - queryAPI := query.FederationSenderInternalAPI{ - DB: federationSenderDB, - } + queryAPI := query.NewFederationSenderInternalAPI( + federationSenderDB, base.Cfg, roomserverProducer, federation, keyRing, + ) queryAPI.SetupHTTP(http.DefaultServeMux) - return &queryAPI + return queryAPI } diff --git a/federationsender/producers/roomserver.go b/federationsender/producers/roomserver.go index 0395f9628..ff4cda5b5 100644 --- a/federationsender/producers/roomserver.go +++ b/federationsender/producers/roomserver.go @@ -54,6 +54,42 @@ func (c *RoomserverProducer) SendInviteResponse( return c.SendInputRoomEvents(ctx, []api.InputRoomEvent{ire}) } +// SendEventWithState writes an event with KindNew to the roomserver input log +// with the state at the event as KindOutlier before it. +func (c *RoomserverProducer) SendEventWithState( + ctx context.Context, state gomatrixserverlib.RespState, event gomatrixserverlib.HeaderedEvent, +) error { + outliers, err := state.Events() + if err != nil { + return err + } + + var ires []api.InputRoomEvent + for _, outlier := range outliers { + ires = append(ires, api.InputRoomEvent{ + Kind: api.KindOutlier, + Event: outlier.Headered(event.RoomVersion), + AuthEventIDs: outlier.AuthEventIDs(), + }) + } + + stateEventIDs := make([]string, len(state.StateEvents)) + for i := range state.StateEvents { + stateEventIDs[i] = state.StateEvents[i].EventID() + } + + ires = append(ires, api.InputRoomEvent{ + Kind: api.KindNew, + Event: event, + AuthEventIDs: event.AuthEventIDs(), + HasState: true, + StateEventIDs: stateEventIDs, + }) + + _, err = c.SendInputRoomEvents(ctx, ires) + return err +} + // SendInputRoomEvents writes the given input room events to the roomserver input API. func (c *RoomserverProducer) SendInputRoomEvents( ctx context.Context, ires []api.InputRoomEvent, diff --git a/federationsender/query/api.go b/federationsender/query/api.go index e33bcc111..e92453f98 100644 --- a/federationsender/query/api.go +++ b/federationsender/query/api.go @@ -5,17 +5,37 @@ import ( "net/http" "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/federationsender/producers" "github.com/matrix-org/dendrite/federationsender/storage" - rsAPI "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) // FederationSenderInternalAPI is an implementation of api.FederationSenderInternalAPI type FederationSenderInternalAPI struct { api.FederationSenderInternalAPI - DB storage.Database - RoomserverInputAPI rsAPI.RoomserverInputAPI + db storage.Database + cfg *config.Dendrite + producer *producers.RoomserverProducer + federation *gomatrixserverlib.FederationClient + keyRing *gomatrixserverlib.KeyRing +} + +func NewFederationSenderInternalAPI( + db storage.Database, cfg *config.Dendrite, + producer *producers.RoomserverProducer, + federation *gomatrixserverlib.FederationClient, + keyRing *gomatrixserverlib.KeyRing, +) *FederationSenderInternalAPI { + return &FederationSenderInternalAPI{ + db: db, + cfg: cfg, + producer: producer, + federation: federation, + keyRing: keyRing, + } } // SetupHTTP adds the FederationSenderInternalAPI handlers to the http.ServeMux. @@ -55,7 +75,7 @@ func (f *FederationSenderInternalAPI) SetupHTTP(servMux *http.ServeMux) { if err := json.NewDecoder(req.Body).Decode(&request); err != nil { return util.MessageResponse(http.StatusBadRequest, err.Error()) } - if err := f.PerformJoinRequest(req.Context(), &request, &response); err != nil { + if err := f.PerformJoin(req.Context(), &request, &response); err != nil { return util.ErrorResponse(err) } return util.JSONResponse{Code: http.StatusOK, JSON: &response} @@ -68,7 +88,7 @@ func (f *FederationSenderInternalAPI) SetupHTTP(servMux *http.ServeMux) { if err := json.NewDecoder(req.Body).Decode(&request); err != nil { return util.MessageResponse(http.StatusBadRequest, err.Error()) } - if err := f.PerformLeaveRequest(req.Context(), &request, &response); err != nil { + if err := f.PerformLeave(req.Context(), &request, &response); err != nil { return util.ErrorResponse(err) } return util.JSONResponse{Code: http.StatusOK, JSON: &response} diff --git a/federationsender/query/perform.go b/federationsender/query/perform.go index 2486873c2..d39fef5ed 100644 --- a/federationsender/query/perform.go +++ b/federationsender/query/perform.go @@ -2,21 +2,116 @@ package query import ( "context" + "fmt" + "time" "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/federationsender/query/perform" + "github.com/matrix-org/dendrite/roomserver/version" + "github.com/matrix-org/gomatrixserverlib" ) // PerformJoinRequest implements api.FederationSenderInternalAPI -func (r *FederationSenderInternalAPI) PerformJoinRequest( +func (r *FederationSenderInternalAPI) PerformJoin( ctx context.Context, request *api.PerformJoinRequest, response *api.PerformJoinResponse, ) (err error) { + // Look up the supported room versions. + var supportedVersions []gomatrixserverlib.RoomVersion + for version := range version.SupportedRoomVersions() { + supportedVersions = append(supportedVersions, version) + } + + // Try to perform a make_join using the information supplied in the + // request. + respMakeJoin, err := r.federation.MakeJoin( + ctx, + request.ServerName, + request.RoomID, + request.UserID, + supportedVersions, + ) + if err != nil { + // TODO: Check if the user was not allowed to join the room. + return fmt.Errorf("r.federation.MakeJoin: %w", err) + } + + // Set all the fields to be what they should be, this should be a no-op + // but it's possible that the remote server returned us something "odd" + respMakeJoin.JoinEvent.Type = "m.room.member" + respMakeJoin.JoinEvent.Sender = request.UserID + respMakeJoin.JoinEvent.StateKey = &request.UserID + respMakeJoin.JoinEvent.RoomID = request.RoomID + respMakeJoin.JoinEvent.Redacts = "" + if request.Content == nil { + request.Content = map[string]interface{}{} + } + request.Content["membership"] = "join" + if err = respMakeJoin.JoinEvent.SetContent(request.Content); err != nil { + return fmt.Errorf("respMakeJoin.JoinEvent.SetContent: %w", err) + } + if err = respMakeJoin.JoinEvent.SetUnsigned(struct{}{}); err != nil { + return fmt.Errorf("respMakeJoin.JoinEvent.SetUnsigned: %w", err) + } + + // Work out if we support the room version that has been supplied in + // the make_join response. + if respMakeJoin.RoomVersion == "" { + respMakeJoin.RoomVersion = gomatrixserverlib.RoomVersionV1 + } + if _, err = respMakeJoin.RoomVersion.EventFormat(); err != nil { + return fmt.Errorf("respMakeJoin.RoomVersion.EventFormat: %w", err) + } + + // Build the join event. + event, err := respMakeJoin.JoinEvent.Build( + time.Now(), + r.cfg.Matrix.ServerName, + r.cfg.Matrix.KeyID, + r.cfg.Matrix.PrivateKey, + respMakeJoin.RoomVersion, + ) + if err != nil { + return fmt.Errorf("respMakeJoin.JoinEvent.Build: %w", err) + } + + // Try to perform a send_join using the newly built event. + respSendJoin, err := r.federation.SendJoin( + ctx, + request.ServerName, + event, + respMakeJoin.RoomVersion, + ) + if err != nil { + return fmt.Errorf("r.federation.SendJoin: %w", err) + } + + // Check that the send_join response was valid. + joinCtx := perform.JoinContext(r.federation, r.keyRing) + if err = joinCtx.CheckSendJoinResponse( + ctx, event, request.ServerName, respMakeJoin, respSendJoin, + ); err != nil { + return fmt.Errorf("perform.JoinRequest.CheckSendJoinResponse: %w", err) + } + + // If we successfully performed a send_join above then the other + // server now thinks we're a part of the room. Send the newly + // returned state to the roomserver to update our local view. + if err = r.producer.SendEventWithState( + ctx, + respSendJoin.ToRespState(), + event.Headered(respMakeJoin.RoomVersion), + ); err != nil { + return fmt.Errorf("r.producer.SendEventWithState: %w", err) + } + + // Everything went to plan. return nil } // PerformLeaveRequest implements api.FederationSenderInternalAPI -func (r *FederationSenderInternalAPI) PerformLeaveRequest( +func (r *FederationSenderInternalAPI) PerformLeave( ctx context.Context, request *api.PerformLeaveRequest, response *api.PerformLeaveResponse, diff --git a/federationsender/query/perform/join.go b/federationsender/query/perform/join.go new file mode 100644 index 000000000..3c7ef0765 --- /dev/null +++ b/federationsender/query/perform/join.go @@ -0,0 +1,70 @@ +package perform + +import ( + "context" + "fmt" + + "github.com/matrix-org/gomatrixserverlib" +) + +// This file contains helpers for the PerformJoin function. + +type joinContext struct { + federation *gomatrixserverlib.FederationClient + keyRing *gomatrixserverlib.KeyRing +} + +// Returns a new join context. +func JoinContext(f *gomatrixserverlib.FederationClient, k *gomatrixserverlib.KeyRing) *joinContext { + return &joinContext{ + federation: f, + keyRing: k, + } +} + +// checkSendJoinResponse checks that all of the signatures are correct +// and that the join is allowed by the supplied state. +func (r joinContext) CheckSendJoinResponse( + ctx context.Context, + event gomatrixserverlib.Event, + server gomatrixserverlib.ServerName, + respMakeJoin gomatrixserverlib.RespMakeJoin, + respSendJoin gomatrixserverlib.RespSendJoin, +) error { + // A list of events that we have retried, if they were not included in + // the auth events supplied in the send_join. + retries := map[string]bool{} + +retryCheck: + // TODO: Can we expand Check here to return a list of missing auth + // events rather than failing one at a time? + if err := respSendJoin.Check(ctx, r.keyRing, event); err != nil { + switch e := err.(type) { + case gomatrixserverlib.MissingAuthEventError: + // Check that we haven't already retried for this event, prevents + // us from ending up in endless loops + if !retries[e.AuthEventID] { + // Ask the server that we're talking to right now for the event + tx, txerr := r.federation.GetEvent(ctx, server, e.AuthEventID) + if txerr != nil { + return fmt.Errorf("r.federation.GetEvent: %w", txerr) + } + // For each event returned, add it to the auth events. + for _, pdu := range tx.PDUs { + ev, everr := gomatrixserverlib.NewEventFromUntrustedJSON(pdu, respMakeJoin.RoomVersion) + if everr != nil { + return fmt.Errorf("gomatrixserverlib.NewEventFromUntrustedJSON: %w", everr) + } + respSendJoin.AuthEvents = append(respSendJoin.AuthEvents, ev) + } + // Mark the event as retried and then give the check another go. + retries[e.AuthEventID] = true + goto retryCheck + } + return fmt.Errorf("respSendJoin (after retries): %w", e) + default: + return fmt.Errorf("respSendJoin: %w", err) + } + } + return nil +} diff --git a/federationsender/query/query.go b/federationsender/query/query.go index ec668204d..004ad156d 100644 --- a/federationsender/query/query.go +++ b/federationsender/query/query.go @@ -13,7 +13,7 @@ func (f *FederationSenderInternalAPI) QueryJoinedHostsInRoom( request *api.QueryJoinedHostsInRoomRequest, response *api.QueryJoinedHostsInRoomResponse, ) (err error) { - response.JoinedHosts, err = f.DB.GetJoinedHosts(ctx, request.RoomID) + response.JoinedHosts, err = f.db.GetJoinedHosts(ctx, request.RoomID) return } @@ -23,7 +23,7 @@ func (f *FederationSenderInternalAPI) QueryJoinedHostServerNamesInRoom( request *api.QueryJoinedHostServerNamesInRoomRequest, response *api.QueryJoinedHostServerNamesInRoomResponse, ) (err error) { - joinedHosts, err := f.DB.GetJoinedHosts(ctx, request.RoomID) + joinedHosts, err := f.db.GetJoinedHosts(ctx, request.RoomID) if err != nil { return } From 458b3647815f0f2c6930611961431a9fb4390fba Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 29 Apr 2020 15:33:17 +0100 Subject: [PATCH 015/200] Update gomatrixserverlib --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 56c58c6c8..5eb54ceb5 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f github.com/matrix-org/go-sqlite3-js v0.0.0-20200325174927-327088cdef10 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200428112024-9f47f9bfa4b2 + github.com/matrix-org/gomatrixserverlib v0.0.0-20200429143250-5df6426424bd github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible diff --git a/go.sum b/go.sum index de844b2a1..1511b927f 100644 --- a/go.sum +++ b/go.sum @@ -367,8 +367,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:kmRjpmFOenVpOaV/DRlo9p6z/IbOKlUC+hhKsAAh8Qg= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200428112024-9f47f9bfa4b2 h1:sy2QOqJhb4WXzq8bJhsCntAUYb64Dl6txsFtXWtxxSg= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200428112024-9f47f9bfa4b2/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200429143250-5df6426424bd h1:YA+1/Y/NK6dHAxRamybQiE6HToTC+5ddPCO4UI7pmH0= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200429143250-5df6426424bd/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y= From 4ad52c67cacc21997f49decd57e3105beb8ab62d Mon Sep 17 00:00:00 2001 From: Kegsay Date: Wed, 29 Apr 2020 18:41:45 +0100 Subject: [PATCH 016/200] Honour history_visibility when backfilling (#990) * Make backfill work for shared history visibility * fetch missing state on backfill to remember snapshots correctly * Fix gmsl to not mux in auth events into room state * Whoops * Linting --- federationapi/routing/events.go | 11 ++- federationapi/routing/routing.go | 2 +- go.mod | 2 +- go.sum | 4 +- roomserver/auth/auth.go | 4 +- roomserver/query/backfill.go | 81 ++++++++++++++-- roomserver/query/query.go | 108 +++++++++++++++++---- roomserver/state/state.go | 5 +- roomserver/storage/sqlite3/events_table.go | 28 +++--- roomserver/storage/sqlite3/storage.go | 2 +- 10 files changed, 195 insertions(+), 52 deletions(-) diff --git a/federationapi/routing/events.go b/federationapi/routing/events.go index a91528b3d..03492db45 100644 --- a/federationapi/routing/events.go +++ b/federationapi/routing/events.go @@ -16,7 +16,9 @@ package routing import ( "context" + "encoding/json" "net/http" + "time" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" @@ -29,13 +31,20 @@ func GetEvent( request *gomatrixserverlib.FederationRequest, query api.RoomserverQueryAPI, eventID string, + origin gomatrixserverlib.ServerName, ) util.JSONResponse { event, err := getEvent(ctx, request, query, eventID) if err != nil { return *err } - return util.JSONResponse{Code: http.StatusOK, JSON: event} + return util.JSONResponse{Code: http.StatusOK, JSON: gomatrixserverlib.Transaction{ + Origin: origin, + OriginServerTS: gomatrixserverlib.AsTimestamp(time.Now()), + PDUs: []json.RawMessage{ + event.JSON(), + }, + }} } // getEvent returns the requested event, diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index ebaeec6e6..e0f842f18 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -126,7 +126,7 @@ func Setup( return util.ErrorResponse(err) } return GetEvent( - httpReq.Context(), request, query, vars["eventID"], + httpReq.Context(), request, query, vars["eventID"], cfg.Matrix.ServerName, ) }, )).Methods(http.MethodGet) diff --git a/go.mod b/go.mod index 5eb54ceb5..d20089e8a 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f github.com/matrix-org/go-sqlite3-js v0.0.0-20200325174927-327088cdef10 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200429143250-5df6426424bd + github.com/matrix-org/gomatrixserverlib v0.0.0-20200429162354-392f0b1b7421 github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible diff --git a/go.sum b/go.sum index 1511b927f..da0e02d6f 100644 --- a/go.sum +++ b/go.sum @@ -367,8 +367,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:kmRjpmFOenVpOaV/DRlo9p6z/IbOKlUC+hhKsAAh8Qg= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200429143250-5df6426424bd h1:YA+1/Y/NK6dHAxRamybQiE6HToTC+5ddPCO4UI7pmH0= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200429143250-5df6426424bd/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200429162354-392f0b1b7421 h1:4zP29YlpfEtJ9a7sZ33Mf0FJInD2N3/KzDcLa62bRKc= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200429162354-392f0b1b7421/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y= diff --git a/roomserver/auth/auth.go b/roomserver/auth/auth.go index 615a94b3c..fdcf9f062 100644 --- a/roomserver/auth/auth.go +++ b/roomserver/auth/auth.go @@ -27,7 +27,7 @@ func IsServerAllowed( serverCurrentlyInRoom bool, authEvents []gomatrixserverlib.Event, ) bool { - historyVisibility := historyVisibilityForRoom(authEvents) + historyVisibility := HistoryVisibilityForRoom(authEvents) // 1. If the history_visibility was set to world_readable, allow. if historyVisibility == "world_readable" { @@ -52,7 +52,7 @@ func IsServerAllowed( return false } -func historyVisibilityForRoom(authEvents []gomatrixserverlib.Event) string { +func HistoryVisibilityForRoom(authEvents []gomatrixserverlib.Event) string { // https://matrix.org/docs/spec/client_server/r0.6.0#id87 // By default if no history_visibility is set, or if the value is not understood, the visibility is assumed to be shared. visibility := "shared" diff --git a/roomserver/query/backfill.go b/roomserver/query/backfill.go index 09a515e99..f518de3e8 100644 --- a/roomserver/query/backfill.go +++ b/roomserver/query/backfill.go @@ -3,6 +3,7 @@ package query import ( "context" + "github.com/matrix-org/dendrite/roomserver/auth" "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" @@ -62,9 +63,9 @@ FederationHit: logrus.WithField("event_id", targetEvent.EventID()).Info("Requesting /state_ids at event") for _, srv := range b.servers { // hit any valid server c := gomatrixserverlib.FederatedStateProvider{ - FedClient: b.fedClient, - AuthEventsOnly: false, - Server: srv, + FedClient: b.fedClient, + RememberAuthEvents: false, + Server: srv, } res, err := c.StateIDsBeforeEvent(ctx, targetEvent) if err != nil { @@ -114,7 +115,9 @@ func (b *backfillRequester) calculateNewStateIDs(targetEvent, prevEvent gomatrix return nil } -func (b *backfillRequester) StateBeforeEvent(ctx context.Context, roomVer gomatrixserverlib.RoomVersion, event gomatrixserverlib.HeaderedEvent, eventIDs []string) (map[string]*gomatrixserverlib.Event, error) { +func (b *backfillRequester) StateBeforeEvent(ctx context.Context, roomVer gomatrixserverlib.RoomVersion, + event gomatrixserverlib.HeaderedEvent, eventIDs []string) (map[string]*gomatrixserverlib.Event, error) { + // try to fetch the events from the database first events, err := b.ProvideEvents(roomVer, eventIDs) if err != nil { @@ -133,9 +136,9 @@ func (b *backfillRequester) StateBeforeEvent(ctx context.Context, roomVer gomatr } c := gomatrixserverlib.FederatedStateProvider{ - FedClient: b.fedClient, - AuthEventsOnly: false, - Server: b.servers[0], + FedClient: b.fedClient, + RememberAuthEvents: false, + Server: b.servers[0], } result, err := c.StateBeforeEvent(ctx, roomVer, event, eventIDs) if err != nil { @@ -160,18 +163,33 @@ func (b *backfillRequester) ServersAtEvent(ctx context.Context, roomID, eventID return } + stateEntries, err := stateBeforeEvent(ctx, b.db, NIDs[eventID]) + if err != nil { + logrus.WithField("event_id", eventID).WithError(err).Error("ServersAtEvent: failed to load state before event") + return + } + + // possibly return all joined servers depending on history visiblity + memberEventsFromVis, err := joinEventsFromHistoryVisibility(ctx, b.db, roomID, stateEntries) + if err != nil { + logrus.WithError(err).Error("ServersAtEvent: failed calculate servers from history visibility rules") + return + } + logrus.Infof("ServersAtEvent including %d current events from history visibility", len(memberEventsFromVis)) + // Retrieve all "m.room.member" state events of "join" membership, which // contains the list of users in the room before the event, therefore all // the servers in it at that moment. - events, err := getMembershipsBeforeEventNID(ctx, b.db, NIDs[eventID], true) + memberEvents, err := getMembershipsAtState(ctx, b.db, stateEntries, true) if err != nil { logrus.WithField("event_id", eventID).WithError(err).Error("ServersAtEvent: failed to get memberships before event") return } + memberEvents = append(memberEvents, memberEventsFromVis...) // Store the server names in a temporary map to avoid duplicates. serverSet := make(map[gomatrixserverlib.ServerName]bool) - for _, event := range events { + for _, event := range memberEvents { serverSet[event.Origin()] = true } for server := range serverSet { @@ -186,7 +204,9 @@ func (b *backfillRequester) ServersAtEvent(ctx context.Context, roomID, eventID // Backfill performs a backfill request to the given server. // https://matrix.org/docs/spec/server_server/latest#get-matrix-federation-v1-backfill-roomid -func (b *backfillRequester) Backfill(ctx context.Context, server gomatrixserverlib.ServerName, roomID string, fromEventIDs []string, limit int) (*gomatrixserverlib.Transaction, error) { +func (b *backfillRequester) Backfill(ctx context.Context, server gomatrixserverlib.ServerName, roomID string, + fromEventIDs []string, limit int) (*gomatrixserverlib.Transaction, error) { + tx, err := b.fedClient.Backfill(ctx, server, roomID, limit, fromEventIDs) return &tx, err } @@ -215,3 +235,44 @@ func (b *backfillRequester) ProvideEvents(roomVer gomatrixserverlib.RoomVersion, } return events, nil } + +// joinEventsFromHistoryVisibility returns all CURRENTLY joined members if the provided state indicated a 'shared' history visibility. +// TODO: Long term we probably want a history_visibility table which stores eventNID | visibility_enum so we can just +// pull all events and then filter by that table. +func joinEventsFromHistoryVisibility( + ctx context.Context, db storage.Database, roomID string, stateEntries []types.StateEntry) ([]types.Event, error) { + + var eventNIDs []types.EventNID + for _, entry := range stateEntries { + // Filter the events to retrieve to only keep the membership events + if entry.EventTypeNID == types.MRoomHistoryVisibilityNID && entry.EventStateKeyNID == types.EmptyStateKeyNID { + eventNIDs = append(eventNIDs, entry.EventNID) + break + } + } + + // Get all of the events in this state + stateEvents, err := db.Events(ctx, eventNIDs) + if err != nil { + return nil, err + } + events := make([]gomatrixserverlib.Event, len(stateEvents)) + for i := range stateEvents { + events[i] = stateEvents[i].Event + } + visibility := auth.HistoryVisibilityForRoom(events) + if visibility != "shared" { + logrus.Infof("ServersAtEvent history visibility not shared: %s", visibility) + return nil, nil + } + // get joined members + roomNID, err := db.RoomNID(ctx, roomID) + if err != nil { + return nil, err + } + joinEventNIDs, err := db.GetMembershipEventNIDsForRoom(ctx, roomNID, true) + if err != nil { + return nil, err + } + return db.Events(ctx, joinEventNIDs) +} diff --git a/roomserver/query/query.go b/roomserver/query/query.go index a54fa58d9..6778ac280 100644 --- a/roomserver/query/query.go +++ b/roomserver/query/query.go @@ -277,6 +277,7 @@ func (r *RoomserverQueryAPI) QueryMembershipsForRoom( response.JoinEvents = []gomatrixserverlib.ClientEvent{} var events []types.Event + var stateEntries []types.StateEntry if stillInRoom { var eventNIDs []types.EventNID eventNIDs, err = r.DB.GetMembershipEventNIDsForRoom(ctx, roomNID, request.JoinedOnly) @@ -286,7 +287,12 @@ func (r *RoomserverQueryAPI) QueryMembershipsForRoom( events, err = r.DB.Events(ctx, eventNIDs) } else { - events, err = getMembershipsBeforeEventNID(ctx, r.DB, membershipEventNID, request.JoinedOnly) + stateEntries, err = stateBeforeEvent(ctx, r.DB, membershipEventNID) + if err != nil { + logrus.WithField("membership_event_nid", membershipEventNID).WithError(err).Error("failed to load state before event") + return err + } + events, err = getMembershipsAtState(ctx, r.DB, stateEntries, request.JoinedOnly) } if err != nil { @@ -301,15 +307,8 @@ func (r *RoomserverQueryAPI) QueryMembershipsForRoom( return nil } -// getMembershipsBeforeEventNID takes the numeric ID of an event and fetches the state -// of the event's room as it was when this event was fired, then filters the state events to -// only keep the "m.room.member" events with a "join" membership. These events are returned. -// Returns an error if there was an issue fetching the events. -func getMembershipsBeforeEventNID( - ctx context.Context, db storage.Database, eventNID types.EventNID, joinedOnly bool, -) ([]types.Event, error) { +func stateBeforeEvent(ctx context.Context, db storage.Database, eventNID types.EventNID) ([]types.StateEntry, error) { roomState := state.NewStateResolution(db) - events := []types.Event{} // Lookup the event NID eIDs, err := db.EventIDs(ctx, []types.EventNID{eventNID}) if err != nil { @@ -323,10 +322,15 @@ func getMembershipsBeforeEventNID( } // Fetch the state as it was when this event was fired - stateEntries, err := roomState.LoadCombinedStateAfterEvents(ctx, prevState) - if err != nil { - return nil, err - } + return roomState.LoadCombinedStateAfterEvents(ctx, prevState) +} + +// getMembershipsAtState filters the state events to +// only keep the "m.room.member" events with a "join" membership. These events are returned. +// Returns an error if there was an issue fetching the events. +func getMembershipsAtState( + ctx context.Context, db storage.Database, stateEntries []types.StateEntry, joinedOnly bool, +) ([]types.Event, error) { var eventNIDs []types.EventNID for _, entry := range stateEntries { @@ -347,6 +351,7 @@ func getMembershipsBeforeEventNID( } // Filter the events to only keep the "join" membership events + var events []types.Event for _, event := range stateEvents { membership, err := event.Membership() if err != nil { @@ -563,20 +568,29 @@ func (r *RoomserverQueryAPI) backfillViaFederation(ctx context.Context, req *api if !ok { // this should be impossible as all events returned must have pass Step 5 of the PDU checks // which requires a list of state IDs. - logrus.WithError(err).WithField("event_id", ev.EventID()).Error("Failed to find state IDs for event which passed auth checks") + logrus.WithError(err).WithField("event_id", ev.EventID()).Error("backfillViaFederation: failed to find state IDs for event which passed auth checks") continue } var entries []types.StateEntry if entries, err = r.DB.StateEntriesForEventIDs(ctx, stateIDs); err != nil { - return err + // attempt to fetch the missing events + r.fetchAndStoreMissingEvents(ctx, roomVer, requester, stateIDs) + // try again + entries, err = r.DB.StateEntriesForEventIDs(ctx, stateIDs) + if err != nil { + logrus.WithError(err).WithField("event_id", ev.EventID()).Error("backfillViaFederation: failed to get state entries for event") + return err + } } var beforeStateSnapshotNID types.StateSnapshotNID if beforeStateSnapshotNID, err = r.DB.AddState(ctx, roomNID, nil, entries); err != nil { + logrus.WithError(err).WithField("event_id", ev.EventID()).Error("backfillViaFederation: failed to persist state entries to get snapshot nid") return err } + util.GetLogger(ctx).Infof("Backfilled event %s (nid=%d) getting snapshot %v with entries %+v", ev.EventID(), ev.EventNID, beforeStateSnapshotNID, entries) if err = r.DB.SetState(ctx, ev.EventNID, beforeStateSnapshotNID); err != nil { - logrus.WithError(err).WithField("event_id", ev.EventID()).Error("Failed to set state before event") + logrus.WithError(err).WithField("event_id", ev.EventID()).Error("backfillViaFederation: failed to persist snapshot nid") } } @@ -608,6 +622,66 @@ func (r *RoomserverQueryAPI) isServerCurrentlyInRoom(ctx context.Context, server return auth.IsAnyUserOnServerWithMembership(serverName, gmslEvents, gomatrixserverlib.Join), nil } +// fetchAndStoreMissingEvents does a best-effort fetch and store of missing events specified in stateIDs. Returns no error as it is just +// best effort. +func (r *RoomserverQueryAPI) fetchAndStoreMissingEvents(ctx context.Context, roomVer gomatrixserverlib.RoomVersion, + backfillRequester *backfillRequester, stateIDs []string) { + + servers := backfillRequester.servers + + // work out which are missing + nidMap, err := r.DB.EventNIDs(ctx, stateIDs) + if err != nil { + util.GetLogger(ctx).WithError(err).Warn("cannot query missing events") + return + } + missingMap := make(map[string]*gomatrixserverlib.HeaderedEvent) // id -> event + for _, id := range stateIDs { + if _, ok := nidMap[id]; !ok { + missingMap[id] = nil + } + } + util.GetLogger(ctx).Infof("Fetching %d missing state events (from %d possible servers)", len(missingMap), len(servers)) + + // fetch the events from federation. Loop the servers first so if we find one that works we stick with them + for _, srv := range servers { + for id, ev := range missingMap { + if ev != nil { + continue // already found + } + logger := util.GetLogger(ctx).WithField("server", srv).WithField("event_id", id) + res, err := r.FedClient.GetEvent(ctx, srv, id) + if err != nil { + logger.WithError(err).Warn("failed to get event from server") + continue + } + loader := gomatrixserverlib.NewEventsLoader(roomVer, r.KeyRing, backfillRequester, backfillRequester.ProvideEvents, false) + result, err := loader.LoadAndVerify(ctx, res.PDUs, gomatrixserverlib.TopologicalOrderByPrevEvents) + if err != nil { + logger.WithError(err).Warn("failed to load and verify event") + continue + } + logger.Infof("returned %d PDUs which made events %+v", len(res.PDUs), result) + for _, res := range result { + if res.Error != nil { + logger.WithError(err).Warn("event failed PDU checks") + continue + } + missingMap[id] = res.Event + } + } + } + + var newEvents []gomatrixserverlib.HeaderedEvent + for _, ev := range missingMap { + if ev != nil { + newEvents = append(newEvents, *ev) + } + } + util.GetLogger(ctx).Infof("Persisting %d new events", len(newEvents)) + persistEvents(ctx, r.DB, newEvents) +} + // TODO: Remove this when we have tests to assert correctness of this function // nolint:gocyclo func (r *RoomserverQueryAPI) scanEventTree( @@ -857,7 +931,7 @@ func persistEvents(ctx context.Context, db storage.Database, events []gomatrixse var stateAtEvent types.StateAtEvent roomNID, stateAtEvent, err = db.StoreEvent(ctx, ev.Unwrap(), nil, authNids) if err != nil { - logrus.WithError(err).WithField("event_id", ev.EventID()).Error("Failed to store backfilled event") + logrus.WithError(err).WithField("event_id", ev.EventID()).Error("Failed to persist event") continue } backfilledEventMap[ev.EventID()] = types.Event{ diff --git a/roomserver/state/state.go b/roomserver/state/state.go index 389c94400..9b005ee6a 100644 --- a/roomserver/state/state.go +++ b/roomserver/state/state.go @@ -86,7 +86,10 @@ func (v StateResolution) LoadStateAtEvent( ) ([]types.StateEntry, error) { snapshotNID, err := v.db.SnapshotNIDFromEventID(ctx, eventID) if err != nil { - return nil, err + return nil, fmt.Errorf("LoadStateAtEvent.SnapshotNIDFromEventID failed for event %s : %s", eventID, err) + } + if snapshotNID == 0 { + return nil, fmt.Errorf("LoadStateAtEvent.SnapshotNIDFromEventID(%s) returned 0 NID, was this event stored?", eventID) } stateEntries, err := v.LoadStateAtSnapshot(ctx, snapshotNID) diff --git a/roomserver/storage/sqlite3/events_table.go b/roomserver/storage/sqlite3/events_table.go index d881fa91f..a63596aeb 100644 --- a/roomserver/storage/sqlite3/events_table.go +++ b/roomserver/storage/sqlite3/events_table.go @@ -48,11 +48,6 @@ const insertEventSQL = ` ON CONFLICT DO NOTHING; ` -const insertEventResultSQL = ` - SELECT event_nid, state_snapshot_nid FROM roomserver_events - WHERE rowid = last_insert_rowid(); -` - const selectEventSQL = "" + "SELECT event_nid, state_snapshot_nid FROM roomserver_events WHERE event_id = $1" @@ -102,7 +97,6 @@ const selectRoomNIDForEventNIDSQL = "" + type eventStatements struct { db *sql.DB insertEventStmt *sql.Stmt - insertEventResultStmt *sql.Stmt selectEventStmt *sql.Stmt bulkSelectStateEventByIDStmt *sql.Stmt bulkSelectStateAtEventByIDStmt *sql.Stmt @@ -126,7 +120,6 @@ func (s *eventStatements) prepare(db *sql.DB) (err error) { return statementList{ {&s.insertEventStmt, insertEventSQL}, - {&s.insertEventResultStmt, insertEventResultSQL}, {&s.selectEventStmt, selectEventSQL}, {&s.bulkSelectStateEventByIDStmt, bulkSelectStateEventByIDSQL}, {&s.bulkSelectStateAtEventByIDStmt, bulkSelectStateAtEventByIDSQL}, @@ -152,19 +145,22 @@ func (s *eventStatements) insertEvent( referenceSHA256 []byte, authEventNIDs []types.EventNID, depth int64, -) (types.EventNID, types.StateSnapshotNID, error) { - var eventNID int64 - var stateNID int64 - var err error +) (types.EventNID, error) { + // attempt to insert: the last_row_id is the event NID insertStmt := common.TxStmt(txn, s.insertEventStmt) - resultStmt := common.TxStmt(txn, s.insertEventResultStmt) - if _, err = insertStmt.ExecContext( + result, err := insertStmt.ExecContext( ctx, int64(roomNID), int64(eventTypeNID), int64(eventStateKeyNID), eventID, referenceSHA256, eventNIDsAsArray(authEventNIDs), depth, - ); err == nil { - err = resultStmt.QueryRowContext(ctx).Scan(&eventNID, &stateNID) + ) + if err != nil { + return 0, err } - return types.EventNID(eventNID), types.StateSnapshotNID(stateNID), err + modified, err := result.RowsAffected() + if modified == 0 && err == nil { + return 0, sql.ErrNoRows + } + eventNID, err := result.LastInsertId() + return types.EventNID(eventNID), err } func (s *eventStatements) selectEvent( diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index 5df9c4e08..b6e846df7 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -124,7 +124,7 @@ func (d *Database) StoreEvent( } } - if eventNID, stateNID, err = d.statements.insertEvent( + if eventNID, err = d.statements.insertEvent( ctx, txn, roomNID, From 77fe509031d578a133e648a9eae3dd727eb2aeb5 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 29 Apr 2020 19:37:00 +0100 Subject: [PATCH 017/200] Enable v5 rooms (#992) * Enable v5 roooms * Update sytest-whitelist * Enable v5 rooms by default, update gomatrixserverlib --- go.mod | 2 +- go.sum | 4 ++-- roomserver/version/version.go | 6 +++--- sytest-whitelist | 6 ++++++ 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index d20089e8a..7cda4fe31 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f github.com/matrix-org/go-sqlite3-js v0.0.0-20200325174927-327088cdef10 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200429162354-392f0b1b7421 + github.com/matrix-org/gomatrixserverlib v0.0.0-20200429173835-c4e77c0d7e72 github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible diff --git a/go.sum b/go.sum index da0e02d6f..5b83ed09d 100644 --- a/go.sum +++ b/go.sum @@ -367,8 +367,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:kmRjpmFOenVpOaV/DRlo9p6z/IbOKlUC+hhKsAAh8Qg= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200429162354-392f0b1b7421 h1:4zP29YlpfEtJ9a7sZ33Mf0FJInD2N3/KzDcLa62bRKc= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200429162354-392f0b1b7421/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200429173835-c4e77c0d7e72 h1:onwidlObCIqpqXpaqU2BISW4Ngq4ETer9AsbCTiR8T4= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200429173835-c4e77c0d7e72/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y= diff --git a/roomserver/version/version.go b/roomserver/version/version.go index f2a67e74d..ddd0f23a6 100644 --- a/roomserver/version/version.go +++ b/roomserver/version/version.go @@ -51,15 +51,15 @@ var roomVersions = map[gomatrixserverlib.RoomVersion]RoomVersionDescription{ Stable: true, }, gomatrixserverlib.RoomVersionV5: RoomVersionDescription{ - Supported: false, - Stable: false, + Supported: true, + Stable: true, }, } // DefaultRoomVersion contains the room version that will, by // default, be used to create new rooms on this server. func DefaultRoomVersion() gomatrixserverlib.RoomVersion { - return gomatrixserverlib.RoomVersionV4 + return gomatrixserverlib.RoomVersionV5 } // RoomVersions returns a map of all known room versions to this diff --git a/sytest-whitelist b/sytest-whitelist index 439c306c7..c957021cb 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -258,3 +258,9 @@ User can invite remote user to room with version 1 User can invite remote user to room with version 2 User can invite remote user to room with version 3 User can invite remote user to room with version 4 +User can create and send/receive messages in a room with version 5 +local user can join room with version 5 +User can invite local user to room with version 5 +remote user can join room with version 5 +User can invite remote user to room with version 5 +Remote user can backfill in a room with version 5 From 540f6fcd9475a9c562837d59cf008ee6d0ce8983 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 30 Apr 2020 13:50:11 +0100 Subject: [PATCH 018/200] Update gmsl for key validity fix --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7cda4fe31..55c1179c2 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f github.com/matrix-org/go-sqlite3-js v0.0.0-20200325174927-327088cdef10 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200429173835-c4e77c0d7e72 + github.com/matrix-org/gomatrixserverlib v0.0.0-20200430104311-8d41c4d924ec github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible diff --git a/go.sum b/go.sum index 5b83ed09d..8c1d15ad7 100644 --- a/go.sum +++ b/go.sum @@ -367,8 +367,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:kmRjpmFOenVpOaV/DRlo9p6z/IbOKlUC+hhKsAAh8Qg= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200429173835-c4e77c0d7e72 h1:onwidlObCIqpqXpaqU2BISW4Ngq4ETer9AsbCTiR8T4= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200429173835-c4e77c0d7e72/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200430104311-8d41c4d924ec h1:9MvZSZzBKvCWqM5KXMGZ1PBDrSLcxs5zfc561UPgcYA= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200430104311-8d41c4d924ec/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y= From ebbfc125920beb321713e28a2a137d768406fa15 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Thu, 30 Apr 2020 17:15:29 +0100 Subject: [PATCH 019/200] Add tests for the storage interface (#995) * Move docs to interface * Add tests around syncing * Add topology token test * Linting --- syncapi/storage/interface.go | 59 ++++- syncapi/storage/postgres/syncserver.go | 54 ----- syncapi/storage/storage_test.go | 313 +++++++++++++++++++++++++ 3 files changed, 371 insertions(+), 55 deletions(-) create mode 100644 syncapi/storage/storage_test.go diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index a3efd8d58..bd9504db8 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -28,26 +28,83 @@ import ( type Database interface { common.PartitionStorer + // AllJoinedUsersInRooms returns a map of room ID to a list of all joined user IDs. AllJoinedUsersInRooms(ctx context.Context) (map[string][]string, error) + // Events lookups a list of event by their event ID. + // Returns a list of events matching the requested IDs found in the database. + // If an event is not found in the database then it will be omitted from the list. + // Returns an error if there was a problem talking with the database. + // Does not include any transaction IDs in the returned events. Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.HeaderedEvent, error) - WriteEvent(context.Context, *gomatrixserverlib.HeaderedEvent, []gomatrixserverlib.HeaderedEvent, []string, []string, *api.TransactionID, bool) (types.StreamPosition, error) + // WriteEvent into the database. It is not safe to call this function from multiple goroutines, as it would create races + // when generating the sync stream position for this event. Returns the sync stream position for the inserted event. + // Returns an error if there was a problem inserting this event. + WriteEvent(ctx context.Context, ev *gomatrixserverlib.HeaderedEvent, addStateEvents []gomatrixserverlib.HeaderedEvent, + addStateEventIDs []string, removeStateEventIDs []string, transactionID *api.TransactionID, excludeFromSync bool) (types.StreamPosition, error) + // GetStateEvent returns the Matrix state event of a given type for a given room with a given state key + // If no event could be found, returns nil + // If there was an issue during the retrieval, returns an error GetStateEvent(ctx context.Context, roomID, evType, stateKey string) (*gomatrixserverlib.HeaderedEvent, error) + // GetStateEventsForRoom fetches the state events for a given room. + // Returns an empty slice if no state events could be found for this room. + // Returns an error if there was an issue with the retrieval. GetStateEventsForRoom(ctx context.Context, roomID string, stateFilterPart *gomatrixserverlib.StateFilter) (stateEvents []gomatrixserverlib.HeaderedEvent, err error) + // SyncPosition returns the latest positions for syncing. SyncPosition(ctx context.Context) (types.PaginationToken, error) + // IncrementalSync returns all the data needed in order to create an incremental + // sync response for the given user. Events returned will include any client + // transaction IDs associated with the given device. These transaction IDs come + // from when the device sent the event via an API that included a transaction + // ID. IncrementalSync(ctx context.Context, device authtypes.Device, fromPos, toPos types.PaginationToken, numRecentEventsPerRoom int, wantFullState bool) (*types.Response, error) + // CompleteSync returns a complete /sync API response for the given user. CompleteSync(ctx context.Context, userID string, numRecentEventsPerRoom int) (*types.Response, error) + // GetAccountDataInRange returns all account data for a given user inserted or + // updated between two given positions + // Returns a map following the format data[roomID] = []dataTypes + // If no data is retrieved, returns an empty map + // If there was an issue with the retrieval, returns an error GetAccountDataInRange(ctx context.Context, userID string, oldPos, newPos types.StreamPosition, accountDataFilterPart *gomatrixserverlib.EventFilter) (map[string][]string, error) + // UpsertAccountData keeps track of new or updated account data, by saving the type + // of the new/updated data, and the user ID and room ID the data is related to (empty) + // room ID means the data isn't specific to any room) + // If no data with the given type, user ID and room ID exists in the database, + // creates a new row, else update the existing one + // Returns an error if there was an issue with the upsert UpsertAccountData(ctx context.Context, userID, roomID, dataType string) (types.StreamPosition, error) + // AddInviteEvent stores a new invite event for a user. + // If the invite was successfully stored this returns the stream ID it was stored at. + // Returns an error if there was a problem communicating with the database. AddInviteEvent(ctx context.Context, inviteEvent gomatrixserverlib.HeaderedEvent) (types.StreamPosition, error) + // RetireInviteEvent removes an old invite event from the database. + // Returns an error if there was a problem communicating with the database. RetireInviteEvent(ctx context.Context, inviteEventID string) error + // SetTypingTimeoutCallback sets a callback function that is called right after + // a user is removed from the typing user list due to timeout. SetTypingTimeoutCallback(fn cache.TimeoutCallbackFn) + // AddTypingUser adds a typing user to the typing cache. + // Returns the newly calculated sync position for typing notifications. AddTypingUser(userID, roomID string, expireTime *time.Time) types.StreamPosition + // RemoveTypingUser removes a typing user from the typing cache. + // Returns the newly calculated sync position for typing notifications. RemoveTypingUser(userID, roomID string) types.StreamPosition + // GetEventsInRange retrieves all of the events on a given ordering using the + // given extremities and limit. GetEventsInRange(ctx context.Context, from, to *types.PaginationToken, roomID string, limit int, backwardOrdering bool) (events []types.StreamEvent, err error) + // EventPositionInTopology returns the depth of the given event. EventPositionInTopology(ctx context.Context, eventID string) (types.StreamPosition, error) + // EventsAtTopologicalPosition returns all of the events matching a given + // position in the topology of a given room. EventsAtTopologicalPosition(ctx context.Context, roomID string, pos types.StreamPosition) ([]types.StreamEvent, error) + // BackwardExtremitiesForRoom returns the event IDs of all of the backward + // extremities we know of for a given room. BackwardExtremitiesForRoom(ctx context.Context, roomID string) (backwardExtremities []string, err error) + // MaxTopologicalPosition returns the highest topological position for a given room. MaxTopologicalPosition(ctx context.Context, roomID string) (types.StreamPosition, error) + // StreamEventsToEvents converts streamEvent to Event. If device is non-nil and + // matches the streamevent.transactionID device then the transaction ID gets + // added to the unsigned section of the output event. StreamEventsToEvents(device *authtypes.Device, in []types.StreamEvent) []gomatrixserverlib.HeaderedEvent + // SyncStreamPosition returns the latest position in the sync stream. Returns 0 if there are no events yet. SyncStreamPosition(ctx context.Context) (types.StreamPosition, error) } diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go index 9d61ccfc3..ef9707397 100644 --- a/syncapi/storage/postgres/syncserver.go +++ b/syncapi/storage/postgres/syncserver.go @@ -93,16 +93,10 @@ func NewSyncServerDatasource(dbDataSourceName string) (*SyncServerDatasource, er return &d, nil } -// AllJoinedUsersInRooms returns a map of room ID to a list of all joined user IDs. func (d *SyncServerDatasource) AllJoinedUsersInRooms(ctx context.Context) (map[string][]string, error) { return d.roomstate.selectJoinedUsers(ctx) } -// Events lookups a list of event by their event ID. -// Returns a list of events matching the requested IDs found in the database. -// If an event is not found in the database then it will be omitted from the list. -// Returns an error if there was a problem talking with the database. -// Does not include any transaction IDs in the returned events. func (d *SyncServerDatasource) Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.HeaderedEvent, error) { streamEvents, err := d.events.selectEvents(ctx, nil, eventIDs) if err != nil { @@ -148,9 +142,6 @@ func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, tx return nil } -// WriteEvent into the database. It is not safe to call this function from multiple goroutines, as it would create races -// when generating the sync stream position for this event. Returns the sync stream position for the inserted event. -// Returns an error if there was a problem inserting this event. func (d *SyncServerDatasource) WriteEvent( ctx context.Context, ev *gomatrixserverlib.HeaderedEvent, @@ -221,18 +212,12 @@ func (d *SyncServerDatasource) updateRoomState( return nil } -// GetStateEvent returns the Matrix state event of a given type for a given room with a given state key -// If no event could be found, returns nil -// If there was an issue during the retrieval, returns an error func (d *SyncServerDatasource) GetStateEvent( ctx context.Context, roomID, evType, stateKey string, ) (*gomatrixserverlib.HeaderedEvent, error) { return d.roomstate.selectStateEvent(ctx, roomID, evType, stateKey) } -// GetStateEventsForRoom fetches the state events for a given room. -// Returns an empty slice if no state events could be found for this room. -// Returns an error if there was an issue with the retrieval. func (d *SyncServerDatasource) GetStateEventsForRoom( ctx context.Context, roomID string, stateFilter *gomatrixserverlib.StateFilter, ) (stateEvents []gomatrixserverlib.HeaderedEvent, err error) { @@ -243,8 +228,6 @@ func (d *SyncServerDatasource) GetStateEventsForRoom( return } -// GetEventsInRange retrieves all of the events on a given ordering using the -// given extremities and limit. func (d *SyncServerDatasource) GetEventsInRange( ctx context.Context, from, to *types.PaginationToken, @@ -306,29 +289,22 @@ func (d *SyncServerDatasource) GetEventsInRange( return } -// SyncPosition returns the latest positions for syncing. func (d *SyncServerDatasource) SyncPosition(ctx context.Context) (types.PaginationToken, error) { return d.syncPositionTx(ctx, nil) } -// BackwardExtremitiesForRoom returns the event IDs of all of the backward -// extremities we know of for a given room. func (d *SyncServerDatasource) BackwardExtremitiesForRoom( ctx context.Context, roomID string, ) (backwardExtremities []string, err error) { return d.backwardExtremities.SelectBackwardExtremitiesForRoom(ctx, roomID) } -// MaxTopologicalPosition returns the highest topological position for a given -// room. func (d *SyncServerDatasource) MaxTopologicalPosition( ctx context.Context, roomID string, ) (types.StreamPosition, error) { return d.topology.selectMaxPositionInTopology(ctx, roomID) } -// EventsAtTopologicalPosition returns all of the events matching a given -// position in the topology of a given room. func (d *SyncServerDatasource) EventsAtTopologicalPosition( ctx context.Context, roomID string, pos types.StreamPosition, ) ([]types.StreamEvent, error) { @@ -346,7 +322,6 @@ func (d *SyncServerDatasource) EventPositionInTopology( return d.topology.selectPositionInTopology(ctx, eventID) } -// SyncStreamPosition returns the latest position in the sync stream. Returns 0 if there are no events yet. func (d *SyncServerDatasource) SyncStreamPosition(ctx context.Context) (types.StreamPosition, error) { return d.syncStreamPositionTx(ctx, nil) } @@ -511,11 +486,6 @@ func (d *SyncServerDatasource) addEDUDeltaToResponse( return } -// IncrementalSync returns all the data needed in order to create an incremental -// sync response for the given user. Events returned will include any client -// transaction IDs associated with the given device. These transaction IDs come -// from when the device sent the event via an API that included a transaction -// ID. func (d *SyncServerDatasource) IncrementalSync( ctx context.Context, device authtypes.Device, @@ -645,7 +615,6 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( return res, toPos, joinedRoomIDs, err } -// CompleteSync returns a complete /sync API response for the given user. func (d *SyncServerDatasource) CompleteSync( ctx context.Context, userID string, numRecentEventsPerRoom int, ) (*types.Response, error) { @@ -677,11 +646,6 @@ var txReadOnlySnapshot = sql.TxOptions{ ReadOnly: true, } -// GetAccountDataInRange returns all account data for a given user inserted or -// updated between two given positions -// Returns a map following the format data[roomID] = []dataTypes -// If no data is retrieved, returns an empty map -// If there was an issue with the retrieval, returns an error func (d *SyncServerDatasource) GetAccountDataInRange( ctx context.Context, userID string, oldPos, newPos types.StreamPosition, accountDataFilterPart *gomatrixserverlib.EventFilter, @@ -689,29 +653,18 @@ func (d *SyncServerDatasource) GetAccountDataInRange( return d.accountData.selectAccountDataInRange(ctx, userID, oldPos, newPos, accountDataFilterPart) } -// UpsertAccountData keeps track of new or updated account data, by saving the type -// of the new/updated data, and the user ID and room ID the data is related to (empty) -// room ID means the data isn't specific to any room) -// If no data with the given type, user ID and room ID exists in the database, -// creates a new row, else update the existing one -// Returns an error if there was an issue with the upsert func (d *SyncServerDatasource) UpsertAccountData( ctx context.Context, userID, roomID, dataType string, ) (types.StreamPosition, error) { return d.accountData.insertAccountData(ctx, userID, roomID, dataType) } -// AddInviteEvent stores a new invite event for a user. -// If the invite was successfully stored this returns the stream ID it was stored at. -// Returns an error if there was a problem communicating with the database. func (d *SyncServerDatasource) AddInviteEvent( ctx context.Context, inviteEvent gomatrixserverlib.HeaderedEvent, ) (types.StreamPosition, error) { return d.invites.insertInviteEvent(ctx, inviteEvent) } -// RetireInviteEvent removes an old invite event from the database. -// Returns an error if there was a problem communicating with the database. func (d *SyncServerDatasource) RetireInviteEvent( ctx context.Context, inviteEventID string, ) error { @@ -725,16 +678,12 @@ func (d *SyncServerDatasource) SetTypingTimeoutCallback(fn cache.TimeoutCallback d.eduCache.SetTimeoutCallback(fn) } -// AddTypingUser adds a typing user to the typing cache. -// Returns the newly calculated sync position for typing notifications. func (d *SyncServerDatasource) AddTypingUser( userID, roomID string, expireTime *time.Time, ) types.StreamPosition { return types.StreamPosition(d.eduCache.AddTypingUser(userID, roomID, expireTime)) } -// RemoveTypingUser removes a typing user from the typing cache. -// Returns the newly calculated sync position for typing notifications. func (d *SyncServerDatasource) RemoveTypingUser( userID, roomID string, ) types.StreamPosition { @@ -1073,9 +1022,6 @@ func (d *SyncServerDatasource) currentStateStreamEventsForRoom( return s, nil } -// StreamEventsToEvents converts streamEvent to Event. If device is non-nil and -// matches the streamevent.transactionID device then the transaction ID gets -// added to the unsigned section of the output event. func (d *SyncServerDatasource) StreamEventsToEvents(device *authtypes.Device, in []types.StreamEvent) []gomatrixserverlib.HeaderedEvent { out := make([]gomatrixserverlib.HeaderedEvent, len(in)) for i := 0; i < len(in); i++ { diff --git a/syncapi/storage/storage_test.go b/syncapi/storage/storage_test.go new file mode 100644 index 000000000..e591e7ed7 --- /dev/null +++ b/syncapi/storage/storage_test.go @@ -0,0 +1,313 @@ +package storage_test + +import ( + "context" + "crypto/ed25519" + "fmt" + "testing" + "time" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/dendrite/syncapi/storage/sqlite3" + "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/gomatrixserverlib" +) + +var ( + ctx = context.Background() + emptyStateKey = "" + testOrigin = gomatrixserverlib.ServerName("hollow.knight") + testRoomID = fmt.Sprintf("!hallownest:%s", testOrigin) + testUserIDA = fmt.Sprintf("@hornet:%s", testOrigin) + testUserIDB = fmt.Sprintf("@paleking:%s", testOrigin) + testUserDeviceA = authtypes.Device{ + UserID: testUserIDA, + ID: "device_id_A", + DisplayName: "Device A", + } + testRoomVersion = gomatrixserverlib.RoomVersionV4 + testKeyID = gomatrixserverlib.KeyID("ed25519:storage_test") + testPrivateKey = ed25519.NewKeyFromSeed([]byte{ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + }) +) + +func MustCreateEvent(t *testing.T, roomID string, prevs []gomatrixserverlib.HeaderedEvent, b *gomatrixserverlib.EventBuilder) gomatrixserverlib.HeaderedEvent { + b.RoomID = roomID + if prevs != nil { + prevIDs := make([]string, len(prevs)) + for i := range prevs { + prevIDs[i] = prevs[i].EventID() + } + b.PrevEvents = prevIDs + } + e, err := b.Build(time.Now(), testOrigin, testKeyID, testPrivateKey, testRoomVersion) + if err != nil { + t.Fatalf("failed to build event: %s", err) + } + return e.Headered(testRoomVersion) +} + +func MustCreateDatabase(t *testing.T) storage.Database { + db, err := sqlite3.NewSyncServerDatasource("file::memory:") + if err != nil { + t.Fatalf("NewSyncServerDatasource returned %s", err) + } + return db +} + +// Create a list of events which include a create event, join event and some messages. +func SimpleRoom(t *testing.T, roomID, userA, userB string) (msgs []gomatrixserverlib.HeaderedEvent, state []gomatrixserverlib.HeaderedEvent) { + var events []gomatrixserverlib.HeaderedEvent + events = append(events, MustCreateEvent(t, roomID, nil, &gomatrixserverlib.EventBuilder{ + Content: []byte(fmt.Sprintf(`{"room_version":"4","creator":"%s"}`, userA)), + Type: "m.room.create", + StateKey: &emptyStateKey, + Sender: userA, + Depth: int64(len(events) + 1), + })) + state = append(state, events[len(events)-1]) + events = append(events, MustCreateEvent(t, roomID, []gomatrixserverlib.HeaderedEvent{events[len(events)-1]}, &gomatrixserverlib.EventBuilder{ + Content: []byte(fmt.Sprintf(`{"membership":"join"}`)), + Type: "m.room.member", + StateKey: &userA, + Sender: userA, + Depth: int64(len(events) + 1), + })) + state = append(state, events[len(events)-1]) + for i := 0; i < 10; i++ { + events = append(events, MustCreateEvent(t, roomID, []gomatrixserverlib.HeaderedEvent{events[len(events)-1]}, &gomatrixserverlib.EventBuilder{ + Content: []byte(fmt.Sprintf(`{"body":"Message A %d"}`, i+1)), + Type: "m.room.message", + Sender: userA, + Depth: int64(len(events) + 1), + })) + } + events = append(events, MustCreateEvent(t, roomID, []gomatrixserverlib.HeaderedEvent{events[len(events)-1]}, &gomatrixserverlib.EventBuilder{ + Content: []byte(fmt.Sprintf(`{"membership":"join"}`)), + Type: "m.room.member", + StateKey: &userB, + Sender: userB, + Depth: int64(len(events) + 1), + })) + state = append(state, events[len(events)-1]) + for i := 0; i < 10; i++ { + events = append(events, MustCreateEvent(t, roomID, []gomatrixserverlib.HeaderedEvent{events[len(events)-1]}, &gomatrixserverlib.EventBuilder{ + Content: []byte(fmt.Sprintf(`{"body":"Message B %d"}`, i+1)), + Type: "m.room.message", + Sender: userB, + Depth: int64(len(events) + 1), + })) + } + + return events, state +} + +func MustWriteEvents(t *testing.T, db storage.Database, events []gomatrixserverlib.HeaderedEvent) (positions []types.StreamPosition) { + for _, ev := range events { + var addStateEvents []gomatrixserverlib.HeaderedEvent + var addStateEventIDs []string + var removeStateEventIDs []string + if ev.StateKey() != nil { + addStateEvents = append(addStateEvents, ev) + addStateEventIDs = append(addStateEventIDs, ev.EventID()) + } + pos, err := db.WriteEvent(ctx, &ev, addStateEvents, addStateEventIDs, removeStateEventIDs, nil, false) + if err != nil { + t.Fatalf("WriteEvent failed: %s", err) + } + positions = append(positions, pos) + } + return +} + +func TestWriteEvents(t *testing.T) { + t.Parallel() + db := MustCreateDatabase(t) + events, _ := SimpleRoom(t, testRoomID, testUserIDA, testUserIDB) + MustWriteEvents(t, db, events) +} + +// These tests assert basic functionality of the IncrementalSync and CompleteSync functions. +func TestSyncResponse(t *testing.T) { + t.Parallel() + db := MustCreateDatabase(t) + events, state := SimpleRoom(t, testRoomID, testUserIDA, testUserIDB) + positions := MustWriteEvents(t, db, events) + latest, err := db.SyncPosition(ctx) + if err != nil { + t.Fatalf("failed to get SyncPosition: %s", err) + } + + testCases := []struct { + Name string + DoSync func() (*types.Response, error) + WantTimeline []gomatrixserverlib.HeaderedEvent + WantState []gomatrixserverlib.HeaderedEvent + }{ + // The purpose of this test is to make sure that incremental syncs are including up to the latest events. + // It's a basic sanity test that sync works. It creates a `since` token that is on the penultimate event. + // It makes sure the response includes the final event. + { + Name: "IncrementalSync penultimate", + DoSync: func() (*types.Response, error) { + from := types.NewPaginationTokenFromTypeAndPosition( // pretend we are at the penultimate event + types.PaginationTokenTypeStream, positions[len(positions)-2], types.StreamPosition(0), + ) + return db.IncrementalSync(ctx, testUserDeviceA, *from, latest, 5, false) + }, + WantTimeline: events[len(events)-1:], + }, + // The purpose of this test is to check that passing a `numRecentEventsPerRoom` correctly limits the + // number of returned events. This is critical for big rooms hence the test here. + { + Name: "IncrementalSync limited", + DoSync: func() (*types.Response, error) { + from := types.NewPaginationTokenFromTypeAndPosition( // pretend we are 10 events behind + types.PaginationTokenTypeStream, positions[len(positions)-11], types.StreamPosition(0), + ) + // limit is set to 5 + return db.IncrementalSync(ctx, testUserDeviceA, *from, latest, 5, false) + }, + // want the last 5 events, NOT the last 10. + WantTimeline: events[len(events)-5:], + }, + // The purpose of this test is to check that CompleteSync returns all the current state as well as + // honouring the `numRecentEventsPerRoom` value + { + Name: "CompleteSync limited", + DoSync: func() (*types.Response, error) { + // limit set to 5 + return db.CompleteSync(ctx, testUserIDA, 5) + }, + // want the last 5 events, NOT the last 10. + WantTimeline: events[len(events)-5:], + // want all state for the room + WantState: state, + }, + // The purpose of this test is to check that CompleteSync can return everything with a high enough + // `numRecentEventsPerRoom`. + { + Name: "CompleteSync", + DoSync: func() (*types.Response, error) { + return db.CompleteSync(ctx, testUserIDA, len(events)+1) + }, + WantTimeline: events, + // We want no state at all as that field in /sync is the delta between the token (beginning of time) + // and the START of the timeline. + }, + } + + for _, tc := range testCases { + t.Run(tc.Name, func(st *testing.T) { + res, err := tc.DoSync() + if err != nil { + st.Fatalf("failed to do sync: %s", err) + } + next := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeStream, latest.PDUPosition, latest.EDUTypingPosition) + if res.NextBatch != next.String() { + st.Errorf("NextBatch got %s want %s", res.NextBatch, next.String()) + } + roomRes, ok := res.Rooms.Join[testRoomID] + if !ok { + st.Fatalf("IncrementalSync response missing room %s - response: %+v", testRoomID, res) + } + assertEventsEqual(st, "state for "+testRoomID, false, roomRes.State.Events, tc.WantState) + assertEventsEqual(st, "timeline for "+testRoomID, false, roomRes.Timeline.Events, tc.WantTimeline) + }) + } +} + +// The purpose of this test is to ensure that backfill does indeed go backwards, using a stream token. +func TestGetEventsInRangeWithStreamToken(t *testing.T) { + t.Parallel() + db := MustCreateDatabase(t) + events, _ := SimpleRoom(t, testRoomID, testUserIDA, testUserIDB) + MustWriteEvents(t, db, events) + latest, err := db.SyncPosition(ctx) + if err != nil { + t.Fatalf("failed to get SyncPosition: %s", err) + } + // head towards the beginning of time + to := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, 0, 0) + + // backpaginate 5 messages starting at the latest position. + paginatedEvents, err := db.GetEventsInRange(ctx, &latest, to, testRoomID, 5, true) + if err != nil { + t.Fatalf("GetEventsInRange returned an error: %s", err) + } + gots := gomatrixserverlib.HeaderedToClientEvents(db.StreamEventsToEvents(&testUserDeviceA, paginatedEvents), gomatrixserverlib.FormatAll) + assertEventsEqual(t, "", true, gots, reversed(events[len(events)-5:])) +} + +// The purpose of this test is to ensure that backfill does indeed go backwards, using a topology token +func TestGetEventsInRangeWithTopologyToken(t *testing.T) { + t.Parallel() + db := MustCreateDatabase(t) + events, _ := SimpleRoom(t, testRoomID, testUserIDA, testUserIDB) + MustWriteEvents(t, db, events) + latest, err := db.MaxTopologicalPosition(ctx, testRoomID) + if err != nil { + t.Fatalf("failed to get MaxTopologicalPosition: %s", err) + } + from := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, latest, 0) + // head towards the beginning of time + to := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, 0, 0) + + // backpaginate 5 messages starting at the latest position. + paginatedEvents, err := db.GetEventsInRange(ctx, from, to, testRoomID, 5, true) + if err != nil { + t.Fatalf("GetEventsInRange returned an error: %s", err) + } + gots := gomatrixserverlib.HeaderedToClientEvents(db.StreamEventsToEvents(&testUserDeviceA, paginatedEvents), gomatrixserverlib.FormatAll) + assertEventsEqual(t, "", true, gots, reversed(events[len(events)-5:])) +} + +func assertEventsEqual(t *testing.T, msg string, checkRoomID bool, gots []gomatrixserverlib.ClientEvent, wants []gomatrixserverlib.HeaderedEvent) { + if len(gots) != len(wants) { + t.Fatalf("%s response returned %d events, want %d", msg, len(gots), len(wants)) + } + for i := range gots { + g := gots[i] + w := wants[i] + if g.EventID != w.EventID() { + t.Errorf("%s event[%d] event_id mismatch: got %s want %s", msg, i, g.EventID, w.EventID()) + } + if g.Sender != w.Sender() { + t.Errorf("%s event[%d] sender mismatch: got %s want %s", msg, i, g.Sender, w.Sender()) + } + if checkRoomID && g.RoomID != w.RoomID() { + t.Errorf("%s event[%d] room_id mismatch: got %s want %s", msg, i, g.RoomID, w.RoomID()) + } + if g.Type != w.Type() { + t.Errorf("%s event[%d] event type mismatch: got %s want %s", msg, i, g.Type, w.Type()) + } + if g.OriginServerTS != w.OriginServerTS() { + t.Errorf("%s event[%d] origin_server_ts mismatch: got %v want %v", msg, i, g.OriginServerTS, w.OriginServerTS()) + } + if string(g.Content) != string(w.Content()) { + t.Errorf("%s event[%d] content mismatch: got %s want %s", msg, i, string(g.Content), string(w.Content())) + } + if string(g.Unsigned) != string(w.Unsigned()) { + t.Errorf("%s event[%d] unsigned mismatch: got %s want %s", msg, i, string(g.Unsigned), string(w.Unsigned())) + } + if (g.StateKey == nil && w.StateKey() != nil) || (g.StateKey != nil && w.StateKey() == nil) { + t.Fatalf("%s event[%d] state_key [not] missing: got %v want %v", msg, i, g.StateKey, w.StateKey()) + } + if g.StateKey != nil { + if !w.StateKeyEquals(*g.StateKey) { + t.Errorf("%s event[%d] state_key mismatch: got %s want %s", msg, i, *g.StateKey, *w.StateKey()) + } + } + } +} + +func reversed(in []gomatrixserverlib.HeaderedEvent) []gomatrixserverlib.HeaderedEvent { + out := make([]gomatrixserverlib.HeaderedEvent, len(in)) + for i := 0; i < len(in); i++ { + out[i] = in[len(in)-i-1] + } + return out +} From e15f6676ac3f76ec2ef679c2df300d6a8e7e668f Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 1 May 2020 10:48:17 +0100 Subject: [PATCH 020/200] Consolidation of roomserver APIs (#994) * Consolidation of roomserver APIs * Comment out alias tests for now, they are broken * Wire AS API into roomserver again * Roomserver didn't take asAPI param before so return to that * Prevent roomserver asking AS API for alias info * Rename some files * Remove alias_test, incoherent tests and unwanted appservice integration * Remove FS API inject on syncapi component --- appservice/appservice.go | 7 +- appservice/consumers/roomserver.go | 13 +- appservice/routing/routing.go | 2 +- clientapi/clientapi.go | 10 +- clientapi/consumers/roomserver.go | 22 +- clientapi/producers/roomserver.go | 12 +- clientapi/routing/capabilities.go | 4 +- clientapi/routing/createroom.go | 8 +- clientapi/routing/directory.go | 6 +- clientapi/routing/getevent.go | 6 +- clientapi/routing/joinroom.go | 14 +- clientapi/routing/membership.go | 16 +- clientapi/routing/memberships.go | 6 +- clientapi/routing/profile.go | 14 +- clientapi/routing/routing.go | 41 ++- clientapi/routing/sendevent.go | 10 +- clientapi/routing/state.go | 8 +- clientapi/threepid/invites.go | 8 +- cmd/dendrite-appservice-server/main.go | 4 +- cmd/dendrite-client-api-server/main.go | 5 +- cmd/dendrite-demo-libp2p/main.go | 27 +- cmd/dendrite-federation-api-server/main.go | 7 +- cmd/dendrite-federation-sender-server/main.go | 9 +- cmd/dendrite-monolith-server/main.go | 28 +- cmd/dendrite-public-rooms-api-server/main.go | 7 +- cmd/dendrite-room-server/main.go | 4 +- cmd/dendrite-sync-api-server/main.go | 4 +- cmd/dendritejs/main.go | 16 +- cmd/roomserver-integration-tests/main.go | 8 +- common/basecomponent/base.go | 20 +- common/events.go | 10 +- federationapi/federationapi.go | 10 +- federationapi/routing/backfill.go | 4 +- federationapi/routing/eventauth.go | 4 +- federationapi/routing/events.go | 10 +- federationapi/routing/join.go | 16 +- federationapi/routing/leave.go | 6 +- federationapi/routing/missingevents.go | 4 +- federationapi/routing/query.go | 4 +- federationapi/routing/routing.go | 29 +- federationapi/routing/send.go | 10 +- federationapi/routing/state.go | 14 +- federationapi/routing/threepid.go | 22 +- federationsender/consumers/roomserver.go | 26 +- federationsender/federationsender.go | 7 +- federationsender/producers/roomserver.go | 6 +- publicroomsapi/consumers/roomserver.go | 20 +- publicroomsapi/directory/directory.go | 6 +- publicroomsapi/publicroomsapi.go | 6 +- publicroomsapi/routing/routing.go | 4 +- roomserver/alias/alias_test.go | 209 ------------- roomserver/api/alias.go | 64 +--- roomserver/api/api.go | 141 +++++++++ roomserver/api/http.go | 41 +++ roomserver/api/input.go | 41 +-- roomserver/api/query.go | 131 +------- roomserver/{alias => internal}/alias.go | 186 ++++-------- roomserver/internal/api.go | 287 ++++++++++++++++++ roomserver/{input => internal}/input.go | 46 +-- .../input_authevents.go} | 2 +- .../input_authevents_test.go} | 2 +- .../events.go => internal/input_events.go} | 2 +- .../input_latest_events.go} | 2 +- .../input_membership.go} | 2 +- roomserver/{query => internal}/query.go | 250 ++------------- .../query_backfill.go} | 2 +- roomserver/{query => internal}/query_test.go | 6 +- roomserver/roomserver.go | 46 +-- syncapi/consumers/roomserver.go | 22 +- syncapi/routing/messages.go | 8 +- syncapi/routing/routing.go | 4 +- syncapi/syncapi.go | 6 +- 72 files changed, 894 insertions(+), 1170 deletions(-) delete mode 100644 roomserver/alias/alias_test.go create mode 100644 roomserver/api/api.go create mode 100644 roomserver/api/http.go rename roomserver/{alias => internal}/alias.go (50%) create mode 100644 roomserver/internal/api.go rename roomserver/{input => internal}/input.go (55%) rename roomserver/{input/authevents.go => internal/input_authevents.go} (99%) rename roomserver/{input/authevents_test.go => internal/input_authevents_test.go} (99%) rename roomserver/{input/events.go => internal/input_events.go} (99%) rename roomserver/{input/latest_events.go => internal/input_latest_events.go} (99%) rename roomserver/{input/membership.go => internal/input_membership.go} (99%) rename roomserver/{query => internal}/query.go (74%) rename roomserver/{query/backfill.go => internal/query_backfill.go} (99%) rename roomserver/{query => internal}/query_test.go (96%) diff --git a/appservice/appservice.go b/appservice/appservice.go index 181799879..71d131998 100644 --- a/appservice/appservice.go +++ b/appservice/appservice.go @@ -44,8 +44,7 @@ func SetupAppServiceAPIComponent( accountsDB accounts.Database, deviceDB devices.Database, federation *gomatrixserverlib.FederationClient, - roomserverAliasAPI roomserverAPI.RoomserverAliasAPI, - roomserverQueryAPI roomserverAPI.RoomserverQueryAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, transactionsCache *transactions.Cache, ) appserviceAPI.AppServiceQueryAPI { // Create a connection to the appservice postgres DB @@ -87,7 +86,7 @@ func SetupAppServiceAPIComponent( consumer := consumers.NewOutputRoomEventConsumer( base.Cfg, base.KafkaConsumer, accountsDB, appserviceDB, - roomserverQueryAPI, roomserverAliasAPI, workerStates, + rsAPI, workerStates, ) if err := consumer.Start(); err != nil { logrus.WithError(err).Panicf("failed to start appservice roomserver consumer") @@ -100,7 +99,7 @@ func SetupAppServiceAPIComponent( // Set up HTTP Endpoints routing.Setup( - base.APIMux, base.Cfg, roomserverQueryAPI, roomserverAliasAPI, + base.APIMux, base.Cfg, rsAPI, accountsDB, federation, transactionsCache, ) diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go index 3bd364c58..b7f689249 100644 --- a/appservice/consumers/roomserver.go +++ b/appservice/consumers/roomserver.go @@ -35,8 +35,7 @@ type OutputRoomEventConsumer struct { roomServerConsumer *common.ContinualConsumer db accounts.Database asDB storage.Database - query api.RoomserverQueryAPI - alias api.RoomserverAliasAPI + rsAPI api.RoomserverInternalAPI serverName string workerStates []types.ApplicationServiceWorkerState } @@ -48,8 +47,7 @@ func NewOutputRoomEventConsumer( kafkaConsumer sarama.Consumer, store accounts.Database, appserviceDB storage.Database, - queryAPI api.RoomserverQueryAPI, - aliasAPI api.RoomserverAliasAPI, + rsAPI api.RoomserverInternalAPI, workerStates []types.ApplicationServiceWorkerState, ) *OutputRoomEventConsumer { consumer := common.ContinualConsumer{ @@ -61,8 +59,7 @@ func NewOutputRoomEventConsumer( roomServerConsumer: &consumer, db: store, asDB: appserviceDB, - query: queryAPI, - alias: aliasAPI, + rsAPI: rsAPI, serverName: string(cfg.Matrix.ServerName), workerStates: workerStates, } @@ -139,7 +136,7 @@ func (s *OutputRoomEventConsumer) lookupMissingStateEvents( // Request the missing events from the roomserver eventReq := api.QueryEventsByIDRequest{EventIDs: missing} var eventResp api.QueryEventsByIDResponse - if err := s.query.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil { + if err := s.rsAPI.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil { return nil, err } @@ -200,7 +197,7 @@ func (s *OutputRoomEventConsumer) appserviceIsInterestedInEvent(ctx context.Cont // Check all known room aliases of the room the event came from queryReq := api.GetAliasesForRoomIDRequest{RoomID: event.RoomID()} var queryRes api.GetAliasesForRoomIDResponse - if err := s.alias.GetAliasesForRoomID(ctx, &queryReq, &queryRes); err == nil { + if err := s.rsAPI.GetAliasesForRoomID(ctx, &queryReq, &queryRes); err == nil { for _, alias := range queryRes.Aliases { if appservice.IsInterestedInRoomAlias(alias) { return true diff --git a/appservice/routing/routing.go b/appservice/routing/routing.go index 42fa80520..9f59e05f7 100644 --- a/appservice/routing/routing.go +++ b/appservice/routing/routing.go @@ -37,7 +37,7 @@ const pathPrefixApp = "/_matrix/app/v1" // nolint: gocyclo func Setup( apiMux *mux.Router, cfg *config.Dendrite, // nolint: unparam - queryAPI api.RoomserverQueryAPI, aliasAPI api.RoomserverAliasAPI, // nolint: unparam + rsAPI api.RoomserverInternalAPI, // nolint: unparam accountDB accounts.Database, // nolint: unparam federation *gomatrixserverlib.FederationClient, // nolint: unparam transactionsCache *transactions.Cache, // nolint: unparam diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go index 6a857e52c..f81e0242c 100644 --- a/clientapi/clientapi.go +++ b/clientapi/clientapi.go @@ -38,15 +38,13 @@ func SetupClientAPIComponent( accountsDB accounts.Database, federation *gomatrixserverlib.FederationClient, keyRing *gomatrixserverlib.KeyRing, - aliasAPI roomserverAPI.RoomserverAliasAPI, - inputAPI roomserverAPI.RoomserverInputAPI, - queryAPI roomserverAPI.RoomserverQueryAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, eduInputAPI eduServerAPI.EDUServerInputAPI, asAPI appserviceAPI.AppServiceQueryAPI, transactionsCache *transactions.Cache, fsAPI federationSenderAPI.FederationSenderInternalAPI, ) { - roomserverProducer := producers.NewRoomserverProducer(inputAPI, queryAPI) + roomserverProducer := producers.NewRoomserverProducer(rsAPI) eduProducer := producers.NewEDUServerProducer(eduInputAPI) userUpdateProducer := &producers.UserUpdateProducer{ @@ -60,14 +58,14 @@ func SetupClientAPIComponent( } consumer := consumers.NewOutputRoomEventConsumer( - base.Cfg, base.KafkaConsumer, accountsDB, queryAPI, + base.Cfg, base.KafkaConsumer, accountsDB, rsAPI, ) if err := consumer.Start(); err != nil { logrus.WithError(err).Panicf("failed to start room server consumer") } routing.Setup( - base.APIMux, base.Cfg, roomserverProducer, queryAPI, aliasAPI, asAPI, + base.APIMux, base.Cfg, roomserverProducer, rsAPI, asAPI, accountsDB, deviceDB, federation, *keyRing, userUpdateProducer, syncProducer, eduProducer, transactionsCache, fsAPI, ) diff --git a/clientapi/consumers/roomserver.go b/clientapi/consumers/roomserver.go index 3c7905721..d0c91e88f 100644 --- a/clientapi/consumers/roomserver.go +++ b/clientapi/consumers/roomserver.go @@ -30,10 +30,10 @@ import ( // OutputRoomEventConsumer consumes events that originated in the room server. type OutputRoomEventConsumer struct { - roomServerConsumer *common.ContinualConsumer - db accounts.Database - query api.RoomserverQueryAPI - serverName string + rsAPI api.RoomserverInternalAPI + rsConsumer *common.ContinualConsumer + db accounts.Database + serverName string } // NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call Start() to begin consuming from room servers. @@ -41,7 +41,7 @@ func NewOutputRoomEventConsumer( cfg *config.Dendrite, kafkaConsumer sarama.Consumer, store accounts.Database, - queryAPI api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, ) *OutputRoomEventConsumer { consumer := common.ContinualConsumer{ @@ -50,10 +50,10 @@ func NewOutputRoomEventConsumer( PartitionStore: store, } s := &OutputRoomEventConsumer{ - roomServerConsumer: &consumer, - db: store, - query: queryAPI, - serverName: string(cfg.Matrix.ServerName), + rsConsumer: &consumer, + db: store, + rsAPI: rsAPI, + serverName: string(cfg.Matrix.ServerName), } consumer.ProcessMessage = s.onMessage @@ -62,7 +62,7 @@ func NewOutputRoomEventConsumer( // Start consuming from room servers func (s *OutputRoomEventConsumer) Start() error { - return s.roomServerConsumer.Start() + return s.rsConsumer.Start() } // onMessage is called when the sync server receives a new event from the room server output log. @@ -134,7 +134,7 @@ func (s *OutputRoomEventConsumer) lookupStateEvents( // Request the missing events from the roomserver eventReq := api.QueryEventsByIDRequest{EventIDs: missing} var eventResp api.QueryEventsByIDResponse - if err := s.query.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil { + if err := s.rsAPI.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil { return nil, err } diff --git a/clientapi/producers/roomserver.go b/clientapi/producers/roomserver.go index fac1e3c7c..a804abfe9 100644 --- a/clientapi/producers/roomserver.go +++ b/clientapi/producers/roomserver.go @@ -23,15 +23,13 @@ import ( // RoomserverProducer produces events for the roomserver to consume. type RoomserverProducer struct { - InputAPI api.RoomserverInputAPI - QueryAPI api.RoomserverQueryAPI + RsAPI api.RoomserverInternalAPI } // NewRoomserverProducer creates a new RoomserverProducer -func NewRoomserverProducer(inputAPI api.RoomserverInputAPI, queryAPI api.RoomserverQueryAPI) *RoomserverProducer { +func NewRoomserverProducer(rsAPI api.RoomserverInternalAPI) *RoomserverProducer { return &RoomserverProducer{ - InputAPI: inputAPI, - QueryAPI: queryAPI, + RsAPI: rsAPI, } } @@ -95,7 +93,7 @@ func (c *RoomserverProducer) SendInputRoomEvents( ) (eventID string, err error) { request := api.InputRoomEventsRequest{InputRoomEvents: ires} var response api.InputRoomEventsResponse - err = c.InputAPI.InputRoomEvents(ctx, &request, &response) + err = c.RsAPI.InputRoomEvents(ctx, &request, &response) eventID = response.EventID return } @@ -118,5 +116,5 @@ func (c *RoomserverProducer) SendInvite( }}, } var response api.InputRoomEventsResponse - return c.InputAPI.InputRoomEvents(ctx, &request, &response) + return c.RsAPI.InputRoomEvents(ctx, &request, &response) } diff --git a/clientapi/routing/capabilities.go b/clientapi/routing/capabilities.go index 1792c6308..199b15240 100644 --- a/clientapi/routing/capabilities.go +++ b/clientapi/routing/capabilities.go @@ -26,11 +26,11 @@ import ( // SendMembership implements PUT /rooms/{roomID}/(join|kick|ban|unban|leave|invite) // by building a m.room.member event then sending it to the room server func GetCapabilities( - req *http.Request, queryAPI roomserverAPI.RoomserverQueryAPI, + req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI, ) util.JSONResponse { roomVersionsQueryReq := roomserverAPI.QueryRoomVersionCapabilitiesRequest{} roomVersionsQueryRes := roomserverAPI.QueryRoomVersionCapabilitiesResponse{} - if err := queryAPI.QueryRoomVersionCapabilities( + if err := rsAPI.QueryRoomVersionCapabilities( req.Context(), &roomVersionsQueryReq, &roomVersionsQueryRes, diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index ef11e8b3e..28e2b1514 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -137,13 +137,13 @@ type fledglingEvent struct { func CreateRoom( req *http.Request, device *authtypes.Device, cfg *config.Dendrite, producer *producers.RoomserverProducer, - accountDB accounts.Database, aliasAPI roomserverAPI.RoomserverAliasAPI, + accountDB accounts.Database, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, ) util.JSONResponse { // TODO (#267): Check room ID doesn't clash with an existing one, and we // probably shouldn't be using pseudo-random strings, maybe GUIDs? roomID := fmt.Sprintf("!%s:%s", util.RandomString(16), cfg.Matrix.ServerName) - return createRoom(req, device, cfg, roomID, producer, accountDB, aliasAPI, asAPI) + return createRoom(req, device, cfg, roomID, producer, accountDB, rsAPI, asAPI) } // createRoom implements /createRoom @@ -151,7 +151,7 @@ func CreateRoom( func createRoom( req *http.Request, device *authtypes.Device, cfg *config.Dendrite, roomID string, producer *producers.RoomserverProducer, - accountDB accounts.Database, aliasAPI roomserverAPI.RoomserverAliasAPI, + accountDB accounts.Database, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, ) util.JSONResponse { logger := util.GetLogger(req.Context()) @@ -340,7 +340,7 @@ func createRoom( } var aliasResp roomserverAPI.SetRoomAliasResponse - err = aliasAPI.SetRoomAlias(req.Context(), &aliasReq, &aliasResp) + err = rsAPI.SetRoomAlias(req.Context(), &aliasReq, &aliasResp) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("aliasAPI.SetRoomAlias failed") return jsonerror.InternalServerError() diff --git a/clientapi/routing/directory.go b/clientapi/routing/directory.go index 101ba11ff..a0a60a47a 100644 --- a/clientapi/routing/directory.go +++ b/clientapi/routing/directory.go @@ -46,7 +46,7 @@ func DirectoryRoom( roomAlias string, federation *gomatrixserverlib.FederationClient, cfg *config.Dendrite, - rsAPI roomserverAPI.RoomserverAliasAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, fedSenderAPI federationSenderAPI.FederationSenderInternalAPI, ) util.JSONResponse { _, domain, err := gomatrixserverlib.SplitID('#', roomAlias) @@ -115,7 +115,7 @@ func SetLocalAlias( device *authtypes.Device, alias string, cfg *config.Dendrite, - aliasAPI roomserverAPI.RoomserverAliasAPI, + aliasAPI roomserverAPI.RoomserverInternalAPI, ) util.JSONResponse { _, domain, err := gomatrixserverlib.SplitID('#', alias) if err != nil { @@ -190,7 +190,7 @@ func RemoveLocalAlias( req *http.Request, device *authtypes.Device, alias string, - aliasAPI roomserverAPI.RoomserverAliasAPI, + aliasAPI roomserverAPI.RoomserverInternalAPI, ) util.JSONResponse { creatorQueryReq := roomserverAPI.GetCreatorIDForAliasRequest{ diff --git a/clientapi/routing/getevent.go b/clientapi/routing/getevent.go index 2d3152510..bf49968de 100644 --- a/clientapi/routing/getevent.go +++ b/clientapi/routing/getevent.go @@ -44,7 +44,7 @@ func GetEvent( roomID string, eventID string, cfg *config.Dendrite, - queryAPI api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, federation *gomatrixserverlib.FederationClient, keyRing gomatrixserverlib.KeyRing, ) util.JSONResponse { @@ -52,7 +52,7 @@ func GetEvent( EventIDs: []string{eventID}, } var eventsResp api.QueryEventsByIDResponse - err := queryAPI.QueryEventsByID(req.Context(), &eventsReq, &eventsResp) + err := rsAPI.QueryEventsByID(req.Context(), &eventsReq, &eventsResp) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("queryAPI.QueryEventsByID failed") return jsonerror.InternalServerError() @@ -88,7 +88,7 @@ func GetEvent( }}, } var stateResp api.QueryStateAfterEventsResponse - if err := queryAPI.QueryStateAfterEvents(req.Context(), &stateReq, &stateResp); err != nil { + if err := rsAPI.QueryStateAfterEvents(req.Context(), &stateReq, &stateResp); err != nil { util.GetLogger(req.Context()).WithError(err).Error("queryAPI.QueryStateAfterEvents failed") return jsonerror.InternalServerError() } diff --git a/clientapi/routing/joinroom.go b/clientapi/routing/joinroom.go index f55d1b6ab..df83c2a9f 100644 --- a/clientapi/routing/joinroom.go +++ b/clientapi/routing/joinroom.go @@ -43,8 +43,7 @@ func JoinRoomByIDOrAlias( cfg *config.Dendrite, federation *gomatrixserverlib.FederationClient, producer *producers.RoomserverProducer, - queryAPI roomserverAPI.RoomserverQueryAPI, - aliasAPI roomserverAPI.RoomserverAliasAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, fsAPI federationSenderAPI.FederationSenderInternalAPI, keyRing gomatrixserverlib.KeyRing, accountDB accounts.Database, @@ -80,7 +79,7 @@ func JoinRoomByIDOrAlias( r := joinRoomReq{ req, evTime, content, device.UserID, cfg, federation, producer, - queryAPI, aliasAPI, fsAPI, keyRing, + rsAPI, fsAPI, keyRing, } if strings.HasPrefix(roomIDOrAlias, "!") { @@ -106,8 +105,7 @@ type joinRoomReq struct { cfg *config.Dendrite federation *gomatrixserverlib.FederationClient producer *producers.RoomserverProducer - queryAPI roomserverAPI.RoomserverQueryAPI - aliasAPI roomserverAPI.RoomserverAliasAPI + rsAPI roomserverAPI.RoomserverInternalAPI fsAPI federationSenderAPI.FederationSenderInternalAPI keyRing gomatrixserverlib.KeyRing } @@ -124,7 +122,7 @@ func (r joinRoomReq) joinRoomByID(roomID string) util.JSONResponse { RoomID: roomID, TargetUserID: r.userID, } var queryRes roomserverAPI.QueryInvitesForUserResponse - if err := r.queryAPI.QueryInvitesForUser(r.req.Context(), &queryReq, &queryRes); err != nil { + if err := r.rsAPI.QueryInvitesForUser(r.req.Context(), &queryReq, &queryRes); err != nil { util.GetLogger(r.req.Context()).WithError(err).Error("r.queryAPI.QueryInvitesForUser failed") return jsonerror.InternalServerError() } @@ -172,7 +170,7 @@ func (r joinRoomReq) joinRoomByAlias(roomAlias string) util.JSONResponse { if domain == r.cfg.Matrix.ServerName { queryReq := roomserverAPI.GetRoomIDForAliasRequest{Alias: roomAlias} var queryRes roomserverAPI.GetRoomIDForAliasResponse - if err = r.aliasAPI.GetRoomIDForAlias(r.req.Context(), &queryReq, &queryRes); err != nil { + if err = r.rsAPI.GetRoomIDForAlias(r.req.Context(), &queryReq, &queryRes); err != nil { util.GetLogger(r.req.Context()).WithError(err).Error("r.aliasAPI.GetRoomIDForAlias failed") return jsonerror.InternalServerError() } @@ -243,7 +241,7 @@ func (r joinRoomReq) joinRoomUsingServers( } queryRes := roomserverAPI.QueryLatestEventsAndStateResponse{} - event, err := common.BuildEvent(r.req.Context(), &eb, r.cfg, r.evTime, r.queryAPI, &queryRes) + event, err := common.BuildEvent(r.req.Context(), &eb, r.cfg, r.evTime, r.rsAPI, &queryRes) if err == nil { // If we have successfully built an event at this point then we can // assert that the room is a local room, as BuildEvent was able to diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index dff194dd3..9030f9f7e 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -45,12 +45,12 @@ var errMissingUserID = errors.New("'user_id' must be supplied") func SendMembership( req *http.Request, accountDB accounts.Database, device *authtypes.Device, roomID string, membership string, cfg *config.Dendrite, - queryAPI roomserverAPI.RoomserverQueryAPI, asAPI appserviceAPI.AppServiceQueryAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, producer *producers.RoomserverProducer, ) util.JSONResponse { verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} verRes := api.QueryRoomVersionForRoomResponse{} - if err := queryAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil { + if err := rsAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.UnsupportedRoomVersion(err.Error()), @@ -71,7 +71,7 @@ func SendMembership( } inviteStored, jsonErrResp := checkAndProcessThreepid( - req, device, &body, cfg, queryAPI, accountDB, producer, + req, device, &body, cfg, rsAPI, accountDB, producer, membership, roomID, evTime, ) if jsonErrResp != nil { @@ -89,7 +89,7 @@ func SendMembership( } event, err := buildMembershipEvent( - req.Context(), body, accountDB, device, membership, roomID, cfg, evTime, queryAPI, asAPI, + req.Context(), body, accountDB, device, membership, roomID, cfg, evTime, rsAPI, asAPI, ) if err == errMissingUserID { return util.JSONResponse{ @@ -153,7 +153,7 @@ func buildMembershipEvent( device *authtypes.Device, membership, roomID string, cfg *config.Dendrite, evTime time.Time, - queryAPI roomserverAPI.RoomserverQueryAPI, asAPI appserviceAPI.AppServiceQueryAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, ) (*gomatrixserverlib.Event, error) { stateKey, reason, err := getMembershipStateKey(body, device, membership) if err != nil { @@ -188,7 +188,7 @@ func buildMembershipEvent( return nil, err } - return common.BuildEvent(ctx, &builder, cfg, evTime, queryAPI, nil) + return common.BuildEvent(ctx, &builder, cfg, evTime, rsAPI, nil) } // loadProfile lookups the profile of a given user from the database and returns @@ -248,7 +248,7 @@ func checkAndProcessThreepid( device *authtypes.Device, body *threepid.MembershipRequest, cfg *config.Dendrite, - queryAPI roomserverAPI.RoomserverQueryAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, accountDB accounts.Database, producer *producers.RoomserverProducer, membership, roomID string, @@ -256,7 +256,7 @@ func checkAndProcessThreepid( ) (inviteStored bool, errRes *util.JSONResponse) { inviteStored, err := threepid.CheckAndProcessInvite( - req.Context(), device, body, cfg, queryAPI, accountDB, producer, + req.Context(), device, body, cfg, rsAPI, accountDB, producer, membership, roomID, evTime, ) if err == threepid.ErrMissingParameter { diff --git a/clientapi/routing/memberships.go b/clientapi/routing/memberships.go index 0b846e5e3..f5d9bc4c5 100644 --- a/clientapi/routing/memberships.go +++ b/clientapi/routing/memberships.go @@ -39,7 +39,7 @@ type getJoinedRoomsResponse struct { func GetMemberships( req *http.Request, device *authtypes.Device, roomID string, joinedOnly bool, _ *config.Dendrite, - queryAPI api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, ) util.JSONResponse { queryReq := api.QueryMembershipsForRoomRequest{ JoinedOnly: joinedOnly, @@ -47,8 +47,8 @@ func GetMemberships( Sender: device.UserID, } var queryRes api.QueryMembershipsForRoomResponse - if err := queryAPI.QueryMembershipsForRoom(req.Context(), &queryReq, &queryRes); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("queryAPI.QueryMembershipsForRoom failed") + if err := rsAPI.QueryMembershipsForRoom(req.Context(), &queryReq, &queryRes); err != nil { + util.GetLogger(req.Context()).WithError(err).Error("rsAPI.QueryMembershipsForRoom failed") return jsonerror.InternalServerError() } diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index a51c55ea5..b51533e45 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -94,7 +94,7 @@ func GetAvatarURL( func SetAvatarURL( req *http.Request, accountDB accounts.Database, device *authtypes.Device, userID string, producer *producers.UserUpdateProducer, cfg *config.Dendrite, - rsProducer *producers.RoomserverProducer, queryAPI api.RoomserverQueryAPI, + rsProducer *producers.RoomserverProducer, rsAPI api.RoomserverInternalAPI, ) util.JSONResponse { if userID != device.UserID { return util.JSONResponse{ @@ -154,7 +154,7 @@ func SetAvatarURL( } events, err := buildMembershipEvents( - req.Context(), memberships, newProfile, userID, cfg, evTime, queryAPI, + req.Context(), memberships, newProfile, userID, cfg, evTime, rsAPI, ) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("buildMembershipEvents failed") @@ -208,7 +208,7 @@ func GetDisplayName( func SetDisplayName( req *http.Request, accountDB accounts.Database, device *authtypes.Device, userID string, producer *producers.UserUpdateProducer, cfg *config.Dendrite, - rsProducer *producers.RoomserverProducer, queryAPI api.RoomserverQueryAPI, + rsProducer *producers.RoomserverProducer, rsAPI api.RoomserverInternalAPI, ) util.JSONResponse { if userID != device.UserID { return util.JSONResponse{ @@ -268,7 +268,7 @@ func SetDisplayName( } events, err := buildMembershipEvents( - req.Context(), memberships, newProfile, userID, cfg, evTime, queryAPI, + req.Context(), memberships, newProfile, userID, cfg, evTime, rsAPI, ) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("buildMembershipEvents failed") @@ -337,14 +337,14 @@ func buildMembershipEvents( ctx context.Context, memberships []authtypes.Membership, newProfile authtypes.Profile, userID string, cfg *config.Dendrite, - evTime time.Time, queryAPI api.RoomserverQueryAPI, + evTime time.Time, rsAPI api.RoomserverInternalAPI, ) ([]gomatrixserverlib.HeaderedEvent, error) { evs := []gomatrixserverlib.HeaderedEvent{} for _, membership := range memberships { verReq := api.QueryRoomVersionForRoomRequest{RoomID: membership.RoomID} verRes := api.QueryRoomVersionForRoomResponse{} - if err := queryAPI.QueryRoomVersionForRoom(ctx, &verReq, &verRes); err != nil { + if err := rsAPI.QueryRoomVersionForRoom(ctx, &verReq, &verRes); err != nil { return []gomatrixserverlib.HeaderedEvent{}, err } @@ -366,7 +366,7 @@ func buildMembershipEvents( return nil, err } - event, err := common.BuildEvent(ctx, &builder, cfg, evTime, queryAPI, nil) + event, err := common.BuildEvent(ctx, &builder, cfg, evTime, rsAPI, nil) if err != nil { return nil, err } diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index e62b51935..42b391de6 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -49,8 +49,7 @@ const pathPrefixUnstable = "/_matrix/client/unstable" func Setup( apiMux *mux.Router, cfg *config.Dendrite, producer *producers.RoomserverProducer, - queryAPI roomserverAPI.RoomserverQueryAPI, - aliasAPI roomserverAPI.RoomserverAliasAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, accountDB accounts.Database, deviceDB devices.Database, @@ -91,7 +90,7 @@ func Setup( r0mux.Handle("/createRoom", common.MakeAuthAPI("createRoom", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - return CreateRoom(req, device, cfg, producer, accountDB, aliasAPI, asAPI) + return CreateRoom(req, device, cfg, producer, accountDB, rsAPI, asAPI) }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/join/{roomIDOrAlias}", @@ -102,7 +101,7 @@ func Setup( } return JoinRoomByIDOrAlias( req, device, vars["roomIDOrAlias"], cfg, federation, producer, - queryAPI, aliasAPI, federationSender, keyRing, accountDB, + rsAPI, federationSender, keyRing, accountDB, ) }), ).Methods(http.MethodPost, http.MethodOptions) @@ -118,7 +117,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SendMembership(req, accountDB, device, vars["roomID"], vars["membership"], cfg, queryAPI, asAPI, producer) + return SendMembership(req, accountDB, device, vars["roomID"], vars["membership"], cfg, rsAPI, asAPI, producer) }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/send/{eventType}", @@ -127,7 +126,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, nil, cfg, queryAPI, producer, nil) + return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, nil, cfg, rsAPI, producer, nil) }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/send/{eventType}/{txnID}", @@ -138,7 +137,7 @@ func Setup( } txnID := vars["txnID"] return SendEvent(req, device, vars["roomID"], vars["eventType"], &txnID, - nil, cfg, queryAPI, producer, transactionsCache) + nil, cfg, rsAPI, producer, transactionsCache) }), ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/event/{eventID}", @@ -147,7 +146,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return GetEvent(req, device, vars["roomID"], vars["eventID"], cfg, queryAPI, federation, keyRing) + return GetEvent(req, device, vars["roomID"], vars["eventID"], cfg, rsAPI, federation, keyRing) }), ).Methods(http.MethodGet, http.MethodOptions) @@ -156,7 +155,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return OnIncomingStateRequest(req.Context(), queryAPI, vars["roomID"]) + return OnIncomingStateRequest(req.Context(), rsAPI, vars["roomID"]) })).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/state/{type}", common.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { @@ -164,7 +163,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return OnIncomingStateTypeRequest(req.Context(), queryAPI, vars["roomID"], vars["type"], "") + return OnIncomingStateTypeRequest(req.Context(), rsAPI, vars["roomID"], vars["type"], "") })).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", common.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { @@ -172,7 +171,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return OnIncomingStateTypeRequest(req.Context(), queryAPI, vars["roomID"], vars["type"], vars["stateKey"]) + return OnIncomingStateTypeRequest(req.Context(), rsAPI, vars["roomID"], vars["type"], vars["stateKey"]) })).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/state/{eventType:[^/]+/?}", @@ -187,7 +186,7 @@ func Setup( if strings.HasSuffix(eventType, "/") { eventType = eventType[:len(eventType)-1] } - return SendEvent(req, device, vars["roomID"], eventType, nil, &emptyString, cfg, queryAPI, producer, nil) + return SendEvent(req, device, vars["roomID"], eventType, nil, &emptyString, cfg, rsAPI, producer, nil) }), ).Methods(http.MethodPut, http.MethodOptions) @@ -198,7 +197,7 @@ func Setup( return util.ErrorResponse(err) } stateKey := vars["stateKey"] - return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, &stateKey, cfg, queryAPI, producer, nil) + return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, &stateKey, cfg, rsAPI, producer, nil) }), ).Methods(http.MethodPut, http.MethodOptions) @@ -220,7 +219,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return DirectoryRoom(req, vars["roomAlias"], federation, cfg, aliasAPI, federationSender) + return DirectoryRoom(req, vars["roomAlias"], federation, cfg, rsAPI, federationSender) }), ).Methods(http.MethodGet, http.MethodOptions) @@ -230,7 +229,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SetLocalAlias(req, device, vars["roomAlias"], cfg, aliasAPI) + return SetLocalAlias(req, device, vars["roomAlias"], cfg, rsAPI) }), ).Methods(http.MethodPut, http.MethodOptions) @@ -240,7 +239,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return RemoveLocalAlias(req, device, vars["roomAlias"], aliasAPI) + return RemoveLocalAlias(req, device, vars["roomAlias"], rsAPI) }), ).Methods(http.MethodDelete, http.MethodOptions) @@ -354,7 +353,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SetAvatarURL(req, accountDB, device, vars["userID"], userUpdateProducer, cfg, producer, queryAPI) + return SetAvatarURL(req, accountDB, device, vars["userID"], userUpdateProducer, cfg, producer, rsAPI) }), ).Methods(http.MethodPut, http.MethodOptions) // Browsers use the OPTIONS HTTP method to check if the CORS policy allows @@ -376,7 +375,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SetDisplayName(req, accountDB, device, vars["userID"], userUpdateProducer, cfg, producer, queryAPI) + return SetDisplayName(req, accountDB, device, vars["userID"], userUpdateProducer, cfg, producer, rsAPI) }), ).Methods(http.MethodPut, http.MethodOptions) // Browsers use the OPTIONS HTTP method to check if the CORS policy allows @@ -489,7 +488,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return GetMemberships(req, device, vars["roomID"], false, cfg, queryAPI) + return GetMemberships(req, device, vars["roomID"], false, cfg, rsAPI) }), ).Methods(http.MethodGet, http.MethodOptions) @@ -499,7 +498,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return GetMemberships(req, device, vars["roomID"], true, cfg, queryAPI) + return GetMemberships(req, device, vars["roomID"], true, cfg, rsAPI) }), ).Methods(http.MethodGet, http.MethodOptions) @@ -603,7 +602,7 @@ func Setup( r0mux.Handle("/capabilities", common.MakeAuthAPI("capabilities", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - return GetCapabilities(req, queryAPI) + return GetCapabilities(req, rsAPI) }), ).Methods(http.MethodGet) } diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go index 5b2cd8ad4..7280dcd94 100644 --- a/clientapi/routing/sendevent.go +++ b/clientapi/routing/sendevent.go @@ -45,13 +45,13 @@ func SendEvent( device *authtypes.Device, roomID, eventType string, txnID, stateKey *string, cfg *config.Dendrite, - queryAPI api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, producer *producers.RoomserverProducer, txnCache *transactions.Cache, ) util.JSONResponse { verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} verRes := api.QueryRoomVersionForRoomResponse{} - if err := queryAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil { + if err := rsAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.UnsupportedRoomVersion(err.Error()), @@ -65,7 +65,7 @@ func SendEvent( } } - e, resErr := generateSendEvent(req, device, roomID, eventType, stateKey, cfg, queryAPI) + e, resErr := generateSendEvent(req, device, roomID, eventType, stateKey, cfg, rsAPI) if resErr != nil { return *resErr } @@ -115,7 +115,7 @@ func generateSendEvent( device *authtypes.Device, roomID, eventType string, stateKey *string, cfg *config.Dendrite, - queryAPI api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, ) (*gomatrixserverlib.Event, *util.JSONResponse) { // parse the incoming http request userID := device.UserID @@ -148,7 +148,7 @@ func generateSendEvent( } var queryRes api.QueryLatestEventsAndStateResponse - e, err := common.BuildEvent(req.Context(), &builder, cfg, evTime, queryAPI, &queryRes) + e, err := common.BuildEvent(req.Context(), &builder, cfg, evTime, rsAPI, &queryRes) if err == common.ErrRoomNoExists { return nil, &util.JSONResponse{ Code: http.StatusNotFound, diff --git a/clientapi/routing/state.go b/clientapi/routing/state.go index c243eec0f..e3e5bdb6d 100644 --- a/clientapi/routing/state.go +++ b/clientapi/routing/state.go @@ -40,7 +40,7 @@ type stateEventInStateResp struct { // TODO: Check if the user is in the room. If not, check if the room's history // is publicly visible. Current behaviour is returning an empty array if the // user cannot see the room's history. -func OnIncomingStateRequest(ctx context.Context, queryAPI api.RoomserverQueryAPI, roomID string) util.JSONResponse { +func OnIncomingStateRequest(ctx context.Context, rsAPI api.RoomserverInternalAPI, roomID string) util.JSONResponse { // TODO(#287): Auth request and handle the case where the user has left (where // we should return the state at the poin they left) stateReq := api.QueryLatestEventsAndStateRequest{ @@ -48,7 +48,7 @@ func OnIncomingStateRequest(ctx context.Context, queryAPI api.RoomserverQueryAPI } stateRes := api.QueryLatestEventsAndStateResponse{} - if err := queryAPI.QueryLatestEventsAndState(ctx, &stateReq, &stateRes); err != nil { + if err := rsAPI.QueryLatestEventsAndState(ctx, &stateReq, &stateRes); err != nil { util.GetLogger(ctx).WithError(err).Error("queryAPI.QueryLatestEventsAndState failed") return jsonerror.InternalServerError() } @@ -98,7 +98,7 @@ func OnIncomingStateRequest(ctx context.Context, queryAPI api.RoomserverQueryAPI // /rooms/{roomID}/state/{type}/{statekey} request. It will look in current // state to see if there is an event with that type and state key, if there // is then (by default) we return the content, otherwise a 404. -func OnIncomingStateTypeRequest(ctx context.Context, queryAPI api.RoomserverQueryAPI, roomID string, evType, stateKey string) util.JSONResponse { +func OnIncomingStateTypeRequest(ctx context.Context, rsAPI api.RoomserverInternalAPI, roomID string, evType, stateKey string) util.JSONResponse { // TODO(#287): Auth request and handle the case where the user has left (where // we should return the state at the poin they left) util.GetLogger(ctx).WithFields(log.Fields{ @@ -118,7 +118,7 @@ func OnIncomingStateTypeRequest(ctx context.Context, queryAPI api.RoomserverQuer } stateRes := api.QueryLatestEventsAndStateResponse{} - if err := queryAPI.QueryLatestEventsAndState(ctx, &stateReq, &stateRes); err != nil { + if err := rsAPI.QueryLatestEventsAndState(ctx, &stateReq, &stateRes); err != nil { util.GetLogger(ctx).WithError(err).Error("queryAPI.QueryLatestEventsAndState failed") return jsonerror.InternalServerError() } diff --git a/clientapi/threepid/invites.go b/clientapi/threepid/invites.go index e34e91b56..5e7e4f2b4 100644 --- a/clientapi/threepid/invites.go +++ b/clientapi/threepid/invites.go @@ -87,7 +87,7 @@ var ( func CheckAndProcessInvite( ctx context.Context, device *authtypes.Device, body *MembershipRequest, cfg *config.Dendrite, - queryAPI api.RoomserverQueryAPI, db accounts.Database, + rsAPI api.RoomserverInternalAPI, db accounts.Database, producer *producers.RoomserverProducer, membership string, roomID string, evTime time.Time, ) (inviteStoredOnIDServer bool, err error) { @@ -112,7 +112,7 @@ func CheckAndProcessInvite( // "m.room.third_party_invite" have to be emitted from the data in // storeInviteRes. err = emit3PIDInviteEvent( - ctx, body, storeInviteRes, device, roomID, cfg, queryAPI, producer, evTime, + ctx, body, storeInviteRes, device, roomID, cfg, rsAPI, producer, evTime, ) inviteStoredOnIDServer = err == nil @@ -331,7 +331,7 @@ func emit3PIDInviteEvent( ctx context.Context, body *MembershipRequest, res *idServerStoreInviteResponse, device *authtypes.Device, roomID string, cfg *config.Dendrite, - queryAPI api.RoomserverQueryAPI, producer *producers.RoomserverProducer, + rsAPI api.RoomserverInternalAPI, producer *producers.RoomserverProducer, evTime time.Time, ) error { builder := &gomatrixserverlib.EventBuilder{ @@ -354,7 +354,7 @@ func emit3PIDInviteEvent( } queryRes := api.QueryLatestEventsAndStateResponse{} - event, err := common.BuildEvent(ctx, builder, cfg, evTime, queryAPI, &queryRes) + event, err := common.BuildEvent(ctx, builder, cfg, evTime, rsAPI, &queryRes) if err != nil { return err } diff --git a/cmd/dendrite-appservice-server/main.go b/cmd/dendrite-appservice-server/main.go index f203969f4..91ca707e7 100644 --- a/cmd/dendrite-appservice-server/main.go +++ b/cmd/dendrite-appservice-server/main.go @@ -28,11 +28,11 @@ func main() { accountDB := base.CreateAccountsDB() deviceDB := base.CreateDeviceDB() federation := base.CreateFederationClient() - alias, _, query := base.CreateHTTPRoomserverAPIs() + rsAPI := base.CreateHTTPRoomserverAPIs() cache := transactions.New() appservice.SetupAppServiceAPIComponent( - base, accountDB, deviceDB, federation, alias, query, cache, + base, accountDB, deviceDB, federation, rsAPI, cache, ) base.SetupAndServeHTTP(string(base.Cfg.Bind.AppServiceAPI), string(base.Cfg.Listen.AppServiceAPI)) diff --git a/cmd/dendrite-client-api-server/main.go b/cmd/dendrite-client-api-server/main.go index c8f629689..1ce66db2c 100644 --- a/cmd/dendrite-client-api-server/main.go +++ b/cmd/dendrite-client-api-server/main.go @@ -36,13 +36,14 @@ func main() { keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) asQuery := base.CreateHTTPAppServiceAPIs() - alias, input, query := base.CreateHTTPRoomserverAPIs() + rsAPI := base.CreateHTTPRoomserverAPIs() fsAPI := base.CreateHTTPFederationSenderAPIs() + rsAPI.SetFederationSenderAPI(fsAPI) eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New()) clientapi.SetupClientAPIComponent( base, deviceDB, accountDB, federation, &keyRing, - alias, input, query, eduInputAPI, asQuery, transactions.New(), fsAPI, + rsAPI, eduInputAPI, asQuery, transactions.New(), fsAPI, ) base.SetupAndServeHTTP(string(base.Cfg.Bind.ClientAPI), string(base.Cfg.Listen.ClientAPI)) diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index a2a4675b3..28c7153fd 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -148,27 +148,34 @@ func main() { federation := createFederationClient(base) keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) - alias, input, query := roomserver.SetupRoomServerComponent(&base.Base, keyRing, federation) - eduInputAPI := eduserver.SetupEDUServerComponent(&base.Base, cache.New()) - asQuery := appservice.SetupAppServiceAPIComponent( - &base.Base, accountDB, deviceDB, federation, alias, query, transactions.New(), + rsAPI := roomserver.SetupRoomServerComponent( + &base.Base, keyRing, federation, ) - fsAPI := federationsender.SetupFederationSenderComponent(&base.Base, federation, query, input, &keyRing) + eduInputAPI := eduserver.SetupEDUServerComponent( + &base.Base, cache.New(), + ) + asAPI := appservice.SetupAppServiceAPIComponent( + &base.Base, accountDB, deviceDB, federation, rsAPI, transactions.New(), + ) + fsAPI := federationsender.SetupFederationSenderComponent( + &base.Base, federation, rsAPI, &keyRing, + ) + rsAPI.SetFederationSenderAPI(fsAPI) clientapi.SetupClientAPIComponent( &base.Base, deviceDB, accountDB, - federation, &keyRing, alias, input, query, - eduInputAPI, asQuery, transactions.New(), fsAPI, + federation, &keyRing, rsAPI, + eduInputAPI, asAPI, transactions.New(), fsAPI, ) eduProducer := producers.NewEDUServerProducer(eduInputAPI) - federationapi.SetupFederationAPIComponent(&base.Base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fsAPI, eduProducer) + federationapi.SetupFederationAPIComponent(&base.Base, accountDB, deviceDB, federation, &keyRing, rsAPI, asAPI, fsAPI, eduProducer) mediaapi.SetupMediaAPIComponent(&base.Base, deviceDB) publicRoomsDB, err := storage.NewPublicRoomsServerDatabaseWithPubSub(string(base.Base.Cfg.Database.PublicRoomsAPI), base.LibP2PPubsub) if err != nil { logrus.WithError(err).Panicf("failed to connect to public rooms db") } - publicroomsapi.SetupPublicRoomsAPIComponent(&base.Base, deviceDB, publicRoomsDB, query, federation, nil) // Check this later - syncapi.SetupSyncAPIComponent(&base.Base, deviceDB, accountDB, query, federation, &cfg) + publicroomsapi.SetupPublicRoomsAPIComponent(&base.Base, deviceDB, publicRoomsDB, rsAPI, federation, nil) // Check this later + syncapi.SetupSyncAPIComponent(&base.Base, deviceDB, accountDB, rsAPI, federation, &cfg) httpHandler := common.WrapHandlerInCORS(base.Base.APIMux) diff --git a/cmd/dendrite-federation-api-server/main.go b/cmd/dendrite-federation-api-server/main.go index 4267cf166..d829326a7 100644 --- a/cmd/dendrite-federation-api-server/main.go +++ b/cmd/dendrite-federation-api-server/main.go @@ -35,14 +35,15 @@ func main() { fsAPI := base.CreateHTTPFederationSenderAPIs() keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) - alias, input, query := base.CreateHTTPRoomserverAPIs() - asQuery := base.CreateHTTPAppServiceAPIs() + rsAPI := base.CreateHTTPRoomserverAPIs() + asAPI := base.CreateHTTPAppServiceAPIs() + rsAPI.SetFederationSenderAPI(fsAPI) eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New()) eduProducer := producers.NewEDUServerProducer(eduInputAPI) federationapi.SetupFederationAPIComponent( base, accountDB, deviceDB, federation, &keyRing, - alias, input, query, asQuery, fsAPI, eduProducer, + rsAPI, asAPI, fsAPI, eduProducer, ) base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationAPI), string(base.Cfg.Listen.FederationAPI)) diff --git a/cmd/dendrite-federation-sender-server/main.go b/cmd/dendrite-federation-sender-server/main.go index f8d43b990..0daac1bcb 100644 --- a/cmd/dendrite-federation-sender-server/main.go +++ b/cmd/dendrite-federation-sender-server/main.go @@ -28,12 +28,11 @@ func main() { federation := base.CreateFederationClient() keyDB := base.CreateKeyDB() keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) - - _, input, query := base.CreateHTTPRoomserverAPIs() - - federationsender.SetupFederationSenderComponent( - base, federation, query, input, &keyRing, + rsAPI := base.CreateHTTPRoomserverAPIs() + fsAPI := federationsender.SetupFederationSenderComponent( + base, federation, rsAPI, &keyRing, ) + rsAPI.SetFederationSenderAPI(fsAPI) base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationSender), string(base.Cfg.Listen.FederationSender)) diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index f43f8b04c..060019711 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -57,28 +57,34 @@ func main() { federation := base.CreateFederationClient() keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) - alias, input, query := roomserver.SetupRoomServerComponent(base, keyRing, federation) - eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New()) - asQuery := appservice.SetupAppServiceAPIComponent( - base, accountDB, deviceDB, federation, alias, query, transactions.New(), + rsAPI := roomserver.SetupRoomServerComponent( + base, keyRing, federation, ) - fsAPI := federationsender.SetupFederationSenderComponent(base, federation, query, input, &keyRing) - input.SetFederationSenderAPI(fsAPI) + eduInputAPI := eduserver.SetupEDUServerComponent( + base, cache.New(), + ) + asAPI := appservice.SetupAppServiceAPIComponent( + base, accountDB, deviceDB, federation, rsAPI, transactions.New(), + ) + fsAPI := federationsender.SetupFederationSenderComponent( + base, federation, rsAPI, &keyRing, + ) + rsAPI.SetFederationSenderAPI(fsAPI) clientapi.SetupClientAPIComponent( base, deviceDB, accountDB, - federation, &keyRing, alias, input, query, - eduInputAPI, asQuery, transactions.New(), fsAPI, + federation, &keyRing, rsAPI, + eduInputAPI, asAPI, transactions.New(), fsAPI, ) eduProducer := producers.NewEDUServerProducer(eduInputAPI) - federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fsAPI, eduProducer) + federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, rsAPI, asAPI, fsAPI, eduProducer) mediaapi.SetupMediaAPIComponent(base, deviceDB) publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI)) if err != nil { logrus.WithError(err).Panicf("failed to connect to public rooms db") } - publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, query, federation, nil) - syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query, federation, cfg) + publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, rsAPI, federation, nil) + syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, rsAPI, federation, cfg) httpHandler := common.WrapHandlerInCORS(base.APIMux) diff --git a/cmd/dendrite-public-rooms-api-server/main.go b/cmd/dendrite-public-rooms-api-server/main.go index f6a782f66..fca39a2fe 100644 --- a/cmd/dendrite-public-rooms-api-server/main.go +++ b/cmd/dendrite-public-rooms-api-server/main.go @@ -28,12 +28,15 @@ func main() { deviceDB := base.CreateDeviceDB() - _, _, query := base.CreateHTTPRoomserverAPIs() + fsAPI := base.CreateHTTPFederationSenderAPIs() + rsAPI := base.CreateHTTPRoomserverAPIs() + rsAPI.SetFederationSenderAPI(fsAPI) + publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI)) if err != nil { logrus.WithError(err).Panicf("failed to connect to public rooms db") } - publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, query, nil, nil) + publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, rsAPI, nil, nil) base.SetupAndServeHTTP(string(base.Cfg.Bind.PublicRoomsAPI), string(base.Cfg.Listen.PublicRoomsAPI)) diff --git a/cmd/dendrite-room-server/main.go b/cmd/dendrite-room-server/main.go index 3f9913e24..41149ad90 100644 --- a/cmd/dendrite-room-server/main.go +++ b/cmd/dendrite-room-server/main.go @@ -31,8 +31,8 @@ func main() { keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) fsAPI := base.CreateHTTPFederationSenderAPIs() - _, input, _ := roomserver.SetupRoomServerComponent(base, keyRing, federation) - input.SetFederationSenderAPI(fsAPI) + rsAPI := roomserver.SetupRoomServerComponent(base, keyRing, federation) + rsAPI.SetFederationSenderAPI(fsAPI) base.SetupAndServeHTTP(string(base.Cfg.Bind.RoomServer), string(base.Cfg.Listen.RoomServer)) diff --git a/cmd/dendrite-sync-api-server/main.go b/cmd/dendrite-sync-api-server/main.go index 55e9faeef..259447af2 100644 --- a/cmd/dendrite-sync-api-server/main.go +++ b/cmd/dendrite-sync-api-server/main.go @@ -28,9 +28,9 @@ func main() { accountDB := base.CreateAccountsDB() federation := base.CreateFederationClient() - _, _, query := base.CreateHTTPRoomserverAPIs() + rsAPI := base.CreateHTTPRoomserverAPIs() - syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query, federation, cfg) + syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, rsAPI, federation, cfg) base.SetupAndServeHTTP(string(base.Cfg.Bind.SyncAPI), string(base.Cfg.Listen.SyncAPI)) diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 1f2f20fb4..5b7ed4807 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -123,28 +123,28 @@ func main() { } p2pPublicRoomProvider := NewLibP2PPublicRoomsProvider(node) - alias, input, query := roomserver.SetupRoomServerComponent(base, keyRing, federation) + rsAPI := roomserver.SetupRoomServerComponent(base, keyRing, federation) eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New()) asQuery := appservice.SetupAppServiceAPIComponent( - base, accountDB, deviceDB, federation, alias, query, transactions.New(), + base, accountDB, deviceDB, federation, rsAPI, transactions.New(), ) - fedSenderAPI := federationsender.SetupFederationSenderComponent(base, federation, query, input, &keyRing) - input.SetFederationSenderAPI(fedSenderAPI) + fedSenderAPI := federationsender.SetupFederationSenderComponent(base, federation, rsAPI, &keyRing) + rsAPI.SetFederationSenderAPI(fedSenderAPI) clientapi.SetupClientAPIComponent( base, deviceDB, accountDB, - federation, &keyRing, alias, input, query, + federation, &keyRing, rsAPI, eduInputAPI, asQuery, transactions.New(), fedSenderAPI, ) eduProducer := producers.NewEDUServerProducer(eduInputAPI) - federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, alias, input, query, asQuery, fedSenderAPI, eduProducer) + federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, rsAPI, asQuery, fedSenderAPI, eduProducer) mediaapi.SetupMediaAPIComponent(base, deviceDB) publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI)) if err != nil { logrus.WithError(err).Panicf("failed to connect to public rooms db") } - publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, query, federation, p2pPublicRoomProvider) - syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, query, federation, cfg) + publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, rsAPI, federation, p2pPublicRoomProvider) + syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, rsAPI, federation, cfg) httpHandler := common.WrapHandlerInCORS(base.APIMux) diff --git a/cmd/roomserver-integration-tests/main.go b/cmd/roomserver-integration-tests/main.go index 682fc6224..7126405e5 100644 --- a/cmd/roomserver-integration-tests/main.go +++ b/cmd/roomserver-integration-tests/main.go @@ -209,7 +209,7 @@ func writeToRoomServer(input []string, roomserverURL string) error { return err } } - x, err := api.NewRoomserverInputAPIHTTP(roomserverURL, &http.Client{Timeout: timeoutHTTP}) + x, err := api.NewRoomserverInternalAPIHTTP(roomserverURL, &http.Client{Timeout: timeoutHTTP}, nil) if err != nil { return err } @@ -225,7 +225,7 @@ func writeToRoomServer(input []string, roomserverURL string) error { // Once those messages have been written it runs the checkQueries function passing // a api.RoomserverQueryAPI client. The caller can use this function to check the // behaviour of the query API. -func testRoomserver(input []string, wantOutput []string, checkQueries func(api.RoomserverQueryAPI)) { +func testRoomserver(input []string, wantOutput []string, checkQueries func(api.RoomserverInternalAPI)) { dir, err := ioutil.TempDir("", "room-server-test") if err != nil { panic(err) @@ -276,7 +276,7 @@ func testRoomserver(input []string, wantOutput []string, checkQueries func(api.R cmd.Args = []string{"dendrite-room-server", "--config", filepath.Join(dir, test.ConfigFile)} gotOutput, err := runAndReadFromTopic(cmd, cfg.RoomServerURL()+"/metrics", doInput, outputTopic, len(wantOutput), func() { - queryAPI, _ := api.NewRoomserverQueryAPIHTTP("http://"+string(cfg.Listen.RoomServer), &http.Client{Timeout: timeoutHTTP}, cache) + queryAPI, _ := api.NewRoomserverInternalAPIHTTP("http://"+string(cfg.Listen.RoomServer), &http.Client{Timeout: timeoutHTTP}, cache) checkQueries(queryAPI) }) if err != nil { @@ -410,7 +410,7 @@ func main() { }}`, } - testRoomserver(input, want, func(q api.RoomserverQueryAPI) { + testRoomserver(input, want, func(q api.RoomserverInternalAPI) { var response api.QueryLatestEventsAndStateResponse if err := q.QueryLatestEventsAndState( context.Background(), diff --git a/common/basecomponent/base.go b/common/basecomponent/base.go index f245dd50b..154acd806 100644 --- a/common/basecomponent/base.go +++ b/common/basecomponent/base.go @@ -119,24 +119,12 @@ func (b *BaseDendrite) CreateHTTPAppServiceAPIs() appserviceAPI.AppServiceQueryA // CreateHTTPRoomserverAPIs returns the AliasAPI, InputAPI and QueryAPI for hitting // the roomserver over HTTP. -func (b *BaseDendrite) CreateHTTPRoomserverAPIs() ( - roomserverAPI.RoomserverAliasAPI, - roomserverAPI.RoomserverInputAPI, - roomserverAPI.RoomserverQueryAPI, -) { - alias, err := roomserverAPI.NewRoomserverAliasAPIHTTP(b.Cfg.RoomServerURL(), b.httpClient) +func (b *BaseDendrite) CreateHTTPRoomserverAPIs() roomserverAPI.RoomserverInternalAPI { + rsAPI, err := roomserverAPI.NewRoomserverInternalAPIHTTP(b.Cfg.RoomServerURL(), b.httpClient, b.ImmutableCache) if err != nil { - logrus.WithError(err).Panic("NewRoomserverAliasAPIHTTP failed") + logrus.WithError(err).Panic("NewRoomserverInternalAPIHTTP failed", b.httpClient) } - input, err := roomserverAPI.NewRoomserverInputAPIHTTP(b.Cfg.RoomServerURL(), b.httpClient) - if err != nil { - logrus.WithError(err).Panic("NewRoomserverInputAPIHTTP failed", b.httpClient) - } - query, err := roomserverAPI.NewRoomserverQueryAPIHTTP(b.Cfg.RoomServerURL(), b.httpClient, b.ImmutableCache) - if err != nil { - logrus.WithError(err).Panic("NewRoomserverQueryAPIHTTP failed", b.httpClient) - } - return alias, input, query + return rsAPI } // CreateHTTPEDUServerAPIs returns eduInputAPI for hitting the EDU diff --git a/common/events.go b/common/events.go index 556b7b671..0c8ead3a7 100644 --- a/common/events.go +++ b/common/events.go @@ -39,13 +39,13 @@ var ErrRoomNoExists = errors.New("Room does not exist") func BuildEvent( ctx context.Context, builder *gomatrixserverlib.EventBuilder, cfg *config.Dendrite, evTime time.Time, - queryAPI api.RoomserverQueryAPI, queryRes *api.QueryLatestEventsAndStateResponse, + rsAPI api.RoomserverInternalAPI, queryRes *api.QueryLatestEventsAndStateResponse, ) (*gomatrixserverlib.Event, error) { if queryRes == nil { queryRes = &api.QueryLatestEventsAndStateResponse{} } - err := AddPrevEventsToEvent(ctx, builder, queryAPI, queryRes) + err := AddPrevEventsToEvent(ctx, builder, rsAPI, queryRes) if err != nil { // This can pass through a ErrRoomNoExists to the caller return nil, err @@ -66,7 +66,7 @@ func BuildEvent( func AddPrevEventsToEvent( ctx context.Context, builder *gomatrixserverlib.EventBuilder, - queryAPI api.RoomserverQueryAPI, queryRes *api.QueryLatestEventsAndStateResponse, + rsAPI api.RoomserverInternalAPI, queryRes *api.QueryLatestEventsAndStateResponse, ) error { eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(builder) if err != nil { @@ -82,8 +82,8 @@ func AddPrevEventsToEvent( RoomID: builder.RoomID, StateToFetch: eventsNeeded.Tuples(), } - if err = queryAPI.QueryLatestEventsAndState(ctx, &queryReq, queryRes); err != nil { - return fmt.Errorf("queryAPI.QueryLatestEventsAndState: %w", err) + if err = rsAPI.QueryLatestEventsAndState(ctx, &queryReq, queryRes); err != nil { + return fmt.Errorf("rsAPI.QueryLatestEventsAndState: %w", err) } if !queryRes.RoomExists { diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go index 72e2b54ac..d458b8537 100644 --- a/federationapi/federationapi.go +++ b/federationapi/federationapi.go @@ -36,18 +36,16 @@ func SetupFederationAPIComponent( deviceDB devices.Database, federation *gomatrixserverlib.FederationClient, keyRing *gomatrixserverlib.KeyRing, - aliasAPI roomserverAPI.RoomserverAliasAPI, - inputAPI roomserverAPI.RoomserverInputAPI, - queryAPI roomserverAPI.RoomserverQueryAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, federationSenderAPI federationSenderAPI.FederationSenderInternalAPI, eduProducer *producers.EDUServerProducer, ) { - roomserverProducer := producers.NewRoomserverProducer(inputAPI, queryAPI) + roomserverProducer := producers.NewRoomserverProducer(rsAPI) routing.Setup( - base.APIMux, base.Cfg, queryAPI, aliasAPI, asAPI, - roomserverProducer, eduProducer, federationSenderAPI, *keyRing, + base.APIMux, base.Cfg, rsAPI, asAPI, roomserverProducer, + eduProducer, federationSenderAPI, *keyRing, federation, accountsDB, deviceDB, ) } diff --git a/federationapi/routing/backfill.go b/federationapi/routing/backfill.go index 6f49b9a81..651a4a2d2 100644 --- a/federationapi/routing/backfill.go +++ b/federationapi/routing/backfill.go @@ -33,7 +33,7 @@ import ( func Backfill( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, roomID string, cfg *config.Dendrite, ) util.JSONResponse { @@ -82,7 +82,7 @@ func Backfill( } // Query the roomserver. - if err = query.QueryBackfill(httpReq.Context(), &req, &res); err != nil { + if err = rsAPI.QueryBackfill(httpReq.Context(), &req, &res); err != nil { util.GetLogger(httpReq.Context()).WithError(err).Error("query.QueryBackfill failed") return jsonerror.InternalServerError() } diff --git a/federationapi/routing/eventauth.go b/federationapi/routing/eventauth.go index 003165c85..34eaad1c5 100644 --- a/federationapi/routing/eventauth.go +++ b/federationapi/routing/eventauth.go @@ -25,13 +25,13 @@ import ( func GetEventAuth( ctx context.Context, request *gomatrixserverlib.FederationRequest, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, roomID string, eventID string, ) util.JSONResponse { // TODO: Optimisation: we shouldn't be querying all the room state // that is in state.StateEvents - we just ignore it. - state, err := getState(ctx, request, query, roomID, eventID) + state, err := getState(ctx, request, rsAPI, roomID, eventID) if err != nil { return *err } diff --git a/federationapi/routing/events.go b/federationapi/routing/events.go index 03492db45..ced9e3d53 100644 --- a/federationapi/routing/events.go +++ b/federationapi/routing/events.go @@ -29,11 +29,11 @@ import ( func GetEvent( ctx context.Context, request *gomatrixserverlib.FederationRequest, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, eventID string, origin gomatrixserverlib.ServerName, ) util.JSONResponse { - event, err := getEvent(ctx, request, query, eventID) + event, err := getEvent(ctx, request, rsAPI, eventID) if err != nil { return *err } @@ -52,11 +52,11 @@ func GetEvent( func getEvent( ctx context.Context, request *gomatrixserverlib.FederationRequest, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, eventID string, ) (*gomatrixserverlib.Event, *util.JSONResponse) { var authResponse api.QueryServerAllowedToSeeEventResponse - err := query.QueryServerAllowedToSeeEvent( + err := rsAPI.QueryServerAllowedToSeeEvent( ctx, &api.QueryServerAllowedToSeeEventRequest{ EventID: eventID, @@ -75,7 +75,7 @@ func getEvent( } var eventsResponse api.QueryEventsByIDResponse - err = query.QueryEventsByID( + err = rsAPI.QueryEventsByID( ctx, &api.QueryEventsByIDRequest{EventIDs: []string{eventID}}, &eventsResponse, diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go index 0c899ab9f..be5e988ab 100644 --- a/federationapi/routing/join.go +++ b/federationapi/routing/join.go @@ -33,13 +33,13 @@ func MakeJoin( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, cfg *config.Dendrite, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, roomID, userID string, remoteVersions []gomatrixserverlib.RoomVersion, ) util.JSONResponse { verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} verRes := api.QueryRoomVersionForRoomResponse{} - if err := query.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil { + if err := rsAPI.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil { return util.JSONResponse{ Code: http.StatusInternalServerError, JSON: jsonerror.InternalServerError(), @@ -97,7 +97,7 @@ func MakeJoin( queryRes := api.QueryLatestEventsAndStateResponse{ RoomVersion: verRes.RoomVersion, } - event, err := common.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), query, &queryRes) + event, err := common.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), rsAPI, &queryRes) if err == common.ErrRoomNoExists { return util.JSONResponse{ Code: http.StatusNotFound, @@ -136,15 +136,15 @@ func SendJoin( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, cfg *config.Dendrite, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, producer *producers.RoomserverProducer, keys gomatrixserverlib.KeyRing, roomID, eventID string, ) util.JSONResponse { verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} verRes := api.QueryRoomVersionForRoomResponse{} - if err := query.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil { - util.GetLogger(httpReq.Context()).WithError(err).Error("query.QueryRoomVersionForRoom failed") + if err := rsAPI.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil { + util.GetLogger(httpReq.Context()).WithError(err).Error("rsAPI.QueryRoomVersionForRoom failed") return util.JSONResponse{ Code: http.StatusInternalServerError, JSON: jsonerror.InternalServerError(), @@ -216,14 +216,14 @@ func SendJoin( // Fetch the state and auth chain. We do this before we send the events // on, in case this fails. var stateAndAuthChainResponse api.QueryStateAndAuthChainResponse - err = query.QueryStateAndAuthChain(httpReq.Context(), &api.QueryStateAndAuthChainRequest{ + err = rsAPI.QueryStateAndAuthChain(httpReq.Context(), &api.QueryStateAndAuthChainRequest{ PrevEventIDs: event.PrevEventIDs(), AuthEventIDs: event.AuthEventIDs(), RoomID: roomID, ResolveState: true, }, &stateAndAuthChainResponse) if err != nil { - util.GetLogger(httpReq.Context()).WithError(err).Error("query.QueryStateAndAuthChain failed") + util.GetLogger(httpReq.Context()).WithError(err).Error("rsAPI.QueryStateAndAuthChain failed") return jsonerror.InternalServerError() } diff --git a/federationapi/routing/leave.go b/federationapi/routing/leave.go index 6fc3b12ed..1124bfa27 100644 --- a/federationapi/routing/leave.go +++ b/federationapi/routing/leave.go @@ -30,7 +30,7 @@ func MakeLeave( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, cfg *config.Dendrite, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, roomID, userID string, ) util.JSONResponse { _, domain, err := gomatrixserverlib.SplitID('@', userID) @@ -61,7 +61,7 @@ func MakeLeave( } var queryRes api.QueryLatestEventsAndStateResponse - event, err := common.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), query, &queryRes) + event, err := common.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), rsAPI, &queryRes) if err == common.ErrRoomNoExists { return util.JSONResponse{ Code: http.StatusNotFound, @@ -102,7 +102,7 @@ func SendLeave( ) util.JSONResponse { verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} verRes := api.QueryRoomVersionForRoomResponse{} - if err := producer.QueryAPI.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil { + if err := producer.RsAPI.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.UnsupportedRoomVersion(err.Error()), diff --git a/federationapi/routing/missingevents.go b/federationapi/routing/missingevents.go index 069bff3dd..ae91c5891 100644 --- a/federationapi/routing/missingevents.go +++ b/federationapi/routing/missingevents.go @@ -34,7 +34,7 @@ type getMissingEventRequest struct { func GetMissingEvents( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, roomID string, ) util.JSONResponse { var gme getMissingEventRequest @@ -46,7 +46,7 @@ func GetMissingEvents( } var eventsResponse api.QueryMissingEventsResponse - if err := query.QueryMissingEvents( + if err := rsAPI.QueryMissingEvents( httpReq.Context(), &api.QueryMissingEventsRequest{ EarliestEvents: gme.EarliestEvents, LatestEvents: gme.LatestEvents, diff --git a/federationapi/routing/query.go b/federationapi/routing/query.go index 13c92451d..c58690c63 100644 --- a/federationapi/routing/query.go +++ b/federationapi/routing/query.go @@ -32,7 +32,7 @@ func RoomAliasToID( httpReq *http.Request, federation *gomatrixserverlib.FederationClient, cfg *config.Dendrite, - aliasAPI roomserverAPI.RoomserverAliasAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, senderAPI federationSenderAPI.FederationSenderInternalAPI, ) util.JSONResponse { roomAlias := httpReq.FormValue("room_alias") @@ -55,7 +55,7 @@ func RoomAliasToID( if domain == cfg.Matrix.ServerName { queryReq := roomserverAPI.GetRoomIDForAliasRequest{Alias: roomAlias} var queryRes roomserverAPI.GetRoomIDForAliasResponse - if err = aliasAPI.GetRoomIDForAlias(httpReq.Context(), &queryReq, &queryRes); err != nil { + if err = rsAPI.GetRoomIDForAlias(httpReq.Context(), &queryReq, &queryRes); err != nil { util.GetLogger(httpReq.Context()).WithError(err).Error("aliasAPI.GetRoomIDForAlias failed") return jsonerror.InternalServerError() } diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index e0f842f18..a5b8ce24e 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -44,8 +44,7 @@ const ( func Setup( apiMux *mux.Router, cfg *config.Dendrite, - query roomserverAPI.RoomserverQueryAPI, - aliasAPI roomserverAPI.RoomserverAliasAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, producer *producers.RoomserverProducer, eduProducer *producers.EDUServerProducer, @@ -80,7 +79,7 @@ func Setup( } return Send( httpReq, request, gomatrixserverlib.TransactionID(vars["txnID"]), - cfg, query, producer, eduProducer, keys, federation, + cfg, rsAPI, producer, eduProducer, keys, federation, ) }, )).Methods(http.MethodPut, http.MethodOptions) @@ -101,7 +100,7 @@ func Setup( v1fedmux.Handle("/3pid/onbind", common.MakeExternalAPI("3pid_onbind", func(req *http.Request) util.JSONResponse { - return CreateInvitesFrom3PIDInvites(req, query, asAPI, cfg, producer, federation, accountDB) + return CreateInvitesFrom3PIDInvites(req, rsAPI, asAPI, cfg, producer, federation, accountDB) }, )).Methods(http.MethodPost, http.MethodOptions) @@ -113,7 +112,7 @@ func Setup( return util.ErrorResponse(err) } return ExchangeThirdPartyInvite( - httpReq, request, vars["roomID"], query, cfg, federation, producer, + httpReq, request, vars["roomID"], rsAPI, cfg, federation, producer, ) }, )).Methods(http.MethodPut, http.MethodOptions) @@ -126,7 +125,7 @@ func Setup( return util.ErrorResponse(err) } return GetEvent( - httpReq.Context(), request, query, vars["eventID"], cfg.Matrix.ServerName, + httpReq.Context(), request, rsAPI, vars["eventID"], cfg.Matrix.ServerName, ) }, )).Methods(http.MethodGet) @@ -139,7 +138,7 @@ func Setup( return util.ErrorResponse(err) } return GetState( - httpReq.Context(), request, query, vars["roomID"], + httpReq.Context(), request, rsAPI, vars["roomID"], ) }, )).Methods(http.MethodGet) @@ -152,7 +151,7 @@ func Setup( return util.ErrorResponse(err) } return GetStateIDs( - httpReq.Context(), request, query, vars["roomID"], + httpReq.Context(), request, rsAPI, vars["roomID"], ) }, )).Methods(http.MethodGet) @@ -162,7 +161,7 @@ func Setup( func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { vars := mux.Vars(httpReq) return GetEventAuth( - httpReq.Context(), request, query, vars["roomID"], vars["eventID"], + httpReq.Context(), request, rsAPI, vars["roomID"], vars["eventID"], ) }, )).Methods(http.MethodGet) @@ -171,7 +170,7 @@ func Setup( "federation_query_room_alias", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { return RoomAliasToID( - httpReq, federation, cfg, aliasAPI, federationSenderAPI, + httpReq, federation, cfg, rsAPI, federationSenderAPI, ) }, )).Methods(http.MethodGet) @@ -222,7 +221,7 @@ func Setup( remoteVersions = append(remoteVersions, gomatrixserverlib.RoomVersionV1) } return MakeJoin( - httpReq, request, cfg, query, roomID, eventID, remoteVersions, + httpReq, request, cfg, rsAPI, roomID, eventID, remoteVersions, ) }, )).Methods(http.MethodGet) @@ -237,7 +236,7 @@ func Setup( roomID := vars["roomID"] eventID := vars["eventID"] return SendJoin( - httpReq, request, cfg, query, producer, keys, roomID, eventID, + httpReq, request, cfg, rsAPI, producer, keys, roomID, eventID, ) }, )).Methods(http.MethodPut) @@ -252,7 +251,7 @@ func Setup( roomID := vars["roomID"] eventID := vars["eventID"] return MakeLeave( - httpReq, request, cfg, query, roomID, eventID, + httpReq, request, cfg, rsAPI, roomID, eventID, ) }, )).Methods(http.MethodGet) @@ -286,7 +285,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return GetMissingEvents(httpReq, request, query, vars["roomID"]) + return GetMissingEvents(httpReq, request, rsAPI, vars["roomID"]) }, )).Methods(http.MethodPost) @@ -297,7 +296,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return Backfill(httpReq, request, query, vars["roomID"], cfg) + return Backfill(httpReq, request, rsAPI, vars["roomID"], cfg) }, )).Methods(http.MethodGet) } diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index 5a9766f81..88411b81a 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -34,7 +34,7 @@ func Send( request *gomatrixserverlib.FederationRequest, txnID gomatrixserverlib.TransactionID, cfg *config.Dendrite, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, producer *producers.RoomserverProducer, eduProducer *producers.EDUServerProducer, keys gomatrixserverlib.KeyRing, @@ -42,7 +42,7 @@ func Send( ) util.JSONResponse { t := txnReq{ context: httpReq.Context(), - query: query, + rsAPI: rsAPI, producer: producer, eduProducer: eduProducer, keys: keys, @@ -99,7 +99,7 @@ func Send( type txnReq struct { gomatrixserverlib.Transaction context context.Context - query api.RoomserverQueryAPI + rsAPI api.RoomserverInternalAPI producer *producers.RoomserverProducer eduProducer *producers.EDUServerProducer keys gomatrixserverlib.KeyRing @@ -120,7 +120,7 @@ func (t *txnReq) processTransaction() (*gomatrixserverlib.RespSend, error) { } verReq := api.QueryRoomVersionForRoomRequest{RoomID: header.RoomID} verRes := api.QueryRoomVersionForRoomResponse{} - if err := t.query.QueryRoomVersionForRoom(t.context, &verReq, &verRes); err != nil { + if err := t.rsAPI.QueryRoomVersionForRoom(t.context, &verReq, &verRes); err != nil { util.GetLogger(t.context).WithError(err).Warn("Transaction: Failed to query room version for room", verReq.RoomID) return nil, roomNotFoundError{verReq.RoomID} } @@ -228,7 +228,7 @@ func (t *txnReq) processEvent(e gomatrixserverlib.Event) error { StateToFetch: needed.Tuples(), } var stateResp api.QueryStateAfterEventsResponse - if err := t.query.QueryStateAfterEvents(t.context, &stateReq, &stateResp); err != nil { + if err := t.rsAPI.QueryStateAfterEvents(t.context, &stateReq, &stateResp); err != nil { return err } diff --git a/federationapi/routing/state.go b/federationapi/routing/state.go index 548598dd7..f90c494c3 100644 --- a/federationapi/routing/state.go +++ b/federationapi/routing/state.go @@ -27,7 +27,7 @@ import ( func GetState( ctx context.Context, request *gomatrixserverlib.FederationRequest, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, roomID string, ) util.JSONResponse { eventID, err := parseEventIDParam(request) @@ -35,7 +35,7 @@ func GetState( return *err } - state, err := getState(ctx, request, query, roomID, eventID) + state, err := getState(ctx, request, rsAPI, roomID, eventID) if err != nil { return *err } @@ -47,7 +47,7 @@ func GetState( func GetStateIDs( ctx context.Context, request *gomatrixserverlib.FederationRequest, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, roomID string, ) util.JSONResponse { eventID, err := parseEventIDParam(request) @@ -55,7 +55,7 @@ func GetStateIDs( return *err } - state, err := getState(ctx, request, query, roomID, eventID) + state, err := getState(ctx, request, rsAPI, roomID, eventID) if err != nil { return *err } @@ -94,11 +94,11 @@ func parseEventIDParam( func getState( ctx context.Context, request *gomatrixserverlib.FederationRequest, - query api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, roomID string, eventID string, ) (*gomatrixserverlib.RespState, *util.JSONResponse) { - event, resErr := getEvent(ctx, request, query, eventID) + event, resErr := getEvent(ctx, request, rsAPI, eventID) if resErr != nil { return nil, resErr } @@ -110,7 +110,7 @@ func getState( authEventIDs := getIDsFromEventRef(event.AuthEvents()) var response api.QueryStateAndAuthChainResponse - err := query.QueryStateAndAuthChain( + err := rsAPI.QueryStateAndAuthChain( ctx, &api.QueryStateAndAuthChainRequest{ RoomID: roomID, diff --git a/federationapi/routing/threepid.go b/federationapi/routing/threepid.go index f93d934ed..3c1d09e11 100644 --- a/federationapi/routing/threepid.go +++ b/federationapi/routing/threepid.go @@ -58,7 +58,7 @@ var ( // CreateInvitesFrom3PIDInvites implements POST /_matrix/federation/v1/3pid/onbind func CreateInvitesFrom3PIDInvites( - req *http.Request, queryAPI roomserverAPI.RoomserverQueryAPI, + req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, cfg *config.Dendrite, producer *producers.RoomserverProducer, federation *gomatrixserverlib.FederationClient, accountDB accounts.Database, @@ -72,7 +72,7 @@ func CreateInvitesFrom3PIDInvites( for _, inv := range body.Invites { verReq := api.QueryRoomVersionForRoomRequest{RoomID: inv.RoomID} verRes := api.QueryRoomVersionForRoomResponse{} - if err := queryAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil { + if err := rsAPI.QueryRoomVersionForRoom(req.Context(), &verReq, &verRes); err != nil { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.UnsupportedRoomVersion(err.Error()), @@ -80,7 +80,7 @@ func CreateInvitesFrom3PIDInvites( } event, err := createInviteFrom3PIDInvite( - req.Context(), queryAPI, asAPI, cfg, inv, federation, accountDB, + req.Context(), rsAPI, asAPI, cfg, inv, federation, accountDB, ) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("createInviteFrom3PIDInvite failed") @@ -108,7 +108,7 @@ func ExchangeThirdPartyInvite( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, roomID string, - queryAPI roomserverAPI.RoomserverQueryAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, cfg *config.Dendrite, federation *gomatrixserverlib.FederationClient, producer *producers.RoomserverProducer, @@ -148,7 +148,7 @@ func ExchangeThirdPartyInvite( verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} verRes := api.QueryRoomVersionForRoomResponse{} - if err = queryAPI.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil { + if err = rsAPI.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.UnsupportedRoomVersion(err.Error()), @@ -156,7 +156,7 @@ func ExchangeThirdPartyInvite( } // Auth and build the event from what the remote server sent us - event, err := buildMembershipEvent(httpReq.Context(), &builder, queryAPI, cfg) + event, err := buildMembershipEvent(httpReq.Context(), &builder, rsAPI, cfg) if err == errNotInRoom { return util.JSONResponse{ Code: http.StatusNotFound, @@ -199,14 +199,14 @@ func ExchangeThirdPartyInvite( // Returns an error if there was a problem building the event or fetching the // necessary data to do so. func createInviteFrom3PIDInvite( - ctx context.Context, queryAPI roomserverAPI.RoomserverQueryAPI, + ctx context.Context, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, cfg *config.Dendrite, inv invite, federation *gomatrixserverlib.FederationClient, accountDB accounts.Database, ) (*gomatrixserverlib.Event, error) { verReq := api.QueryRoomVersionForRoomRequest{RoomID: inv.RoomID} verRes := api.QueryRoomVersionForRoomResponse{} - if err := queryAPI.QueryRoomVersionForRoom(ctx, &verReq, &verRes); err != nil { + if err := rsAPI.QueryRoomVersionForRoom(ctx, &verReq, &verRes); err != nil { return nil, err } @@ -245,7 +245,7 @@ func createInviteFrom3PIDInvite( return nil, err } - event, err := buildMembershipEvent(ctx, builder, queryAPI, cfg) + event, err := buildMembershipEvent(ctx, builder, rsAPI, cfg) if err == errNotInRoom { return nil, sendToRemoteServer(ctx, inv, federation, cfg, *builder) } @@ -263,7 +263,7 @@ func createInviteFrom3PIDInvite( // Returns an error if something failed during the process. func buildMembershipEvent( ctx context.Context, - builder *gomatrixserverlib.EventBuilder, queryAPI roomserverAPI.RoomserverQueryAPI, + builder *gomatrixserverlib.EventBuilder, rsAPI roomserverAPI.RoomserverInternalAPI, cfg *config.Dendrite, ) (*gomatrixserverlib.Event, error) { eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(builder) @@ -281,7 +281,7 @@ func buildMembershipEvent( StateToFetch: eventsNeeded.Tuples(), } var queryRes roomserverAPI.QueryLatestEventsAndStateResponse - if err = queryAPI.QueryLatestEventsAndState(ctx, &queryReq, &queryRes); err != nil { + if err = rsAPI.QueryLatestEventsAndState(ctx, &queryReq, &queryRes); err != nil { return nil, err } diff --git a/federationsender/consumers/roomserver.go b/federationsender/consumers/roomserver.go index 18c8324b4..67d08b339 100644 --- a/federationsender/consumers/roomserver.go +++ b/federationsender/consumers/roomserver.go @@ -33,11 +33,11 @@ import ( // OutputRoomEventConsumer consumes events that originated in the room server. type OutputRoomEventConsumer struct { - cfg *config.Dendrite - roomServerConsumer *common.ContinualConsumer - db storage.Database - queues *queue.OutgoingQueues - query api.RoomserverQueryAPI + cfg *config.Dendrite + rsAPI api.RoomserverInternalAPI + rsConsumer *common.ContinualConsumer + db storage.Database + queues *queue.OutgoingQueues } // NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call Start() to begin consuming from room servers. @@ -46,7 +46,7 @@ func NewOutputRoomEventConsumer( kafkaConsumer sarama.Consumer, queues *queue.OutgoingQueues, store storage.Database, - queryAPI api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, ) *OutputRoomEventConsumer { consumer := common.ContinualConsumer{ Topic: string(cfg.Kafka.Topics.OutputRoomEvent), @@ -54,11 +54,11 @@ func NewOutputRoomEventConsumer( PartitionStore: store, } s := &OutputRoomEventConsumer{ - cfg: cfg, - roomServerConsumer: &consumer, - db: store, - queues: queues, - query: queryAPI, + cfg: cfg, + rsConsumer: &consumer, + db: store, + queues: queues, + rsAPI: rsAPI, } consumer.ProcessMessage = s.onMessage @@ -67,7 +67,7 @@ func NewOutputRoomEventConsumer( // Start consuming from room servers func (s *OutputRoomEventConsumer) Start() error { - return s.roomServerConsumer.Start() + return s.rsConsumer.Start() } // onMessage is called when the federation server receives a new event from the room server output log. @@ -369,7 +369,7 @@ func (s *OutputRoomEventConsumer) lookupStateEvents( // from the roomserver using the query API. eventReq := api.QueryEventsByIDRequest{EventIDs: missing} var eventResp api.QueryEventsByIDResponse - if err := s.query.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil { + if err := s.rsAPI.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil { return nil, err } diff --git a/federationsender/federationsender.go b/federationsender/federationsender.go index aa9a7bc9c..bf9d326bb 100644 --- a/federationsender/federationsender.go +++ b/federationsender/federationsender.go @@ -34,8 +34,7 @@ import ( func SetupFederationSenderComponent( base *basecomponent.BaseDendrite, federation *gomatrixserverlib.FederationClient, - rsQueryAPI roomserverAPI.RoomserverQueryAPI, - rsInputAPI roomserverAPI.RoomserverInputAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, keyRing *gomatrixserverlib.KeyRing, ) api.FederationSenderInternalAPI { federationSenderDB, err := storage.NewDatabase(string(base.Cfg.Database.FederationSender)) @@ -43,13 +42,13 @@ func SetupFederationSenderComponent( logrus.WithError(err).Panic("failed to connect to federation sender db") } - roomserverProducer := producers.NewRoomserverProducer(rsInputAPI, base.Cfg.Matrix.ServerName) + roomserverProducer := producers.NewRoomserverProducer(rsAPI, base.Cfg.Matrix.ServerName) queues := queue.NewOutgoingQueues(base.Cfg.Matrix.ServerName, federation, roomserverProducer) rsConsumer := consumers.NewOutputRoomEventConsumer( base.Cfg, base.KafkaConsumer, queues, - federationSenderDB, rsQueryAPI, + federationSenderDB, rsAPI, ) if err = rsConsumer.Start(); err != nil { logrus.WithError(err).Panic("failed to start room server consumer") diff --git a/federationsender/producers/roomserver.go b/federationsender/producers/roomserver.go index ff4cda5b5..48aeed8cc 100644 --- a/federationsender/producers/roomserver.go +++ b/federationsender/producers/roomserver.go @@ -23,16 +23,16 @@ import ( // RoomserverProducer produces events for the roomserver to consume. type RoomserverProducer struct { - InputAPI api.RoomserverInputAPI + InputAPI api.RoomserverInternalAPI serverName gomatrixserverlib.ServerName } // NewRoomserverProducer creates a new RoomserverProducer func NewRoomserverProducer( - inputAPI api.RoomserverInputAPI, serverName gomatrixserverlib.ServerName, + rsAPI api.RoomserverInternalAPI, serverName gomatrixserverlib.ServerName, ) *RoomserverProducer { return &RoomserverProducer{ - InputAPI: inputAPI, + InputAPI: rsAPI, serverName: serverName, } } diff --git a/publicroomsapi/consumers/roomserver.go b/publicroomsapi/consumers/roomserver.go index 853761c36..efd093b7e 100644 --- a/publicroomsapi/consumers/roomserver.go +++ b/publicroomsapi/consumers/roomserver.go @@ -29,9 +29,9 @@ import ( // OutputRoomEventConsumer consumes events that originated in the room server. type OutputRoomEventConsumer struct { - roomServerConsumer *common.ContinualConsumer - db storage.Database - query api.RoomserverQueryAPI + rsAPI api.RoomserverInternalAPI + rsConsumer *common.ContinualConsumer + db storage.Database } // NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call Start() to begin consuming from room servers. @@ -39,7 +39,7 @@ func NewOutputRoomEventConsumer( cfg *config.Dendrite, kafkaConsumer sarama.Consumer, store storage.Database, - queryAPI api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, ) *OutputRoomEventConsumer { consumer := common.ContinualConsumer{ Topic: string(cfg.Kafka.Topics.OutputRoomEvent), @@ -47,9 +47,9 @@ func NewOutputRoomEventConsumer( PartitionStore: store, } s := &OutputRoomEventConsumer{ - roomServerConsumer: &consumer, - db: store, - query: queryAPI, + rsConsumer: &consumer, + db: store, + rsAPI: rsAPI, } consumer.ProcessMessage = s.onMessage @@ -58,7 +58,7 @@ func NewOutputRoomEventConsumer( // Start consuming from room servers func (s *OutputRoomEventConsumer) Start() error { - return s.roomServerConsumer.Start() + return s.rsConsumer.Start() } // onMessage is called when the sync server receives a new event from the room server output log. @@ -87,14 +87,14 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { addQueryReq := api.QueryEventsByIDRequest{EventIDs: output.NewRoomEvent.AddsStateEventIDs} var addQueryRes api.QueryEventsByIDResponse - if err := s.query.QueryEventsByID(context.TODO(), &addQueryReq, &addQueryRes); err != nil { + if err := s.rsAPI.QueryEventsByID(context.TODO(), &addQueryReq, &addQueryRes); err != nil { log.Warn(err) return err } remQueryReq := api.QueryEventsByIDRequest{EventIDs: output.NewRoomEvent.RemovesStateEventIDs} var remQueryRes api.QueryEventsByIDResponse - if err := s.query.QueryEventsByID(context.TODO(), &remQueryReq, &remQueryRes); err != nil { + if err := s.rsAPI.QueryEventsByID(context.TODO(), &remQueryReq, &remQueryRes); err != nil { log.Warn(err) return err } diff --git a/publicroomsapi/directory/directory.go b/publicroomsapi/directory/directory.go index 837018e64..fe7a67932 100644 --- a/publicroomsapi/directory/directory.go +++ b/publicroomsapi/directory/directory.go @@ -59,7 +59,7 @@ func GetVisibility( // SetVisibility implements PUT /directory/list/room/{roomID} // TODO: Allow admin users to edit the room visibility func SetVisibility( - req *http.Request, publicRoomsDatabase storage.Database, queryAPI api.RoomserverQueryAPI, dev *authtypes.Device, + req *http.Request, publicRoomsDatabase storage.Database, rsAPI api.RoomserverInternalAPI, dev *authtypes.Device, roomID string, ) util.JSONResponse { queryMembershipReq := api.QueryMembershipForUserRequest{ @@ -67,7 +67,7 @@ func SetVisibility( UserID: dev.UserID, } var queryMembershipRes api.QueryMembershipForUserResponse - err := queryAPI.QueryMembershipForUser(req.Context(), &queryMembershipReq, &queryMembershipRes) + err := rsAPI.QueryMembershipForUser(req.Context(), &queryMembershipReq, &queryMembershipRes) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("could not query membership for user") return jsonerror.InternalServerError() @@ -87,7 +87,7 @@ func SetVisibility( }}, } var queryEventsRes api.QueryLatestEventsAndStateResponse - err = queryAPI.QueryLatestEventsAndState(req.Context(), &queryEventsReq, &queryEventsRes) + err = rsAPI.QueryLatestEventsAndState(req.Context(), &queryEventsReq, &queryEventsRes) if err != nil || len(queryEventsRes.StateEvents) == 0 { util.GetLogger(req.Context()).WithError(err).Error("could not query events from room") return jsonerror.InternalServerError() diff --git a/publicroomsapi/publicroomsapi.go b/publicroomsapi/publicroomsapi.go index 6efb54bd9..6a4e65674 100644 --- a/publicroomsapi/publicroomsapi.go +++ b/publicroomsapi/publicroomsapi.go @@ -32,16 +32,16 @@ func SetupPublicRoomsAPIComponent( base *basecomponent.BaseDendrite, deviceDB devices.Database, publicRoomsDB storage.Database, - rsQueryAPI roomserverAPI.RoomserverQueryAPI, + rsAPI roomserverAPI.RoomserverInternalAPI, fedClient *gomatrixserverlib.FederationClient, extRoomsProvider types.ExternalPublicRoomsProvider, ) { rsConsumer := consumers.NewOutputRoomEventConsumer( - base.Cfg, base.KafkaConsumer, publicRoomsDB, rsQueryAPI, + base.Cfg, base.KafkaConsumer, publicRoomsDB, rsAPI, ) if err := rsConsumer.Start(); err != nil { logrus.WithError(err).Panic("failed to start public rooms server consumer") } - routing.Setup(base.APIMux, deviceDB, publicRoomsDB, rsQueryAPI, fedClient, extRoomsProvider) + routing.Setup(base.APIMux, deviceDB, publicRoomsDB, rsAPI, fedClient, extRoomsProvider) } diff --git a/publicroomsapi/routing/routing.go b/publicroomsapi/routing/routing.go index da5ea90d6..09a8eff76 100644 --- a/publicroomsapi/routing/routing.go +++ b/publicroomsapi/routing/routing.go @@ -39,7 +39,7 @@ const pathPrefixR0 = "/_matrix/client/r0" // applied: // nolint: gocyclo func Setup( - apiMux *mux.Router, deviceDB devices.Database, publicRoomsDB storage.Database, queryAPI api.RoomserverQueryAPI, + apiMux *mux.Router, deviceDB devices.Database, publicRoomsDB storage.Database, rsAPI api.RoomserverInternalAPI, fedClient *gomatrixserverlib.FederationClient, extRoomsProvider types.ExternalPublicRoomsProvider, ) { r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter() @@ -66,7 +66,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return directory.SetVisibility(req, publicRoomsDB, queryAPI, device, vars["roomID"]) + return directory.SetVisibility(req, publicRoomsDB, rsAPI, device, vars["roomID"]) }), ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/publicRooms", diff --git a/roomserver/alias/alias_test.go b/roomserver/alias/alias_test.go deleted file mode 100644 index 0aefa19d9..000000000 --- a/roomserver/alias/alias_test.go +++ /dev/null @@ -1,209 +0,0 @@ -// Copyright 2019 Serra Allgood -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package alias - -import ( - "context" - "fmt" - "strings" - "testing" - - appserviceAPI "github.com/matrix-org/dendrite/appservice/api" - roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/gomatrixserverlib" -) - -type MockRoomserverAliasAPIDatabase struct { - mode string - attempts int -} - -// These methods can be essentially noop -func (db MockRoomserverAliasAPIDatabase) SetRoomAlias(ctx context.Context, alias string, roomID string, creatorUserID string) error { - return nil -} - -func (db MockRoomserverAliasAPIDatabase) GetAliasesForRoomID(ctx context.Context, roomID string) ([]string, error) { - aliases := make([]string, 0) - return aliases, nil -} - -func (db MockRoomserverAliasAPIDatabase) RemoveRoomAlias(ctx context.Context, alias string) error { - return nil -} - -func (db *MockRoomserverAliasAPIDatabase) GetCreatorIDForAlias( - ctx context.Context, alias string, -) (string, error) { - return "", nil -} - -func (db *MockRoomserverAliasAPIDatabase) GetRoomVersionForRoom( - ctx context.Context, roomID string, -) (gomatrixserverlib.RoomVersion, error) { - return gomatrixserverlib.RoomVersionV1, nil -} - -// This method needs to change depending on test case -func (db *MockRoomserverAliasAPIDatabase) GetRoomIDForAlias( - ctx context.Context, - alias string, -) (string, error) { - switch db.mode { - case "empty": - return "", nil - case "error": - return "", fmt.Errorf("found an error from GetRoomIDForAlias") - case "found": - return "123", nil - case "emptyFound": - switch db.attempts { - case 0: - db.attempts = 1 - return "", nil - case 1: - db.attempts = 0 - return "123", nil - default: - return "", nil - } - default: - return "", fmt.Errorf("unknown option used") - } -} - -type MockAppServiceQueryAPI struct { - mode string -} - -// This method can be noop -func (q MockAppServiceQueryAPI) UserIDExists( - ctx context.Context, - req *appserviceAPI.UserIDExistsRequest, - resp *appserviceAPI.UserIDExistsResponse, -) error { - return nil -} - -func (q MockAppServiceQueryAPI) RoomAliasExists( - ctx context.Context, - req *appserviceAPI.RoomAliasExistsRequest, - resp *appserviceAPI.RoomAliasExistsResponse, -) error { - switch q.mode { - case "error": - return fmt.Errorf("found an error from RoomAliasExists") - case "found": - resp.AliasExists = true - return nil - case "empty": - resp.AliasExists = false - return nil - default: - return fmt.Errorf("Unknown option used") - } -} - -func TestGetRoomIDForAlias(t *testing.T) { - type arguments struct { - ctx context.Context - request *roomserverAPI.GetRoomIDForAliasRequest - response *roomserverAPI.GetRoomIDForAliasResponse - } - args := arguments{ - context.Background(), - &roomserverAPI.GetRoomIDForAliasRequest{}, - &roomserverAPI.GetRoomIDForAliasResponse{}, - } - type testCase struct { - name string - dbMode string - queryMode string - wantError bool - errorMsg string - want string - } - tt := []testCase{ - { - "found local alias", - "found", - "error", - false, - "", - "123", - }, - { - "found appservice alias", - "emptyFound", - "found", - false, - "", - "123", - }, - { - "error returned from DB", - "error", - "", - true, - "GetRoomIDForAlias", - "", - }, - { - "error returned from appserviceAPI", - "empty", - "error", - true, - "RoomAliasExists", - "", - }, - { - "no errors but no alias", - "empty", - "empty", - false, - "", - "", - }, - } - - setup := func(dbMode, queryMode string) *RoomserverAliasAPI { - mockAliasAPIDB := &MockRoomserverAliasAPIDatabase{dbMode, 0} - mockAppServiceQueryAPI := MockAppServiceQueryAPI{queryMode} - - return &RoomserverAliasAPI{ - DB: mockAliasAPIDB, - AppserviceAPI: mockAppServiceQueryAPI, - } - } - - for _, tc := range tt { - t.Run(tc.name, func(t *testing.T) { - aliasAPI := setup(tc.dbMode, tc.queryMode) - - err := aliasAPI.GetRoomIDForAlias(args.ctx, args.request, args.response) - if tc.wantError { - if err == nil { - t.Fatalf("Got no error; wanted error from %s", tc.errorMsg) - } else if !strings.Contains(err.Error(), tc.errorMsg) { - t.Fatalf("Got %s; wanted error from %s", err, tc.errorMsg) - } - } else if err != nil { - t.Fatalf("Got %s; wanted no error", err) - } else if args.response.RoomID != tc.want { - t.Errorf("Got '%s'; wanted '%s'", args.response.RoomID, tc.want) - } - }) - } -} diff --git a/roomserver/api/alias.go b/roomserver/api/alias.go index ad375a830..488e99ab8 100644 --- a/roomserver/api/alias.go +++ b/roomserver/api/alias.go @@ -16,8 +16,6 @@ package api import ( "context" - "errors" - "net/http" commonHTTP "github.com/matrix-org/dendrite/common/http" opentracing "github.com/opentracing/opentracing-go" @@ -86,44 +84,6 @@ type RemoveRoomAliasRequest struct { // RemoveRoomAliasResponse is a response to RemoveRoomAlias type RemoveRoomAliasResponse struct{} -// RoomserverAliasAPI is used to save, lookup or remove a room alias -type RoomserverAliasAPI interface { - // Set a room alias - SetRoomAlias( - ctx context.Context, - req *SetRoomAliasRequest, - response *SetRoomAliasResponse, - ) error - - // Get the room ID for an alias - GetRoomIDForAlias( - ctx context.Context, - req *GetRoomIDForAliasRequest, - response *GetRoomIDForAliasResponse, - ) error - - // Get all known aliases for a room ID - GetAliasesForRoomID( - ctx context.Context, - req *GetAliasesForRoomIDRequest, - response *GetAliasesForRoomIDResponse, - ) error - - // Get the user ID of the creator of an alias - GetCreatorIDForAlias( - ctx context.Context, - req *GetCreatorIDForAliasRequest, - response *GetCreatorIDForAliasResponse, - ) error - - // Remove a room alias - RemoveRoomAlias( - ctx context.Context, - req *RemoveRoomAliasRequest, - response *RemoveRoomAliasResponse, - ) error -} - // RoomserverSetRoomAliasPath is the HTTP path for the SetRoomAlias API. const RoomserverSetRoomAliasPath = "/api/roomserver/setRoomAlias" @@ -139,22 +99,8 @@ const RoomserverGetCreatorIDForAliasPath = "/api/roomserver/GetCreatorIDForAlias // RoomserverRemoveRoomAliasPath is the HTTP path for the RemoveRoomAlias API. const RoomserverRemoveRoomAliasPath = "/api/roomserver/removeRoomAlias" -// NewRoomserverAliasAPIHTTP creates a RoomserverAliasAPI implemented by talking to a HTTP POST API. -// If httpClient is nil an error is returned -func NewRoomserverAliasAPIHTTP(roomserverURL string, httpClient *http.Client) (RoomserverAliasAPI, error) { - if httpClient == nil { - return nil, errors.New("NewRoomserverAliasAPIHTTP: httpClient is ") - } - return &httpRoomserverAliasAPI{roomserverURL, httpClient}, nil -} - -type httpRoomserverAliasAPI struct { - roomserverURL string - httpClient *http.Client -} - // SetRoomAlias implements RoomserverAliasAPI -func (h *httpRoomserverAliasAPI) SetRoomAlias( +func (h *httpRoomserverInternalAPI) SetRoomAlias( ctx context.Context, request *SetRoomAliasRequest, response *SetRoomAliasResponse, @@ -167,7 +113,7 @@ func (h *httpRoomserverAliasAPI) SetRoomAlias( } // GetRoomIDForAlias implements RoomserverAliasAPI -func (h *httpRoomserverAliasAPI) GetRoomIDForAlias( +func (h *httpRoomserverInternalAPI) GetRoomIDForAlias( ctx context.Context, request *GetRoomIDForAliasRequest, response *GetRoomIDForAliasResponse, @@ -180,7 +126,7 @@ func (h *httpRoomserverAliasAPI) GetRoomIDForAlias( } // GetAliasesForRoomID implements RoomserverAliasAPI -func (h *httpRoomserverAliasAPI) GetAliasesForRoomID( +func (h *httpRoomserverInternalAPI) GetAliasesForRoomID( ctx context.Context, request *GetAliasesForRoomIDRequest, response *GetAliasesForRoomIDResponse, @@ -193,7 +139,7 @@ func (h *httpRoomserverAliasAPI) GetAliasesForRoomID( } // GetCreatorIDForAlias implements RoomserverAliasAPI -func (h *httpRoomserverAliasAPI) GetCreatorIDForAlias( +func (h *httpRoomserverInternalAPI) GetCreatorIDForAlias( ctx context.Context, request *GetCreatorIDForAliasRequest, response *GetCreatorIDForAliasResponse, @@ -206,7 +152,7 @@ func (h *httpRoomserverAliasAPI) GetCreatorIDForAlias( } // RemoveRoomAlias implements RoomserverAliasAPI -func (h *httpRoomserverAliasAPI) RemoveRoomAlias( +func (h *httpRoomserverInternalAPI) RemoveRoomAlias( ctx context.Context, request *RemoveRoomAliasRequest, response *RemoveRoomAliasResponse, diff --git a/roomserver/api/api.go b/roomserver/api/api.go new file mode 100644 index 000000000..c12dbddde --- /dev/null +++ b/roomserver/api/api.go @@ -0,0 +1,141 @@ +package api + +import ( + "context" + + fsAPI "github.com/matrix-org/dendrite/federationsender/api" +) + +// RoomserverInputAPI is used to write events to the room server. +type RoomserverInternalAPI interface { + // needed to avoid chicken and egg scenario when setting up the + // interdependencies between the roomserver and other input APIs + SetFederationSenderAPI(fsAPI fsAPI.FederationSenderInternalAPI) + + InputRoomEvents( + ctx context.Context, + request *InputRoomEventsRequest, + response *InputRoomEventsResponse, + ) error + + // Query the latest events and state for a room from the room server. + QueryLatestEventsAndState( + ctx context.Context, + request *QueryLatestEventsAndStateRequest, + response *QueryLatestEventsAndStateResponse, + ) error + + // Query the state after a list of events in a room from the room server. + QueryStateAfterEvents( + ctx context.Context, + request *QueryStateAfterEventsRequest, + response *QueryStateAfterEventsResponse, + ) error + + // Query a list of events by event ID. + QueryEventsByID( + ctx context.Context, + request *QueryEventsByIDRequest, + response *QueryEventsByIDResponse, + ) error + + // Query the membership event for an user for a room. + QueryMembershipForUser( + ctx context.Context, + request *QueryMembershipForUserRequest, + response *QueryMembershipForUserResponse, + ) error + + // Query a list of membership events for a room + QueryMembershipsForRoom( + ctx context.Context, + request *QueryMembershipsForRoomRequest, + response *QueryMembershipsForRoomResponse, + ) error + + // Query a list of invite event senders for a user in a room. + QueryInvitesForUser( + ctx context.Context, + request *QueryInvitesForUserRequest, + response *QueryInvitesForUserResponse, + ) error + + // Query whether a server is allowed to see an event + QueryServerAllowedToSeeEvent( + ctx context.Context, + request *QueryServerAllowedToSeeEventRequest, + response *QueryServerAllowedToSeeEventResponse, + ) error + + // Query missing events for a room from roomserver + QueryMissingEvents( + ctx context.Context, + request *QueryMissingEventsRequest, + response *QueryMissingEventsResponse, + ) error + + // Query to get state and auth chain for a (potentially hypothetical) event. + // Takes lists of PrevEventIDs and AuthEventsIDs and uses them to calculate + // the state and auth chain to return. + QueryStateAndAuthChain( + ctx context.Context, + request *QueryStateAndAuthChainRequest, + response *QueryStateAndAuthChainResponse, + ) error + + // Query a given amount (or less) of events prior to a given set of events. + QueryBackfill( + ctx context.Context, + request *QueryBackfillRequest, + response *QueryBackfillResponse, + ) error + + // Asks for the default room version as preferred by the server. + QueryRoomVersionCapabilities( + ctx context.Context, + request *QueryRoomVersionCapabilitiesRequest, + response *QueryRoomVersionCapabilitiesResponse, + ) error + + // Asks for the room version for a given room. + QueryRoomVersionForRoom( + ctx context.Context, + request *QueryRoomVersionForRoomRequest, + response *QueryRoomVersionForRoomResponse, + ) error + + // Set a room alias + SetRoomAlias( + ctx context.Context, + req *SetRoomAliasRequest, + response *SetRoomAliasResponse, + ) error + + // Get the room ID for an alias + GetRoomIDForAlias( + ctx context.Context, + req *GetRoomIDForAliasRequest, + response *GetRoomIDForAliasResponse, + ) error + + // Get all known aliases for a room ID + GetAliasesForRoomID( + ctx context.Context, + req *GetAliasesForRoomIDRequest, + response *GetAliasesForRoomIDResponse, + ) error + + // Get the user ID of the creator of an alias + GetCreatorIDForAlias( + ctx context.Context, + req *GetCreatorIDForAliasRequest, + response *GetCreatorIDForAliasResponse, + ) error + + // Remove a room alias + RemoveRoomAlias( + ctx context.Context, + req *RemoveRoomAliasRequest, + response *RemoveRoomAliasResponse, + ) error +} diff --git a/roomserver/api/http.go b/roomserver/api/http.go new file mode 100644 index 000000000..d643526bd --- /dev/null +++ b/roomserver/api/http.go @@ -0,0 +1,41 @@ +package api + +import ( + "errors" + "net/http" + + "github.com/matrix-org/dendrite/common/caching" + fsInputAPI "github.com/matrix-org/dendrite/federationsender/api" +) + +type httpRoomserverInternalAPI struct { + roomserverURL string + httpClient *http.Client + fsAPI fsInputAPI.FederationSenderInternalAPI + immutableCache caching.ImmutableCache +} + +// NewRoomserverInputAPIHTTP creates a RoomserverInputAPI implemented by talking to a HTTP POST API. +// If httpClient is nil an error is returned +func NewRoomserverInternalAPIHTTP( + roomserverURL string, + httpClient *http.Client, + //fsInputAPI fsAPI.FederationSenderInternalAPI, + immutableCache caching.ImmutableCache, +) (RoomserverInternalAPI, error) { + if httpClient == nil { + return nil, errors.New("NewRoomserverInternalAPIHTTP: httpClient is ") + } + return &httpRoomserverInternalAPI{ + roomserverURL: roomserverURL, + httpClient: httpClient, + immutableCache: immutableCache, + }, nil +} + +// SetFederationSenderInputAPI passes in a federation sender input API reference +// so that we can avoid the chicken-and-egg problem of both the roomserver input API +// and the federation sender input API being interdependent. +func (h *httpRoomserverInternalAPI) SetFederationSenderAPI(fsAPI fsInputAPI.FederationSenderInternalAPI) { + h.fsAPI = fsAPI +} diff --git a/roomserver/api/input.go b/roomserver/api/input.go index d9cffad27..8e8fdae4e 100644 --- a/roomserver/api/input.go +++ b/roomserver/api/input.go @@ -17,11 +17,8 @@ package api import ( "context" - "errors" - "net/http" commonHTTP "github.com/matrix-org/dendrite/common/http" - fsAPI "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/gomatrixserverlib" opentracing "github.com/opentracing/opentracing-go" ) @@ -105,47 +102,11 @@ type InputRoomEventsResponse struct { EventID string `json:"event_id"` } -// RoomserverInputAPI is used to write events to the room server. -type RoomserverInputAPI interface { - // needed to avoid chicken and egg scenario when setting up the - // interdependencies between the roomserver and the FS input API - SetFederationSenderAPI(fsInputAPI fsAPI.FederationSenderInternalAPI) - InputRoomEvents( - ctx context.Context, - request *InputRoomEventsRequest, - response *InputRoomEventsResponse, - ) error -} - // RoomserverInputRoomEventsPath is the HTTP path for the InputRoomEvents API. const RoomserverInputRoomEventsPath = "/api/roomserver/inputRoomEvents" -// NewRoomserverInputAPIHTTP creates a RoomserverInputAPI implemented by talking to a HTTP POST API. -// If httpClient is nil an error is returned -func NewRoomserverInputAPIHTTP(roomserverURL string, httpClient *http.Client) (RoomserverInputAPI, error) { - if httpClient == nil { - return nil, errors.New("NewRoomserverInputAPIHTTP: httpClient is ") - } - return &httpRoomserverInputAPI{roomserverURL, httpClient, nil}, nil -} - -type httpRoomserverInputAPI struct { - roomserverURL string - httpClient *http.Client - // The federation sender API allows us to send federation - // requests from the new perform input requests, still TODO. - fsInputAPI fsAPI.FederationSenderInternalAPI -} - -// SetFederationSenderInputAPI passes in a federation sender input API reference -// so that we can avoid the chicken-and-egg problem of both the roomserver input API -// and the federation sender input API being interdependent. -func (h *httpRoomserverInputAPI) SetFederationSenderAPI(fsInputAPI fsAPI.FederationSenderInternalAPI) { - h.fsInputAPI = fsInputAPI -} - // InputRoomEvents implements RoomserverInputAPI -func (h *httpRoomserverInputAPI) InputRoomEvents( +func (h *httpRoomserverInternalAPI) InputRoomEvents( ctx context.Context, request *InputRoomEventsRequest, response *InputRoomEventsResponse, diff --git a/roomserver/api/query.go b/roomserver/api/query.go index 11fa5c9ca..cb7cbb86c 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -18,10 +18,7 @@ package api import ( "context" - "errors" - "net/http" - "github.com/matrix-org/dendrite/common/caching" commonHTTP "github.com/matrix-org/dendrite/common/http" "github.com/matrix-org/gomatrixserverlib" opentracing "github.com/opentracing/opentracing-go" @@ -264,95 +261,6 @@ type QueryRoomVersionForRoomResponse struct { RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"` } -// RoomserverQueryAPI is used to query information from the room server. -type RoomserverQueryAPI interface { - // Query the latest events and state for a room from the room server. - QueryLatestEventsAndState( - ctx context.Context, - request *QueryLatestEventsAndStateRequest, - response *QueryLatestEventsAndStateResponse, - ) error - - // Query the state after a list of events in a room from the room server. - QueryStateAfterEvents( - ctx context.Context, - request *QueryStateAfterEventsRequest, - response *QueryStateAfterEventsResponse, - ) error - - // Query a list of events by event ID. - QueryEventsByID( - ctx context.Context, - request *QueryEventsByIDRequest, - response *QueryEventsByIDResponse, - ) error - - // Query the membership event for an user for a room. - QueryMembershipForUser( - ctx context.Context, - request *QueryMembershipForUserRequest, - response *QueryMembershipForUserResponse, - ) error - - // Query a list of membership events for a room - QueryMembershipsForRoom( - ctx context.Context, - request *QueryMembershipsForRoomRequest, - response *QueryMembershipsForRoomResponse, - ) error - - // Query a list of invite event senders for a user in a room. - QueryInvitesForUser( - ctx context.Context, - request *QueryInvitesForUserRequest, - response *QueryInvitesForUserResponse, - ) error - - // Query whether a server is allowed to see an event - QueryServerAllowedToSeeEvent( - ctx context.Context, - request *QueryServerAllowedToSeeEventRequest, - response *QueryServerAllowedToSeeEventResponse, - ) error - - // Query missing events for a room from roomserver - QueryMissingEvents( - ctx context.Context, - request *QueryMissingEventsRequest, - response *QueryMissingEventsResponse, - ) error - - // Query to get state and auth chain for a (potentially hypothetical) event. - // Takes lists of PrevEventIDs and AuthEventsIDs and uses them to calculate - // the state and auth chain to return. - QueryStateAndAuthChain( - ctx context.Context, - request *QueryStateAndAuthChainRequest, - response *QueryStateAndAuthChainResponse, - ) error - - // Query a given amount (or less) of events prior to a given set of events. - QueryBackfill( - ctx context.Context, - request *QueryBackfillRequest, - response *QueryBackfillResponse, - ) error - - // Asks for the default room version as preferred by the server. - QueryRoomVersionCapabilities( - ctx context.Context, - request *QueryRoomVersionCapabilitiesRequest, - response *QueryRoomVersionCapabilitiesResponse, - ) error - - // Asks for the room version for a given room. - QueryRoomVersionForRoom( - ctx context.Context, - request *QueryRoomVersionForRoomRequest, - response *QueryRoomVersionForRoomResponse, - ) error -} - // RoomserverQueryLatestEventsAndStatePath is the HTTP path for the QueryLatestEventsAndState API. const RoomserverQueryLatestEventsAndStatePath = "/api/roomserver/queryLatestEventsAndState" @@ -389,23 +297,8 @@ const RoomserverQueryRoomVersionCapabilitiesPath = "/api/roomserver/queryRoomVer // RoomserverQueryRoomVersionForRoomPath is the HTTP path for the QueryRoomVersionForRoom API const RoomserverQueryRoomVersionForRoomPath = "/api/roomserver/queryRoomVersionForRoom" -// NewRoomserverQueryAPIHTTP creates a RoomserverQueryAPI implemented by talking to a HTTP POST API. -// If httpClient is nil an error is returned -func NewRoomserverQueryAPIHTTP(roomserverURL string, httpClient *http.Client, cache caching.ImmutableCache) (RoomserverQueryAPI, error) { - if httpClient == nil { - return nil, errors.New("NewRoomserverQueryAPIHTTP: httpClient is ") - } - return &httpRoomserverQueryAPI{roomserverURL, httpClient, cache}, nil -} - -type httpRoomserverQueryAPI struct { - roomserverURL string - httpClient *http.Client - immutableCache caching.ImmutableCache -} - // QueryLatestEventsAndState implements RoomserverQueryAPI -func (h *httpRoomserverQueryAPI) QueryLatestEventsAndState( +func (h *httpRoomserverInternalAPI) QueryLatestEventsAndState( ctx context.Context, request *QueryLatestEventsAndStateRequest, response *QueryLatestEventsAndStateResponse, @@ -418,7 +311,7 @@ func (h *httpRoomserverQueryAPI) QueryLatestEventsAndState( } // QueryStateAfterEvents implements RoomserverQueryAPI -func (h *httpRoomserverQueryAPI) QueryStateAfterEvents( +func (h *httpRoomserverInternalAPI) QueryStateAfterEvents( ctx context.Context, request *QueryStateAfterEventsRequest, response *QueryStateAfterEventsResponse, @@ -431,7 +324,7 @@ func (h *httpRoomserverQueryAPI) QueryStateAfterEvents( } // QueryEventsByID implements RoomserverQueryAPI -func (h *httpRoomserverQueryAPI) QueryEventsByID( +func (h *httpRoomserverInternalAPI) QueryEventsByID( ctx context.Context, request *QueryEventsByIDRequest, response *QueryEventsByIDResponse, @@ -444,7 +337,7 @@ func (h *httpRoomserverQueryAPI) QueryEventsByID( } // QueryMembershipForUser implements RoomserverQueryAPI -func (h *httpRoomserverQueryAPI) QueryMembershipForUser( +func (h *httpRoomserverInternalAPI) QueryMembershipForUser( ctx context.Context, request *QueryMembershipForUserRequest, response *QueryMembershipForUserResponse, @@ -457,7 +350,7 @@ func (h *httpRoomserverQueryAPI) QueryMembershipForUser( } // QueryMembershipsForRoom implements RoomserverQueryAPI -func (h *httpRoomserverQueryAPI) QueryMembershipsForRoom( +func (h *httpRoomserverInternalAPI) QueryMembershipsForRoom( ctx context.Context, request *QueryMembershipsForRoomRequest, response *QueryMembershipsForRoomResponse, @@ -470,7 +363,7 @@ func (h *httpRoomserverQueryAPI) QueryMembershipsForRoom( } // QueryInvitesForUser implements RoomserverQueryAPI -func (h *httpRoomserverQueryAPI) QueryInvitesForUser( +func (h *httpRoomserverInternalAPI) QueryInvitesForUser( ctx context.Context, request *QueryInvitesForUserRequest, response *QueryInvitesForUserResponse, @@ -483,7 +376,7 @@ func (h *httpRoomserverQueryAPI) QueryInvitesForUser( } // QueryServerAllowedToSeeEvent implements RoomserverQueryAPI -func (h *httpRoomserverQueryAPI) QueryServerAllowedToSeeEvent( +func (h *httpRoomserverInternalAPI) QueryServerAllowedToSeeEvent( ctx context.Context, request *QueryServerAllowedToSeeEventRequest, response *QueryServerAllowedToSeeEventResponse, @@ -496,7 +389,7 @@ func (h *httpRoomserverQueryAPI) QueryServerAllowedToSeeEvent( } // QueryMissingEvents implements RoomServerQueryAPI -func (h *httpRoomserverQueryAPI) QueryMissingEvents( +func (h *httpRoomserverInternalAPI) QueryMissingEvents( ctx context.Context, request *QueryMissingEventsRequest, response *QueryMissingEventsResponse, @@ -509,7 +402,7 @@ func (h *httpRoomserverQueryAPI) QueryMissingEvents( } // QueryStateAndAuthChain implements RoomserverQueryAPI -func (h *httpRoomserverQueryAPI) QueryStateAndAuthChain( +func (h *httpRoomserverInternalAPI) QueryStateAndAuthChain( ctx context.Context, request *QueryStateAndAuthChainRequest, response *QueryStateAndAuthChainResponse, @@ -522,7 +415,7 @@ func (h *httpRoomserverQueryAPI) QueryStateAndAuthChain( } // QueryBackfill implements RoomServerQueryAPI -func (h *httpRoomserverQueryAPI) QueryBackfill( +func (h *httpRoomserverInternalAPI) QueryBackfill( ctx context.Context, request *QueryBackfillRequest, response *QueryBackfillResponse, @@ -535,7 +428,7 @@ func (h *httpRoomserverQueryAPI) QueryBackfill( } // QueryRoomVersionCapabilities implements RoomServerQueryAPI -func (h *httpRoomserverQueryAPI) QueryRoomVersionCapabilities( +func (h *httpRoomserverInternalAPI) QueryRoomVersionCapabilities( ctx context.Context, request *QueryRoomVersionCapabilitiesRequest, response *QueryRoomVersionCapabilitiesResponse, @@ -548,7 +441,7 @@ func (h *httpRoomserverQueryAPI) QueryRoomVersionCapabilities( } // QueryRoomVersionForRoom implements RoomServerQueryAPI -func (h *httpRoomserverQueryAPI) QueryRoomVersionForRoom( +func (h *httpRoomserverInternalAPI) QueryRoomVersionForRoom( ctx context.Context, request *QueryRoomVersionForRoomRequest, response *QueryRoomVersionForRoomResponse, diff --git a/roomserver/alias/alias.go b/roomserver/internal/alias.go similarity index 50% rename from roomserver/alias/alias.go rename to roomserver/internal/alias.go index eb606e5cd..4139582b6 100644 --- a/roomserver/alias/alias.go +++ b/roomserver/internal/alias.go @@ -12,25 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. -package alias +package internal import ( "context" "encoding/json" "errors" - "net/http" "time" - appserviceAPI "github.com/matrix-org/dendrite/appservice/api" - "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/common/config" - roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" ) -// RoomserverAliasAPIDatabase has the storage APIs needed to implement the alias API. -type RoomserverAliasAPIDatabase interface { +// RoomserverInternalAPIDatabase has the storage APIs needed to implement the alias API. +type RoomserverInternalAPIDatabase interface { // Save a given room alias with the room ID it refers to. // Returns an error if there was a problem talking to the database. SetRoomAlias(ctx context.Context, alias string, roomID string, creatorUserID string) error @@ -52,20 +47,11 @@ type RoomserverAliasAPIDatabase interface { ) (gomatrixserverlib.RoomVersion, error) } -// RoomserverAliasAPI is an implementation of alias.RoomserverAliasAPI -type RoomserverAliasAPI struct { - DB RoomserverAliasAPIDatabase - Cfg *config.Dendrite - InputAPI roomserverAPI.RoomserverInputAPI - QueryAPI roomserverAPI.RoomserverQueryAPI - AppserviceAPI appserviceAPI.AppServiceQueryAPI -} - -// SetRoomAlias implements alias.RoomserverAliasAPI -func (r *RoomserverAliasAPI) SetRoomAlias( +// SetRoomAlias implements alias.RoomserverInternalAPI +func (r *RoomserverInternalAPI) SetRoomAlias( ctx context.Context, - request *roomserverAPI.SetRoomAliasRequest, - response *roomserverAPI.SetRoomAliasResponse, + request *api.SetRoomAliasRequest, + response *api.SetRoomAliasResponse, ) error { // Check if the alias isn't already referring to a room roomID, err := r.DB.GetRoomIDForAlias(ctx, request.Alias) @@ -91,11 +77,11 @@ func (r *RoomserverAliasAPI) SetRoomAlias( return r.sendUpdatedAliasesEvent(context.TODO(), request.UserID, request.RoomID) } -// GetRoomIDForAlias implements alias.RoomserverAliasAPI -func (r *RoomserverAliasAPI) GetRoomIDForAlias( +// GetRoomIDForAlias implements alias.RoomserverInternalAPI +func (r *RoomserverInternalAPI) GetRoomIDForAlias( ctx context.Context, - request *roomserverAPI.GetRoomIDForAliasRequest, - response *roomserverAPI.GetRoomIDForAliasResponse, + request *api.GetRoomIDForAliasRequest, + response *api.GetRoomIDForAliasResponse, ) error { // Look up the room ID in the database roomID, err := r.DB.GetRoomIDForAlias(ctx, request.Alias) @@ -103,32 +89,38 @@ func (r *RoomserverAliasAPI) GetRoomIDForAlias( return err } - if roomID == "" { - // No room found locally, try our application services by making a call to - // the appservice component - aliasReq := appserviceAPI.RoomAliasExistsRequest{Alias: request.Alias} - var aliasResp appserviceAPI.RoomAliasExistsResponse - if err = r.AppserviceAPI.RoomAliasExists(ctx, &aliasReq, &aliasResp); err != nil { - return err - } + /* + TODO: Why is this here? It creates an unnecessary dependency + from the roomserver to the appservice component, which should be + altogether optional. - if aliasResp.AliasExists { - roomID, err = r.DB.GetRoomIDForAlias(ctx, request.Alias) - if err != nil { + if roomID == "" { + // No room found locally, try our application services by making a call to + // the appservice component + aliasReq := appserviceAPI.RoomAliasExistsRequest{Alias: request.Alias} + var aliasResp appserviceAPI.RoomAliasExistsResponse + if err = r.AppserviceAPI.RoomAliasExists(ctx, &aliasReq, &aliasResp); err != nil { return err } + + if aliasResp.AliasExists { + roomID, err = r.DB.GetRoomIDForAlias(ctx, request.Alias) + if err != nil { + return err + } + } } - } + */ response.RoomID = roomID return nil } -// GetAliasesForRoomID implements alias.RoomserverAliasAPI -func (r *RoomserverAliasAPI) GetAliasesForRoomID( +// GetAliasesForRoomID implements alias.RoomserverInternalAPI +func (r *RoomserverInternalAPI) GetAliasesForRoomID( ctx context.Context, - request *roomserverAPI.GetAliasesForRoomIDRequest, - response *roomserverAPI.GetAliasesForRoomIDResponse, + request *api.GetAliasesForRoomIDRequest, + response *api.GetAliasesForRoomIDResponse, ) error { // Look up the aliases in the database for the given RoomID aliases, err := r.DB.GetAliasesForRoomID(ctx, request.RoomID) @@ -140,11 +132,11 @@ func (r *RoomserverAliasAPI) GetAliasesForRoomID( return nil } -// GetCreatorIDForAlias implements alias.RoomserverAliasAPI -func (r *RoomserverAliasAPI) GetCreatorIDForAlias( +// GetCreatorIDForAlias implements alias.RoomserverInternalAPI +func (r *RoomserverInternalAPI) GetCreatorIDForAlias( ctx context.Context, - request *roomserverAPI.GetCreatorIDForAliasRequest, - response *roomserverAPI.GetCreatorIDForAliasResponse, + request *api.GetCreatorIDForAliasRequest, + response *api.GetCreatorIDForAliasResponse, ) error { // Look up the aliases in the database for the given RoomID creatorID, err := r.DB.GetCreatorIDForAlias(ctx, request.Alias) @@ -156,11 +148,11 @@ func (r *RoomserverAliasAPI) GetCreatorIDForAlias( return nil } -// RemoveRoomAlias implements alias.RoomserverAliasAPI -func (r *RoomserverAliasAPI) RemoveRoomAlias( +// RemoveRoomAlias implements alias.RoomserverInternalAPI +func (r *RoomserverInternalAPI) RemoveRoomAlias( ctx context.Context, - request *roomserverAPI.RemoveRoomAliasRequest, - response *roomserverAPI.RemoveRoomAliasResponse, + request *api.RemoveRoomAliasRequest, + response *api.RemoveRoomAliasResponse, ) error { // Look up the room ID in the database roomID, err := r.DB.GetRoomIDForAlias(ctx, request.Alias) @@ -186,7 +178,7 @@ type roomAliasesContent struct { // Build the updated m.room.aliases event to send to the room after addition or // removal of an alias -func (r *RoomserverAliasAPI) sendUpdatedAliasesEvent( +func (r *RoomserverInternalAPI) sendUpdatedAliasesEvent( ctx context.Context, userID string, roomID string, ) error { serverName := string(r.Cfg.Matrix.ServerName) @@ -222,12 +214,12 @@ func (r *RoomserverAliasAPI) sendUpdatedAliasesEvent( if len(eventsNeeded.Tuples()) == 0 { return errors.New("expecting state tuples for event builder, got none") } - req := roomserverAPI.QueryLatestEventsAndStateRequest{ + req := api.QueryLatestEventsAndStateRequest{ RoomID: roomID, StateToFetch: eventsNeeded.Tuples(), } - var res roomserverAPI.QueryLatestEventsAndStateResponse - if err = r.QueryAPI.QueryLatestEventsAndState(ctx, &req, &res); err != nil { + var res api.QueryLatestEventsAndStateResponse + if err = r.QueryLatestEventsAndState(ctx, &req, &res); err != nil { return err } builder.Depth = res.Depth @@ -263,91 +255,17 @@ func (r *RoomserverAliasAPI) sendUpdatedAliasesEvent( } // Create the request - ire := roomserverAPI.InputRoomEvent{ - Kind: roomserverAPI.KindNew, + ire := api.InputRoomEvent{ + Kind: api.KindNew, Event: event.Headered(roomVersion), AuthEventIDs: event.AuthEventIDs(), SendAsServer: serverName, } - inputReq := roomserverAPI.InputRoomEventsRequest{ - InputRoomEvents: []roomserverAPI.InputRoomEvent{ire}, + inputReq := api.InputRoomEventsRequest{ + InputRoomEvents: []api.InputRoomEvent{ire}, } - var inputRes roomserverAPI.InputRoomEventsResponse + var inputRes api.InputRoomEventsResponse // Send the request - return r.InputAPI.InputRoomEvents(ctx, &inputReq, &inputRes) -} - -// SetupHTTP adds the RoomserverAliasAPI handlers to the http.ServeMux. -func (r *RoomserverAliasAPI) SetupHTTP(servMux *http.ServeMux) { - servMux.Handle( - roomserverAPI.RoomserverSetRoomAliasPath, - common.MakeInternalAPI("setRoomAlias", func(req *http.Request) util.JSONResponse { - var request roomserverAPI.SetRoomAliasRequest - var response roomserverAPI.SetRoomAliasResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.SetRoomAlias(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - roomserverAPI.RoomserverGetRoomIDForAliasPath, - common.MakeInternalAPI("GetRoomIDForAlias", func(req *http.Request) util.JSONResponse { - var request roomserverAPI.GetRoomIDForAliasRequest - var response roomserverAPI.GetRoomIDForAliasResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.GetRoomIDForAlias(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - roomserverAPI.RoomserverGetCreatorIDForAliasPath, - common.MakeInternalAPI("GetCreatorIDForAlias", func(req *http.Request) util.JSONResponse { - var request roomserverAPI.GetCreatorIDForAliasRequest - var response roomserverAPI.GetCreatorIDForAliasResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.GetCreatorIDForAlias(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - roomserverAPI.RoomserverGetAliasesForRoomIDPath, - common.MakeInternalAPI("getAliasesForRoomID", func(req *http.Request) util.JSONResponse { - var request roomserverAPI.GetAliasesForRoomIDRequest - var response roomserverAPI.GetAliasesForRoomIDResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.GetAliasesForRoomID(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - roomserverAPI.RoomserverRemoveRoomAliasPath, - common.MakeInternalAPI("removeRoomAlias", func(req *http.Request) util.JSONResponse { - var request roomserverAPI.RemoveRoomAliasRequest - var response roomserverAPI.RemoveRoomAliasResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.RemoveRoomAlias(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) + return r.InputRoomEvents(ctx, &inputReq, &inputRes) } diff --git a/roomserver/internal/api.go b/roomserver/internal/api.go new file mode 100644 index 000000000..d1c443f24 --- /dev/null +++ b/roomserver/internal/api.go @@ -0,0 +1,287 @@ +package internal + +import ( + "encoding/json" + "net/http" + "sync" + + "github.com/Shopify/sarama" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/common/caching" + "github.com/matrix-org/dendrite/common/config" + fsAPI "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/storage" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" +) + +// RoomserverInternalAPI is an implementation of api.RoomserverInternalAPI +type RoomserverInternalAPI struct { + DB storage.Database + Cfg *config.Dendrite + Producer sarama.SyncProducer + ImmutableCache caching.ImmutableCache + ServerName gomatrixserverlib.ServerName + KeyRing gomatrixserverlib.JSONVerifier + FedClient *gomatrixserverlib.FederationClient + OutputRoomEventTopic string // Kafka topic for new output room events + mutex sync.Mutex // Protects calls to processRoomEvent + fsAPI fsAPI.FederationSenderInternalAPI +} + +// SetupHTTP adds the RoomserverInternalAPI handlers to the http.ServeMux. +// nolint: gocyclo +func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { + servMux.Handle(api.RoomserverInputRoomEventsPath, + common.MakeInternalAPI("inputRoomEvents", func(req *http.Request) util.JSONResponse { + var request api.InputRoomEventsRequest + var response api.InputRoomEventsResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := r.InputRoomEvents(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryLatestEventsAndStatePath, + common.MakeInternalAPI("queryLatestEventsAndState", func(req *http.Request) util.JSONResponse { + var request api.QueryLatestEventsAndStateRequest + var response api.QueryLatestEventsAndStateResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryLatestEventsAndState(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryStateAfterEventsPath, + common.MakeInternalAPI("queryStateAfterEvents", func(req *http.Request) util.JSONResponse { + var request api.QueryStateAfterEventsRequest + var response api.QueryStateAfterEventsResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryStateAfterEvents(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryEventsByIDPath, + common.MakeInternalAPI("queryEventsByID", func(req *http.Request) util.JSONResponse { + var request api.QueryEventsByIDRequest + var response api.QueryEventsByIDResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryEventsByID(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryMembershipForUserPath, + common.MakeInternalAPI("QueryMembershipForUser", func(req *http.Request) util.JSONResponse { + var request api.QueryMembershipForUserRequest + var response api.QueryMembershipForUserResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryMembershipForUser(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryMembershipsForRoomPath, + common.MakeInternalAPI("queryMembershipsForRoom", func(req *http.Request) util.JSONResponse { + var request api.QueryMembershipsForRoomRequest + var response api.QueryMembershipsForRoomResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryMembershipsForRoom(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryInvitesForUserPath, + common.MakeInternalAPI("queryInvitesForUser", func(req *http.Request) util.JSONResponse { + var request api.QueryInvitesForUserRequest + var response api.QueryInvitesForUserResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryInvitesForUser(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryServerAllowedToSeeEventPath, + common.MakeInternalAPI("queryServerAllowedToSeeEvent", func(req *http.Request) util.JSONResponse { + var request api.QueryServerAllowedToSeeEventRequest + var response api.QueryServerAllowedToSeeEventResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryServerAllowedToSeeEvent(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryMissingEventsPath, + common.MakeInternalAPI("queryMissingEvents", func(req *http.Request) util.JSONResponse { + var request api.QueryMissingEventsRequest + var response api.QueryMissingEventsResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryMissingEvents(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryStateAndAuthChainPath, + common.MakeInternalAPI("queryStateAndAuthChain", func(req *http.Request) util.JSONResponse { + var request api.QueryStateAndAuthChainRequest + var response api.QueryStateAndAuthChainResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryStateAndAuthChain(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryBackfillPath, + common.MakeInternalAPI("QueryBackfill", func(req *http.Request) util.JSONResponse { + var request api.QueryBackfillRequest + var response api.QueryBackfillResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryBackfill(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryRoomVersionCapabilitiesPath, + common.MakeInternalAPI("QueryRoomVersionCapabilities", func(req *http.Request) util.JSONResponse { + var request api.QueryRoomVersionCapabilitiesRequest + var response api.QueryRoomVersionCapabilitiesResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryRoomVersionCapabilities(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverQueryRoomVersionForRoomPath, + common.MakeInternalAPI("QueryRoomVersionForRoom", func(req *http.Request) util.JSONResponse { + var request api.QueryRoomVersionForRoomRequest + var response api.QueryRoomVersionForRoomResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryRoomVersionForRoom(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverSetRoomAliasPath, + common.MakeInternalAPI("setRoomAlias", func(req *http.Request) util.JSONResponse { + var request api.SetRoomAliasRequest + var response api.SetRoomAliasResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.SetRoomAlias(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverGetRoomIDForAliasPath, + common.MakeInternalAPI("GetRoomIDForAlias", func(req *http.Request) util.JSONResponse { + var request api.GetRoomIDForAliasRequest + var response api.GetRoomIDForAliasResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.GetRoomIDForAlias(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverGetCreatorIDForAliasPath, + common.MakeInternalAPI("GetCreatorIDForAlias", func(req *http.Request) util.JSONResponse { + var request api.GetCreatorIDForAliasRequest + var response api.GetCreatorIDForAliasResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.GetCreatorIDForAlias(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverGetAliasesForRoomIDPath, + common.MakeInternalAPI("getAliasesForRoomID", func(req *http.Request) util.JSONResponse { + var request api.GetAliasesForRoomIDRequest + var response api.GetAliasesForRoomIDResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.GetAliasesForRoomID(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + servMux.Handle( + api.RoomserverRemoveRoomAliasPath, + common.MakeInternalAPI("removeRoomAlias", func(req *http.Request) util.JSONResponse { + var request api.RemoveRoomAliasRequest + var response api.RemoveRoomAliasResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.RemoveRoomAlias(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) +} diff --git a/roomserver/input/input.go b/roomserver/internal/input.go similarity index 55% rename from roomserver/input/input.go rename to roomserver/internal/input.go index 20b6afc4b..19ebea660 100644 --- a/roomserver/input/input.go +++ b/roomserver/internal/input.go @@ -13,46 +13,27 @@ // limitations under the License. // Package input contains the code processes new room events -package input +package internal import ( "context" "encoding/json" - "net/http" - "sync" "github.com/Shopify/sarama" - "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/roomserver/storage" - "github.com/matrix-org/util" fsAPI "github.com/matrix-org/dendrite/federationsender/api" ) -// RoomserverInputAPI implements api.RoomserverInputAPI -type RoomserverInputAPI struct { - DB storage.Database - Producer sarama.SyncProducer - // The kafkaesque topic to output new room events to. - // This is the name used in kafka to identify the stream to write events to. - OutputRoomEventTopic string - // Protects calls to processRoomEvent - mutex sync.Mutex - // The federation sender API allows us to send federation - // requests from the new perform input requests, still TODO. - fsAPI fsAPI.FederationSenderInternalAPI -} - // SetFederationSenderInputAPI passes in a federation sender input API reference // so that we can avoid the chicken-and-egg problem of both the roomserver input API // and the federation sender input API being interdependent. -func (r *RoomserverInputAPI) SetFederationSenderAPI(fsAPI fsAPI.FederationSenderInternalAPI) { +func (r *RoomserverInternalAPI) SetFederationSenderAPI(fsAPI fsAPI.FederationSenderInternalAPI) { r.fsAPI = fsAPI } // WriteOutputEvents implements OutputRoomEventWriter -func (r *RoomserverInputAPI) WriteOutputEvents(roomID string, updates []api.OutputEvent) error { +func (r *RoomserverInternalAPI) WriteOutputEvents(roomID string, updates []api.OutputEvent) error { messages := make([]*sarama.ProducerMessage, len(updates)) for i := range updates { value, err := json.Marshal(updates[i]) @@ -68,8 +49,8 @@ func (r *RoomserverInputAPI) WriteOutputEvents(roomID string, updates []api.Outp return r.Producer.SendMessages(messages) } -// InputRoomEvents implements api.RoomserverInputAPI -func (r *RoomserverInputAPI) InputRoomEvents( +// InputRoomEvents implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) InputRoomEvents( ctx context.Context, request *api.InputRoomEventsRequest, response *api.InputRoomEventsResponse, @@ -89,20 +70,3 @@ func (r *RoomserverInputAPI) InputRoomEvents( } return nil } - -// SetupHTTP adds the RoomserverInputAPI handlers to the http.ServeMux. -func (r *RoomserverInputAPI) SetupHTTP(servMux *http.ServeMux) { - servMux.Handle(api.RoomserverInputRoomEventsPath, - common.MakeInternalAPI("inputRoomEvents", func(req *http.Request) util.JSONResponse { - var request api.InputRoomEventsRequest - var response api.InputRoomEventsResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.MessageResponse(http.StatusBadRequest, err.Error()) - } - if err := r.InputRoomEvents(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) -} diff --git a/roomserver/input/authevents.go b/roomserver/internal/input_authevents.go similarity index 99% rename from roomserver/input/authevents.go rename to roomserver/internal/input_authevents.go index 2c2e14b3a..e3828f566 100644 --- a/roomserver/input/authevents.go +++ b/roomserver/internal/input_authevents.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package input +package internal import ( "context" diff --git a/roomserver/input/authevents_test.go b/roomserver/internal/input_authevents_test.go similarity index 99% rename from roomserver/input/authevents_test.go rename to roomserver/internal/input_authevents_test.go index 0621a0842..6b981571b 100644 --- a/roomserver/input/authevents_test.go +++ b/roomserver/internal/input_authevents_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package input +package internal import ( "testing" diff --git a/roomserver/input/events.go b/roomserver/internal/input_events.go similarity index 99% rename from roomserver/input/events.go rename to roomserver/internal/input_events.go index 69828d9f9..6da63716c 100644 --- a/roomserver/input/events.go +++ b/roomserver/internal/input_events.go @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package input +package internal import ( "context" diff --git a/roomserver/input/latest_events.go b/roomserver/internal/input_latest_events.go similarity index 99% rename from roomserver/input/latest_events.go rename to roomserver/internal/input_latest_events.go index cac3968d9..42be0f408 100644 --- a/roomserver/input/latest_events.go +++ b/roomserver/internal/input_latest_events.go @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package input +package internal import ( "bytes" diff --git a/roomserver/input/membership.go b/roomserver/internal/input_membership.go similarity index 99% rename from roomserver/input/membership.go rename to roomserver/internal/input_membership.go index 351e63d61..cba75b4fc 100644 --- a/roomserver/input/membership.go +++ b/roomserver/internal/input_membership.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package input +package internal import ( "context" diff --git a/roomserver/query/query.go b/roomserver/internal/query.go similarity index 74% rename from roomserver/query/query.go rename to roomserver/internal/query.go index 6778ac280..98adc24b3 100644 --- a/roomserver/query/query.go +++ b/roomserver/internal/query.go @@ -14,16 +14,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -package query +package internal import ( "context" - "encoding/json" "fmt" - "net/http" - "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/common/caching" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/auth" "github.com/matrix-org/dendrite/roomserver/state" @@ -35,17 +31,8 @@ import ( "github.com/sirupsen/logrus" ) -// RoomserverQueryAPI is an implementation of api.RoomserverQueryAPI -type RoomserverQueryAPI struct { - DB storage.Database - ImmutableCache caching.ImmutableCache - ServerName gomatrixserverlib.ServerName - KeyRing gomatrixserverlib.JSONVerifier - FedClient *gomatrixserverlib.FederationClient -} - -// QueryLatestEventsAndState implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryLatestEventsAndState( +// QueryLatestEventsAndState implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) QueryLatestEventsAndState( ctx context.Context, request *api.QueryLatestEventsAndStateRequest, response *api.QueryLatestEventsAndStateResponse, @@ -104,8 +91,8 @@ func (r *RoomserverQueryAPI) QueryLatestEventsAndState( return nil } -// QueryStateAfterEvents implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryStateAfterEvents( +// QueryStateAfterEvents implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) QueryStateAfterEvents( ctx context.Context, request *api.QueryStateAfterEventsRequest, response *api.QueryStateAfterEventsResponse, @@ -160,8 +147,8 @@ func (r *RoomserverQueryAPI) QueryStateAfterEvents( return nil } -// QueryEventsByID implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryEventsByID( +// QueryEventsByID implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) QueryEventsByID( ctx context.Context, request *api.QueryEventsByIDRequest, response *api.QueryEventsByIDResponse, @@ -195,7 +182,7 @@ func (r *RoomserverQueryAPI) QueryEventsByID( return nil } -func (r *RoomserverQueryAPI) loadStateEvents( +func (r *RoomserverInternalAPI) loadStateEvents( ctx context.Context, stateEntries []types.StateEntry, ) ([]gomatrixserverlib.Event, error) { eventNIDs := make([]types.EventNID, len(stateEntries)) @@ -205,7 +192,7 @@ func (r *RoomserverQueryAPI) loadStateEvents( return r.loadEvents(ctx, eventNIDs) } -func (r *RoomserverQueryAPI) loadEvents( +func (r *RoomserverInternalAPI) loadEvents( ctx context.Context, eventNIDs []types.EventNID, ) ([]gomatrixserverlib.Event, error) { stateEvents, err := r.DB.Events(ctx, eventNIDs) @@ -220,8 +207,8 @@ func (r *RoomserverQueryAPI) loadEvents( return result, nil } -// QueryMembershipForUser implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryMembershipForUser( +// QueryMembershipForUser implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) QueryMembershipForUser( ctx context.Context, request *api.QueryMembershipForUserRequest, response *api.QueryMembershipForUserResponse, @@ -251,8 +238,8 @@ func (r *RoomserverQueryAPI) QueryMembershipForUser( return nil } -// QueryMembershipsForRoom implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryMembershipsForRoom( +// QueryMembershipsForRoom implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) QueryMembershipsForRoom( ctx context.Context, request *api.QueryMembershipsForRoomRequest, response *api.QueryMembershipsForRoomResponse, @@ -366,8 +353,8 @@ func getMembershipsAtState( return events, nil } -// QueryInvitesForUser implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryInvitesForUser( +// QueryInvitesForUser implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) QueryInvitesForUser( ctx context.Context, request *api.QueryInvitesForUserRequest, response *api.QueryInvitesForUserResponse, @@ -400,8 +387,8 @@ func (r *RoomserverQueryAPI) QueryInvitesForUser( return nil } -// QueryServerAllowedToSeeEvent implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryServerAllowedToSeeEvent( +// QueryServerAllowedToSeeEvent implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) QueryServerAllowedToSeeEvent( ctx context.Context, request *api.QueryServerAllowedToSeeEventRequest, response *api.QueryServerAllowedToSeeEventResponse, @@ -424,7 +411,7 @@ func (r *RoomserverQueryAPI) QueryServerAllowedToSeeEvent( return } -func (r *RoomserverQueryAPI) checkServerAllowedToSeeEvent( +func (r *RoomserverInternalAPI) checkServerAllowedToSeeEvent( ctx context.Context, eventID string, serverName gomatrixserverlib.ServerName, isServerInRoom bool, ) (bool, error) { roomState := state.NewStateResolution(r.DB) @@ -443,8 +430,8 @@ func (r *RoomserverQueryAPI) checkServerAllowedToSeeEvent( return auth.IsServerAllowed(serverName, isServerInRoom, stateAtEvent), nil } -// QueryMissingEvents implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryMissingEvents( +// QueryMissingEvents implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) QueryMissingEvents( ctx context.Context, request *api.QueryMissingEventsRequest, response *api.QueryMissingEventsResponse, @@ -489,7 +476,7 @@ func (r *RoomserverQueryAPI) QueryMissingEvents( } // QueryBackfill implements api.RoomServerQueryAPI -func (r *RoomserverQueryAPI) QueryBackfill( +func (r *RoomserverInternalAPI) QueryBackfill( ctx context.Context, request *api.QueryBackfillRequest, response *api.QueryBackfillResponse, @@ -542,7 +529,7 @@ func (r *RoomserverQueryAPI) QueryBackfill( return err } -func (r *RoomserverQueryAPI) backfillViaFederation(ctx context.Context, req *api.QueryBackfillRequest, res *api.QueryBackfillResponse) error { +func (r *RoomserverInternalAPI) backfillViaFederation(ctx context.Context, req *api.QueryBackfillRequest, res *api.QueryBackfillResponse) error { roomVer, err := r.DB.GetRoomVersionForRoom(ctx, req.RoomID) if err != nil { return fmt.Errorf("backfillViaFederation: unknown room version for room %s : %w", req.RoomID, err) @@ -600,7 +587,7 @@ func (r *RoomserverQueryAPI) backfillViaFederation(ctx context.Context, req *api return nil } -func (r *RoomserverQueryAPI) isServerCurrentlyInRoom(ctx context.Context, serverName gomatrixserverlib.ServerName, roomID string) (bool, error) { +func (r *RoomserverInternalAPI) isServerCurrentlyInRoom(ctx context.Context, serverName gomatrixserverlib.ServerName, roomID string) (bool, error) { roomNID, err := r.DB.RoomNID(ctx, roomID) if err != nil { return false, err @@ -624,7 +611,7 @@ func (r *RoomserverQueryAPI) isServerCurrentlyInRoom(ctx context.Context, server // fetchAndStoreMissingEvents does a best-effort fetch and store of missing events specified in stateIDs. Returns no error as it is just // best effort. -func (r *RoomserverQueryAPI) fetchAndStoreMissingEvents(ctx context.Context, roomVer gomatrixserverlib.RoomVersion, +func (r *RoomserverInternalAPI) fetchAndStoreMissingEvents(ctx context.Context, roomVer gomatrixserverlib.RoomVersion, backfillRequester *backfillRequester, stateIDs []string) { servers := backfillRequester.servers @@ -684,7 +671,7 @@ func (r *RoomserverQueryAPI) fetchAndStoreMissingEvents(ctx context.Context, roo // TODO: Remove this when we have tests to assert correctness of this function // nolint:gocyclo -func (r *RoomserverQueryAPI) scanEventTree( +func (r *RoomserverInternalAPI) scanEventTree( ctx context.Context, front []string, visited map[string]bool, limit int, serverName gomatrixserverlib.ServerName, ) ([]types.EventNID, error) { @@ -777,8 +764,8 @@ BFSLoop: return resultNIDs, err } -// QueryStateAndAuthChain implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryStateAndAuthChain( +// QueryStateAndAuthChain implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) QueryStateAndAuthChain( ctx context.Context, request *api.QueryStateAndAuthChainRequest, response *api.QueryStateAndAuthChainResponse, @@ -837,7 +824,7 @@ func (r *RoomserverQueryAPI) QueryStateAndAuthChain( return err } -func (r *RoomserverQueryAPI) loadStateAtEventIDs(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.Event, error) { +func (r *RoomserverInternalAPI) loadStateAtEventIDs(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.Event, error) { roomState := state.NewStateResolution(r.DB) prevStates, err := r.DB.StateAtEventIDs(ctx, eventIDs) if err != nil { @@ -942,8 +929,8 @@ func persistEvents(ctx context.Context, db storage.Database, events []gomatrixse return roomNID, backfilledEventMap } -// QueryRoomVersionCapabilities implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryRoomVersionCapabilities( +// QueryRoomVersionCapabilities implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) QueryRoomVersionCapabilities( ctx context.Context, request *api.QueryRoomVersionCapabilitiesRequest, response *api.QueryRoomVersionCapabilitiesResponse, @@ -960,8 +947,8 @@ func (r *RoomserverQueryAPI) QueryRoomVersionCapabilities( return nil } -// QueryRoomVersionCapabilities implements api.RoomserverQueryAPI -func (r *RoomserverQueryAPI) QueryRoomVersionForRoom( +// QueryRoomVersionCapabilities implements api.RoomserverInternalAPI +func (r *RoomserverInternalAPI) QueryRoomVersionForRoom( ctx context.Context, request *api.QueryRoomVersionForRoomRequest, response *api.QueryRoomVersionForRoomResponse, @@ -979,176 +966,3 @@ func (r *RoomserverQueryAPI) QueryRoomVersionForRoom( r.ImmutableCache.StoreRoomVersion(request.RoomID, response.RoomVersion) return nil } - -// SetupHTTP adds the RoomserverQueryAPI handlers to the http.ServeMux. -// nolint: gocyclo -func (r *RoomserverQueryAPI) SetupHTTP(servMux *http.ServeMux) { - servMux.Handle( - api.RoomserverQueryLatestEventsAndStatePath, - common.MakeInternalAPI("queryLatestEventsAndState", func(req *http.Request) util.JSONResponse { - var request api.QueryLatestEventsAndStateRequest - var response api.QueryLatestEventsAndStateResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryLatestEventsAndState(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryStateAfterEventsPath, - common.MakeInternalAPI("queryStateAfterEvents", func(req *http.Request) util.JSONResponse { - var request api.QueryStateAfterEventsRequest - var response api.QueryStateAfterEventsResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryStateAfterEvents(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryEventsByIDPath, - common.MakeInternalAPI("queryEventsByID", func(req *http.Request) util.JSONResponse { - var request api.QueryEventsByIDRequest - var response api.QueryEventsByIDResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryEventsByID(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryMembershipForUserPath, - common.MakeInternalAPI("QueryMembershipForUser", func(req *http.Request) util.JSONResponse { - var request api.QueryMembershipForUserRequest - var response api.QueryMembershipForUserResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryMembershipForUser(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryMembershipsForRoomPath, - common.MakeInternalAPI("queryMembershipsForRoom", func(req *http.Request) util.JSONResponse { - var request api.QueryMembershipsForRoomRequest - var response api.QueryMembershipsForRoomResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryMembershipsForRoom(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryInvitesForUserPath, - common.MakeInternalAPI("queryInvitesForUser", func(req *http.Request) util.JSONResponse { - var request api.QueryInvitesForUserRequest - var response api.QueryInvitesForUserResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryInvitesForUser(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryServerAllowedToSeeEventPath, - common.MakeInternalAPI("queryServerAllowedToSeeEvent", func(req *http.Request) util.JSONResponse { - var request api.QueryServerAllowedToSeeEventRequest - var response api.QueryServerAllowedToSeeEventResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryServerAllowedToSeeEvent(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryMissingEventsPath, - common.MakeInternalAPI("queryMissingEvents", func(req *http.Request) util.JSONResponse { - var request api.QueryMissingEventsRequest - var response api.QueryMissingEventsResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryMissingEvents(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryStateAndAuthChainPath, - common.MakeInternalAPI("queryStateAndAuthChain", func(req *http.Request) util.JSONResponse { - var request api.QueryStateAndAuthChainRequest - var response api.QueryStateAndAuthChainResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryStateAndAuthChain(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryBackfillPath, - common.MakeInternalAPI("QueryBackfill", func(req *http.Request) util.JSONResponse { - var request api.QueryBackfillRequest - var response api.QueryBackfillResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryBackfill(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryRoomVersionCapabilitiesPath, - common.MakeInternalAPI("QueryRoomVersionCapabilities", func(req *http.Request) util.JSONResponse { - var request api.QueryRoomVersionCapabilitiesRequest - var response api.QueryRoomVersionCapabilitiesResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryRoomVersionCapabilities(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - servMux.Handle( - api.RoomserverQueryRoomVersionForRoomPath, - common.MakeInternalAPI("QueryRoomVersionForRoom", func(req *http.Request) util.JSONResponse { - var request api.QueryRoomVersionForRoomRequest - var response api.QueryRoomVersionForRoomResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryRoomVersionForRoom(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) -} diff --git a/roomserver/query/backfill.go b/roomserver/internal/query_backfill.go similarity index 99% rename from roomserver/query/backfill.go rename to roomserver/internal/query_backfill.go index f518de3e8..d42038e74 100644 --- a/roomserver/query/backfill.go +++ b/roomserver/internal/query_backfill.go @@ -1,4 +1,4 @@ -package query +package internal import ( "context" diff --git a/roomserver/query/query_test.go b/roomserver/internal/query_test.go similarity index 96% rename from roomserver/query/query_test.go rename to roomserver/internal/query_test.go index 8fb6a082e..211ab5083 100644 --- a/roomserver/query/query_test.go +++ b/roomserver/internal/query_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package query +package internal import ( "context" @@ -24,7 +24,7 @@ import ( "github.com/matrix-org/gomatrixserverlib" ) -// used to implement RoomserverQueryAPIEventDB to test getAuthChain +// used to implement RoomserverInternalAPIEventDB to test getAuthChain type getEventDB struct { eventMap map[string]gomatrixserverlib.Event } @@ -79,7 +79,7 @@ func (db *getEventDB) addFakeEvents(graph map[string][]string) error { return nil } -// EventsFromIDs implements RoomserverQueryAPIEventDB +// EventsFromIDs implements RoomserverInternalAPIEventDB func (db *getEventDB) EventsFromIDs(ctx context.Context, eventIDs []string) (res []types.Event, err error) { for _, evID := range eventIDs { res = append(res, types.Event{ diff --git a/roomserver/roomserver.go b/roomserver/roomserver.go index 6fb2caff9..916e25fbd 100644 --- a/roomserver/roomserver.go +++ b/roomserver/roomserver.go @@ -20,12 +20,8 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" - asQuery "github.com/matrix-org/dendrite/appservice/query" - "github.com/matrix-org/dendrite/common/basecomponent" - "github.com/matrix-org/dendrite/roomserver/alias" - "github.com/matrix-org/dendrite/roomserver/input" - "github.com/matrix-org/dendrite/roomserver/query" + "github.com/matrix-org/dendrite/roomserver/internal" "github.com/matrix-org/dendrite/roomserver/storage" "github.com/sirupsen/logrus" ) @@ -35,45 +31,27 @@ import ( // allowing other components running in the same process to hit the query the // APIs directly instead of having to use HTTP. func SetupRoomServerComponent( - base *basecomponent.BaseDendrite, keyRing gomatrixserverlib.JSONVerifier, + base *basecomponent.BaseDendrite, + keyRing gomatrixserverlib.JSONVerifier, fedClient *gomatrixserverlib.FederationClient, -) (api.RoomserverAliasAPI, api.RoomserverInputAPI, api.RoomserverQueryAPI) { +) api.RoomserverInternalAPI { roomserverDB, err := storage.Open(string(base.Cfg.Database.RoomServer)) if err != nil { logrus.WithError(err).Panicf("failed to connect to room server db") } - inputAPI := input.RoomserverInputAPI{ + internalAPI := internal.RoomserverInternalAPI{ DB: roomserverDB, + Cfg: base.Cfg, Producer: base.KafkaProducer, OutputRoomEventTopic: string(base.Cfg.Kafka.Topics.OutputRoomEvent), + ImmutableCache: base.ImmutableCache, + ServerName: base.Cfg.Matrix.ServerName, + FedClient: fedClient, + KeyRing: keyRing, } - inputAPI.SetupHTTP(http.DefaultServeMux) + internalAPI.SetupHTTP(http.DefaultServeMux) - queryAPI := query.RoomserverQueryAPI{ - DB: roomserverDB, - ImmutableCache: base.ImmutableCache, - ServerName: base.Cfg.Matrix.ServerName, - FedClient: fedClient, - // TODO: We should have a key server so we don't keep adding components - // which talk to the same DB. - KeyRing: keyRing, - } - - queryAPI.SetupHTTP(http.DefaultServeMux) - - asAPI := asQuery.AppServiceQueryAPI{Cfg: base.Cfg} - - aliasAPI := alias.RoomserverAliasAPI{ - DB: roomserverDB, - Cfg: base.Cfg, - InputAPI: &inputAPI, - QueryAPI: &queryAPI, - AppserviceAPI: &asAPI, - } - - aliasAPI.SetupHTTP(http.DefaultServeMux) - - return &aliasAPI, &inputAPI, &queryAPI + return &internalAPI } diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index 1d512972d..987cc5df6 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -32,10 +32,10 @@ import ( // OutputRoomEventConsumer consumes events that originated in the room server. type OutputRoomEventConsumer struct { - roomServerConsumer *common.ContinualConsumer - db storage.Database - notifier *sync.Notifier - query api.RoomserverQueryAPI + rsAPI api.RoomserverInternalAPI + rsConsumer *common.ContinualConsumer + db storage.Database + notifier *sync.Notifier } // NewOutputRoomEventConsumer creates a new OutputRoomEventConsumer. Call Start() to begin consuming from room servers. @@ -44,7 +44,7 @@ func NewOutputRoomEventConsumer( kafkaConsumer sarama.Consumer, n *sync.Notifier, store storage.Database, - queryAPI api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, ) *OutputRoomEventConsumer { consumer := common.ContinualConsumer{ @@ -53,10 +53,10 @@ func NewOutputRoomEventConsumer( PartitionStore: store, } s := &OutputRoomEventConsumer{ - roomServerConsumer: &consumer, - db: store, - notifier: n, - query: queryAPI, + rsConsumer: &consumer, + db: store, + notifier: n, + rsAPI: rsAPI, } consumer.ProcessMessage = s.onMessage @@ -65,7 +65,7 @@ func NewOutputRoomEventConsumer( // Start consuming from room servers func (s *OutputRoomEventConsumer) Start() error { - return s.roomServerConsumer.Start() + return s.rsConsumer.Start() } // onMessage is called when the sync server receives a new event from the room server output log. @@ -226,7 +226,7 @@ func (s *OutputRoomEventConsumer) lookupStateEvents( // from the roomserver using the query API. eventReq := api.QueryEventsByIDRequest{EventIDs: missing} var eventResp api.QueryEventsByIDResponse - if err := s.query.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil { + if err := s.rsAPI.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil { return nil, err } diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index c48414ab6..5105e224a 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -34,7 +34,7 @@ import ( type messagesReq struct { ctx context.Context db storage.Database - queryAPI api.RoomserverQueryAPI + rsAPI api.RoomserverInternalAPI federation *gomatrixserverlib.FederationClient cfg *config.Dendrite roomID string @@ -59,7 +59,7 @@ const defaultMessagesLimit = 10 func OnIncomingMessagesRequest( req *http.Request, db storage.Database, roomID string, federation *gomatrixserverlib.FederationClient, - queryAPI api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, cfg *config.Dendrite, ) util.JSONResponse { var err error @@ -135,7 +135,7 @@ func OnIncomingMessagesRequest( mReq := messagesReq{ ctx: req.Context(), db: db, - queryAPI: queryAPI, + rsAPI: rsAPI, federation: federation, cfg: cfg, roomID: roomID, @@ -360,7 +360,7 @@ func (r *messagesReq) handleNonEmptyEventsSlice(streamEvents []types.StreamEvent // the room or sending the request. func (r *messagesReq) backfill(roomID string, fromEventIDs []string, limit int) ([]gomatrixserverlib.HeaderedEvent, error) { var res api.QueryBackfillResponse - err := r.queryAPI.QueryBackfill(context.Background(), &api.QueryBackfillRequest{ + err := r.rsAPI.QueryBackfill(context.Background(), &api.QueryBackfillRequest{ RoomID: roomID, EarliestEventsIDs: fromEventIDs, Limit: limit, diff --git a/syncapi/routing/routing.go b/syncapi/routing/routing.go index 9078b87ff..5a36a279f 100644 --- a/syncapi/routing/routing.go +++ b/syncapi/routing/routing.go @@ -40,7 +40,7 @@ const pathPrefixR0 = "/_matrix/client/r0" func Setup( apiMux *mux.Router, srp *sync.RequestPool, syncDB storage.Database, deviceDB devices.Database, federation *gomatrixserverlib.FederationClient, - queryAPI api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, cfg *config.Dendrite, ) { r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter() @@ -61,6 +61,6 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return OnIncomingMessagesRequest(req, syncDB, vars["roomID"], federation, queryAPI, cfg) + return OnIncomingMessagesRequest(req, syncDB, vars["roomID"], federation, rsAPI, cfg) })).Methods(http.MethodGet, http.MethodOptions) } diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go index 1535d2b13..5ab1ec7c8 100644 --- a/syncapi/syncapi.go +++ b/syncapi/syncapi.go @@ -38,7 +38,7 @@ func SetupSyncAPIComponent( base *basecomponent.BaseDendrite, deviceDB devices.Database, accountsDB accounts.Database, - queryAPI api.RoomserverQueryAPI, + rsAPI api.RoomserverInternalAPI, federation *gomatrixserverlib.FederationClient, cfg *config.Dendrite, ) { @@ -61,7 +61,7 @@ func SetupSyncAPIComponent( requestPool := sync.NewRequestPool(syncDB, notifier, accountsDB) roomConsumer := consumers.NewOutputRoomEventConsumer( - base.Cfg, base.KafkaConsumer, notifier, syncDB, queryAPI, + base.Cfg, base.KafkaConsumer, notifier, syncDB, rsAPI, ) if err = roomConsumer.Start(); err != nil { logrus.WithError(err).Panicf("failed to start room server consumer") @@ -81,5 +81,5 @@ func SetupSyncAPIComponent( logrus.WithError(err).Panicf("failed to start typing server consumer") } - routing.Setup(base.APIMux, requestPool, syncDB, deviceDB, federation, queryAPI, cfg) + routing.Setup(base.APIMux, requestPool, syncDB, deviceDB, federation, rsAPI, cfg) } From b28674435e3024bf0e4723a9cf53180607f2045e Mon Sep 17 00:00:00 2001 From: Kegsay Date: Fri, 1 May 2020 11:01:34 +0100 Subject: [PATCH 021/200] Correctly generate backpagination tokens for events which have the same depth (#996) * Correctly generate backpagination tokens for events which have the same depth With tests. Unfortunately the code around here is hard to understand. There will be a subsequent PR which fixes this up now that we have a test harness in place. * Add postgres impl * More linting * Fix psql statement so it actually works --- syncapi/routing/messages.go | 14 +-- syncapi/storage/interface.go | 6 +- .../postgres/output_room_events_table.go | 27 +++-- .../output_room_events_topology_table.go | 41 ++++--- syncapi/storage/postgres/syncserver.go | 18 ++- .../sqlite3/output_room_events_table.go | 29 +++-- .../output_room_events_topology_table.go | 43 ++++--- syncapi/storage/sqlite3/syncserver.go | 22 ++-- syncapi/storage/storage_test.go | 108 +++++++++++++++++- syncapi/types/types.go | 10 +- 10 files changed, 238 insertions(+), 80 deletions(-) diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index 5105e224a..d7e39704d 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -229,14 +229,14 @@ func (r *messagesReq) retrieveEvents() ( // change the way topological positions are defined (as depth isn't the most // reliable way to define it), it would be easier and less troublesome to // only have to change it in one place, i.e. the database. - startPos, err := r.db.EventPositionInTopology( + startPos, startStreamPos, err := r.db.EventPositionInTopology( r.ctx, events[0].EventID(), ) if err != nil { err = fmt.Errorf("EventPositionInTopology: for start event %s: %w", events[0].EventID(), err) return } - endPos, err := r.db.EventPositionInTopology( + endPos, endStreamPos, err := r.db.EventPositionInTopology( r.ctx, events[len(events)-1].EventID(), ) if err != nil { @@ -246,10 +246,10 @@ func (r *messagesReq) retrieveEvents() ( // Generate pagination tokens to send to the client using the positions // retrieved previously. start = types.NewPaginationTokenFromTypeAndPosition( - types.PaginationTokenTypeTopology, startPos, 0, + types.PaginationTokenTypeTopology, startPos, startStreamPos, ) end = types.NewPaginationTokenFromTypeAndPosition( - types.PaginationTokenTypeTopology, endPos, 0, + types.PaginationTokenTypeTopology, endPos, endStreamPos, ) if r.backwardOrdering { @@ -407,13 +407,13 @@ func setToDefault( // go 1 earlier than the first event so we correctly fetch the earliest event to = types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, 0, 0) } else { - var pos types.StreamPosition - pos, err = db.MaxTopologicalPosition(ctx, roomID) + var pos, stream types.StreamPosition + pos, stream, err = db.MaxTopologicalPosition(ctx, roomID) if err != nil { return } - to = types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, pos, 0) + to = types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, pos, stream) } return diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index bd9504db8..7d6376438 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -91,8 +91,8 @@ type Database interface { // GetEventsInRange retrieves all of the events on a given ordering using the // given extremities and limit. GetEventsInRange(ctx context.Context, from, to *types.PaginationToken, roomID string, limit int, backwardOrdering bool) (events []types.StreamEvent, err error) - // EventPositionInTopology returns the depth of the given event. - EventPositionInTopology(ctx context.Context, eventID string) (types.StreamPosition, error) + // EventPositionInTopology returns the depth and stream position of the given event. + EventPositionInTopology(ctx context.Context, eventID string) (depth types.StreamPosition, stream types.StreamPosition, err error) // EventsAtTopologicalPosition returns all of the events matching a given // position in the topology of a given room. EventsAtTopologicalPosition(ctx context.Context, roomID string, pos types.StreamPosition) ([]types.StreamEvent, error) @@ -100,7 +100,7 @@ type Database interface { // extremities we know of for a given room. BackwardExtremitiesForRoom(ctx context.Context, roomID string) (backwardExtremities []string, err error) // MaxTopologicalPosition returns the highest topological position for a given room. - MaxTopologicalPosition(ctx context.Context, roomID string) (types.StreamPosition, error) + MaxTopologicalPosition(ctx context.Context, roomID string) (depth types.StreamPosition, stream types.StreamPosition, err error) // StreamEventsToEvents converts streamEvent to Event. If device is non-nil and // matches the streamevent.transactionID device then the transaction ID gets // added to the unsigned section of the output event. diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go index 0b53dfa9e..769823e92 100644 --- a/syncapi/storage/postgres/output_room_events_table.go +++ b/syncapi/storage/postgres/output_room_events_table.go @@ -94,6 +94,9 @@ const selectEarlyEventsSQL = "" + " WHERE room_id = $1 AND id > $2 AND id <= $3" + " ORDER BY id ASC LIMIT $4" +const selectStreamPositionForEventIDSQL = "" + + "SELECT id FROM syncapi_output_room_events WHERE event_id = $1" + const selectMaxEventIDSQL = "" + "SELECT MAX(id) FROM syncapi_output_room_events" @@ -111,13 +114,14 @@ const selectStateInRangeSQL = "" + " LIMIT $8" type outputRoomEventsStatements struct { - insertEventStmt *sql.Stmt - selectEventsStmt *sql.Stmt - selectMaxEventIDStmt *sql.Stmt - selectRecentEventsStmt *sql.Stmt - selectRecentEventsForSyncStmt *sql.Stmt - selectEarlyEventsStmt *sql.Stmt - selectStateInRangeStmt *sql.Stmt + insertEventStmt *sql.Stmt + selectEventsStmt *sql.Stmt + selectMaxEventIDStmt *sql.Stmt + selectRecentEventsStmt *sql.Stmt + selectRecentEventsForSyncStmt *sql.Stmt + selectEarlyEventsStmt *sql.Stmt + selectStateInRangeStmt *sql.Stmt + selectStreamPositionForEventIDStmt *sql.Stmt } func (s *outputRoomEventsStatements) prepare(db *sql.DB) (err error) { @@ -146,9 +150,18 @@ func (s *outputRoomEventsStatements) prepare(db *sql.DB) (err error) { if s.selectStateInRangeStmt, err = db.Prepare(selectStateInRangeSQL); err != nil { return } + if s.selectStreamPositionForEventIDStmt, err = db.Prepare(selectStreamPositionForEventIDSQL); err != nil { + return + } return } +func (s *outputRoomEventsStatements) selectStreamPositionForEventID(ctx context.Context, eventID string) (types.StreamPosition, error) { + var id int64 + err := s.selectStreamPositionForEventIDStmt.QueryRowContext(ctx, eventID).Scan(&id) + return types.StreamPosition(id), err +} + // selectStateInRange returns the state events between the two given PDU stream positions, exclusive of oldPos, inclusive of newPos. // Results are bucketed based on the room ID. If the same state is overwritten multiple times between the // two positions, only the most recent state is returned. diff --git a/syncapi/storage/postgres/output_room_events_topology_table.go b/syncapi/storage/postgres/output_room_events_topology_table.go index 280d4ec39..f77365c8d 100644 --- a/syncapi/storage/postgres/output_room_events_topology_table.go +++ b/syncapi/storage/postgres/output_room_events_topology_table.go @@ -32,35 +32,44 @@ CREATE TABLE IF NOT EXISTS syncapi_output_room_events_topology ( -- The place of the event in the room's topology. This can usually be determined -- from the event's depth. topological_position BIGINT NOT NULL, + stream_position BIGINT NOT NULL, -- The 'room_id' key for the event. room_id TEXT NOT NULL ); -- The topological order will be used in events selection and ordering -CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_topological_position_idx ON syncapi_output_room_events_topology(topological_position, room_id); +CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_topological_position_idx ON syncapi_output_room_events_topology(topological_position, stream_position, room_id); ` const insertEventInTopologySQL = "" + - "INSERT INTO syncapi_output_room_events_topology (event_id, topological_position, room_id)" + - " VALUES ($1, $2, $3)" + - " ON CONFLICT (topological_position, room_id) DO UPDATE SET event_id = $1" + "INSERT INTO syncapi_output_room_events_topology (event_id, topological_position, room_id, stream_position)" + + " VALUES ($1, $2, $3, $4)" + + " ON CONFLICT (topological_position, stream_position, room_id) DO UPDATE SET event_id = $1" const selectEventIDsInRangeASCSQL = "" + "SELECT event_id FROM syncapi_output_room_events_topology" + - " WHERE room_id = $1 AND topological_position > $2 AND topological_position <= $3" + - " ORDER BY topological_position ASC LIMIT $4" + " WHERE room_id = $1 AND" + + "(topological_position > $2 AND topological_position < $3) OR" + + "(topological_position = $4 AND stream_position <= $5)" + + " ORDER BY topological_position ASC, stream_position ASC LIMIT $6" const selectEventIDsInRangeDESCSQL = "" + "SELECT event_id FROM syncapi_output_room_events_topology" + - " WHERE room_id = $1 AND topological_position > $2 AND topological_position <= $3" + - " ORDER BY topological_position DESC LIMIT $4" + " WHERE room_id = $1 AND" + + "(topological_position > $2 AND topological_position < $3) OR" + + "(topological_position = $4 AND stream_position <= $5)" + + " ORDER BY topological_position DESC, stream_position DESC LIMIT $6" const selectPositionInTopologySQL = "" + "SELECT topological_position FROM syncapi_output_room_events_topology" + " WHERE event_id = $1" + // Select the max topological position for the room, then sort by stream position and take the highest, + // returning both topological and stream positions. const selectMaxPositionInTopologySQL = "" + - "SELECT MAX(topological_position) FROM syncapi_output_room_events_topology" + - " WHERE room_id = $1" + "SELECT topological_position, stream_position FROM syncapi_output_room_events_topology" + + " WHERE topological_position=(" + + "SELECT MAX(topological_position) FROM syncapi_output_room_events_topology WHERE room_id=$1" + + ") ORDER BY stream_position DESC LIMIT 1" const selectEventIDsFromPositionSQL = "" + "SELECT event_id FROM syncapi_output_room_events_topology" + @@ -104,10 +113,10 @@ func (s *outputRoomEventsTopologyStatements) prepare(db *sql.DB) (err error) { // insertEventInTopology inserts the given event in the room's topology, based // on the event's depth. func (s *outputRoomEventsTopologyStatements) insertEventInTopology( - ctx context.Context, event *gomatrixserverlib.HeaderedEvent, + ctx context.Context, event *gomatrixserverlib.HeaderedEvent, pos types.StreamPosition, ) (err error) { _, err = s.insertEventInTopologyStmt.ExecContext( - ctx, event.EventID(), event.Depth(), event.RoomID(), + ctx, event.EventID(), event.Depth(), event.RoomID(), pos, ) return } @@ -116,7 +125,7 @@ func (s *outputRoomEventsTopologyStatements) insertEventInTopology( // given range in a given room's topological order. // Returns an empty slice if no events match the given range. func (s *outputRoomEventsTopologyStatements) selectEventIDsInRange( - ctx context.Context, roomID string, fromPos, toPos types.StreamPosition, + ctx context.Context, roomID string, fromPos, toPos, toMicroPos types.StreamPosition, limit int, chronologicalOrder bool, ) (eventIDs []string, err error) { // Decide on the selection's order according to whether chronological order @@ -129,7 +138,7 @@ func (s *outputRoomEventsTopologyStatements) selectEventIDsInRange( } // Query the event IDs. - rows, err := stmt.QueryContext(ctx, roomID, fromPos, toPos, limit) + rows, err := stmt.QueryContext(ctx, roomID, fromPos, toPos, toPos, toMicroPos, limit) if err == sql.ErrNoRows { // If no event matched the request, return an empty slice. return []string{}, nil @@ -161,8 +170,8 @@ func (s *outputRoomEventsTopologyStatements) selectPositionInTopology( func (s *outputRoomEventsTopologyStatements) selectMaxPositionInTopology( ctx context.Context, roomID string, -) (pos types.StreamPosition, err error) { - err = s.selectMaxPositionInTopologyStmt.QueryRowContext(ctx, roomID).Scan(&pos) +) (pos types.StreamPosition, spos types.StreamPosition, err error) { + err = s.selectMaxPositionInTopologyStmt.QueryRowContext(ctx, roomID).Scan(&pos, &spos) return } diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go index ef9707397..744ae7b8d 100644 --- a/syncapi/storage/postgres/syncserver.go +++ b/syncapi/storage/postgres/syncserver.go @@ -159,7 +159,7 @@ func (d *SyncServerDatasource) WriteEvent( } pduPosition = pos - if err = d.topology.insertEventInTopology(ctx, ev); err != nil { + if err = d.topology.insertEventInTopology(ctx, ev, pos); err != nil { return err } @@ -240,12 +240,13 @@ func (d *SyncServerDatasource) GetEventsInRange( if from.Type == types.PaginationTokenTypeTopology { // Determine the backward and forward limit, i.e. the upper and lower // limits to the selection in the room's topology, from the direction. - var backwardLimit, forwardLimit types.StreamPosition + var backwardLimit, forwardLimit, forwardMicroLimit types.StreamPosition if backwardOrdering { // Backward ordering is antichronological (latest event to oldest // one). backwardLimit = to.PDUPosition forwardLimit = from.PDUPosition + forwardMicroLimit = from.EDUTypingPosition } else { // Forward ordering is chronological (oldest event to latest one). backwardLimit = from.PDUPosition @@ -255,7 +256,7 @@ func (d *SyncServerDatasource) GetEventsInRange( // Select the event IDs from the defined range. var eIDs []string eIDs, err = d.topology.selectEventIDsInRange( - ctx, roomID, backwardLimit, forwardLimit, limit, !backwardOrdering, + ctx, roomID, backwardLimit, forwardLimit, forwardMicroLimit, limit, !backwardOrdering, ) if err != nil { return @@ -301,7 +302,7 @@ func (d *SyncServerDatasource) BackwardExtremitiesForRoom( func (d *SyncServerDatasource) MaxTopologicalPosition( ctx context.Context, roomID string, -) (types.StreamPosition, error) { +) (depth types.StreamPosition, stream types.StreamPosition, err error) { return d.topology.selectMaxPositionInTopology(ctx, roomID) } @@ -318,8 +319,13 @@ func (d *SyncServerDatasource) EventsAtTopologicalPosition( func (d *SyncServerDatasource) EventPositionInTopology( ctx context.Context, eventID string, -) (types.StreamPosition, error) { - return d.topology.selectPositionInTopology(ctx, eventID) +) (depth types.StreamPosition, stream types.StreamPosition, err error) { + depth, err = d.topology.selectPositionInTopology(ctx, eventID) + if err != nil { + return + } + stream, err = d.events.selectStreamPositionForEventID(ctx, eventID) + return } func (d *SyncServerDatasource) SyncStreamPosition(ctx context.Context) (types.StreamPosition, error) { diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index 08299f64b..83d7940ad 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -74,6 +74,9 @@ const selectEarlyEventsSQL = "" + const selectMaxEventIDSQL = "" + "SELECT MAX(id) FROM syncapi_output_room_events" +const selectStreamPositionForEventIDSQL = "" + + "SELECT id FROM syncapi_output_room_events WHERE event_id = $1" + // In order for us to apply the state updates correctly, rows need to be ordered in the order they were received (id). /* $1 = oldPos, @@ -99,14 +102,15 @@ const selectStateInRangeSQL = "" + " LIMIT $8" // limit type outputRoomEventsStatements struct { - streamIDStatements *streamIDStatements - insertEventStmt *sql.Stmt - selectEventsStmt *sql.Stmt - selectMaxEventIDStmt *sql.Stmt - selectRecentEventsStmt *sql.Stmt - selectRecentEventsForSyncStmt *sql.Stmt - selectEarlyEventsStmt *sql.Stmt - selectStateInRangeStmt *sql.Stmt + streamIDStatements *streamIDStatements + insertEventStmt *sql.Stmt + selectEventsStmt *sql.Stmt + selectMaxEventIDStmt *sql.Stmt + selectRecentEventsStmt *sql.Stmt + selectRecentEventsForSyncStmt *sql.Stmt + selectEarlyEventsStmt *sql.Stmt + selectStateInRangeStmt *sql.Stmt + selectStreamPositionForEventIDStmt *sql.Stmt } func (s *outputRoomEventsStatements) prepare(db *sql.DB, streamID *streamIDStatements) (err error) { @@ -136,9 +140,18 @@ func (s *outputRoomEventsStatements) prepare(db *sql.DB, streamID *streamIDState if s.selectStateInRangeStmt, err = db.Prepare(selectStateInRangeSQL); err != nil { return } + if s.selectStreamPositionForEventIDStmt, err = db.Prepare(selectStreamPositionForEventIDSQL); err != nil { + return + } return } +func (s *outputRoomEventsStatements) selectStreamPositionForEventID(ctx context.Context, eventID string) (types.StreamPosition, error) { + var id int64 + err := s.selectStreamPositionForEventIDStmt.QueryRowContext(ctx, eventID).Scan(&id) + return types.StreamPosition(id), err +} + // selectStateInRange returns the state events between the two given PDU stream positions, exclusive of oldPos, inclusive of newPos. // Results are bucketed based on the room ID. If the same state is overwritten multiple times between the // two positions, only the most recent state is returned. diff --git a/syncapi/storage/sqlite3/output_room_events_topology_table.go b/syncapi/storage/sqlite3/output_room_events_topology_table.go index a2944c2f9..d64894663 100644 --- a/syncapi/storage/sqlite3/output_room_events_topology_table.go +++ b/syncapi/storage/sqlite3/output_room_events_topology_table.go @@ -27,37 +27,42 @@ const outputRoomEventsTopologySchema = ` -- Stores output room events received from the roomserver. CREATE TABLE IF NOT EXISTS syncapi_output_room_events_topology ( event_id TEXT PRIMARY KEY, - topological_position BIGINT NOT NULL, + topological_position BIGINT NOT NULL, + stream_position BIGINT NOT NULL, room_id TEXT NOT NULL, - UNIQUE(topological_position, room_id) + UNIQUE(topological_position, room_id, stream_position) ); -- The topological order will be used in events selection and ordering --- CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_topological_position_idx ON syncapi_output_room_events_topology(topological_position, room_id); +-- CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_topological_position_idx ON syncapi_output_room_events_topology(topological_position, stream_position, room_id); ` const insertEventInTopologySQL = "" + - "INSERT INTO syncapi_output_room_events_topology (event_id, topological_position, room_id)" + - " VALUES ($1, $2, $3)" + - " ON CONFLICT (topological_position, room_id) DO UPDATE SET event_id = $1" + "INSERT INTO syncapi_output_room_events_topology (event_id, topological_position, room_id, stream_position)" + + " VALUES ($1, $2, $3, $4)" + + " ON CONFLICT DO NOTHING" const selectEventIDsInRangeASCSQL = "" + "SELECT event_id FROM syncapi_output_room_events_topology" + - " WHERE room_id = $1 AND topological_position > $2 AND topological_position <= $3" + - " ORDER BY topological_position ASC LIMIT $4" + " WHERE room_id = $1 AND" + + "(topological_position > $2 AND topological_position < $3) OR" + + "(topological_position = $4 AND stream_position <= $5)" + + " ORDER BY topological_position ASC, stream_position ASC LIMIT $6" const selectEventIDsInRangeDESCSQL = "" + - "SELECT event_id FROM syncapi_output_room_events_topology" + - " WHERE room_id = $1 AND topological_position > $2 AND topological_position <= $3" + - " ORDER BY topological_position DESC LIMIT $4" + "SELECT event_id FROM syncapi_output_room_events_topology" + + " WHERE room_id = $1 AND" + + "(topological_position > $2 AND topological_position < $3) OR" + + "(topological_position = $4 AND stream_position <= $5)" + + " ORDER BY topological_position DESC, stream_position DESC LIMIT $6" const selectPositionInTopologySQL = "" + "SELECT topological_position FROM syncapi_output_room_events_topology" + " WHERE event_id = $1" const selectMaxPositionInTopologySQL = "" + - "SELECT MAX(topological_position) FROM syncapi_output_room_events_topology" + - " WHERE room_id = $1" + "SELECT MAX(topological_position), stream_position FROM syncapi_output_room_events_topology" + + " WHERE room_id = $1 ORDER BY stream_position DESC" const selectEventIDsFromPositionSQL = "" + "SELECT event_id FROM syncapi_output_room_events_topology" + @@ -101,11 +106,11 @@ func (s *outputRoomEventsTopologyStatements) prepare(db *sql.DB) (err error) { // insertEventInTopology inserts the given event in the room's topology, based // on the event's depth. func (s *outputRoomEventsTopologyStatements) insertEventInTopology( - ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.HeaderedEvent, + ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.HeaderedEvent, pos types.StreamPosition, ) (err error) { stmt := common.TxStmt(txn, s.insertEventInTopologyStmt) _, err = stmt.ExecContext( - ctx, event.EventID(), event.Depth(), event.RoomID(), + ctx, event.EventID(), event.Depth(), event.RoomID(), pos, ) return } @@ -115,7 +120,7 @@ func (s *outputRoomEventsTopologyStatements) insertEventInTopology( // Returns an empty slice if no events match the given range. func (s *outputRoomEventsTopologyStatements) selectEventIDsInRange( ctx context.Context, txn *sql.Tx, roomID string, - fromPos, toPos types.StreamPosition, + fromPos, toPos, toMicroPos types.StreamPosition, limit int, chronologicalOrder bool, ) (eventIDs []string, err error) { // Decide on the selection's order according to whether chronological order @@ -128,7 +133,7 @@ func (s *outputRoomEventsTopologyStatements) selectEventIDsInRange( } // Query the event IDs. - rows, err := stmt.QueryContext(ctx, roomID, fromPos, toPos, limit) + rows, err := stmt.QueryContext(ctx, roomID, fromPos, toPos, toPos, toMicroPos, limit) if err == sql.ErrNoRows { // If no event matched the request, return an empty slice. return []string{}, nil @@ -160,9 +165,9 @@ func (s *outputRoomEventsTopologyStatements) selectPositionInTopology( func (s *outputRoomEventsTopologyStatements) selectMaxPositionInTopology( ctx context.Context, txn *sql.Tx, roomID string, -) (pos types.StreamPosition, err error) { +) (pos types.StreamPosition, spos types.StreamPosition, err error) { stmt := common.TxStmt(txn, s.selectMaxPositionInTopologyStmt) - err = stmt.QueryRowContext(ctx, roomID).Scan(&pos) + err = stmt.QueryRowContext(ctx, roomID).Scan(&pos, &spos) return } diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index 35774830e..bdf943e08 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -194,7 +194,7 @@ func (d *SyncServerDatasource) WriteEvent( } pduPosition = pos - if err = d.topology.insertEventInTopology(ctx, txn, ev); err != nil { + if err = d.topology.insertEventInTopology(ctx, txn, ev, pos); err != nil { return err } @@ -281,14 +281,16 @@ func (d *SyncServerDatasource) GetEventsInRange( // events must be retrieved from the rooms' topology table rather than the // table contaning the syncapi server's whole stream of events. if from.Type == types.PaginationTokenTypeTopology { + // TODO: ARGH CONFUSING // Determine the backward and forward limit, i.e. the upper and lower // limits to the selection in the room's topology, from the direction. - var backwardLimit, forwardLimit types.StreamPosition + var backwardLimit, forwardLimit, forwardMicroLimit types.StreamPosition if backwardOrdering { // Backward ordering is antichronological (latest event to oldest // one). backwardLimit = to.PDUPosition forwardLimit = from.PDUPosition + forwardMicroLimit = from.EDUTypingPosition } else { // Forward ordering is chronological (oldest event to latest one). backwardLimit = from.PDUPosition @@ -298,7 +300,7 @@ func (d *SyncServerDatasource) GetEventsInRange( // Select the event IDs from the defined range. var eIDs []string eIDs, err = d.topology.selectEventIDsInRange( - ctx, nil, roomID, backwardLimit, forwardLimit, limit, !backwardOrdering, + ctx, nil, roomID, backwardLimit, forwardLimit, forwardMicroLimit, limit, !backwardOrdering, ) if err != nil { return @@ -328,8 +330,7 @@ func (d *SyncServerDatasource) GetEventsInRange( return } } - - return + return events, err } // SyncPosition returns the latest positions for syncing. @@ -353,7 +354,7 @@ func (d *SyncServerDatasource) BackwardExtremitiesForRoom( // room. func (d *SyncServerDatasource) MaxTopologicalPosition( ctx context.Context, roomID string, -) (types.StreamPosition, error) { +) (types.StreamPosition, types.StreamPosition, error) { return d.topology.selectMaxPositionInTopology(ctx, nil, roomID) } @@ -372,8 +373,13 @@ func (d *SyncServerDatasource) EventsAtTopologicalPosition( func (d *SyncServerDatasource) EventPositionInTopology( ctx context.Context, eventID string, -) (types.StreamPosition, error) { - return d.topology.selectPositionInTopology(ctx, nil, eventID) +) (depth types.StreamPosition, stream types.StreamPosition, err error) { + depth, err = d.topology.selectPositionInTopology(ctx, nil, eventID) + if err != nil { + return + } + stream, err = d.events.selectStreamPositionForEventID(ctx, eventID) + return } // SyncStreamPosition returns the latest position in the sync stream. Returns 0 if there are no events yet. diff --git a/syncapi/storage/storage_test.go b/syncapi/storage/storage_test.go index e591e7ed7..a57d59176 100644 --- a/syncapi/storage/storage_test.go +++ b/syncapi/storage/storage_test.go @@ -182,7 +182,7 @@ func TestSyncResponse(t *testing.T) { // limit set to 5 return db.CompleteSync(ctx, testUserIDA, 5) }, - // want the last 5 events, NOT the last 10. + // want the last 5 events WantTimeline: events[len(events)-5:], // want all state for the room WantState: state, @@ -248,11 +248,11 @@ func TestGetEventsInRangeWithTopologyToken(t *testing.T) { db := MustCreateDatabase(t) events, _ := SimpleRoom(t, testRoomID, testUserIDA, testUserIDB) MustWriteEvents(t, db, events) - latest, err := db.MaxTopologicalPosition(ctx, testRoomID) + latest, latestStream, err := db.MaxTopologicalPosition(ctx, testRoomID) if err != nil { t.Fatalf("failed to get MaxTopologicalPosition: %s", err) } - from := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, latest, 0) + from := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, latest, latestStream) // head towards the beginning of time to := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, 0, 0) @@ -265,6 +265,105 @@ func TestGetEventsInRangeWithTopologyToken(t *testing.T) { assertEventsEqual(t, "", true, gots, reversed(events[len(events)-5:])) } +// The purpose of this test is to make sure that backpagination returns all events, even if some events have the same depth. +// For cases where events have the same depth, the streaming token should be used to tie break so events written via WriteEvent +// will appear FIRST when going backwards. This test creates a DAG like: +// .-----> Message ---. +// Create -> Membership --------> Message -------> Message +// `-----> Message ---` +// depth 1 2 3 4 +// +// With a total depth of 4. It tests that: +// - Backpagination over the whole fork should include all messages and not leave any out. +// - Backpagination from the middle of the fork should not return duplicates (things later than the token). +func TestGetEventsInRangeWithEventsSameDepth(t *testing.T) { + t.Parallel() + db := MustCreateDatabase(t) + + var events []gomatrixserverlib.HeaderedEvent + events = append(events, MustCreateEvent(t, testRoomID, nil, &gomatrixserverlib.EventBuilder{ + Content: []byte(fmt.Sprintf(`{"room_version":"4","creator":"%s"}`, testUserIDA)), + Type: "m.room.create", + StateKey: &emptyStateKey, + Sender: testUserIDA, + Depth: int64(len(events) + 1), + })) + events = append(events, MustCreateEvent(t, testRoomID, []gomatrixserverlib.HeaderedEvent{events[len(events)-1]}, &gomatrixserverlib.EventBuilder{ + Content: []byte(fmt.Sprintf(`{"membership":"join"}`)), + Type: "m.room.member", + StateKey: &testUserIDA, + Sender: testUserIDA, + Depth: int64(len(events) + 1), + })) + // fork the dag into three, same prev_events and depth + parent := []gomatrixserverlib.HeaderedEvent{events[len(events)-1]} + depth := int64(len(events) + 1) + for i := 0; i < 3; i++ { + events = append(events, MustCreateEvent(t, testRoomID, parent, &gomatrixserverlib.EventBuilder{ + Content: []byte(fmt.Sprintf(`{"body":"Message A %d"}`, i+1)), + Type: "m.room.message", + Sender: testUserIDA, + Depth: depth, + })) + } + // merge the fork, prev_events are all 3 messages, depth is increased by 1. + events = append(events, MustCreateEvent(t, testRoomID, events[len(events)-3:], &gomatrixserverlib.EventBuilder{ + Content: []byte(fmt.Sprintf(`{"body":"Message merge"}`)), + Type: "m.room.message", + Sender: testUserIDA, + Depth: depth + 1, + })) + MustWriteEvents(t, db, events) + latestPos, latestStreamPos, err := db.EventPositionInTopology(ctx, events[len(events)-1].EventID()) + if err != nil { + t.Fatalf("failed to get EventPositionInTopology: %s", err) + } + topoPos, streamPos, err := db.EventPositionInTopology(ctx, events[len(events)-3].EventID()) // Message 2 + if err != nil { + t.Fatalf("failed to get EventPositionInTopology for event: %s", err) + } + fromLatest := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, latestPos, latestStreamPos) + fromFork := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, topoPos, streamPos) + // head towards the beginning of time + to := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, 0, 0) + + testCases := []struct { + Name string + From *types.PaginationToken + Limit int + Wants []gomatrixserverlib.HeaderedEvent + }{ + { + Name: "Pagination over the whole fork", + From: fromLatest, + Limit: 5, + Wants: reversed(events[len(events)-5:]), + }, + { + Name: "Paginating to the middle of the fork", + From: fromLatest, + Limit: 2, + Wants: reversed(events[len(events)-2:]), + }, + { + Name: "Pagination FROM the middle of the fork", + From: fromFork, + Limit: 3, + Wants: reversed(events[len(events)-5 : len(events)-2]), + }, + } + + for _, tc := range testCases { + // backpaginate messages starting at the latest position. + paginatedEvents, err := db.GetEventsInRange(ctx, tc.From, to, testRoomID, tc.Limit, true) + if err != nil { + t.Fatalf("%s GetEventsInRange returned an error: %s", tc.Name, err) + } + gots := gomatrixserverlib.HeaderedToClientEvents(db.StreamEventsToEvents(&testUserDeviceA, paginatedEvents), gomatrixserverlib.FormatAll) + assertEventsEqual(t, tc.Name, true, gots, tc.Wants) + } +} + func assertEventsEqual(t *testing.T, msg string, checkRoomID bool, gots []gomatrixserverlib.ClientEvent, wants []gomatrixserverlib.HeaderedEvent) { if len(gots) != len(wants) { t.Fatalf("%s response returned %d events, want %d", msg, len(gots), len(wants)) @@ -294,7 +393,8 @@ func assertEventsEqual(t *testing.T, msg string, checkRoomID bool, gots []gomatr t.Errorf("%s event[%d] unsigned mismatch: got %s want %s", msg, i, string(g.Unsigned), string(w.Unsigned())) } if (g.StateKey == nil && w.StateKey() != nil) || (g.StateKey != nil && w.StateKey() == nil) { - t.Fatalf("%s event[%d] state_key [not] missing: got %v want %v", msg, i, g.StateKey, w.StateKey()) + t.Errorf("%s event[%d] state_key [not] missing: got %v want %v", msg, i, g.StateKey, w.StateKey()) + continue } if g.StateKey != nil { if !w.StateKeyEquals(*g.StateKey) { diff --git a/syncapi/types/types.go b/syncapi/types/types.go index cfd49ff1f..c04fe5219 100644 --- a/syncapi/types/types.go +++ b/syncapi/types/types.go @@ -64,8 +64,14 @@ const ( // /sync or /messages, for example. type PaginationToken struct { //Position StreamPosition - Type PaginationTokenType - PDUPosition StreamPosition + Type PaginationTokenType + // For /sync, this is the PDU position. For /messages, this is the topological position (depth). + // TODO: Given how different the positions are depending on the token type, they should probably be renamed + // or use different structs altogether. + PDUPosition StreamPosition + // For /sync, this is the EDU position. For /messages, this is the stream (PDU) position. + // TODO: Given how different the positions are depending on the token type, they should probably be renamed + // or use different structs altogether. EDUTypingPosition StreamPosition } From 17e046f18fc182731c112e1dd653cc7021ebd422 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Fri, 1 May 2020 12:41:38 +0100 Subject: [PATCH 022/200] Fix prev_batch tokens (#999) --- .../postgres/output_room_events_table.go | 27 +++--------- .../output_room_events_topology_table.go | 6 +-- syncapi/storage/postgres/syncserver.go | 23 ++++------ .../sqlite3/output_room_events_table.go | 29 ++++--------- .../output_room_events_topology_table.go | 6 +-- syncapi/storage/sqlite3/syncserver.go | 24 +++++------ syncapi/storage/storage_test.go | 43 +++++++++++++++++++ 7 files changed, 83 insertions(+), 75 deletions(-) diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go index 769823e92..0b53dfa9e 100644 --- a/syncapi/storage/postgres/output_room_events_table.go +++ b/syncapi/storage/postgres/output_room_events_table.go @@ -94,9 +94,6 @@ const selectEarlyEventsSQL = "" + " WHERE room_id = $1 AND id > $2 AND id <= $3" + " ORDER BY id ASC LIMIT $4" -const selectStreamPositionForEventIDSQL = "" + - "SELECT id FROM syncapi_output_room_events WHERE event_id = $1" - const selectMaxEventIDSQL = "" + "SELECT MAX(id) FROM syncapi_output_room_events" @@ -114,14 +111,13 @@ const selectStateInRangeSQL = "" + " LIMIT $8" type outputRoomEventsStatements struct { - insertEventStmt *sql.Stmt - selectEventsStmt *sql.Stmt - selectMaxEventIDStmt *sql.Stmt - selectRecentEventsStmt *sql.Stmt - selectRecentEventsForSyncStmt *sql.Stmt - selectEarlyEventsStmt *sql.Stmt - selectStateInRangeStmt *sql.Stmt - selectStreamPositionForEventIDStmt *sql.Stmt + insertEventStmt *sql.Stmt + selectEventsStmt *sql.Stmt + selectMaxEventIDStmt *sql.Stmt + selectRecentEventsStmt *sql.Stmt + selectRecentEventsForSyncStmt *sql.Stmt + selectEarlyEventsStmt *sql.Stmt + selectStateInRangeStmt *sql.Stmt } func (s *outputRoomEventsStatements) prepare(db *sql.DB) (err error) { @@ -150,18 +146,9 @@ func (s *outputRoomEventsStatements) prepare(db *sql.DB) (err error) { if s.selectStateInRangeStmt, err = db.Prepare(selectStateInRangeSQL); err != nil { return } - if s.selectStreamPositionForEventIDStmt, err = db.Prepare(selectStreamPositionForEventIDSQL); err != nil { - return - } return } -func (s *outputRoomEventsStatements) selectStreamPositionForEventID(ctx context.Context, eventID string) (types.StreamPosition, error) { - var id int64 - err := s.selectStreamPositionForEventIDStmt.QueryRowContext(ctx, eventID).Scan(&id) - return types.StreamPosition(id), err -} - // selectStateInRange returns the state events between the two given PDU stream positions, exclusive of oldPos, inclusive of newPos. // Results are bucketed based on the room ID. If the same state is overwritten multiple times between the // two positions, only the most recent state is returned. diff --git a/syncapi/storage/postgres/output_room_events_topology_table.go b/syncapi/storage/postgres/output_room_events_topology_table.go index f77365c8d..51cbd50d0 100644 --- a/syncapi/storage/postgres/output_room_events_topology_table.go +++ b/syncapi/storage/postgres/output_room_events_topology_table.go @@ -60,7 +60,7 @@ const selectEventIDsInRangeDESCSQL = "" + " ORDER BY topological_position DESC, stream_position DESC LIMIT $6" const selectPositionInTopologySQL = "" + - "SELECT topological_position FROM syncapi_output_room_events_topology" + + "SELECT topological_position, stream_position FROM syncapi_output_room_events_topology" + " WHERE event_id = $1" // Select the max topological position for the room, then sort by stream position and take the highest, @@ -163,8 +163,8 @@ func (s *outputRoomEventsTopologyStatements) selectEventIDsInRange( // topology of the room it belongs to. func (s *outputRoomEventsTopologyStatements) selectPositionInTopology( ctx context.Context, eventID string, -) (pos types.StreamPosition, err error) { - err = s.selectPositionInTopologyStmt.QueryRowContext(ctx, eventID).Scan(&pos) +) (pos, spos types.StreamPosition, err error) { + err = s.selectPositionInTopologyStmt.QueryRowContext(ctx, eventID).Scan(&pos, &spos) return } diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go index 744ae7b8d..a6de15178 100644 --- a/syncapi/storage/postgres/syncserver.go +++ b/syncapi/storage/postgres/syncserver.go @@ -320,12 +320,7 @@ func (d *SyncServerDatasource) EventsAtTopologicalPosition( func (d *SyncServerDatasource) EventPositionInTopology( ctx context.Context, eventID string, ) (depth types.StreamPosition, stream types.StreamPosition, err error) { - depth, err = d.topology.selectPositionInTopology(ctx, eventID) - if err != nil { - return - } - stream, err = d.events.selectStreamPositionForEventID(ctx, eventID) - return + return d.topology.selectPositionInTopology(ctx, eventID) } func (d *SyncServerDatasource) SyncStreamPosition(ctx context.Context) (types.StreamPosition, error) { @@ -591,8 +586,8 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( // Retrieve the backward topology position, i.e. the position of the // oldest event in the room's topology. - var backwardTopologyPos types.StreamPosition - backwardTopologyPos, err = d.topology.selectPositionInTopology(ctx, recentStreamEvents[0].EventID()) + var backwardTopologyPos, backwardStreamPos types.StreamPosition + backwardTopologyPos, backwardStreamPos, err = d.topology.selectPositionInTopology(ctx, recentStreamEvents[0].EventID()) if backwardTopologyPos-1 <= 0 { backwardTopologyPos = types.StreamPosition(1) } else { @@ -605,7 +600,7 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( stateEvents = removeDuplicates(stateEvents, recentEvents) jr := types.NewJoinResponse() jr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition( - types.PaginationTokenTypeTopology, backwardTopologyPos, 0, + types.PaginationTokenTypeTopology, backwardTopologyPos, backwardStreamPos, ).String() jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = true @@ -720,9 +715,9 @@ func (d *SyncServerDatasource) addInvitesToResponse( func (d *SyncServerDatasource) getBackwardTopologyPos( ctx context.Context, events []types.StreamEvent, -) (pos types.StreamPosition) { +) (pos, spos types.StreamPosition) { if len(events) > 0 { - pos, _ = d.topology.selectPositionInTopology(ctx, events[0].EventID()) + pos, spos, _ = d.topology.selectPositionInTopology(ctx, events[0].EventID()) } if pos-1 <= 0 { pos = types.StreamPosition(1) @@ -761,14 +756,14 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( } recentEvents := d.StreamEventsToEvents(device, recentStreamEvents) delta.stateEvents = removeDuplicates(delta.stateEvents, recentEvents) // roll back - backwardTopologyPos := d.getBackwardTopologyPos(ctx, recentStreamEvents) + backwardTopologyPos, backwardStreamPos := d.getBackwardTopologyPos(ctx, recentStreamEvents) switch delta.membership { case gomatrixserverlib.Join: jr := types.NewJoinResponse() jr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition( - types.PaginationTokenTypeTopology, backwardTopologyPos, 0, + types.PaginationTokenTypeTopology, backwardTopologyPos, backwardStreamPos, ).String() jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true @@ -781,7 +776,7 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( // no longer in the room. lr := types.NewLeaveResponse() lr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition( - types.PaginationTokenTypeTopology, backwardTopologyPos, 0, + types.PaginationTokenTypeTopology, backwardTopologyPos, backwardStreamPos, ).String() lr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) lr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index 83d7940ad..08299f64b 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -74,9 +74,6 @@ const selectEarlyEventsSQL = "" + const selectMaxEventIDSQL = "" + "SELECT MAX(id) FROM syncapi_output_room_events" -const selectStreamPositionForEventIDSQL = "" + - "SELECT id FROM syncapi_output_room_events WHERE event_id = $1" - // In order for us to apply the state updates correctly, rows need to be ordered in the order they were received (id). /* $1 = oldPos, @@ -102,15 +99,14 @@ const selectStateInRangeSQL = "" + " LIMIT $8" // limit type outputRoomEventsStatements struct { - streamIDStatements *streamIDStatements - insertEventStmt *sql.Stmt - selectEventsStmt *sql.Stmt - selectMaxEventIDStmt *sql.Stmt - selectRecentEventsStmt *sql.Stmt - selectRecentEventsForSyncStmt *sql.Stmt - selectEarlyEventsStmt *sql.Stmt - selectStateInRangeStmt *sql.Stmt - selectStreamPositionForEventIDStmt *sql.Stmt + streamIDStatements *streamIDStatements + insertEventStmt *sql.Stmt + selectEventsStmt *sql.Stmt + selectMaxEventIDStmt *sql.Stmt + selectRecentEventsStmt *sql.Stmt + selectRecentEventsForSyncStmt *sql.Stmt + selectEarlyEventsStmt *sql.Stmt + selectStateInRangeStmt *sql.Stmt } func (s *outputRoomEventsStatements) prepare(db *sql.DB, streamID *streamIDStatements) (err error) { @@ -140,18 +136,9 @@ func (s *outputRoomEventsStatements) prepare(db *sql.DB, streamID *streamIDState if s.selectStateInRangeStmt, err = db.Prepare(selectStateInRangeSQL); err != nil { return } - if s.selectStreamPositionForEventIDStmt, err = db.Prepare(selectStreamPositionForEventIDSQL); err != nil { - return - } return } -func (s *outputRoomEventsStatements) selectStreamPositionForEventID(ctx context.Context, eventID string) (types.StreamPosition, error) { - var id int64 - err := s.selectStreamPositionForEventIDStmt.QueryRowContext(ctx, eventID).Scan(&id) - return types.StreamPosition(id), err -} - // selectStateInRange returns the state events between the two given PDU stream positions, exclusive of oldPos, inclusive of newPos. // Results are bucketed based on the room ID. If the same state is overwritten multiple times between the // two positions, only the most recent state is returned. diff --git a/syncapi/storage/sqlite3/output_room_events_topology_table.go b/syncapi/storage/sqlite3/output_room_events_topology_table.go index d64894663..0d313d7c6 100644 --- a/syncapi/storage/sqlite3/output_room_events_topology_table.go +++ b/syncapi/storage/sqlite3/output_room_events_topology_table.go @@ -57,7 +57,7 @@ const selectEventIDsInRangeDESCSQL = "" + " ORDER BY topological_position DESC, stream_position DESC LIMIT $6" const selectPositionInTopologySQL = "" + - "SELECT topological_position FROM syncapi_output_room_events_topology" + + "SELECT topological_position, stream_position FROM syncapi_output_room_events_topology" + " WHERE event_id = $1" const selectMaxPositionInTopologySQL = "" + @@ -157,9 +157,9 @@ func (s *outputRoomEventsTopologyStatements) selectEventIDsInRange( // topology of the room it belongs to. func (s *outputRoomEventsTopologyStatements) selectPositionInTopology( ctx context.Context, txn *sql.Tx, eventID string, -) (pos types.StreamPosition, err error) { +) (pos types.StreamPosition, spos types.StreamPosition, err error) { stmt := common.TxStmt(txn, s.selectPositionInTopologyStmt) - err = stmt.QueryRowContext(ctx, eventID).Scan(&pos) + err = stmt.QueryRowContext(ctx, eventID).Scan(&pos, &spos) return } diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index bdf943e08..7e8e4ff5d 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -374,12 +374,7 @@ func (d *SyncServerDatasource) EventsAtTopologicalPosition( func (d *SyncServerDatasource) EventPositionInTopology( ctx context.Context, eventID string, ) (depth types.StreamPosition, stream types.StreamPosition, err error) { - depth, err = d.topology.selectPositionInTopology(ctx, nil, eventID) - if err != nil { - return - } - stream, err = d.events.selectStreamPositionForEventID(ctx, eventID) - return + return d.topology.selectPositionInTopology(ctx, nil, eventID) } // SyncStreamPosition returns the latest position in the sync stream. Returns 0 if there are no events yet. @@ -657,8 +652,8 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( // Retrieve the backward topology position, i.e. the position of the // oldest event in the room's topology. - var backwardTopologyPos types.StreamPosition - backwardTopologyPos, err = d.topology.selectPositionInTopology(ctx, txn, recentStreamEvents[0].EventID()) + var backwardTopologyPos, backwardTopologyStreamPos types.StreamPosition + backwardTopologyPos, backwardTopologyStreamPos, err = d.topology.selectPositionInTopology(ctx, txn, recentStreamEvents[0].EventID()) if backwardTopologyPos-1 <= 0 { backwardTopologyPos = types.StreamPosition(1) } else { @@ -671,7 +666,7 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( stateEvents = removeDuplicates(stateEvents, recentEvents) jr := types.NewJoinResponse() jr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition( - types.PaginationTokenTypeTopology, backwardTopologyPos, 0, + types.PaginationTokenTypeTopology, backwardTopologyPos, backwardTopologyStreamPos, ).String() jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = true @@ -818,10 +813,11 @@ func (d *SyncServerDatasource) addInvitesToResponse( func (d *SyncServerDatasource) getBackwardTopologyPos( ctx context.Context, txn *sql.Tx, events []types.StreamEvent, -) (pos types.StreamPosition) { +) (pos, spos types.StreamPosition) { if len(events) > 0 { - pos, _ = d.topology.selectPositionInTopology(ctx, txn, events[0].EventID()) + pos, spos, _ = d.topology.selectPositionInTopology(ctx, txn, events[0].EventID()) } + // TODO: I have no idea what this is doing. if pos-1 <= 0 { pos = types.StreamPosition(1) } else { @@ -859,14 +855,14 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( } recentEvents := d.StreamEventsToEvents(device, recentStreamEvents) delta.stateEvents = removeDuplicates(delta.stateEvents, recentEvents) - backwardTopologyPos := d.getBackwardTopologyPos(ctx, txn, recentStreamEvents) + backwardTopologyPos, backwardStreamPos := d.getBackwardTopologyPos(ctx, txn, recentStreamEvents) switch delta.membership { case gomatrixserverlib.Join: jr := types.NewJoinResponse() jr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition( - types.PaginationTokenTypeTopology, backwardTopologyPos, 0, + types.PaginationTokenTypeTopology, backwardTopologyPos, backwardStreamPos, ).String() jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true @@ -879,7 +875,7 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( // no longer in the room. lr := types.NewLeaveResponse() lr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition( - types.PaginationTokenTypeTopology, backwardTopologyPos, 0, + types.PaginationTokenTypeTopology, backwardTopologyPos, backwardStreamPos, ).String() lr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) lr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true diff --git a/syncapi/storage/storage_test.go b/syncapi/storage/storage_test.go index a57d59176..378c1fe35 100644 --- a/syncapi/storage/storage_test.go +++ b/syncapi/storage/storage_test.go @@ -220,6 +220,49 @@ func TestSyncResponse(t *testing.T) { } } +func TestGetEventsInRangeWithPrevBatch(t *testing.T) { + t.Parallel() + db := MustCreateDatabase(t) + events, _ := SimpleRoom(t, testRoomID, testUserIDA, testUserIDB) + positions := MustWriteEvents(t, db, events) + latest, err := db.SyncPosition(ctx) + if err != nil { + t.Fatalf("failed to get SyncPosition: %s", err) + } + from := types.NewPaginationTokenFromTypeAndPosition( + types.PaginationTokenTypeStream, positions[len(positions)-2], types.StreamPosition(0), + ) + + res, err := db.IncrementalSync(ctx, testUserDeviceA, *from, latest, 5, false) + if err != nil { + t.Fatalf("failed to IncrementalSync with latest token") + } + roomRes, ok := res.Rooms.Join[testRoomID] + if !ok { + t.Fatalf("IncrementalSync response missing room %s - response: %+v", testRoomID, res) + } + // returns the last event "Message 10" + assertEventsEqual(t, "IncrementalSync Timeline", false, roomRes.Timeline.Events, reversed(events[len(events)-1:])) + + prev := roomRes.Timeline.PrevBatch + if prev == "" { + t.Fatalf("IncrementalSync expected prev_batch token") + } + prevBatchToken, err := types.NewPaginationTokenFromString(prev) + if err != nil { + t.Fatalf("failed to NewPaginationTokenFromString : %s", err) + } + // backpaginate 5 messages starting at the latest position. + // head towards the beginning of time + to := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, 0, 0) + paginatedEvents, err := db.GetEventsInRange(ctx, prevBatchToken, to, testRoomID, 5, true) + if err != nil { + t.Fatalf("GetEventsInRange returned an error: %s", err) + } + gots := gomatrixserverlib.HeaderedToClientEvents(db.StreamEventsToEvents(&testUserDeviceA, paginatedEvents), gomatrixserverlib.FormatAll) + assertEventsEqual(t, "", true, gots, reversed(events[len(events)-6:len(events)-1])) +} + // The purpose of this test is to ensure that backfill does indeed go backwards, using a stream token. func TestGetEventsInRangeWithStreamToken(t *testing.T) { t.Parallel() From 908108c23e47df5fbf0d6629b676e105abd2a564 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 1 May 2020 13:01:50 +0100 Subject: [PATCH 023/200] Rename FS queue package to internal (#997) --- federationsender/federationsender.go | 4 ++-- federationsender/{query => internal}/api.go | 2 +- federationsender/{query => internal}/perform.go | 4 ++-- federationsender/{query => internal}/perform/join.go | 0 federationsender/{query => internal}/query.go | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) rename federationsender/{query => internal}/api.go (99%) rename federationsender/{query => internal}/perform.go (97%) rename federationsender/{query => internal}/perform/join.go (100%) rename federationsender/{query => internal}/query.go (98%) diff --git a/federationsender/federationsender.go b/federationsender/federationsender.go index bf9d326bb..64de4fd27 100644 --- a/federationsender/federationsender.go +++ b/federationsender/federationsender.go @@ -20,8 +20,8 @@ import ( "github.com/matrix-org/dendrite/common/basecomponent" "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/federationsender/consumers" + "github.com/matrix-org/dendrite/federationsender/internal" "github.com/matrix-org/dendrite/federationsender/producers" - "github.com/matrix-org/dendrite/federationsender/query" "github.com/matrix-org/dendrite/federationsender/queue" "github.com/matrix-org/dendrite/federationsender/storage" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" @@ -61,7 +61,7 @@ func SetupFederationSenderComponent( logrus.WithError(err).Panic("failed to start typing server consumer") } - queryAPI := query.NewFederationSenderInternalAPI( + queryAPI := internal.NewFederationSenderInternalAPI( federationSenderDB, base.Cfg, roomserverProducer, federation, keyRing, ) queryAPI.SetupHTTP(http.DefaultServeMux) diff --git a/federationsender/query/api.go b/federationsender/internal/api.go similarity index 99% rename from federationsender/query/api.go rename to federationsender/internal/api.go index e92453f98..89a1fda40 100644 --- a/federationsender/query/api.go +++ b/federationsender/internal/api.go @@ -1,4 +1,4 @@ -package query +package internal import ( "encoding/json" diff --git a/federationsender/query/perform.go b/federationsender/internal/perform.go similarity index 97% rename from federationsender/query/perform.go rename to federationsender/internal/perform.go index d39fef5ed..961d80276 100644 --- a/federationsender/query/perform.go +++ b/federationsender/internal/perform.go @@ -1,4 +1,4 @@ -package query +package internal import ( "context" @@ -6,7 +6,7 @@ import ( "time" "github.com/matrix-org/dendrite/federationsender/api" - "github.com/matrix-org/dendrite/federationsender/query/perform" + "github.com/matrix-org/dendrite/federationsender/internal/perform" "github.com/matrix-org/dendrite/roomserver/version" "github.com/matrix-org/gomatrixserverlib" ) diff --git a/federationsender/query/perform/join.go b/federationsender/internal/perform/join.go similarity index 100% rename from federationsender/query/perform/join.go rename to federationsender/internal/perform/join.go diff --git a/federationsender/query/query.go b/federationsender/internal/query.go similarity index 98% rename from federationsender/query/query.go rename to federationsender/internal/query.go index 004ad156d..88dd50a62 100644 --- a/federationsender/query/query.go +++ b/federationsender/internal/query.go @@ -1,4 +1,4 @@ -package query +package internal import ( "context" From f7cfa758864cc9849ce8b0539895283417b75eb0 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 1 May 2020 13:34:53 +0100 Subject: [PATCH 024/200] Limit database connections (#980, #564) (#998) * Limit database connections (#564) - Add new options to the config file database: max_open_conns: 100 max_idle_conns: 2 conn_max_lifetime: -1 - Implement connection parameter setup on the *DB (database/sql) in internal/sqlutil/trace.go:Open() - Propagate the values in the form of DbProperties interface via all the Open() and NewDatabase() functions Signed-off-by: Tomas Jirka * Fix wasm builds * Remove file accidentally added from working tree Co-authored-by: Tomas Jirka --- appservice/appservice.go | 2 +- appservice/storage/postgres/storage.go | 5 ++- appservice/storage/sqlite3/storage.go | 2 +- appservice/storage/storage.go | 11 +++-- appservice/storage/storage_wasm.go | 6 ++- .../auth/storage/accounts/postgres/storage.go | 4 +- .../auth/storage/accounts/sqlite3/storage.go | 2 +- clientapi/auth/storage/accounts/storage.go | 11 +++-- .../auth/storage/accounts/storage_wasm.go | 7 +++- .../auth/storage/devices/postgres/storage.go | 4 +- .../auth/storage/devices/sqlite3/storage.go | 2 +- clientapi/auth/storage/devices/storage.go | 11 +++-- .../auth/storage/devices/storage_wasm.go | 7 +++- cmd/create-account/main.go | 4 +- cmd/dendrite-demo-libp2p/main.go | 1 + .../storage/postgreswithdht/storage.go | 2 +- .../storage/postgreswithpubsub/storage.go | 2 +- cmd/dendrite-monolith-server/main.go | 2 +- cmd/dendrite-public-rooms-api-server/main.go | 2 +- common/basecomponent/base.go | 9 ++-- common/config/config.go | 42 +++++++++++++++++++ common/keydb/keydb.go | 8 ++-- common/keydb/keydb_wasm.go | 2 + common/keydb/postgres/keydb.go | 4 +- common/keydb/sqlite3/keydb.go | 2 +- common/sql.go | 8 ++++ dendrite-config.yaml | 3 ++ federationsender/federationsender.go | 2 +- federationsender/storage/postgres/storage.go | 4 +- federationsender/storage/sqlite3/storage.go | 2 +- federationsender/storage/storage.go | 9 ++-- federationsender/storage/storage_wasm.go | 6 ++- internal/sqlutil/trace.go | 21 +++++++++- mediaapi/mediaapi.go | 2 +- mediaapi/storage/postgres/storage.go | 5 ++- mediaapi/storage/sqlite3/storage.go | 2 +- mediaapi/storage/storage.go | 9 ++-- mediaapi/storage/storage_wasm.go | 6 ++- publicroomsapi/storage/postgres/storage.go | 4 +- publicroomsapi/storage/sqlite3/storage.go | 2 +- publicroomsapi/storage/storage.go | 9 ++-- roomserver/roomserver.go | 2 +- roomserver/storage/postgres/storage.go | 5 ++- roomserver/storage/sqlite3/storage.go | 4 +- roomserver/storage/storage.go | 11 ++--- roomserver/storage/storage_wasm.go | 6 ++- syncapi/storage/postgres/syncserver.go | 4 +- syncapi/storage/sqlite3/syncserver.go | 2 +- syncapi/storage/storage.go | 11 ++--- syncapi/storage/storage_wasm.go | 6 ++- syncapi/syncapi.go | 2 +- 51 files changed, 213 insertions(+), 88 deletions(-) diff --git a/appservice/appservice.go b/appservice/appservice.go index 71d131998..e52db2c28 100644 --- a/appservice/appservice.go +++ b/appservice/appservice.go @@ -48,7 +48,7 @@ func SetupAppServiceAPIComponent( transactionsCache *transactions.Cache, ) appserviceAPI.AppServiceQueryAPI { // Create a connection to the appservice postgres DB - appserviceDB, err := storage.NewDatabase(string(base.Cfg.Database.AppService)) + appserviceDB, err := storage.NewDatabase(string(base.Cfg.Database.AppService), base.Cfg.DbProperties()) if err != nil { logrus.WithError(err).Panicf("failed to connect to appservice db") } diff --git a/appservice/storage/postgres/storage.go b/appservice/storage/postgres/storage.go index e145eeee2..475db6fc2 100644 --- a/appservice/storage/postgres/storage.go +++ b/appservice/storage/postgres/storage.go @@ -21,6 +21,7 @@ import ( // Import postgres database driver _ "github.com/lib/pq" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) @@ -33,10 +34,10 @@ type Database struct { } // NewDatabase opens a new database -func NewDatabase(dataSourceName string) (*Database, error) { +func NewDatabase(dataSourceName string, dbProperties common.DbProperties) (*Database, error) { var result Database var err error - if result.db, err = sqlutil.Open("postgres", dataSourceName); err != nil { + if result.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { return nil, err } if err = result.prepare(); err != nil { diff --git a/appservice/storage/sqlite3/storage.go b/appservice/storage/sqlite3/storage.go index 0cd1e4abc..0b0590f6d 100644 --- a/appservice/storage/sqlite3/storage.go +++ b/appservice/storage/sqlite3/storage.go @@ -37,7 +37,7 @@ type Database struct { func NewDatabase(dataSourceName string) (*Database, error) { var result Database var err error - if result.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil { + if result.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil); err != nil { return nil, err } if err = result.prepare(); err != nil { diff --git a/appservice/storage/storage.go b/appservice/storage/storage.go index 9fbd2a1f3..bf0a9b0c8 100644 --- a/appservice/storage/storage.go +++ b/appservice/storage/storage.go @@ -21,19 +21,22 @@ import ( "github.com/matrix-org/dendrite/appservice/storage/postgres" "github.com/matrix-org/dendrite/appservice/storage/sqlite3" + "github.com/matrix-org/dendrite/common" ) -func NewDatabase(dataSourceName string) (Database, error) { +// NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme) +// and sets DB connection parameters +func NewDatabase(dataSourceName string, dbProperties common.DbProperties) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { - return postgres.NewDatabase(dataSourceName) + return postgres.NewDatabase(dataSourceName, dbProperties) } switch uri.Scheme { case "postgres": - return postgres.NewDatabase(dataSourceName) + return postgres.NewDatabase(dataSourceName, dbProperties) case "file": return sqlite3.NewDatabase(dataSourceName) default: - return postgres.NewDatabase(dataSourceName) + return postgres.NewDatabase(dataSourceName, dbProperties) } } diff --git a/appservice/storage/storage_wasm.go b/appservice/storage/storage_wasm.go index 2bd1433f9..ff0330ae0 100644 --- a/appservice/storage/storage_wasm.go +++ b/appservice/storage/storage_wasm.go @@ -19,9 +19,13 @@ import ( "net/url" "github.com/matrix-org/dendrite/appservice/storage/sqlite3" + "github.com/matrix-org/dendrite/common" ) -func NewDatabase(dataSourceName string) (Database, error) { +func NewDatabase( + dataSourceName string, + dbProperties common.DbProperties, // nolint:unparam +) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return nil, fmt.Errorf("Cannot use postgres implementation") diff --git a/clientapi/auth/storage/accounts/postgres/storage.go b/clientapi/auth/storage/accounts/postgres/storage.go index 8ce367a3e..7e2073ec0 100644 --- a/clientapi/auth/storage/accounts/postgres/storage.go +++ b/clientapi/auth/storage/accounts/postgres/storage.go @@ -44,10 +44,10 @@ type Database struct { } // NewDatabase creates a new accounts and profiles database -func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) { +func NewDatabase(dataSourceName string, dbProperties common.DbProperties, serverName gomatrixserverlib.ServerName) (*Database, error) { var db *sql.DB var err error - if db, err = sqlutil.Open("postgres", dataSourceName); err != nil { + if db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { return nil, err } partitions := common.PartitionOffsetStatements{} diff --git a/clientapi/auth/storage/accounts/sqlite3/storage.go b/clientapi/auth/storage/accounts/sqlite3/storage.go index e190ba6c2..30a93e7ed 100644 --- a/clientapi/auth/storage/accounts/sqlite3/storage.go +++ b/clientapi/auth/storage/accounts/sqlite3/storage.go @@ -50,7 +50,7 @@ type Database struct { func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) { var db *sql.DB var err error - if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil { + if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil); err != nil { return nil, err } partitions := common.PartitionOffsetStatements{} diff --git a/clientapi/auth/storage/accounts/storage.go b/clientapi/auth/storage/accounts/storage.go index c643a4d0a..394cc5e16 100644 --- a/clientapi/auth/storage/accounts/storage.go +++ b/clientapi/auth/storage/accounts/storage.go @@ -21,20 +21,23 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/postgres" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/sqlite3" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/gomatrixserverlib" ) -func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (Database, error) { +// NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme) +// and sets postgres connection parameters +func NewDatabase(dataSourceName string, dbProperties common.DbProperties, serverName gomatrixserverlib.ServerName) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { - return postgres.NewDatabase(dataSourceName, serverName) + return postgres.NewDatabase(dataSourceName, dbProperties, serverName) } switch uri.Scheme { case "postgres": - return postgres.NewDatabase(dataSourceName, serverName) + return postgres.NewDatabase(dataSourceName, dbProperties, serverName) case "file": return sqlite3.NewDatabase(dataSourceName, serverName) default: - return postgres.NewDatabase(dataSourceName, serverName) + return postgres.NewDatabase(dataSourceName, dbProperties, serverName) } } diff --git a/clientapi/auth/storage/accounts/storage_wasm.go b/clientapi/auth/storage/accounts/storage_wasm.go index 828afc6b4..61f9d6997 100644 --- a/clientapi/auth/storage/accounts/storage_wasm.go +++ b/clientapi/auth/storage/accounts/storage_wasm.go @@ -19,10 +19,15 @@ import ( "net/url" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/sqlite3" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/gomatrixserverlib" ) -func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (Database, error) { +func NewDatabase( + dataSourceName string, + dbProperties common.DbProperties, // nolint:unparam + serverName gomatrixserverlib.ServerName, +) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return nil, fmt.Errorf("Cannot use postgres implementation") diff --git a/clientapi/auth/storage/devices/postgres/storage.go b/clientapi/auth/storage/devices/postgres/storage.go index 3f613cf32..57c268df8 100644 --- a/clientapi/auth/storage/devices/postgres/storage.go +++ b/clientapi/auth/storage/devices/postgres/storage.go @@ -36,10 +36,10 @@ type Database struct { } // NewDatabase creates a new device database -func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) { +func NewDatabase(dataSourceName string, dbProperties common.DbProperties, serverName gomatrixserverlib.ServerName) (*Database, error) { var db *sql.DB var err error - if db, err = sqlutil.Open("postgres", dataSourceName); err != nil { + if db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { return nil, err } d := devicesStatements{} diff --git a/clientapi/auth/storage/devices/sqlite3/storage.go b/clientapi/auth/storage/devices/sqlite3/storage.go index 85a8def2c..b69d6278c 100644 --- a/clientapi/auth/storage/devices/sqlite3/storage.go +++ b/clientapi/auth/storage/devices/sqlite3/storage.go @@ -41,7 +41,7 @@ type Database struct { func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) { var db *sql.DB var err error - if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil { + if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil); err != nil { return nil, err } d := devicesStatements{} diff --git a/clientapi/auth/storage/devices/storage.go b/clientapi/auth/storage/devices/storage.go index 99211db28..ec47a3272 100644 --- a/clientapi/auth/storage/devices/storage.go +++ b/clientapi/auth/storage/devices/storage.go @@ -21,20 +21,23 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/storage/devices/postgres" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices/sqlite3" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/gomatrixserverlib" ) -func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (Database, error) { +// NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme) +// and sets postgres connection parameters +func NewDatabase(dataSourceName string, dbProperties common.DbProperties, serverName gomatrixserverlib.ServerName) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { - return postgres.NewDatabase(dataSourceName, serverName) + return postgres.NewDatabase(dataSourceName, dbProperties, serverName) } switch uri.Scheme { case "postgres": - return postgres.NewDatabase(dataSourceName, serverName) + return postgres.NewDatabase(dataSourceName, dbProperties, serverName) case "file": return sqlite3.NewDatabase(dataSourceName, serverName) default: - return postgres.NewDatabase(dataSourceName, serverName) + return postgres.NewDatabase(dataSourceName, dbProperties, serverName) } } diff --git a/clientapi/auth/storage/devices/storage_wasm.go b/clientapi/auth/storage/devices/storage_wasm.go index 322852888..e25b7c640 100644 --- a/clientapi/auth/storage/devices/storage_wasm.go +++ b/clientapi/auth/storage/devices/storage_wasm.go @@ -19,10 +19,15 @@ import ( "net/url" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices/sqlite3" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/gomatrixserverlib" ) -func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (Database, error) { +func NewDatabase( + dataSourceName string, + dbProperties common.DbProperties, // nolint:unparam + serverName gomatrixserverlib.ServerName, +) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return nil, fmt.Errorf("Cannot use postgres implementation") diff --git a/cmd/create-account/main.go b/cmd/create-account/main.go index fc51a5bb6..eca9b2fe6 100644 --- a/cmd/create-account/main.go +++ b/cmd/create-account/main.go @@ -63,7 +63,7 @@ func main() { serverName := gomatrixserverlib.ServerName(*serverNameStr) - accountDB, err := accounts.NewDatabase(*database, serverName) + accountDB, err := accounts.NewDatabase(*database, nil, serverName) if err != nil { fmt.Println(err.Error()) os.Exit(1) @@ -78,7 +78,7 @@ func main() { os.Exit(1) } - deviceDB, err := devices.NewDatabase(*database, serverName) + deviceDB, err := devices.NewDatabase(*database, nil, serverName) if err != nil { fmt.Println(err.Error()) os.Exit(1) diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index 28c7153fd..0ff610a5b 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -56,6 +56,7 @@ func createKeyDB( ) keydb.Database { db, err := keydb.NewDatabase( string(base.Base.Cfg.Database.ServerKey), + base.Base.Cfg.DbProperties(), base.Base.Cfg.Matrix.ServerName, base.Base.Cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey), base.Base.Cfg.Matrix.KeyID, diff --git a/cmd/dendrite-demo-libp2p/storage/postgreswithdht/storage.go b/cmd/dendrite-demo-libp2p/storage/postgreswithdht/storage.go index 819469ee8..cd2804c97 100644 --- a/cmd/dendrite-demo-libp2p/storage/postgreswithdht/storage.go +++ b/cmd/dendrite-demo-libp2p/storage/postgreswithdht/storage.go @@ -45,7 +45,7 @@ type PublicRoomsServerDatabase struct { // NewPublicRoomsServerDatabase creates a new public rooms server database. func NewPublicRoomsServerDatabase(dataSourceName string, dht *dht.IpfsDHT) (*PublicRoomsServerDatabase, error) { - pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName) + pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName, nil) if err != nil { return nil, err } diff --git a/cmd/dendrite-demo-libp2p/storage/postgreswithpubsub/storage.go b/cmd/dendrite-demo-libp2p/storage/postgreswithpubsub/storage.go index 661192243..e4372c64f 100644 --- a/cmd/dendrite-demo-libp2p/storage/postgreswithpubsub/storage.go +++ b/cmd/dendrite-demo-libp2p/storage/postgreswithpubsub/storage.go @@ -48,7 +48,7 @@ type PublicRoomsServerDatabase struct { // NewPublicRoomsServerDatabase creates a new public rooms server database. func NewPublicRoomsServerDatabase(dataSourceName string, pubsub *pubsub.PubSub) (*PublicRoomsServerDatabase, error) { - pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName) + pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName, nil) if err != nil { return nil, err } diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 060019711..e004bc12e 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -79,7 +79,7 @@ func main() { eduProducer := producers.NewEDUServerProducer(eduInputAPI) federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, rsAPI, asAPI, fsAPI, eduProducer) mediaapi.SetupMediaAPIComponent(base, deviceDB) - publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI)) + publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties()) if err != nil { logrus.WithError(err).Panicf("failed to connect to public rooms db") } diff --git a/cmd/dendrite-public-rooms-api-server/main.go b/cmd/dendrite-public-rooms-api-server/main.go index fca39a2fe..c3b49f4f7 100644 --- a/cmd/dendrite-public-rooms-api-server/main.go +++ b/cmd/dendrite-public-rooms-api-server/main.go @@ -32,7 +32,7 @@ func main() { rsAPI := base.CreateHTTPRoomserverAPIs() rsAPI.SetFederationSenderAPI(fsAPI) - publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI)) + publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties()) if err != nil { logrus.WithError(err).Panicf("failed to connect to public rooms db") } diff --git a/common/basecomponent/base.go b/common/basecomponent/base.go index 154acd806..a7e6736a6 100644 --- a/common/basecomponent/base.go +++ b/common/basecomponent/base.go @@ -150,7 +150,7 @@ func (b *BaseDendrite) CreateHTTPFederationSenderAPIs() federationSenderAPI.Fede // CreateDeviceDB creates a new instance of the device database. Should only be // called once per component. func (b *BaseDendrite) CreateDeviceDB() devices.Database { - db, err := devices.NewDatabase(string(b.Cfg.Database.Device), b.Cfg.Matrix.ServerName) + db, err := devices.NewDatabase(string(b.Cfg.Database.Device), b.Cfg.DbProperties(), b.Cfg.Matrix.ServerName) if err != nil { logrus.WithError(err).Panicf("failed to connect to devices db") } @@ -161,7 +161,7 @@ func (b *BaseDendrite) CreateDeviceDB() devices.Database { // CreateAccountsDB creates a new instance of the accounts database. Should only // be called once per component. func (b *BaseDendrite) CreateAccountsDB() accounts.Database { - db, err := accounts.NewDatabase(string(b.Cfg.Database.Account), b.Cfg.Matrix.ServerName) + db, err := accounts.NewDatabase(string(b.Cfg.Database.Account), b.Cfg.DbProperties(), b.Cfg.Matrix.ServerName) if err != nil { logrus.WithError(err).Panicf("failed to connect to accounts db") } @@ -174,6 +174,7 @@ func (b *BaseDendrite) CreateAccountsDB() accounts.Database { func (b *BaseDendrite) CreateKeyDB() keydb.Database { db, err := keydb.NewDatabase( string(b.Cfg.Database.ServerKey), + b.Cfg.DbProperties(), b.Cfg.Matrix.ServerName, b.Cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey), b.Cfg.Matrix.KeyID, @@ -244,7 +245,7 @@ func setupNaffka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) { uri, err := url.Parse(string(cfg.Database.Naffka)) if err != nil || uri.Scheme == "file" { - db, err = sqlutil.Open(common.SQLiteDriverName(), string(cfg.Database.Naffka)) + db, err = sqlutil.Open(common.SQLiteDriverName(), string(cfg.Database.Naffka), nil) if err != nil { logrus.WithError(err).Panic("Failed to open naffka database") } @@ -254,7 +255,7 @@ func setupNaffka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) { logrus.WithError(err).Panic("Failed to setup naffka database") } } else { - db, err = sqlutil.Open("postgres", string(cfg.Database.Naffka)) + db, err = sqlutil.Open("postgres", string(cfg.Database.Naffka), nil) if err != nil { logrus.WithError(err).Panic("Failed to open naffka database") } diff --git a/common/config/config.go b/common/config/config.go index 6b61fda7c..9a29186a6 100644 --- a/common/config/config.go +++ b/common/config/config.go @@ -188,6 +188,12 @@ type Dendrite struct { PublicRoomsAPI DataSource `yaml:"public_rooms_api"` // The Naffka database is used internally by the naffka library, if used. Naffka DataSource `yaml:"naffka,omitempty"` + // Maximum open connections to the DB (0 = use default, negative means unlimited) + MaxOpenConns int `yaml:"max_open_conns"` + // Maximum idle connections to the DB (0 = use default, negative means unlimited) + MaxIdleConns int `yaml:"max_idle_conns"` + // maximum amount of time (in seconds) a connection may be reused (<= 0 means unlimited) + ConnMaxLifetimeSec int `yaml:"conn_max_lifetime"` } `yaml:"database"` // TURN Server Config @@ -484,6 +490,15 @@ func (config *Dendrite) SetDefaults() { defaultMaxFileSizeBytes := FileSizeBytes(10485760) config.Media.MaxFileSizeBytes = &defaultMaxFileSizeBytes } + + if config.Database.MaxIdleConns == 0 { + config.Database.MaxIdleConns = 2 + } + + if config.Database.MaxOpenConns == 0 { + config.Database.MaxOpenConns = 100 + } + } // Error returns a string detailing how many errors were contained within a @@ -746,6 +761,33 @@ func (config *Dendrite) SetupTracing(serviceName string) (closer io.Closer, err ) } +// MaxIdleConns returns maximum idle connections to the DB +func (config Dendrite) MaxIdleConns() int { + return config.Database.MaxIdleConns +} + +// MaxOpenConns returns maximum open connections to the DB +func (config Dendrite) MaxOpenConns() int { + return config.Database.MaxOpenConns +} + +// ConnMaxLifetime returns maximum amount of time a connection may be reused +func (config Dendrite) ConnMaxLifetime() time.Duration { + return time.Duration(config.Database.ConnMaxLifetimeSec) * time.Second +} + +// DbProperties functions return properties used by database/sql/DB +type DbProperties interface { + MaxIdleConns() int + MaxOpenConns() int + ConnMaxLifetime() time.Duration +} + +// DbProperties returns cfg as a DbProperties interface +func (config Dendrite) DbProperties() DbProperties { + return config +} + // logrusLogger is a small wrapper that implements jaeger.Logger using logrus. type logrusLogger struct { l *logrus.Logger diff --git a/common/keydb/keydb.go b/common/keydb/keydb.go index fe6d87fc8..397d7849c 100644 --- a/common/keydb/keydb.go +++ b/common/keydb/keydb.go @@ -21,6 +21,7 @@ import ( "golang.org/x/crypto/ed25519" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/keydb/postgres" "github.com/matrix-org/dendrite/common/keydb/sqlite3" "github.com/matrix-org/gomatrixserverlib" @@ -29,20 +30,21 @@ import ( // NewDatabase opens a database connection. func NewDatabase( dataSourceName string, + dbProperties common.DbProperties, serverName gomatrixserverlib.ServerName, serverKey ed25519.PublicKey, serverKeyID gomatrixserverlib.KeyID, ) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { - return postgres.NewDatabase(dataSourceName, serverName, serverKey, serverKeyID) + return postgres.NewDatabase(dataSourceName, dbProperties, serverName, serverKey, serverKeyID) } switch uri.Scheme { case "postgres": - return postgres.NewDatabase(dataSourceName, serverName, serverKey, serverKeyID) + return postgres.NewDatabase(dataSourceName, dbProperties, serverName, serverKey, serverKeyID) case "file": return sqlite3.NewDatabase(dataSourceName, serverName, serverKey, serverKeyID) default: - return postgres.NewDatabase(dataSourceName, serverName, serverKey, serverKeyID) + return postgres.NewDatabase(dataSourceName, dbProperties, serverName, serverKey, serverKeyID) } } diff --git a/common/keydb/keydb_wasm.go b/common/keydb/keydb_wasm.go index 807ed40b4..38e595820 100644 --- a/common/keydb/keydb_wasm.go +++ b/common/keydb/keydb_wasm.go @@ -20,6 +20,7 @@ import ( "golang.org/x/crypto/ed25519" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/keydb/sqlite3" "github.com/matrix-org/gomatrixserverlib" ) @@ -27,6 +28,7 @@ import ( // NewDatabase opens a database connection. func NewDatabase( dataSourceName string, + dbProperties common.DbProperties, // nolint:unparam serverName gomatrixserverlib.ServerName, serverKey ed25519.PublicKey, serverKeyID gomatrixserverlib.KeyID, diff --git a/common/keydb/postgres/keydb.go b/common/keydb/postgres/keydb.go index 2879683e0..6149d8778 100644 --- a/common/keydb/postgres/keydb.go +++ b/common/keydb/postgres/keydb.go @@ -21,6 +21,7 @@ import ( "golang.org/x/crypto/ed25519" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) @@ -37,11 +38,12 @@ type Database struct { // Returns an error if there was a problem talking to the database. func NewDatabase( dataSourceName string, + dbProperties common.DbProperties, serverName gomatrixserverlib.ServerName, serverKey ed25519.PublicKey, serverKeyID gomatrixserverlib.KeyID, ) (*Database, error) { - db, err := sqlutil.Open("postgres", dataSourceName) + db, err := sqlutil.Open("postgres", dataSourceName, dbProperties) if err != nil { return nil, err } diff --git a/common/keydb/sqlite3/keydb.go b/common/keydb/sqlite3/keydb.go index 82d2a491f..1405836a4 100644 --- a/common/keydb/sqlite3/keydb.go +++ b/common/keydb/sqlite3/keydb.go @@ -44,7 +44,7 @@ func NewDatabase( serverKey ed25519.PublicKey, serverKeyID gomatrixserverlib.KeyID, ) (*Database, error) { - db, err := sqlutil.Open(common.SQLiteDriverName(), dataSourceName) + db, err := sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil) if err != nil { return nil, err } diff --git a/common/sql.go b/common/sql.go index f50a58969..e93ff1c2f 100644 --- a/common/sql.go +++ b/common/sql.go @@ -18,6 +18,7 @@ import ( "database/sql" "fmt" "runtime" + "time" ) // A Transaction is something that can be committed or rolledback. @@ -99,3 +100,10 @@ func SQLiteDriverName() string { } return "sqlite3" } + +// DbProperties functions return properties used by database/sql/DB +type DbProperties interface { + MaxIdleConns() int + MaxOpenConns() int + ConnMaxLifetime() time.Duration +} diff --git a/dendrite-config.yaml b/dendrite-config.yaml index bed78a5af..8c8fba390 100644 --- a/dendrite-config.yaml +++ b/dendrite-config.yaml @@ -116,6 +116,9 @@ database: federation_sender: "postgres://dendrite:itsasecret@localhost/dendrite_federationsender?sslmode=disable" appservice: "postgres://dendrite:itsasecret@localhost/dendrite_appservice?sslmode=disable" public_rooms_api: "postgres://dendrite:itsasecret@localhost/dendrite_publicroomsapi?sslmode=disable" + max_open_conns: 100 + max_idle_conns: 2 + conn_max_lifetime: -1 # If using naffka you need to specify a naffka database # naffka: "postgres://dendrite:itsasecret@localhost/dendrite_naffka?sslmode=disable" diff --git a/federationsender/federationsender.go b/federationsender/federationsender.go index 64de4fd27..cf4395527 100644 --- a/federationsender/federationsender.go +++ b/federationsender/federationsender.go @@ -37,7 +37,7 @@ func SetupFederationSenderComponent( rsAPI roomserverAPI.RoomserverInternalAPI, keyRing *gomatrixserverlib.KeyRing, ) api.FederationSenderInternalAPI { - federationSenderDB, err := storage.NewDatabase(string(base.Cfg.Database.FederationSender)) + federationSenderDB, err := storage.NewDatabase(string(base.Cfg.Database.FederationSender), base.Cfg.DbProperties()) if err != nil { logrus.WithError(err).Panic("failed to connect to federation sender db") } diff --git a/federationsender/storage/postgres/storage.go b/federationsender/storage/postgres/storage.go index b909a189b..c3892ac10 100644 --- a/federationsender/storage/postgres/storage.go +++ b/federationsender/storage/postgres/storage.go @@ -33,10 +33,10 @@ type Database struct { } // NewDatabase opens a new database -func NewDatabase(dataSourceName string) (*Database, error) { +func NewDatabase(dataSourceName string, dbProperties common.DbProperties) (*Database, error) { var result Database var err error - if result.db, err = sqlutil.Open("postgres", dataSourceName); err != nil { + if result.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { return nil, err } if err = result.prepare(); err != nil { diff --git a/federationsender/storage/sqlite3/storage.go b/federationsender/storage/sqlite3/storage.go index 458d7d7e5..772744474 100644 --- a/federationsender/storage/sqlite3/storage.go +++ b/federationsender/storage/sqlite3/storage.go @@ -38,7 +38,7 @@ type Database struct { func NewDatabase(dataSourceName string) (*Database, error) { var result Database var err error - if result.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil { + if result.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil); err != nil { return nil, err } if err = result.prepare(); err != nil { diff --git a/federationsender/storage/storage.go b/federationsender/storage/storage.go index 2f018dff1..d481e58a2 100644 --- a/federationsender/storage/storage.go +++ b/federationsender/storage/storage.go @@ -19,22 +19,23 @@ package storage import ( "net/url" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/federationsender/storage/postgres" "github.com/matrix-org/dendrite/federationsender/storage/sqlite3" ) // NewDatabase opens a new database -func NewDatabase(dataSourceName string) (Database, error) { +func NewDatabase(dataSourceName string, dbProperties common.DbProperties) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { - return postgres.NewDatabase(dataSourceName) + return postgres.NewDatabase(dataSourceName, dbProperties) } switch uri.Scheme { case "file": return sqlite3.NewDatabase(dataSourceName) case "postgres": - return postgres.NewDatabase(dataSourceName) + return postgres.NewDatabase(dataSourceName, dbProperties) default: - return postgres.NewDatabase(dataSourceName) + return postgres.NewDatabase(dataSourceName, dbProperties) } } diff --git a/federationsender/storage/storage_wasm.go b/federationsender/storage/storage_wasm.go index f2c8ae1b4..44d4c8060 100644 --- a/federationsender/storage/storage_wasm.go +++ b/federationsender/storage/storage_wasm.go @@ -18,11 +18,15 @@ import ( "fmt" "net/url" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/federationsender/storage/sqlite3" ) // NewDatabase opens a new database -func NewDatabase(dataSourceName string) (Database, error) { +func NewDatabase( + dataSourceName string, + dbProperties common.DbProperties, // nolint:unparam +) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return nil, fmt.Errorf("Cannot use postgres implementation") diff --git a/internal/sqlutil/trace.go b/internal/sqlutil/trace.go index 3d5fa7dc7..42ac4e582 100644 --- a/internal/sqlutil/trace.go +++ b/internal/sqlutil/trace.go @@ -21,9 +21,11 @@ import ( "fmt" "io" "os" + "regexp" "strings" "time" + "github.com/matrix-org/dendrite/common" "github.com/ngrok/sqlmw" "github.com/sirupsen/logrus" ) @@ -76,12 +78,27 @@ func (in *traceInterceptor) RowsNext(c context.Context, rows driver.Rows, dest [ // Open opens a database specified by its database driver name and a driver-specific data source name, // usually consisting of at least a database name and connection information. Includes tracing driver // if DENDRITE_TRACE_SQL=1 -func Open(driverName, dsn string) (*sql.DB, error) { +func Open(driverName, dsn string, dbProperties common.DbProperties) (*sql.DB, error) { if tracingEnabled { // install the wrapped driver driverName += "-trace" } - return sql.Open(driverName, dsn) + db, err := sql.Open(driverName, dsn) + if err != nil { + return nil, err + } + if driverName != common.SQLiteDriverName() && dbProperties != nil { + logrus.WithFields(logrus.Fields{ + "MaxOpenConns": dbProperties.MaxOpenConns(), + "MaxIdleConns": dbProperties.MaxIdleConns(), + "ConnMaxLifetime": dbProperties.ConnMaxLifetime(), + "dataSourceName": regexp.MustCompile(`://[^@]*@`).ReplaceAllLiteralString(dsn, "://"), + }).Debug("Setting DB connection limits") + db.SetMaxOpenConns(dbProperties.MaxOpenConns()) + db.SetMaxIdleConns(dbProperties.MaxIdleConns()) + db.SetConnMaxLifetime(dbProperties.ConnMaxLifetime()) + } + return db, nil } func init() { diff --git a/mediaapi/mediaapi.go b/mediaapi/mediaapi.go index f2e614c17..4a0f5d188 100644 --- a/mediaapi/mediaapi.go +++ b/mediaapi/mediaapi.go @@ -29,7 +29,7 @@ func SetupMediaAPIComponent( base *basecomponent.BaseDendrite, deviceDB devices.Database, ) { - mediaDB, err := storage.Open(string(base.Cfg.Database.MediaAPI)) + mediaDB, err := storage.Open(string(base.Cfg.Database.MediaAPI), base.Cfg.DbProperties()) if err != nil { logrus.WithError(err).Panicf("failed to connect to media db") } diff --git a/mediaapi/storage/postgres/storage.go b/mediaapi/storage/postgres/storage.go index 18126b151..4ddfc8fdf 100644 --- a/mediaapi/storage/postgres/storage.go +++ b/mediaapi/storage/postgres/storage.go @@ -21,6 +21,7 @@ import ( // Import the postgres database driver. _ "github.com/lib/pq" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/mediaapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -33,10 +34,10 @@ type Database struct { } // Open opens a postgres database. -func Open(dataSourceName string) (*Database, error) { +func Open(dataSourceName string, dbProperties common.DbProperties) (*Database, error) { var d Database var err error - if d.db, err = sqlutil.Open("postgres", dataSourceName); err != nil { + if d.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { return nil, err } if err = d.statements.prepare(d.db); err != nil { diff --git a/mediaapi/storage/sqlite3/storage.go b/mediaapi/storage/sqlite3/storage.go index abafecf20..8fa2e5375 100644 --- a/mediaapi/storage/sqlite3/storage.go +++ b/mediaapi/storage/sqlite3/storage.go @@ -37,7 +37,7 @@ type Database struct { func Open(dataSourceName string) (*Database, error) { var d Database var err error - if d.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil { + if d.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil); err != nil { return nil, err } if err = d.statements.prepare(d.db); err != nil { diff --git a/mediaapi/storage/storage.go b/mediaapi/storage/storage.go index c533477cd..6589c8304 100644 --- a/mediaapi/storage/storage.go +++ b/mediaapi/storage/storage.go @@ -19,22 +19,23 @@ package storage import ( "net/url" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/mediaapi/storage/postgres" "github.com/matrix-org/dendrite/mediaapi/storage/sqlite3" ) // Open opens a postgres database. -func Open(dataSourceName string) (Database, error) { +func Open(dataSourceName string, dbProperties common.DbProperties) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { - return postgres.Open(dataSourceName) + return postgres.Open(dataSourceName, dbProperties) } switch uri.Scheme { case "postgres": - return postgres.Open(dataSourceName) + return postgres.Open(dataSourceName, dbProperties) case "file": return sqlite3.Open(dataSourceName) default: - return postgres.Open(dataSourceName) + return postgres.Open(dataSourceName, dbProperties) } } diff --git a/mediaapi/storage/storage_wasm.go b/mediaapi/storage/storage_wasm.go index 92f0ad134..3c39e5d31 100644 --- a/mediaapi/storage/storage_wasm.go +++ b/mediaapi/storage/storage_wasm.go @@ -18,11 +18,15 @@ import ( "fmt" "net/url" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/mediaapi/storage/sqlite3" ) // Open opens a postgres database. -func Open(dataSourceName string) (Database, error) { +func Open( + dataSourceName string, + dbProperties common.DbProperties, // nolint:unparam +) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return nil, fmt.Errorf("Cannot use postgres implementation") diff --git a/publicroomsapi/storage/postgres/storage.go b/publicroomsapi/storage/postgres/storage.go index 8c4660cca..6242c9d54 100644 --- a/publicroomsapi/storage/postgres/storage.go +++ b/publicroomsapi/storage/postgres/storage.go @@ -36,10 +36,10 @@ type PublicRoomsServerDatabase struct { type attributeValue interface{} // NewPublicRoomsServerDatabase creates a new public rooms server database. -func NewPublicRoomsServerDatabase(dataSourceName string) (*PublicRoomsServerDatabase, error) { +func NewPublicRoomsServerDatabase(dataSourceName string, dbProperties common.DbProperties) (*PublicRoomsServerDatabase, error) { var db *sql.DB var err error - if db, err = sqlutil.Open("postgres", dataSourceName); err != nil { + if db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { return nil, err } storage := PublicRoomsServerDatabase{ diff --git a/publicroomsapi/storage/sqlite3/storage.go b/publicroomsapi/storage/sqlite3/storage.go index 121601628..efe35bdde 100644 --- a/publicroomsapi/storage/sqlite3/storage.go +++ b/publicroomsapi/storage/sqlite3/storage.go @@ -41,7 +41,7 @@ type attributeValue interface{} func NewPublicRoomsServerDatabase(dataSourceName string) (*PublicRoomsServerDatabase, error) { var db *sql.DB var err error - if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName); err != nil { + if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil); err != nil { return nil, err } storage := PublicRoomsServerDatabase{ diff --git a/publicroomsapi/storage/storage.go b/publicroomsapi/storage/storage.go index e674514aa..7dcfe5633 100644 --- a/publicroomsapi/storage/storage.go +++ b/publicroomsapi/storage/storage.go @@ -19,6 +19,7 @@ package storage import ( "net/url" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/publicroomsapi/storage/postgres" "github.com/matrix-org/dendrite/publicroomsapi/storage/sqlite3" ) @@ -27,17 +28,17 @@ const schemePostgres = "postgres" const schemeFile = "file" // NewPublicRoomsServerDatabase opens a database connection. -func NewPublicRoomsServerDatabase(dataSourceName string) (Database, error) { +func NewPublicRoomsServerDatabase(dataSourceName string, dbProperties common.DbProperties) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { - return postgres.NewPublicRoomsServerDatabase(dataSourceName) + return postgres.NewPublicRoomsServerDatabase(dataSourceName, dbProperties) } switch uri.Scheme { case schemePostgres: - return postgres.NewPublicRoomsServerDatabase(dataSourceName) + return postgres.NewPublicRoomsServerDatabase(dataSourceName, dbProperties) case schemeFile: return sqlite3.NewPublicRoomsServerDatabase(dataSourceName) default: - return postgres.NewPublicRoomsServerDatabase(dataSourceName) + return postgres.NewPublicRoomsServerDatabase(dataSourceName, dbProperties) } } diff --git a/roomserver/roomserver.go b/roomserver/roomserver.go index 916e25fbd..450da5bb6 100644 --- a/roomserver/roomserver.go +++ b/roomserver/roomserver.go @@ -35,7 +35,7 @@ func SetupRoomServerComponent( keyRing gomatrixserverlib.JSONVerifier, fedClient *gomatrixserverlib.FederationClient, ) api.RoomserverInternalAPI { - roomserverDB, err := storage.Open(string(base.Cfg.Database.RoomServer)) + roomserverDB, err := storage.Open(string(base.Cfg.Database.RoomServer), base.Cfg.DbProperties()) if err != nil { logrus.WithError(err).Panicf("failed to connect to room server db") } diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go index 5b5c61b0f..1d825ecc2 100644 --- a/roomserver/storage/postgres/storage.go +++ b/roomserver/storage/postgres/storage.go @@ -20,6 +20,7 @@ import ( "database/sql" "encoding/json" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/internal/sqlutil" // Import the postgres database driver. @@ -36,10 +37,10 @@ type Database struct { } // Open a postgres database. -func Open(dataSourceName string) (*Database, error) { +func Open(dataSourceName string, dbProperties common.DbProperties) (*Database, error) { var d Database var err error - if d.db, err = sqlutil.Open("postgres", dataSourceName); err != nil { + if d.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { return nil, err } if err = d.statements.prepare(d.db); err != nil { diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index b6e846df7..e77fea9cf 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -37,7 +37,7 @@ type Database struct { db *sql.DB } -// Open a postgres database. +// Open a sqlite database. func Open(dataSourceName string) (*Database, error) { var d Database uri, err := url.Parse(dataSourceName) @@ -52,7 +52,7 @@ func Open(dataSourceName string) (*Database, error) { } else { return nil, errors.New("no filename or path in connect string") } - if d.db, err = sqlutil.Open(common.SQLiteDriverName(), cs); err != nil { + if d.db, err = sqlutil.Open(common.SQLiteDriverName(), cs, nil); err != nil { return nil, err } //d.db.Exec("PRAGMA journal_mode=WAL;") diff --git a/roomserver/storage/storage.go b/roomserver/storage/storage.go index 7b9109aa0..99e99a008 100644 --- a/roomserver/storage/storage.go +++ b/roomserver/storage/storage.go @@ -19,22 +19,23 @@ package storage import ( "net/url" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/storage/postgres" "github.com/matrix-org/dendrite/roomserver/storage/sqlite3" ) -// NewPublicRoomsServerDatabase opens a database connection. -func Open(dataSourceName string) (Database, error) { +// Open opens a database connection. +func Open(dataSourceName string, dbProperties common.DbProperties) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { - return postgres.Open(dataSourceName) + return postgres.Open(dataSourceName, dbProperties) } switch uri.Scheme { case "postgres": - return postgres.Open(dataSourceName) + return postgres.Open(dataSourceName, dbProperties) case "file": return sqlite3.Open(dataSourceName) default: - return postgres.Open(dataSourceName) + return postgres.Open(dataSourceName, dbProperties) } } diff --git a/roomserver/storage/storage_wasm.go b/roomserver/storage/storage_wasm.go index d7fc352e8..5fa48bc95 100644 --- a/roomserver/storage/storage_wasm.go +++ b/roomserver/storage/storage_wasm.go @@ -18,11 +18,15 @@ import ( "fmt" "net/url" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/storage/sqlite3" ) // NewPublicRoomsServerDatabase opens a database connection. -func Open(dataSourceName string) (Database, error) { +func Open( + dataSourceName string, + dbProperties common.DbProperties, // nolint:unparam +) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return nil, fmt.Errorf("Cannot use postgres implementation") diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go index a6de15178..1845ac386 100644 --- a/syncapi/storage/postgres/syncserver.go +++ b/syncapi/storage/postgres/syncserver.go @@ -61,10 +61,10 @@ type SyncServerDatasource struct { } // NewSyncServerDatasource creates a new sync server database -func NewSyncServerDatasource(dbDataSourceName string) (*SyncServerDatasource, error) { +func NewSyncServerDatasource(dbDataSourceName string, dbProperties common.DbProperties) (*SyncServerDatasource, error) { var d SyncServerDatasource var err error - if d.db, err = sqlutil.Open("postgres", dbDataSourceName); err != nil { + if d.db, err = sqlutil.Open("postgres", dbDataSourceName, dbProperties); err != nil { return nil, err } if err = d.PartitionOffsetStatements.Prepare(d.db, "syncapi"); err != nil { diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index 7e8e4ff5d..425073a7b 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -80,7 +80,7 @@ func NewSyncServerDatasource(dataSourceName string) (*SyncServerDatasource, erro } else { return nil, errors.New("no filename or path in connect string") } - if d.db, err = sqlutil.Open(common.SQLiteDriverName(), cs); err != nil { + if d.db, err = sqlutil.Open(common.SQLiteDriverName(), cs, nil); err != nil { return nil, err } if err = d.prepare(); err != nil { diff --git a/syncapi/storage/storage.go b/syncapi/storage/storage.go index c56db0635..9574f37bd 100644 --- a/syncapi/storage/storage.go +++ b/syncapi/storage/storage.go @@ -19,22 +19,23 @@ package storage import ( "net/url" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/syncapi/storage/postgres" "github.com/matrix-org/dendrite/syncapi/storage/sqlite3" ) -// NewPublicRoomsServerDatabase opens a database connection. -func NewSyncServerDatasource(dataSourceName string) (Database, error) { +// NewSyncServerDatasource opens a database connection. +func NewSyncServerDatasource(dataSourceName string, dbProperties common.DbProperties) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { - return postgres.NewSyncServerDatasource(dataSourceName) + return postgres.NewSyncServerDatasource(dataSourceName, dbProperties) } switch uri.Scheme { case "postgres": - return postgres.NewSyncServerDatasource(dataSourceName) + return postgres.NewSyncServerDatasource(dataSourceName, dbProperties) case "file": return sqlite3.NewSyncServerDatasource(dataSourceName) default: - return postgres.NewSyncServerDatasource(dataSourceName) + return postgres.NewSyncServerDatasource(dataSourceName, dbProperties) } } diff --git a/syncapi/storage/storage_wasm.go b/syncapi/storage/storage_wasm.go index 43806a012..84bd9df96 100644 --- a/syncapi/storage/storage_wasm.go +++ b/syncapi/storage/storage_wasm.go @@ -18,11 +18,15 @@ import ( "fmt" "net/url" + "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/syncapi/storage/sqlite3" ) // NewPublicRoomsServerDatabase opens a database connection. -func NewSyncServerDatasource(dataSourceName string) (Database, error) { +func NewSyncServerDatasource( + dataSourceName string, + dbProperties common.DbProperties, // nolint:unparam +) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return nil, fmt.Errorf("Cannot use postgres implementation") diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go index 5ab1ec7c8..4219d5603 100644 --- a/syncapi/syncapi.go +++ b/syncapi/syncapi.go @@ -42,7 +42,7 @@ func SetupSyncAPIComponent( federation *gomatrixserverlib.FederationClient, cfg *config.Dendrite, ) { - syncDB, err := storage.NewSyncServerDatasource(string(base.Cfg.Database.SyncAPI)) + syncDB, err := storage.NewSyncServerDatasource(string(base.Cfg.Database.SyncAPI), base.Cfg.DbProperties()) if err != nil { logrus.WithError(err).Panicf("failed to connect to sync db") } From 36bbb255614d289a3417194d4d2ac129e339f531 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Fri, 1 May 2020 16:41:13 +0100 Subject: [PATCH 025/200] Fix ordering when backfilling (#1000) * Fix ordering when backfilling The problem was that we weren't sorting the returned events by depth when sending them back to the caller, instead we were sorting by prev_events which is not the same thing. * Fixup tests --- syncapi/routing/messages.go | 40 +++++++++++---- syncapi/storage/sqlite3/syncserver.go | 6 ++- syncapi/storage/storage_test.go | 74 +++++++++++++++++++++++++++ 3 files changed, 110 insertions(+), 10 deletions(-) diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index d7e39704d..270b0ee95 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -210,15 +210,16 @@ func (r *messagesReq) retrieveEvents() ( } // Sort the events to ensure we send them in the right order. - events = gomatrixserverlib.HeaderedReverseTopologicalOrdering( - events, - gomatrixserverlib.TopologicalOrderByPrevEvents, - ) if r.backwardOrdering { // This reverses the array from old->new to new->old - sort.SliceStable(events, func(i, j int) bool { - return true - }) + reversed := func(in []gomatrixserverlib.HeaderedEvent) []gomatrixserverlib.HeaderedEvent { + out := make([]gomatrixserverlib.HeaderedEvent, len(in)) + for i := 0; i < len(in); i++ { + out[i] = in[len(in)-i-1] + } + return out + } + events = reversed(events) } // Convert all of the events into client events. @@ -259,6 +260,7 @@ func (r *messagesReq) retrieveEvents() ( // to them by the event on their left, therefore we need to decrement the // end position we send in the response if we're going backward. end.PDUPosition-- + end.EDUTypingPosition += 1000 } // The lowest token value is 1, therefore we need to manually set it to that @@ -345,10 +347,23 @@ func (r *messagesReq) handleNonEmptyEventsSlice(streamEvents []types.StreamEvent // Append the events ve previously retrieved locally. events = append(events, r.db.StreamEventsToEvents(nil, streamEvents)...) + sort.Sort(eventsByDepth(events)) return } +type eventsByDepth []gomatrixserverlib.HeaderedEvent + +func (e eventsByDepth) Len() int { + return len(e) +} +func (e eventsByDepth) Swap(i, j int) { + e[i], e[j] = e[j], e[i] +} +func (e eventsByDepth) Less(i, j int) bool { + return e[i].Depth() < e[j].Depth() +} + // backfill performs a backfill request over the federation on another // homeserver in the room. // See: https://matrix.org/docs/spec/server_server/latest#get-matrix-federation-v1-backfill-roomid @@ -375,17 +390,24 @@ func (r *messagesReq) backfill(roomID string, fromEventIDs []string, limit int) // Currently, this can race with live events for the room and cause problems. It's also just a bit unclear // when you have multiple entry points to write events. + // we have to order these by depth, starting with the lowest because otherwise the topology tokens + // will skip over events that have the same depth but different stream positions due to the query which is: + // - anything less than the depth OR + // - anything with the same depth and a lower stream position. + sort.Sort(eventsByDepth(res.Events)) + // Store the events in the database, while marking them as unfit to show // up in responses to sync requests. for i := range res.Events { - if _, err = r.db.WriteEvent( + _, err = r.db.WriteEvent( r.ctx, &res.Events[i], []gomatrixserverlib.HeaderedEvent{}, []string{}, []string{}, nil, true, - ); err != nil { + ) + if err != nil { return nil, err } } diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index 425073a7b..314ea2aa3 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -434,6 +434,7 @@ func (d *SyncServerDatasource) syncPositionTx( } sp.PDUPosition = types.StreamPosition(maxEventID) sp.EDUTypingPosition = types.StreamPosition(d.eduCache.GetLatestSyncPosition()) + sp.Type = types.PaginationTokenTypeStream return } @@ -658,6 +659,7 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( backwardTopologyPos = types.StreamPosition(1) } else { backwardTopologyPos-- + backwardTopologyStreamPos += 1000 // this has to be bigger than the number of events we backfill per request } // We don't include a device here as we don't need to send down @@ -817,11 +819,13 @@ func (d *SyncServerDatasource) getBackwardTopologyPos( if len(events) > 0 { pos, spos, _ = d.topology.selectPositionInTopology(ctx, txn, events[0].EventID()) } - // TODO: I have no idea what this is doing. + // go to the previous position so we don't pull out the same event twice + // FIXME: This could be done more nicely by being explicit with inclusive/exclusive rules if pos-1 <= 0 { pos = types.StreamPosition(1) } else { pos = pos - 1 + spos += 1000 // this has to be bigger than the number of events we backfill per request } return } diff --git a/syncapi/storage/storage_test.go b/syncapi/storage/storage_test.go index 378c1fe35..b951efa45 100644 --- a/syncapi/storage/storage_test.go +++ b/syncapi/storage/storage_test.go @@ -118,6 +118,7 @@ func MustWriteEvents(t *testing.T, db storage.Database, events []gomatrixserverl if err != nil { t.Fatalf("WriteEvent failed: %s", err) } + fmt.Println("Event ID", ev.EventID(), " spos=", pos, "depth=", ev.Depth()) positions = append(positions, pos) } return @@ -407,6 +408,64 @@ func TestGetEventsInRangeWithEventsSameDepth(t *testing.T) { } } +// The purpose of this test is to make sure that events are returned in the right *order* when they have been inserted in a manner similar to +// how any kind of backfill operation will insert the events. This test inserts the SimpleRoom events in a manner similar to how backfill over +// federation would: +// - First inserts join event of test user C +// - Inserts chunks of history in strata e.g (25-30, 20-25, 15-20, 10-15, 5-10, 0-5). +// The test then does a backfill to ensure that the response is ordered correctly according to depth. +func TestGetEventsInRangeWithEventsInsertedLikeBackfill(t *testing.T) { + t.Parallel() + db := MustCreateDatabase(t) + events, _ := SimpleRoom(t, testRoomID, testUserIDA, testUserIDB) + + // "federation" join + userC := fmt.Sprintf("@radiance:%s", testOrigin) + joinEvent := MustCreateEvent(t, testRoomID, []gomatrixserverlib.HeaderedEvent{events[len(events)-1]}, &gomatrixserverlib.EventBuilder{ + Content: []byte(fmt.Sprintf(`{"membership":"join"}`)), + Type: "m.room.member", + StateKey: &userC, + Sender: userC, + Depth: int64(len(events) + 1), + }) + MustWriteEvents(t, db, []gomatrixserverlib.HeaderedEvent{joinEvent}) + + // Sync will return this for the prev_batch + from := topologyTokenBefore(t, db, joinEvent.EventID()) + + // inject events in batches as if they were from backfill + // e.g [1,2,3,4,5,6] => [4,5,6] , [1,2,3] + chunkSize := 5 + for i := len(events); i >= 0; i -= chunkSize { + start := i - chunkSize + if start < 0 { + start = 0 + } + backfill := events[start:i] + MustWriteEvents(t, db, backfill) + } + + // head towards the beginning of time + to := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, 0, 0) + + // starting at `from`, backpaginate to the beginning of time, asserting as we go. + chunkSize = 3 + events = reversed(events) + for i := 0; i < len(events); i += chunkSize { + paginatedEvents, err := db.GetEventsInRange(ctx, from, to, testRoomID, chunkSize, true) + if err != nil { + t.Fatalf("GetEventsInRange returned an error: %s", err) + } + gots := gomatrixserverlib.HeaderedToClientEvents(db.StreamEventsToEvents(&testUserDeviceA, paginatedEvents), gomatrixserverlib.FormatAll) + endi := i + chunkSize + if endi > len(events) { + endi = len(events) + } + assertEventsEqual(t, from.String(), true, gots, events[i:endi]) + from = topologyTokenBefore(t, db, paginatedEvents[len(paginatedEvents)-1].EventID()) + } +} + func assertEventsEqual(t *testing.T, msg string, checkRoomID bool, gots []gomatrixserverlib.ClientEvent, wants []gomatrixserverlib.HeaderedEvent) { if len(gots) != len(wants) { t.Fatalf("%s response returned %d events, want %d", msg, len(gots), len(wants)) @@ -447,6 +506,21 @@ func assertEventsEqual(t *testing.T, msg string, checkRoomID bool, gots []gomatr } } +func topologyTokenBefore(t *testing.T, db storage.Database, eventID string) *types.PaginationToken { + pos, spos, err := db.EventPositionInTopology(ctx, eventID) + if err != nil { + t.Fatalf("failed to get EventPositionInTopology: %s", err) + } + + if pos-1 <= 0 { + pos = types.StreamPosition(1) + } else { + pos = pos - 1 + spos += 1000 // this has to be bigger than the chunk limit + } + return types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, pos, spos) +} + func reversed(in []gomatrixserverlib.HeaderedEvent) []gomatrixserverlib.HeaderedEvent { out := make([]gomatrixserverlib.HeaderedEvent, len(in)) for i := 0; i < len(in); i++ { From 5c894efd0ee67bf911b9c3b5428a61ddc58819de Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 4 May 2020 13:53:47 +0100 Subject: [PATCH 026/200] Roomserver perform join (#1001) * Add PerformJoin template * Try roomserver perform join * Send correct server name to FS API * Pass through content, try to handle multiple server names * Fix local server checks * Don't refer to non-existent error * Add directory lookups of aliases * Remove unneeded parameters * Don't repeat join events into the roomserver * Unmarshal the content, that would help * Check if the user is already in the room in the fedeationapi too * Return incompatible room version error * Use Membership, don't try more servers than needed * Review comments, make FS API take list of servernames, dedupe them, break out of loop properly on success * Tweaks --- clientapi/jsonerror/jsonerror.go | 7 + clientapi/routing/joinroom.go | 320 ++------------------------- clientapi/routing/routing.go | 3 +- federationapi/routing/join.go | 53 +++-- federationsender/api/api.go | 6 + federationsender/api/perform.go | 35 ++- federationsender/internal/perform.go | 205 ++++++++++------- federationsender/types/types.go | 6 + roomserver/api/api.go | 6 + roomserver/api/perform.go | 59 +++++ roomserver/internal/api.go | 13 ++ roomserver/internal/perform_join.go | 199 +++++++++++++++++ 12 files changed, 506 insertions(+), 406 deletions(-) create mode 100644 roomserver/api/perform.go create mode 100644 roomserver/internal/perform_join.go diff --git a/clientapi/jsonerror/jsonerror.go b/clientapi/jsonerror/jsonerror.go index 735de5bea..85e887aec 100644 --- a/clientapi/jsonerror/jsonerror.go +++ b/clientapi/jsonerror/jsonerror.go @@ -18,6 +18,7 @@ import ( "fmt" "net/http" + "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -124,6 +125,12 @@ func GuestAccessForbidden(msg string) *MatrixError { return &MatrixError{"M_GUEST_ACCESS_FORBIDDEN", msg} } +// IncompatibleRoomVersion is an error which is returned when the client +// requests a room with a version that is unsupported. +func IncompatibleRoomVersion(roomVersion gomatrixserverlib.RoomVersion) *MatrixError { + return &MatrixError{"M_INCOMPATIBLE_ROOM_VERSION", string(roomVersion)} +} + // UnsupportedRoomVersion is an error which is returned when the client // requests a room with a version that is unsupported. func UnsupportedRoomVersion(msg string) *MatrixError { diff --git a/clientapi/routing/joinroom.go b/clientapi/routing/joinroom.go index df83c2a9f..48e42214d 100644 --- a/clientapi/routing/joinroom.go +++ b/clientapi/routing/joinroom.go @@ -15,332 +15,48 @@ package routing import ( - "fmt" "net/http" - "strings" - "time" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/clientapi/producers" - "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/common/config" - federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/gomatrix" - "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) -// JoinRoomByIDOrAlias implements the "/join/{roomIDOrAlias}" API. -// https://matrix.org/docs/spec/client_server/r0.2.0.html#post-matrix-client-r0-join-roomidoralias func JoinRoomByIDOrAlias( req *http.Request, device *authtypes.Device, - roomIDOrAlias string, - cfg *config.Dendrite, - federation *gomatrixserverlib.FederationClient, - producer *producers.RoomserverProducer, rsAPI roomserverAPI.RoomserverInternalAPI, - fsAPI federationSenderAPI.FederationSenderInternalAPI, - keyRing gomatrixserverlib.KeyRing, - accountDB accounts.Database, + roomIDOrAlias string, ) util.JSONResponse { - var content map[string]interface{} // must be a JSON object - if resErr := httputil.UnmarshalJSONRequest(req, &content); resErr != nil { - return *resErr + // Prepare to ask the roomserver to perform the room join. + joinReq := roomserverAPI.PerformJoinRequest{ + RoomIDOrAlias: roomIDOrAlias, + UserID: device.UserID, + } + joinRes := roomserverAPI.PerformJoinResponse{} + + // If content was provided in the request then incude that + // in the request. It'll get used as a part of the membership + // event content. + if err := httputil.UnmarshalJSONRequest(req, &joinReq.Content); err != nil { + return *err } - evTime, err := httputil.ParseTSParam(req) - if err != nil { + // Ask the roomserver to perform the join. + if err := rsAPI.PerformJoin(req.Context(), &joinReq, &joinRes); err != nil { return util.JSONResponse{ Code: http.StatusBadRequest, - JSON: jsonerror.InvalidArgumentValue(err.Error()), + JSON: jsonerror.Unknown(err.Error()), } } - localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) - if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") - return jsonerror.InternalServerError() - } - - profile, err := accountDB.GetProfileByLocalpart(req.Context(), localpart) - if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("accountDB.GetProfileByLocalpart failed") - return jsonerror.InternalServerError() - } - - content["membership"] = gomatrixserverlib.Join - content["displayname"] = profile.DisplayName - content["avatar_url"] = profile.AvatarURL - - r := joinRoomReq{ - req, evTime, content, device.UserID, cfg, federation, producer, - rsAPI, fsAPI, keyRing, - } - - if strings.HasPrefix(roomIDOrAlias, "!") { - return r.joinRoomByID(roomIDOrAlias) - } - if strings.HasPrefix(roomIDOrAlias, "#") { - return r.joinRoomByAlias(roomIDOrAlias) - } return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.BadJSON( - fmt.Sprintf("Invalid first character '%s' for room ID or alias", - string([]rune(roomIDOrAlias)[0])), // Wrapping with []rune makes this call UTF-8 safe - ), - } -} - -type joinRoomReq struct { - req *http.Request - evTime time.Time - content map[string]interface{} - userID string - cfg *config.Dendrite - federation *gomatrixserverlib.FederationClient - producer *producers.RoomserverProducer - rsAPI roomserverAPI.RoomserverInternalAPI - fsAPI federationSenderAPI.FederationSenderInternalAPI - keyRing gomatrixserverlib.KeyRing -} - -// joinRoomByID joins a room by room ID -func (r joinRoomReq) joinRoomByID(roomID string) util.JSONResponse { - // A client should only join a room by room ID when it has an invite - // to the room. If the server is already in the room then we can - // lookup the invite and process the request as a normal state event. - // If the server is not in the room the we will need to look up the - // remote server the invite came from in order to request a join event - // from that server. - queryReq := roomserverAPI.QueryInvitesForUserRequest{ - RoomID: roomID, TargetUserID: r.userID, - } - var queryRes roomserverAPI.QueryInvitesForUserResponse - if err := r.rsAPI.QueryInvitesForUser(r.req.Context(), &queryReq, &queryRes); err != nil { - util.GetLogger(r.req.Context()).WithError(err).Error("r.queryAPI.QueryInvitesForUser failed") - return jsonerror.InternalServerError() - } - - servers := []gomatrixserverlib.ServerName{} - seenInInviterIDs := map[gomatrixserverlib.ServerName]bool{} - for _, userID := range queryRes.InviteSenderUserIDs { - _, domain, err := gomatrixserverlib.SplitID('@', userID) - if err != nil { - util.GetLogger(r.req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") - return jsonerror.InternalServerError() - } - if !seenInInviterIDs[domain] { - servers = append(servers, domain) - seenInInviterIDs[domain] = true - } - } - - // Also add the domain extracted from the roomID as a last resort to join - // in case the client is erroneously trying to join by ID without an invite - // or all previous attempts at domains extracted from the inviter IDs fail - // Note: It's no guarantee we'll succeed because a room isn't bound to the domain in its ID - _, domain, err := gomatrixserverlib.SplitID('!', roomID) - if err != nil { - util.GetLogger(r.req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") - return jsonerror.InternalServerError() - } - if domain != r.cfg.Matrix.ServerName && !seenInInviterIDs[domain] { - servers = append(servers, domain) - } - - return r.joinRoomUsingServers(roomID, servers) - -} - -// joinRoomByAlias joins a room using a room alias. -func (r joinRoomReq) joinRoomByAlias(roomAlias string) util.JSONResponse { - _, domain, err := gomatrixserverlib.SplitID('#', roomAlias) - if err != nil { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.BadJSON("Room alias must be in the form '#localpart:domain'"), - } - } - if domain == r.cfg.Matrix.ServerName { - queryReq := roomserverAPI.GetRoomIDForAliasRequest{Alias: roomAlias} - var queryRes roomserverAPI.GetRoomIDForAliasResponse - if err = r.rsAPI.GetRoomIDForAlias(r.req.Context(), &queryReq, &queryRes); err != nil { - util.GetLogger(r.req.Context()).WithError(err).Error("r.aliasAPI.GetRoomIDForAlias failed") - return jsonerror.InternalServerError() - } - - if len(queryRes.RoomID) > 0 { - return r.joinRoomUsingServers(queryRes.RoomID, []gomatrixserverlib.ServerName{r.cfg.Matrix.ServerName}) - } - // If the response doesn't contain a non-empty string, return an error - return util.JSONResponse{ - Code: http.StatusNotFound, - JSON: jsonerror.NotFound("Room alias " + roomAlias + " not found."), - } - } - // If the room isn't local, use federation to join - return r.joinRoomByRemoteAlias(domain, roomAlias) -} - -func (r joinRoomReq) joinRoomByRemoteAlias( - domain gomatrixserverlib.ServerName, roomAlias string, -) util.JSONResponse { - resp, err := r.federation.LookupRoomAlias(r.req.Context(), domain, roomAlias) - if err != nil { - switch x := err.(type) { - case gomatrix.HTTPError: - if x.Code == http.StatusNotFound { - return util.JSONResponse{ - Code: http.StatusNotFound, - JSON: jsonerror.NotFound("Room alias not found"), - } - } - } - util.GetLogger(r.req.Context()).WithError(err).Error("r.federation.LookupRoomAlias failed") - return jsonerror.InternalServerError() - } - - return r.joinRoomUsingServers(resp.RoomID, resp.Servers) -} - -func (r joinRoomReq) writeToBuilder(eb *gomatrixserverlib.EventBuilder, roomID string) error { - eb.Type = "m.room.member" - - err := eb.SetContent(r.content) - if err != nil { - return err - } - - err = eb.SetUnsigned(struct{}{}) - if err != nil { - return err - } - - eb.Sender = r.userID - eb.StateKey = &r.userID - eb.RoomID = roomID - eb.Redacts = "" - - return nil -} - -func (r joinRoomReq) joinRoomUsingServers( - roomID string, servers []gomatrixserverlib.ServerName, -) util.JSONResponse { - var eb gomatrixserverlib.EventBuilder - err := r.writeToBuilder(&eb, roomID) - if err != nil { - util.GetLogger(r.req.Context()).WithError(err).Error("r.writeToBuilder failed") - return jsonerror.InternalServerError() - } - - queryRes := roomserverAPI.QueryLatestEventsAndStateResponse{} - event, err := common.BuildEvent(r.req.Context(), &eb, r.cfg, r.evTime, r.rsAPI, &queryRes) - if err == nil { - // If we have successfully built an event at this point then we can - // assert that the room is a local room, as BuildEvent was able to - // add prev_events etc successfully. - if _, err = r.producer.SendEvents( - r.req.Context(), - []gomatrixserverlib.HeaderedEvent{ - (*event).Headered(queryRes.RoomVersion), - }, - r.cfg.Matrix.ServerName, - nil, - ); err != nil { - util.GetLogger(r.req.Context()).WithError(err).Error("r.producer.SendEvents failed") - return jsonerror.InternalServerError() - } - return util.JSONResponse{ - Code: http.StatusOK, - JSON: struct { - RoomID string `json:"room_id"` - }{roomID}, - } - } - - // Otherwise, if we've reached here, then we haven't been able to populate - // prev_events etc for the room, therefore the room is probably federated. - - // TODO: This needs to be re-thought, as in the case of an invite, the room - // will exist in the database in roomserver_rooms but won't have any state - // events, therefore this below check fails. - if err != common.ErrRoomNoExists { - util.GetLogger(r.req.Context()).WithError(err).Error("common.BuildEvent failed") - return jsonerror.InternalServerError() - } - - if len(servers) == 0 { - return util.JSONResponse{ - Code: http.StatusNotFound, - JSON: jsonerror.NotFound("No candidate servers found for room"), - } - } - - var lastErr error - for _, server := range servers { - var response *util.JSONResponse - response, lastErr = r.joinRoomUsingServer(roomID, server) - if lastErr != nil { - // There was a problem talking to one of the servers. - util.GetLogger(r.req.Context()).WithError(lastErr).WithField("server", server).Warn("Failed to join room using server") - // Try the next server. - if r.req.Context().Err() != nil { - // The request context has expired so don't bother trying any - // more servers - they will immediately fail due to the expired - // context. - break - } else { - // The request context hasn't expired yet so try the next server. - continue - } - } - return *response - } - - // Every server we tried to join through resulted in an error. - // We return the error from the last server. - - // TODO: Generate the correct HTTP status code for all different - // kinds of errors that could have happened. - // The possible errors include: - // 1) We can't connect to the remote servers. - // 2) None of the servers we could connect to think we are allowed - // to join the room. - // 3) The remote server returned something invalid. - // 4) We couldn't fetch the public keys needed to verify the - // signatures on the state events. - // 5) ... - util.GetLogger(r.req.Context()).WithError(lastErr).Error("failed to join through any server") - return jsonerror.InternalServerError() -} - -// joinRoomUsingServer tries to join a remote room using a given matrix server. -// If there was a failure communicating with the server or the response from the -// server was invalid this returns an error. -// Otherwise this returns a JSONResponse. -func (r joinRoomReq) joinRoomUsingServer(roomID string, server gomatrixserverlib.ServerName) (*util.JSONResponse, error) { - fedJoinReq := federationSenderAPI.PerformJoinRequest{ - RoomID: roomID, - UserID: r.userID, - ServerName: server, - } - fedJoinRes := federationSenderAPI.PerformJoinResponse{} - if err := r.fsAPI.PerformJoin(r.req.Context(), &fedJoinReq, &fedJoinRes); err != nil { - return nil, err - } - - return &util.JSONResponse{ Code: http.StatusOK, // TODO: Put the response struct somewhere common. JSON: struct { RoomID string `json:"room_id"` - }{roomID}, - }, nil + }{joinReq.RoomIDOrAlias}, + } } diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 42b391de6..3ceefa078 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -100,8 +100,7 @@ func Setup( return util.ErrorResponse(err) } return JoinRoomByIDOrAlias( - req, device, vars["roomIDOrAlias"], cfg, federation, producer, - rsAPI, federationSender, keyRing, accountDB, + req, device, rsAPI, vars["roomIDOrAlias"], ) }), ).Methods(http.MethodPost, http.MethodOptions) diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go index be5e988ab..6cadbd759 100644 --- a/federationapi/routing/join.go +++ b/federationapi/routing/join.go @@ -61,9 +61,7 @@ func MakeJoin( if !remoteSupportsVersion { return util.JSONResponse{ Code: http.StatusBadRequest, - JSON: jsonerror.UnsupportedRoomVersion( - fmt.Sprintf("Joining server does not support room version %s", verRes.RoomVersion), - ), + JSON: jsonerror.IncompatibleRoomVersion(verRes.RoomVersion), } } @@ -132,6 +130,9 @@ func MakeJoin( } // SendJoin implements the /send_join API +// The make-join send-join dance makes much more sense as a single +// flow so the cyclomatic complexity is high: +// nolint:gocyclo func SendJoin( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, @@ -159,6 +160,16 @@ func SendJoin( } } + // Check that a state key is provided. + if event.StateKey() == nil || (event.StateKey() != nil && *event.StateKey() == "") { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.BadJSON( + fmt.Sprintf("No state key was provided in the join event."), + ), + } + } + // Check that the room ID is correct. if event.RoomID() != roomID { return util.JSONResponse{ @@ -234,20 +245,34 @@ func SendJoin( } } + // Check if the user is already in the room. If they're already in then + // there isn't much point in sending another join event into the room. + alreadyJoined := false + for _, se := range stateAndAuthChainResponse.StateEvents { + if membership, merr := se.Membership(); merr == nil { + if se.StateKey() != nil && *se.StateKey() == *event.StateKey() { + alreadyJoined = (membership == "join") + break + } + } + } + // Send the events to the room server. // We are responsible for notifying other servers that the user has joined // the room, so set SendAsServer to cfg.Matrix.ServerName - _, err = producer.SendEvents( - httpReq.Context(), - []gomatrixserverlib.HeaderedEvent{ - event.Headered(stateAndAuthChainResponse.RoomVersion), - }, - cfg.Matrix.ServerName, - nil, - ) - if err != nil { - util.GetLogger(httpReq.Context()).WithError(err).Error("producer.SendEvents failed") - return jsonerror.InternalServerError() + if !alreadyJoined { + _, err = producer.SendEvents( + httpReq.Context(), + []gomatrixserverlib.HeaderedEvent{ + event.Headered(stateAndAuthChainResponse.RoomVersion), + }, + cfg.Matrix.ServerName, + nil, + ) + if err != nil { + util.GetLogger(httpReq.Context()).WithError(err).Error("producer.SendEvents failed") + return jsonerror.InternalServerError() + } } return util.JSONResponse{ diff --git a/federationsender/api/api.go b/federationsender/api/api.go index 10dc66da2..678f02e68 100644 --- a/federationsender/api/api.go +++ b/federationsender/api/api.go @@ -8,6 +8,12 @@ import ( // FederationSenderInternalAPI is used to query information from the federation sender. type FederationSenderInternalAPI interface { + // PerformDirectoryLookup looks up a remote room ID from a room alias. + PerformDirectoryLookup( + ctx context.Context, + request *PerformDirectoryLookupRequest, + response *PerformDirectoryLookupResponse, + ) error // Query the joined hosts and the membership events accounting for their participation in a room. // Note that if a server has multiple users in the room, it will have multiple entries in the returned slice. // See `QueryJoinedHostServerNamesInRoom` for a de-duplicated version. diff --git a/federationsender/api/perform.go b/federationsender/api/perform.go index 87736f294..a7b12adc7 100644 --- a/federationsender/api/perform.go +++ b/federationsender/api/perform.go @@ -4,11 +4,15 @@ import ( "context" commonHTTP "github.com/matrix-org/dendrite/common/http" + "github.com/matrix-org/dendrite/federationsender/types" "github.com/matrix-org/gomatrixserverlib" "github.com/opentracing/opentracing-go" ) const ( + // FederationSenderPerformJoinRequestPath is the HTTP path for the PerformJoinRequest API. + FederationSenderPerformDirectoryLookupRequestPath = "/api/federationsender/performDirectoryLookup" + // FederationSenderPerformJoinRequestPath is the HTTP path for the PerformJoinRequest API. FederationSenderPerformJoinRequestPath = "/api/federationsender/performJoinRequest" @@ -16,11 +20,34 @@ const ( FederationSenderPerformLeaveRequestPath = "/api/federationsender/performLeaveRequest" ) -type PerformJoinRequest struct { - RoomID string `json:"room_id"` - UserID string `json:"user_id"` +type PerformDirectoryLookupRequest struct { + RoomAlias string `json:"room_alias"` ServerName gomatrixserverlib.ServerName `json:"server_name"` - Content map[string]interface{} `json:"content"` +} + +type PerformDirectoryLookupResponse struct { + RoomID string `json:"room_id"` + ServerNames []gomatrixserverlib.ServerName `json:"server_names"` +} + +// Handle an instruction to make_join & send_join with a remote server. +func (h *httpFederationSenderInternalAPI) PerformDirectoryLookup( + ctx context.Context, + request *PerformDirectoryLookupRequest, + response *PerformDirectoryLookupResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "PerformDirectoryLookup") + defer span.Finish() + + apiURL := h.federationSenderURL + FederationSenderPerformDirectoryLookupRequestPath + return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +type PerformJoinRequest struct { + RoomID string `json:"room_id"` + UserID string `json:"user_id"` + ServerNames types.ServerNames `json:"server_names"` + Content map[string]interface{} `json:"content"` } type PerformJoinResponse struct { diff --git a/federationsender/internal/perform.go b/federationsender/internal/perform.go index 961d80276..161b689e1 100644 --- a/federationsender/internal/perform.go +++ b/federationsender/internal/perform.go @@ -9,8 +9,29 @@ import ( "github.com/matrix-org/dendrite/federationsender/internal/perform" "github.com/matrix-org/dendrite/roomserver/version" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" + "github.com/sirupsen/logrus" ) +// PerformLeaveRequest implements api.FederationSenderInternalAPI +func (r *FederationSenderInternalAPI) PerformDirectoryLookup( + ctx context.Context, + request *api.PerformDirectoryLookupRequest, + response *api.PerformDirectoryLookupResponse, +) (err error) { + dir, err := r.federation.LookupRoomAlias( + ctx, + request.ServerName, + request.RoomAlias, + ) + if err != nil { + return err + } + response.RoomID = dir.RoomID + response.ServerNames = dir.Servers + return nil +} + // PerformJoinRequest implements api.FederationSenderInternalAPI func (r *FederationSenderInternalAPI) PerformJoin( ctx context.Context, @@ -23,91 +44,107 @@ func (r *FederationSenderInternalAPI) PerformJoin( supportedVersions = append(supportedVersions, version) } - // Try to perform a make_join using the information supplied in the - // request. - respMakeJoin, err := r.federation.MakeJoin( - ctx, - request.ServerName, - request.RoomID, - request.UserID, - supportedVersions, + // Deduplicate the server names we were provided. + util.Unique(request.ServerNames) + + // Try each server that we were provided until we land on one that + // successfully completes the make-join send-join dance. + for _, serverName := range request.ServerNames { + // Try to perform a make_join using the information supplied in the + // request. + respMakeJoin, err := r.federation.MakeJoin( + ctx, + serverName, + request.RoomID, + request.UserID, + supportedVersions, + ) + if err != nil { + // TODO: Check if the user was not allowed to join the room. + return fmt.Errorf("r.federation.MakeJoin: %w", err) + } + + // Set all the fields to be what they should be, this should be a no-op + // but it's possible that the remote server returned us something "odd" + respMakeJoin.JoinEvent.Type = gomatrixserverlib.MRoomMember + respMakeJoin.JoinEvent.Sender = request.UserID + respMakeJoin.JoinEvent.StateKey = &request.UserID + respMakeJoin.JoinEvent.RoomID = request.RoomID + respMakeJoin.JoinEvent.Redacts = "" + if request.Content == nil { + request.Content = map[string]interface{}{} + } + request.Content["membership"] = "join" + if err = respMakeJoin.JoinEvent.SetContent(request.Content); err != nil { + return fmt.Errorf("respMakeJoin.JoinEvent.SetContent: %w", err) + } + if err = respMakeJoin.JoinEvent.SetUnsigned(struct{}{}); err != nil { + return fmt.Errorf("respMakeJoin.JoinEvent.SetUnsigned: %w", err) + } + + // Work out if we support the room version that has been supplied in + // the make_join response. + if respMakeJoin.RoomVersion == "" { + respMakeJoin.RoomVersion = gomatrixserverlib.RoomVersionV1 + } + if _, err = respMakeJoin.RoomVersion.EventFormat(); err != nil { + return fmt.Errorf("respMakeJoin.RoomVersion.EventFormat: %w", err) + } + + // Build the join event. + event, err := respMakeJoin.JoinEvent.Build( + time.Now(), + r.cfg.Matrix.ServerName, + r.cfg.Matrix.KeyID, + r.cfg.Matrix.PrivateKey, + respMakeJoin.RoomVersion, + ) + if err != nil { + return fmt.Errorf("respMakeJoin.JoinEvent.Build: %w", err) + } + + // Try to perform a send_join using the newly built event. + respSendJoin, err := r.federation.SendJoin( + ctx, + serverName, + event, + respMakeJoin.RoomVersion, + ) + if err != nil { + logrus.WithError(err).Warnf("r.federation.SendJoin failed") + continue + } + + // Check that the send_join response was valid. + joinCtx := perform.JoinContext(r.federation, r.keyRing) + if err = joinCtx.CheckSendJoinResponse( + ctx, event, serverName, respMakeJoin, respSendJoin, + ); err != nil { + logrus.WithError(err).Warnf("joinCtx.CheckSendJoinResponse failed") + continue + } + + // If we successfully performed a send_join above then the other + // server now thinks we're a part of the room. Send the newly + // returned state to the roomserver to update our local view. + if err = r.producer.SendEventWithState( + ctx, + respSendJoin.ToRespState(), + event.Headered(respMakeJoin.RoomVersion), + ); err != nil { + logrus.WithError(err).Warnf("r.producer.SendEventWithState failed") + continue + } + + // We're all good. + return nil + } + + // If we reach here then we didn't complete a join for some reason. + return fmt.Errorf( + "failed to join user %q to room %q through %d server(s)", + request.UserID, request.RoomID, len(request.ServerNames), ) - if err != nil { - // TODO: Check if the user was not allowed to join the room. - return fmt.Errorf("r.federation.MakeJoin: %w", err) - } - - // Set all the fields to be what they should be, this should be a no-op - // but it's possible that the remote server returned us something "odd" - respMakeJoin.JoinEvent.Type = "m.room.member" - respMakeJoin.JoinEvent.Sender = request.UserID - respMakeJoin.JoinEvent.StateKey = &request.UserID - respMakeJoin.JoinEvent.RoomID = request.RoomID - respMakeJoin.JoinEvent.Redacts = "" - if request.Content == nil { - request.Content = map[string]interface{}{} - } - request.Content["membership"] = "join" - if err = respMakeJoin.JoinEvent.SetContent(request.Content); err != nil { - return fmt.Errorf("respMakeJoin.JoinEvent.SetContent: %w", err) - } - if err = respMakeJoin.JoinEvent.SetUnsigned(struct{}{}); err != nil { - return fmt.Errorf("respMakeJoin.JoinEvent.SetUnsigned: %w", err) - } - - // Work out if we support the room version that has been supplied in - // the make_join response. - if respMakeJoin.RoomVersion == "" { - respMakeJoin.RoomVersion = gomatrixserverlib.RoomVersionV1 - } - if _, err = respMakeJoin.RoomVersion.EventFormat(); err != nil { - return fmt.Errorf("respMakeJoin.RoomVersion.EventFormat: %w", err) - } - - // Build the join event. - event, err := respMakeJoin.JoinEvent.Build( - time.Now(), - r.cfg.Matrix.ServerName, - r.cfg.Matrix.KeyID, - r.cfg.Matrix.PrivateKey, - respMakeJoin.RoomVersion, - ) - if err != nil { - return fmt.Errorf("respMakeJoin.JoinEvent.Build: %w", err) - } - - // Try to perform a send_join using the newly built event. - respSendJoin, err := r.federation.SendJoin( - ctx, - request.ServerName, - event, - respMakeJoin.RoomVersion, - ) - if err != nil { - return fmt.Errorf("r.federation.SendJoin: %w", err) - } - - // Check that the send_join response was valid. - joinCtx := perform.JoinContext(r.federation, r.keyRing) - if err = joinCtx.CheckSendJoinResponse( - ctx, event, request.ServerName, respMakeJoin, respSendJoin, - ); err != nil { - return fmt.Errorf("perform.JoinRequest.CheckSendJoinResponse: %w", err) - } - - // If we successfully performed a send_join above then the other - // server now thinks we're a part of the room. Send the newly - // returned state to the roomserver to update our local view. - if err = r.producer.SendEventWithState( - ctx, - respSendJoin.ToRespState(), - event.Headered(respMakeJoin.RoomVersion), - ); err != nil { - return fmt.Errorf("r.producer.SendEventWithState: %w", err) - } - - // Everything went to plan. - return nil } // PerformLeaveRequest implements api.FederationSenderInternalAPI diff --git a/federationsender/types/types.go b/federationsender/types/types.go index 05ba92f77..398d32677 100644 --- a/federationsender/types/types.go +++ b/federationsender/types/types.go @@ -28,6 +28,12 @@ type JoinedHost struct { ServerName gomatrixserverlib.ServerName } +type ServerNames []gomatrixserverlib.ServerName + +func (s ServerNames) Len() int { return len(s) } +func (s ServerNames) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s ServerNames) Less(i, j int) bool { return s[i] < s[j] } + // A EventIDMismatchError indicates that we have got out of sync with the // room server. type EventIDMismatchError struct { diff --git a/roomserver/api/api.go b/roomserver/api/api.go index c12dbddde..ae4beab21 100644 --- a/roomserver/api/api.go +++ b/roomserver/api/api.go @@ -18,6 +18,12 @@ type RoomserverInternalAPI interface { response *InputRoomEventsResponse, ) error + PerformJoin( + ctx context.Context, + req *PerformJoinRequest, + res *PerformJoinResponse, + ) error + // Query the latest events and state for a room from the room server. QueryLatestEventsAndState( ctx context.Context, diff --git a/roomserver/api/perform.go b/roomserver/api/perform.go new file mode 100644 index 000000000..e60c078bc --- /dev/null +++ b/roomserver/api/perform.go @@ -0,0 +1,59 @@ +package api + +import ( + "context" + + commonHTTP "github.com/matrix-org/dendrite/common/http" + "github.com/matrix-org/gomatrixserverlib" + "github.com/opentracing/opentracing-go" +) + +const ( + // RoomserverPerformJoinPath is the HTTP path for the PerformJoin API. + RoomserverPerformJoinPath = "/api/roomserver/performJoin" + + // RoomserverPerformLeavePath is the HTTP path for the PerformLeave API. + RoomserverPerformLeavePath = "/api/roomserver/performLeave" +) + +type PerformJoinRequest struct { + RoomIDOrAlias string `json:"room_id_or_alias"` + UserID string `json:"user_id"` + Content map[string]interface{} `json:"content"` + ServerNames []gomatrixserverlib.ServerName `json:"server_names"` +} + +type PerformJoinResponse struct { +} + +func (h *httpRoomserverInternalAPI) PerformJoin( + ctx context.Context, + request *PerformJoinRequest, + response *PerformJoinResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "PerformJoin") + defer span.Finish() + + apiURL := h.roomserverURL + RoomserverPerformJoinPath + return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +type PerformLeaveRequest struct { + RoomID string `json:"room_id"` + UserID string `json:"user_id"` +} + +type PerformLeaveResponse struct { +} + +func (h *httpRoomserverInternalAPI) PerformLeave( + ctx context.Context, + request *PerformLeaveRequest, + response *PerformLeaveResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "PerformLeave") + defer span.Finish() + + apiURL := h.roomserverURL + RoomserverPerformLeavePath + return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} diff --git a/roomserver/internal/api.go b/roomserver/internal/api.go index d1c443f24..1dc985efa 100644 --- a/roomserver/internal/api.go +++ b/roomserver/internal/api.go @@ -46,6 +46,19 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) + servMux.Handle(api.RoomserverPerformJoinPath, + common.MakeInternalAPI("performJoin", func(req *http.Request) util.JSONResponse { + var request api.PerformJoinRequest + var response api.PerformJoinResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := r.PerformJoin(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) servMux.Handle( api.RoomserverQueryLatestEventsAndStatePath, common.MakeInternalAPI("queryLatestEventsAndState", func(req *http.Request) util.JSONResponse { diff --git a/roomserver/internal/perform_join.go b/roomserver/internal/perform_join.go new file mode 100644 index 000000000..3dfa118fd --- /dev/null +++ b/roomserver/internal/perform_join.go @@ -0,0 +1,199 @@ +package internal + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/matrix-org/dendrite/common" + fsAPI "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/gomatrixserverlib" + "github.com/sirupsen/logrus" +) + +// WriteOutputEvents implements OutputRoomEventWriter +func (r *RoomserverInternalAPI) PerformJoin( + ctx context.Context, + req *api.PerformJoinRequest, + res *api.PerformJoinResponse, +) error { + _, domain, err := gomatrixserverlib.SplitID('@', req.UserID) + if err != nil { + return fmt.Errorf("Supplied user ID %q in incorrect format", req.UserID) + } + if domain != r.Cfg.Matrix.ServerName { + return fmt.Errorf("User %q does not belong to this homeserver", req.UserID) + } + if strings.HasPrefix(req.RoomIDOrAlias, "!") { + return r.performJoinRoomByID(ctx, req, res) + } + if strings.HasPrefix(req.RoomIDOrAlias, "#") { + return r.performJoinRoomByAlias(ctx, req, res) + } + return fmt.Errorf("Room ID or alias %q is invalid", req.RoomIDOrAlias) +} + +func (r *RoomserverInternalAPI) performJoinRoomByAlias( + ctx context.Context, + req *api.PerformJoinRequest, + res *api.PerformJoinResponse, +) error { + // Get the domain part of the room alias. + _, domain, err := gomatrixserverlib.SplitID('#', req.RoomIDOrAlias) + if err != nil { + return fmt.Errorf("Alias %q is not in the correct format", req.RoomIDOrAlias) + } + req.ServerNames = append(req.ServerNames, domain) + + // Check if this alias matches our own server configuration. If it + // doesn't then we'll need to try a federated join. + var roomID string + if domain != r.Cfg.Matrix.ServerName { + // The alias isn't owned by us, so we will need to try joining using + // a remote server. + dirReq := fsAPI.PerformDirectoryLookupRequest{ + RoomAlias: req.RoomIDOrAlias, // the room alias to lookup + ServerName: domain, // the server to ask + } + dirRes := fsAPI.PerformDirectoryLookupResponse{} + err = r.fsAPI.PerformDirectoryLookup(ctx, &dirReq, &dirRes) + if err != nil { + logrus.WithError(err).Errorf("error looking up alias %q", req.RoomIDOrAlias) + return fmt.Errorf("Looking up alias %q over federation failed: %w", req.RoomIDOrAlias, err) + } + roomID = dirRes.RoomID + req.ServerNames = append(req.ServerNames, dirRes.ServerNames...) + } else { + // Otherwise, look up if we know this room alias locally. + roomID, err = r.DB.GetRoomIDForAlias(ctx, req.RoomIDOrAlias) + if err != nil { + return fmt.Errorf("Lookup room alias %q failed: %w", req.RoomIDOrAlias, err) + } + } + + // If the room ID is empty then we failed to look up the alias. + if roomID == "" { + return fmt.Errorf("Alias %q not found", req.RoomIDOrAlias) + } + + // If we do, then pluck out the room ID and continue the join. + req.RoomIDOrAlias = roomID + return r.performJoinRoomByID(ctx, req, res) +} + +// TODO: Break this function up a bit +// nolint:gocyclo +func (r *RoomserverInternalAPI) performJoinRoomByID( + ctx context.Context, + req *api.PerformJoinRequest, + res *api.PerformJoinResponse, // nolint:unparam +) error { + // Get the domain part of the room ID. + _, domain, err := gomatrixserverlib.SplitID('!', req.RoomIDOrAlias) + if err != nil { + return fmt.Errorf("Room ID %q is invalid", req.RoomIDOrAlias) + } + req.ServerNames = append(req.ServerNames, domain) + + // Prepare the template for the join event. + userID := req.UserID + eb := gomatrixserverlib.EventBuilder{ + Type: gomatrixserverlib.MRoomMember, + Sender: userID, + StateKey: &userID, + RoomID: req.RoomIDOrAlias, + Redacts: "", + } + if err = eb.SetUnsigned(struct{}{}); err != nil { + return fmt.Errorf("eb.SetUnsigned: %w", err) + } + + // It is possible for the request to include some "content" for the + // event. We'll always overwrite the "membership" key, but the rest, + // like "display_name" or "avatar_url", will be kept if supplied. + if req.Content == nil { + req.Content = map[string]interface{}{} + } + req.Content["membership"] = "join" + if err = eb.SetContent(req.Content); err != nil { + return fmt.Errorf("eb.SetContent: %w", err) + } + + // Try to construct an actual join event from the template. + // If this succeeds then it is a sign that the room already exists + // locally on the homeserver. + // TODO: Check what happens if the room exists on the server + // but everyone has since left. I suspect it does the wrong thing. + buildRes := api.QueryLatestEventsAndStateResponse{} + event, err := common.BuildEvent( + ctx, // the request context + &eb, // the template join event + r.Cfg, // the server configuration + time.Now(), // the event timestamp to use + r, // the roomserver API to use + &buildRes, // the query response + ) + + switch err { + case nil: + // The room join is local. Send the new join event into the + // roomserver. First of all check that the user isn't already + // a member of the room. + alreadyJoined := false + for _, se := range buildRes.StateEvents { + if membership, merr := se.Membership(); merr == nil { + if se.StateKey() != nil && *se.StateKey() == *event.StateKey() { + alreadyJoined = (membership == "join") + break + } + } + } + + // If we haven't already joined the room then send an event + // into the room changing our membership status. + if !alreadyJoined { + inputReq := api.InputRoomEventsRequest{ + InputRoomEvents: []api.InputRoomEvent{ + api.InputRoomEvent{ + Kind: api.KindNew, + Event: event.Headered(buildRes.RoomVersion), + AuthEventIDs: event.AuthEventIDs(), + SendAsServer: string(r.Cfg.Matrix.ServerName), + }, + }, + } + inputRes := api.InputRoomEventsResponse{} + if err = r.InputRoomEvents(ctx, &inputReq, &inputRes); err != nil { + return fmt.Errorf("r.InputRoomEvents: %w", err) + } + } + + case common.ErrRoomNoExists: + // The room doesn't exist. First of all check if the room is a local + // room. If it is then there's nothing more to do - the room just + // hasn't been created yet. + if domain == r.Cfg.Matrix.ServerName { + return fmt.Errorf("Room ID %q does not exist", req.RoomIDOrAlias) + } + + // Try joining by all of the supplied server names. + fedReq := fsAPI.PerformJoinRequest{ + RoomID: req.RoomIDOrAlias, // the room ID to try and join + UserID: req.UserID, // the user ID joining the room + ServerNames: req.ServerNames, // the server to try joining with + Content: req.Content, // the membership event content + } + fedRes := fsAPI.PerformJoinResponse{} + err = r.fsAPI.PerformJoin(ctx, &fedReq, &fedRes) + if err != nil { + return fmt.Errorf("Error joining federated room: %q", err) + } + + default: + return fmt.Errorf("Error joining room %q: %w", req.RoomIDOrAlias, err) + } + + return nil +} From 65c6fbddeb313732d65744c29721e50eaf0de889 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Mon, 4 May 2020 14:48:49 +0100 Subject: [PATCH 027/200] Fix newlines between white/blacklist test names in buildkite annotations (#1003) --- show-expected-fail-tests.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/show-expected-fail-tests.sh b/show-expected-fail-tests.sh index 0a4c7be87..320d4ebd3 100755 --- a/show-expected-fail-tests.sh +++ b/show-expected-fail-tests.sh @@ -80,8 +80,8 @@ done <<< "${passed_but_expected_fail}" # TODO: Check that the same test doesn't appear twice in the whitelist|blacklist # Trim test output strings -tests_to_add=$(echo -e $tests_to_add | xargs -d '\n') -already_in_whitelist=$(echo -e $already_in_whitelist | xargs -d '\n') +tests_to_add=$(IFS=$'\n' echo "${tests_to_add[*]%%'\n'}") +already_in_whitelist=$(IFS=$'\n' echo "${already_in_whitelist[*]%%'\n'}") # Format output with markdown for buildkite annotation rendering purposes if [ -n "${tests_to_add}" ] && [ -n "${already_in_whitelist}" ]; then @@ -91,14 +91,14 @@ fi if [ -n "${tests_to_add}" ]; then echo "**ERROR**: The following tests passed but are not present in \`$2\`. Please append them to the file:" echo "\`\`\`" - echo -e "${tests_to_add}" + echo -e "${tests_to_add}" echo "\`\`\`" fi if [ -n "${already_in_whitelist}" ]; then echo "**WARN**: Tests in the whitelist still marked as **expected fail**:" echo "\`\`\`" - echo -e "${already_in_whitelist}" + echo -e "${already_in_whitelist}" echo "\`\`\`" fi From df80e424081ff962a137b05b977d465f1c97a481 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 4 May 2020 15:30:36 +0100 Subject: [PATCH 028/200] Update gomatrixserverlib --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 55c1179c2..019edbf31 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f github.com/matrix-org/go-sqlite3-js v0.0.0-20200325174927-327088cdef10 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200430104311-8d41c4d924ec + github.com/matrix-org/gomatrixserverlib v0.0.0-20200504142819-073764319c0f github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible diff --git a/go.sum b/go.sum index 8c1d15ad7..c9f89c6d1 100644 --- a/go.sum +++ b/go.sum @@ -367,8 +367,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:kmRjpmFOenVpOaV/DRlo9p6z/IbOKlUC+hhKsAAh8Qg= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200430104311-8d41c4d924ec h1:9MvZSZzBKvCWqM5KXMGZ1PBDrSLcxs5zfc561UPgcYA= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200430104311-8d41c4d924ec/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200504142819-073764319c0f h1:RiQ+YLu/S5Oi2Tm2QpBfO3bNxinhFtpZiar13kswLmY= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200504142819-073764319c0f/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y= From 9b1b095b495a5cd4a6494bc309dd8c4e92d8661f Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 4 May 2020 18:34:09 +0100 Subject: [PATCH 029/200] Roomserver perform leave (#1004) * First pass at PerformLeave * Fix SQLite bulkSelectEventStateKey * Update gomatrixserverlib * Fix bugs * Tidy a bit * Satisfy King Linter * Review comments * Review comments * Fix constants in SQLite event state keys table --- clientapi/routing/leaveroom.go | 51 +++++ clientapi/routing/routing.go | 14 +- federationapi/routing/leave.go | 14 +- federationsender/api/perform.go | 4 +- federationsender/internal/perform.go | 80 ++++++- go.mod | 2 +- go.sum | 4 +- roomserver/api/api.go | 6 + roomserver/internal/api.go | 13 ++ roomserver/internal/perform_join.go | 6 +- roomserver/internal/perform_leave.go | 208 ++++++++++++++++++ .../storage/sqlite3/event_state_keys_table.go | 6 +- 12 files changed, 394 insertions(+), 14 deletions(-) create mode 100644 clientapi/routing/leaveroom.go create mode 100644 roomserver/internal/perform_leave.go diff --git a/clientapi/routing/leaveroom.go b/clientapi/routing/leaveroom.go new file mode 100644 index 000000000..bd7696181 --- /dev/null +++ b/clientapi/routing/leaveroom.go @@ -0,0 +1,51 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routing + +import ( + "net/http" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/clientapi/jsonerror" + roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/util" +) + +func LeaveRoomByID( + req *http.Request, + device *authtypes.Device, + rsAPI roomserverAPI.RoomserverInternalAPI, + roomID string, +) util.JSONResponse { + // Prepare to ask the roomserver to perform the room join. + leaveReq := roomserverAPI.PerformLeaveRequest{ + RoomID: roomID, + UserID: device.UserID, + } + leaveRes := roomserverAPI.PerformLeaveResponse{} + + // Ask the roomserver to perform the leave. + if err := rsAPI.PerformLeave(req.Context(), &leaveReq, &leaveRes); err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.Unknown(err.Error()), + } + } + + return util.JSONResponse{ + Code: http.StatusOK, + JSON: struct{}{}, + } +} diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 3ceefa078..ead8a4c85 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -109,8 +109,18 @@ func Setup( return GetJoinedRooms(req, device, accountDB) }), ).Methods(http.MethodGet, http.MethodOptions) - - r0mux.Handle("/rooms/{roomID}/{membership:(?:join|kick|ban|unban|leave|invite)}", + r0mux.Handle("/rooms/{roomID}/leave", + common.MakeAuthAPI("membership", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := common.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return LeaveRoomByID( + req, device, rsAPI, vars["roomID"], + ) + }), + ).Methods(http.MethodPost, http.MethodOptions) + r0mux.Handle("/rooms/{roomID}/{membership:(?:join|kick|ban|unban|invite)}", common.MakeAuthAPI("membership", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { vars, err := common.URLDecodeMapValues(mux.Vars(req)) if err != nil { diff --git a/federationapi/routing/leave.go b/federationapi/routing/leave.go index 1124bfa27..bab3fd0b5 100644 --- a/federationapi/routing/leave.go +++ b/federationapi/routing/leave.go @@ -33,6 +33,15 @@ func MakeLeave( rsAPI api.RoomserverInternalAPI, roomID, userID string, ) util.JSONResponse { + verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} + verRes := api.QueryRoomVersionForRoomResponse{} + if err := rsAPI.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil { + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: jsonerror.InternalServerError(), + } + } + _, domain, err := gomatrixserverlib.SplitID('@', userID) if err != nil { return util.JSONResponse{ @@ -87,7 +96,10 @@ func MakeLeave( return util.JSONResponse{ Code: http.StatusOK, - JSON: map[string]interface{}{"event": builder}, + JSON: map[string]interface{}{ + "room_version": verRes.RoomVersion, + "event": builder, + }, } } diff --git a/federationsender/api/perform.go b/federationsender/api/perform.go index a7b12adc7..2f643e5cf 100644 --- a/federationsender/api/perform.go +++ b/federationsender/api/perform.go @@ -67,7 +67,9 @@ func (h *httpFederationSenderInternalAPI) PerformJoin( } type PerformLeaveRequest struct { - RoomID string `json:"room_id"` + RoomID string `json:"room_id"` + UserID string `json:"user_id"` + ServerNames types.ServerNames `json:"server_names"` } type PerformLeaveResponse struct { diff --git a/federationsender/internal/perform.go b/federationsender/internal/perform.go index 161b689e1..ff7f821c1 100644 --- a/federationsender/internal/perform.go +++ b/federationsender/internal/perform.go @@ -153,5 +153,83 @@ func (r *FederationSenderInternalAPI) PerformLeave( request *api.PerformLeaveRequest, response *api.PerformLeaveResponse, ) (err error) { - return nil + // Deduplicate the server names we were provided. + util.Unique(request.ServerNames) + + // Try each server that we were provided until we land on one that + // successfully completes the make-leave send-leave dance. + for _, serverName := range request.ServerNames { + // Try to perform a make_leave using the information supplied in the + // request. + respMakeLeave, err := r.federation.MakeLeave( + ctx, + serverName, + request.RoomID, + request.UserID, + ) + if err != nil { + // TODO: Check if the user was not allowed to leave the room. + logrus.WithError(err).Warnf("r.federation.MakeLeave failed") + continue + } + + // Set all the fields to be what they should be, this should be a no-op + // but it's possible that the remote server returned us something "odd" + respMakeLeave.LeaveEvent.Type = gomatrixserverlib.MRoomMember + respMakeLeave.LeaveEvent.Sender = request.UserID + respMakeLeave.LeaveEvent.StateKey = &request.UserID + respMakeLeave.LeaveEvent.RoomID = request.RoomID + respMakeLeave.LeaveEvent.Redacts = "" + if respMakeLeave.LeaveEvent.Content == nil { + content := map[string]interface{}{ + "membership": "leave", + } + if err = respMakeLeave.LeaveEvent.SetContent(content); err != nil { + logrus.WithError(err).Warnf("respMakeLeave.LeaveEvent.SetContent failed") + continue + } + } + if err = respMakeLeave.LeaveEvent.SetUnsigned(struct{}{}); err != nil { + logrus.WithError(err).Warnf("respMakeLeave.LeaveEvent.SetUnsigned failed") + continue + } + + // Work out if we support the room version that has been supplied in + // the make_leave response. + if _, err = respMakeLeave.RoomVersion.EventFormat(); err != nil { + return gomatrixserverlib.UnsupportedRoomVersionError{} + } + + // Build the leave event. + event, err := respMakeLeave.LeaveEvent.Build( + time.Now(), + r.cfg.Matrix.ServerName, + r.cfg.Matrix.KeyID, + r.cfg.Matrix.PrivateKey, + respMakeLeave.RoomVersion, + ) + if err != nil { + logrus.WithError(err).Warnf("respMakeLeave.LeaveEvent.Build failed") + continue + } + + // Try to perform a send_leave using the newly built event. + err = r.federation.SendLeave( + ctx, + serverName, + event, + ) + if err != nil { + logrus.WithError(err).Warnf("r.federation.SendLeave failed") + continue + } + + return nil + } + + // If we reach here then we didn't complete a leave for some reason. + return fmt.Errorf( + "Failed to leave room %q through %d server(s)", + request.RoomID, len(request.ServerNames), + ) } diff --git a/go.mod b/go.mod index 019edbf31..5188a7f59 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f github.com/matrix-org/go-sqlite3-js v0.0.0-20200325174927-327088cdef10 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200504142819-073764319c0f + github.com/matrix-org/gomatrixserverlib v0.0.0-20200504153202-7542702abea6 github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible diff --git a/go.sum b/go.sum index c9f89c6d1..80839ead8 100644 --- a/go.sum +++ b/go.sum @@ -367,8 +367,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:kmRjpmFOenVpOaV/DRlo9p6z/IbOKlUC+hhKsAAh8Qg= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200504142819-073764319c0f h1:RiQ+YLu/S5Oi2Tm2QpBfO3bNxinhFtpZiar13kswLmY= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200504142819-073764319c0f/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200504153202-7542702abea6 h1:CnU+0kV1xzpvzEkFr1tX7c9BTWCTFOIlBPM1XD9I++c= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200504153202-7542702abea6/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y= diff --git a/roomserver/api/api.go b/roomserver/api/api.go index ae4beab21..aefe55bcd 100644 --- a/roomserver/api/api.go +++ b/roomserver/api/api.go @@ -24,6 +24,12 @@ type RoomserverInternalAPI interface { res *PerformJoinResponse, ) error + PerformLeave( + ctx context.Context, + req *PerformLeaveRequest, + res *PerformLeaveResponse, + ) error + // Query the latest events and state for a room from the room server. QueryLatestEventsAndState( ctx context.Context, diff --git a/roomserver/internal/api.go b/roomserver/internal/api.go index 1dc985efa..62389c237 100644 --- a/roomserver/internal/api.go +++ b/roomserver/internal/api.go @@ -59,6 +59,19 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) + servMux.Handle(api.RoomserverPerformLeavePath, + common.MakeInternalAPI("performLeave", func(req *http.Request) util.JSONResponse { + var request api.PerformLeaveRequest + var response api.PerformLeaveResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := r.PerformLeave(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) servMux.Handle( api.RoomserverQueryLatestEventsAndStatePath, common.MakeInternalAPI("queryLatestEventsAndState", func(req *http.Request) util.JSONResponse { diff --git a/roomserver/internal/perform_join.go b/roomserver/internal/perform_join.go index 3dfa118fd..99e10d974 100644 --- a/roomserver/internal/perform_join.go +++ b/roomserver/internal/perform_join.go @@ -116,7 +116,7 @@ func (r *RoomserverInternalAPI) performJoinRoomByID( if req.Content == nil { req.Content = map[string]interface{}{} } - req.Content["membership"] = "join" + req.Content["membership"] = gomatrixserverlib.Join if err = eb.SetContent(req.Content); err != nil { return fmt.Errorf("eb.SetContent: %w", err) } @@ -145,7 +145,7 @@ func (r *RoomserverInternalAPI) performJoinRoomByID( for _, se := range buildRes.StateEvents { if membership, merr := se.Membership(); merr == nil { if se.StateKey() != nil && *se.StateKey() == *event.StateKey() { - alreadyJoined = (membership == "join") + alreadyJoined = (membership == gomatrixserverlib.Join) break } } @@ -156,7 +156,7 @@ func (r *RoomserverInternalAPI) performJoinRoomByID( if !alreadyJoined { inputReq := api.InputRoomEventsRequest{ InputRoomEvents: []api.InputRoomEvent{ - api.InputRoomEvent{ + { Kind: api.KindNew, Event: event.Headered(buildRes.RoomVersion), AuthEventIDs: event.AuthEventIDs(), diff --git a/roomserver/internal/perform_leave.go b/roomserver/internal/perform_leave.go new file mode 100644 index 000000000..422748e6a --- /dev/null +++ b/roomserver/internal/perform_leave.go @@ -0,0 +1,208 @@ +package internal + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/matrix-org/dendrite/common" + fsAPI "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/gomatrixserverlib" +) + +// WriteOutputEvents implements OutputRoomEventWriter +func (r *RoomserverInternalAPI) PerformLeave( + ctx context.Context, + req *api.PerformLeaveRequest, + res *api.PerformLeaveResponse, +) error { + _, domain, err := gomatrixserverlib.SplitID('@', req.UserID) + if err != nil { + return fmt.Errorf("Supplied user ID %q in incorrect format", req.UserID) + } + if domain != r.Cfg.Matrix.ServerName { + return fmt.Errorf("User %q does not belong to this homeserver", req.UserID) + } + if strings.HasPrefix(req.RoomID, "!") { + return r.performLeaveRoomByID(ctx, req, res) + } + return fmt.Errorf("Room ID %q is invalid", req.RoomID) +} + +func (r *RoomserverInternalAPI) performLeaveRoomByID( + ctx context.Context, + req *api.PerformLeaveRequest, + res *api.PerformLeaveResponse, // nolint:unparam +) error { + // If there's an invite outstanding for the room then respond to + // that. + isInvitePending, senderUser, err := r.isInvitePending(ctx, req, res) + if err == nil && isInvitePending { + return r.performRejectInvite(ctx, req, res, senderUser) + } + + // There's no invite pending, so first of all we want to find out + // if the room exists and if the user is actually in it. + latestReq := api.QueryLatestEventsAndStateRequest{ + RoomID: req.RoomID, + StateToFetch: []gomatrixserverlib.StateKeyTuple{ + { + EventType: gomatrixserverlib.MRoomMember, + StateKey: req.UserID, + }, + }, + } + latestRes := api.QueryLatestEventsAndStateResponse{} + if err = r.QueryLatestEventsAndState(ctx, &latestReq, &latestRes); err != nil { + return err + } + if !latestRes.RoomExists { + return fmt.Errorf("Room %q does not exist", req.RoomID) + } + + // Now let's see if the user is in the room. + if len(latestRes.StateEvents) == 0 { + return fmt.Errorf("User %q is not a member of room %q", req.UserID, req.RoomID) + } + membership, err := latestRes.StateEvents[0].Membership() + if err != nil { + return fmt.Errorf("Error getting membership: %w", err) + } + if membership != gomatrixserverlib.Join { + // TODO: should be able to handle "invite" in this case too, if + // it's a case of kicking or banning or such + return fmt.Errorf("User %q is not joined to the room (membership is %q)", req.UserID, membership) + } + + // Prepare the template for the leave event. + userID := req.UserID + eb := gomatrixserverlib.EventBuilder{ + Type: gomatrixserverlib.MRoomMember, + Sender: userID, + StateKey: &userID, + RoomID: req.RoomID, + Redacts: "", + } + if err = eb.SetContent(map[string]interface{}{"membership": "leave"}); err != nil { + return fmt.Errorf("eb.SetContent: %w", err) + } + if err = eb.SetUnsigned(struct{}{}); err != nil { + return fmt.Errorf("eb.SetUnsigned: %w", err) + } + + // We know that the user is in the room at this point so let's build + // a leave event. + // TODO: Check what happens if the room exists on the server + // but everyone has since left. I suspect it does the wrong thing. + buildRes := api.QueryLatestEventsAndStateResponse{} + event, err := common.BuildEvent( + ctx, // the request context + &eb, // the template leave event + r.Cfg, // the server configuration + time.Now(), // the event timestamp to use + r, // the roomserver API to use + &buildRes, // the query response + ) + if err != nil { + return fmt.Errorf("common.BuildEvent: %w", err) + } + + // Give our leave event to the roomserver input stream. The + // roomserver will process the membership change and notify + // downstream automatically. + inputReq := api.InputRoomEventsRequest{ + InputRoomEvents: []api.InputRoomEvent{ + { + Kind: api.KindNew, + Event: event.Headered(buildRes.RoomVersion), + AuthEventIDs: event.AuthEventIDs(), + SendAsServer: string(r.Cfg.Matrix.ServerName), + }, + }, + } + inputRes := api.InputRoomEventsResponse{} + if err = r.InputRoomEvents(ctx, &inputReq, &inputRes); err != nil { + return fmt.Errorf("r.InputRoomEvents: %w", err) + } + + return nil +} + +func (r *RoomserverInternalAPI) performRejectInvite( + ctx context.Context, + req *api.PerformLeaveRequest, + res *api.PerformLeaveResponse, // nolint:unparam + senderUser string, +) error { + _, domain, err := gomatrixserverlib.SplitID('@', senderUser) + if err != nil { + return fmt.Errorf("User ID %q invalid: %w", senderUser, err) + } + + // Ask the federation sender to perform a federated leave for us. + leaveReq := fsAPI.PerformLeaveRequest{ + RoomID: req.RoomID, + UserID: req.UserID, + ServerNames: []gomatrixserverlib.ServerName{domain}, + } + leaveRes := fsAPI.PerformLeaveResponse{} + if err := r.fsAPI.PerformLeave(ctx, &leaveReq, &leaveRes); err != nil { + return err + } + + // TODO: Withdraw the invite, so that the sync API etc are + // notified that we rejected it. + + return nil +} + +func (r *RoomserverInternalAPI) isInvitePending( + ctx context.Context, + req *api.PerformLeaveRequest, + res *api.PerformLeaveResponse, // nolint:unparam +) (bool, string, error) { + // Look up the room NID for the supplied room ID. + roomNID, err := r.DB.RoomNID(ctx, req.RoomID) + if err != nil { + return false, "", fmt.Errorf("r.DB.RoomNID: %w", err) + } + + // Look up the state key NID for the supplied user ID. + targetUserNIDs, err := r.DB.EventStateKeyNIDs(ctx, []string{req.UserID}) + if err != nil { + return false, "", fmt.Errorf("r.DB.EventStateKeyNIDs: %w", err) + } + targetUserNID, targetUserFound := targetUserNIDs[req.UserID] + if !targetUserFound { + return false, "", fmt.Errorf("missing NID for user %q (%+v)", req.UserID, targetUserNIDs) + } + + // Let's see if we have an event active for the user in the room. If + // we do then it will contain a server name that we can direct the + // send_leave to. + senderUserNIDs, err := r.DB.GetInvitesForUser(ctx, roomNID, targetUserNID) + if err != nil { + return false, "", fmt.Errorf("r.DB.GetInvitesForUser: %w", err) + } + if len(senderUserNIDs) == 0 { + return false, "", nil + } + + // Look up the user ID from the NID. + senderUsers, err := r.DB.EventStateKeys(ctx, senderUserNIDs) + if err != nil { + return false, "", fmt.Errorf("r.DB.EventStateKeys: %w", err) + } + if len(senderUsers) == 0 { + return false, "", fmt.Errorf("no senderUsers") + } + + senderUser, senderUserFound := senderUsers[senderUserNIDs[0]] + if !senderUserFound { + return false, "", fmt.Errorf("missing user for NID %d (%+v)", senderUserNIDs[0], senderUsers) + } + + return true, senderUser, nil +} diff --git a/roomserver/storage/sqlite3/event_state_keys_table.go b/roomserver/storage/sqlite3/event_state_keys_table.go index fa8fc57eb..204b4eb62 100644 --- a/roomserver/storage/sqlite3/event_state_keys_table.go +++ b/roomserver/storage/sqlite3/event_state_keys_table.go @@ -47,14 +47,14 @@ const selectEventStateKeyNIDSQL = ` // Bulk lookup from string state key to numeric ID for that state key. // Takes an array of strings as the query parameter. -const bulkSelectEventStateKeyNIDSQL = ` +const bulkSelectEventStateKeySQL = ` SELECT event_state_key, event_state_key_nid FROM roomserver_event_state_keys WHERE event_state_key IN ($1) ` // Bulk lookup from numeric ID to string state key for that state key. // Takes an array of strings as the query parameter. -const bulkSelectEventStateKeySQL = ` +const bulkSelectEventStateKeyNIDSQL = ` SELECT event_state_key, event_state_key_nid FROM roomserver_event_state_keys WHERE event_state_key_nid IN ($1) ` @@ -110,7 +110,7 @@ func (s *eventStateKeyStatements) bulkSelectEventStateKeyNID( for k, v := range eventStateKeys { iEventStateKeys[k] = v } - selectOrig := strings.Replace(bulkSelectEventStateKeyNIDSQL, "($1)", common.QueryVariadic(len(eventStateKeys)), 1) + selectOrig := strings.Replace(bulkSelectEventStateKeySQL, "($1)", common.QueryVariadic(len(eventStateKeys)), 1) rows, err := txn.QueryContext(ctx, selectOrig, iEventStateKeys...) if err != nil { From 9d15312ef641e871a2958e2f7037c50902dbfcfe Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 5 May 2020 10:53:38 +0100 Subject: [PATCH 030/200] Fix RespState/RespSendJoin (#1005) * Update gmsl * Update gomatrixserverlib * Add link to spec --- federationapi/routing/join.go | 9 ++++----- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go index 6cadbd759..ada89e4a6 100644 --- a/federationapi/routing/join.go +++ b/federationapi/routing/join.go @@ -275,14 +275,13 @@ func SendJoin( } } + // https://matrix.org/docs/spec/server_server/latest#put-matrix-federation-v1-send-join-roomid-eventid return util.JSONResponse{ Code: http.StatusOK, JSON: gomatrixserverlib.RespSendJoin{ - RespState: gomatrixserverlib.RespState{ - StateEvents: gomatrixserverlib.UnwrapEventHeaders(stateAndAuthChainResponse.StateEvents), - AuthEvents: gomatrixserverlib.UnwrapEventHeaders(stateAndAuthChainResponse.AuthChainEvents), - }, - Origin: cfg.Matrix.ServerName, + StateEvents: gomatrixserverlib.UnwrapEventHeaders(stateAndAuthChainResponse.StateEvents), + AuthEvents: gomatrixserverlib.UnwrapEventHeaders(stateAndAuthChainResponse.AuthChainEvents), + Origin: cfg.Matrix.ServerName, }, } } diff --git a/go.mod b/go.mod index 5188a7f59..fd1bb3d70 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f github.com/matrix-org/go-sqlite3-js v0.0.0-20200325174927-327088cdef10 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200504153202-7542702abea6 + github.com/matrix-org/gomatrixserverlib v0.0.0-20200505092542-ef8abbde3f6b github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible diff --git a/go.sum b/go.sum index 80839ead8..7ab035364 100644 --- a/go.sum +++ b/go.sum @@ -367,8 +367,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:kmRjpmFOenVpOaV/DRlo9p6z/IbOKlUC+hhKsAAh8Qg= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200504153202-7542702abea6 h1:CnU+0kV1xzpvzEkFr1tX7c9BTWCTFOIlBPM1XD9I++c= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200504153202-7542702abea6/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200505092542-ef8abbde3f6b h1:gxLun/noFJ7DplX7rqT8E4v4NkeDJ45tqW7LXC6k4C4= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200505092542-ef8abbde3f6b/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y= From 31d3b0d4a52aa52b05f85069ae3f08f9fc8b64ab Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 5 May 2020 15:48:37 +0100 Subject: [PATCH 031/200] Prefer /state_ids when missing state across federation (#1008) * Prefer /state_ids when missing state across federation * Linting * Better logging --- clientapi/producers/roomserver.go | 2 +- federationapi/routing/send.go | 134 +++++++++++++++++++++++++++--- roomserver/api/query.go | 2 +- 3 files changed, 126 insertions(+), 12 deletions(-) diff --git a/clientapi/producers/roomserver.go b/clientapi/producers/roomserver.go index a804abfe9..7eee83f5e 100644 --- a/clientapi/producers/roomserver.go +++ b/clientapi/producers/roomserver.go @@ -54,7 +54,7 @@ func (c *RoomserverProducer) SendEvents( // SendEventWithState writes an event with KindNew to the roomserver input log // with the state at the event as KindOutlier before it. func (c *RoomserverProducer) SendEventWithState( - ctx context.Context, state gomatrixserverlib.RespState, event gomatrixserverlib.HeaderedEvent, + ctx context.Context, state *gomatrixserverlib.RespState, event gomatrixserverlib.HeaderedEvent, ) error { outliers, err := state.Events() if err != nil { diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index 88411b81a..a6072c66c 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -26,6 +26,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" + "github.com/sirupsen/logrus" ) // Send implements /_matrix/federation/v1/send/{txnID} @@ -297,22 +298,27 @@ func (t *txnReq) processEventWithMissingState(e gomatrixserverlib.Event, roomVer // However not all version of synapse support /state_ids so you may // need to fallback to /state. // TODO: Attempt to fill in the gap using /get_missing_events - // TODO: Attempt to fetch the state using /state_ids and /events - state, err := t.federation.LookupState(t.context, t.Origin, e.RoomID(), e.EventID(), roomVersion) + + // Attempt to fetch the missing state using /state_ids and /events + var respState *gomatrixserverlib.RespState + var err error + respState, err = t.lookupMissingStateViaStateIDs(e, roomVersion) if err != nil { - return err - } - // Check that the returned state is valid. - if err := state.Check(t.context, t.keys); err != nil { - return err + // Fallback to /state + util.GetLogger(t.context).WithError(err).Warn("processEventWithMissingState failed to /state_ids, falling back to /state") + respState, err = t.lookupMissingStateViaState(e, roomVersion) + if err != nil { + return err + } } + // Check that the event is allowed by the state. retryAllowedState: - if err := checkAllowedByState(e, state.StateEvents); err != nil { + if err := checkAllowedByState(e, respState.StateEvents); err != nil { switch missing := err.(type) { case gomatrixserverlib.MissingAuthEventError: // An auth event was missing so let's look up that event over federation - for _, s := range state.StateEvents { + for _, s := range respState.StateEvents { if s.EventID() != missing.AuthEventID { continue } @@ -329,5 +335,113 @@ retryAllowedState: } // pass the event along with the state to the roomserver - return t.producer.SendEventWithState(t.context, state, e.Headered(roomVersion)) + return t.producer.SendEventWithState(t.context, respState, e.Headered(roomVersion)) +} + +func (t *txnReq) lookupMissingStateViaState(e gomatrixserverlib.Event, roomVersion gomatrixserverlib.RoomVersion) ( + respState *gomatrixserverlib.RespState, err error) { + state, err := t.federation.LookupState(t.context, t.Origin, e.RoomID(), e.EventID(), roomVersion) + if err != nil { + return nil, err + } + // Check that the returned state is valid. + if err := state.Check(t.context, t.keys); err != nil { + return nil, err + } + return &state, nil +} + +func (t *txnReq) lookupMissingStateViaStateIDs(e gomatrixserverlib.Event, roomVersion gomatrixserverlib.RoomVersion) ( + *gomatrixserverlib.RespState, error) { + + // fetch all the state events we do know about: the current state + queryReq := api.QueryLatestEventsAndStateRequest{ + RoomID: e.RoomID(), + StateToFetch: nil, // fetch all state + } + var queryRes api.QueryLatestEventsAndStateResponse + if err := t.rsAPI.QueryLatestEventsAndState(t.context, &queryReq, &queryRes); err != nil { + return nil, err + } + if !queryRes.RoomExists { + return nil, fmt.Errorf("room %s doesn't exist", e.RoomID()) + } + // allow indexing of current state by event ID + haveEventMap := make(map[string]*gomatrixserverlib.HeaderedEvent, len(queryRes.StateEvents)) + for i := range queryRes.StateEvents { + haveEventMap[queryRes.StateEvents[i].EventID()] = &queryRes.StateEvents[i] + } + + // fetch the state event IDs at the time of the event + stateIDs, err := t.federation.LookupStateIDs(t.context, t.Origin, e.RoomID(), e.EventID()) + if err != nil { + return nil, err + } + // work out which auth/state IDs are missing + wantIDs := append(stateIDs.StateEventIDs, stateIDs.AuthEventIDs...) + missing := make(map[string]bool) + for _, sid := range wantIDs { + if _, ok := haveEventMap[sid]; !ok { + missing[sid] = true + } + } + util.GetLogger(t.context).WithFields(logrus.Fields{ + "missing": len(missing), + "event_id": e.EventID(), + "room_id": e.RoomID(), + "already_have": len(queryRes.StateEvents), + "total_state": len(stateIDs.StateEventIDs), + "total_auth_events": len(stateIDs.AuthEventIDs), + }).Info("Fetching missing state at event") + + for missingEventID := range missing { + txn, err := t.federation.GetEvent(t.context, t.Origin, missingEventID) + if err != nil { + util.GetLogger(t.context).WithError(err).WithField("event_id", missingEventID).Warn("failed to get missing /event for event ID") + return nil, err + } + for _, pdu := range txn.PDUs { + event, err := gomatrixserverlib.NewEventFromUntrustedJSON(pdu, roomVersion) + if err != nil { + util.GetLogger(t.context).WithError(err).Warnf("Transaction: Failed to parse event JSON of event %q", event.EventID()) + return nil, unmarshalError{err} + } + if err := gomatrixserverlib.VerifyAllEventSignatures(t.context, []gomatrixserverlib.Event{event}, t.keys); err != nil { + util.GetLogger(t.context).WithError(err).Warnf("Transaction: Couldn't validate signature of event %q", event.EventID()) + return nil, verifySigError{event.EventID(), err} + } + h := event.Headered(roomVersion) + haveEventMap[event.EventID()] = &h + } + } + return t.createRespStateFromStateIDs(stateIDs, haveEventMap) +} + +func (t *txnReq) createRespStateFromStateIDs(stateIDs gomatrixserverlib.RespStateIDs, haveEventMap map[string]*gomatrixserverlib.HeaderedEvent) ( + *gomatrixserverlib.RespState, error) { + // create a RespState response using the response to /state_ids as a guide + respState := gomatrixserverlib.RespState{ + AuthEvents: make([]gomatrixserverlib.Event, len(stateIDs.AuthEventIDs)), + StateEvents: make([]gomatrixserverlib.Event, len(stateIDs.StateEventIDs)), + } + + for i := range stateIDs.StateEventIDs { + ev, ok := haveEventMap[stateIDs.StateEventIDs[i]] + if !ok { + return nil, fmt.Errorf("missing state event %s", stateIDs.StateEventIDs[i]) + } + respState.StateEvents[i] = ev.Unwrap() + } + for i := range stateIDs.AuthEventIDs { + ev, ok := haveEventMap[stateIDs.AuthEventIDs[i]] + if !ok { + return nil, fmt.Errorf("missing auth event %s", stateIDs.AuthEventIDs[i]) + } + respState.AuthEvents[i] = ev.Unwrap() + } + // Check that the returned state is valid. + if err := respState.Check(t.context, t.keys); err != nil { + return nil, err + } + return &respState, nil } diff --git a/roomserver/api/query.go b/roomserver/api/query.go index cb7cbb86c..9afc51f4e 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -29,7 +29,7 @@ type QueryLatestEventsAndStateRequest struct { // The room ID to query the latest events for. RoomID string `json:"room_id"` // The state key tuples to fetch from the room current state. - // If this list is empty or nil then no state events are returned. + // If this list is empty or nil then *ALL* current state events are returned. StateToFetch []gomatrixserverlib.StateKeyTuple `json:"state_to_fetch"` } From a06511cae8663a11dbd195f31007eab7e96715fe Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 5 May 2020 15:54:12 +0100 Subject: [PATCH 032/200] Remove debug line --- roomserver/internal/query.go | 1 - 1 file changed, 1 deletion(-) diff --git a/roomserver/internal/query.go b/roomserver/internal/query.go index 98adc24b3..8a8c8e7d0 100644 --- a/roomserver/internal/query.go +++ b/roomserver/internal/query.go @@ -575,7 +575,6 @@ func (r *RoomserverInternalAPI) backfillViaFederation(ctx context.Context, req * logrus.WithError(err).WithField("event_id", ev.EventID()).Error("backfillViaFederation: failed to persist state entries to get snapshot nid") return err } - util.GetLogger(ctx).Infof("Backfilled event %s (nid=%d) getting snapshot %v with entries %+v", ev.EventID(), ev.EventNID, beforeStateSnapshotNID, entries) if err = r.DB.SetState(ctx, ev.EventNID, beforeStateSnapshotNID); err != nil { logrus.WithError(err).WithField("event_id", ev.EventID()).Error("backfillViaFederation: failed to persist snapshot nid") } From 1db5dfe4d08b437bfd788eb4ffec09b61b27fd2f Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 5 May 2020 16:46:22 +0100 Subject: [PATCH 033/200] Fetch events by ID rather than use current state as this includes auth events (#1009) --- federationapi/routing/send.go | 38 +++++++++++++++++------------------ 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index a6072c66c..0af3cd762 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -354,29 +354,29 @@ func (t *txnReq) lookupMissingStateViaState(e gomatrixserverlib.Event, roomVersi func (t *txnReq) lookupMissingStateViaStateIDs(e gomatrixserverlib.Event, roomVersion gomatrixserverlib.RoomVersion) ( *gomatrixserverlib.RespState, error) { - // fetch all the state events we do know about: the current state - queryReq := api.QueryLatestEventsAndStateRequest{ - RoomID: e.RoomID(), - StateToFetch: nil, // fetch all state - } - var queryRes api.QueryLatestEventsAndStateResponse - if err := t.rsAPI.QueryLatestEventsAndState(t.context, &queryReq, &queryRes); err != nil { - return nil, err - } - if !queryRes.RoomExists { - return nil, fmt.Errorf("room %s doesn't exist", e.RoomID()) - } - // allow indexing of current state by event ID - haveEventMap := make(map[string]*gomatrixserverlib.HeaderedEvent, len(queryRes.StateEvents)) - for i := range queryRes.StateEvents { - haveEventMap[queryRes.StateEvents[i].EventID()] = &queryRes.StateEvents[i] - } - // fetch the state event IDs at the time of the event stateIDs, err := t.federation.LookupStateIDs(t.context, t.Origin, e.RoomID(), e.EventID()) if err != nil { return nil, err } + + // fetch as many as we can from the roomserver, do them as 2 calls rather than + // 1 to try to reduce the number of parameters in the bulk query this will use + haveEventMap := make(map[string]*gomatrixserverlib.HeaderedEvent, len(stateIDs.StateEventIDs)) + for _, eventList := range [][]string{stateIDs.StateEventIDs, stateIDs.AuthEventIDs} { + queryReq := api.QueryEventsByIDRequest{ + EventIDs: eventList, + } + var queryRes api.QueryEventsByIDResponse + if err := t.rsAPI.QueryEventsByID(t.context, &queryReq, &queryRes); err != nil { + return nil, err + } + // allow indexing of current state by event ID + for i := range queryRes.Events { + haveEventMap[queryRes.Events[i].EventID()] = &queryRes.Events[i] + } + } + // work out which auth/state IDs are missing wantIDs := append(stateIDs.StateEventIDs, stateIDs.AuthEventIDs...) missing := make(map[string]bool) @@ -389,7 +389,7 @@ func (t *txnReq) lookupMissingStateViaStateIDs(e gomatrixserverlib.Event, roomVe "missing": len(missing), "event_id": e.EventID(), "room_id": e.RoomID(), - "already_have": len(queryRes.StateEvents), + "already_have": len(haveEventMap), "total_state": len(stateIDs.StateEventIDs), "total_auth_events": len(stateIDs.AuthEventIDs), }).Info("Fetching missing state at event") From 12948522703570e25cd7bacdfdaa81deba988131 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Wed, 6 May 2020 14:27:02 +0100 Subject: [PATCH 034/200] Add tests around federationapi's txnReq (#1010) * Add necessary stubs for testing txnReq * Add basic tests --- federationapi/routing/send.go | 13 +- federationapi/routing/send_test.go | 399 +++++++++++++++++++++++++++++ 2 files changed, 410 insertions(+), 2 deletions(-) create mode 100644 federationapi/routing/send_test.go diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index 0af3cd762..10210db64 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -103,8 +103,17 @@ type txnReq struct { rsAPI api.RoomserverInternalAPI producer *producers.RoomserverProducer eduProducer *producers.EDUServerProducer - keys gomatrixserverlib.KeyRing - federation *gomatrixserverlib.FederationClient + keys gomatrixserverlib.JSONVerifier + federation txnFederationClient +} + +// A subset of FederationClient functionality that txn requires. Useful for testing. +type txnFederationClient interface { + LookupState(ctx context.Context, s gomatrixserverlib.ServerName, roomID string, eventID string, roomVersion gomatrixserverlib.RoomVersion) ( + res gomatrixserverlib.RespState, err error, + ) + LookupStateIDs(ctx context.Context, s gomatrixserverlib.ServerName, roomID string, eventID string) (res gomatrixserverlib.RespStateIDs, err error) + GetEvent(ctx context.Context, s gomatrixserverlib.ServerName, eventID string) (res gomatrixserverlib.Transaction, err error) } func (t *txnReq) processTransaction() (*gomatrixserverlib.RespSend, error) { diff --git a/federationapi/routing/send_test.go b/federationapi/routing/send_test.go new file mode 100644 index 000000000..5e8e503a8 --- /dev/null +++ b/federationapi/routing/send_test.go @@ -0,0 +1,399 @@ +package routing + +import ( + "context" + "encoding/json" + "fmt" + "testing" + "time" + + "github.com/matrix-org/dendrite/clientapi/producers" + eduAPI "github.com/matrix-org/dendrite/eduserver/api" + fsAPI "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/gomatrixserverlib" +) + +const ( + testOrigin = gomatrixserverlib.ServerName("kaer.morhen") + testDestination = gomatrixserverlib.ServerName("white.orchard") +) + +var ( + testRoomVersion = gomatrixserverlib.RoomVersionV1 + testData = []json.RawMessage{ + []byte(`{"auth_events":[],"content":{"creator":"@userid:kaer.morhen"},"depth":0,"event_id":"$0ok8ynDp7kjc95e3:kaer.morhen","hashes":{"sha256":"17kPoH+h0Dk4Omn7Sus0qMb6+oGcf+CZFEgDhv7UKWs"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[],"prev_state":[],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"jP4a04f5/F10Pw95FPpdCyKAO44JOwUQ/MZOOeA/RTU1Dn+AHPMzGSaZnuGjRr/xQuADt+I3ctb5ZQfLKNzHDw"}},"state_key":"","type":"m.room.create"}`), + []byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}]],"content":{"membership":"join"},"depth":1,"event_id":"$LEwEu0kxrtu5fOiS:kaer.morhen","hashes":{"sha256":"B7M88PhXf3vd1LaFtjQutFu4x/w7fHD28XKZ4sAsJTo"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}]],"prev_state":[],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"p2vqmuJn7ZBRImctSaKbXCAxCcBlIjPH9JHte1ouIUGy84gpu4eLipOvSBCLL26hXfC0Zrm4WUto6Hr+ohdrCg"}},"state_key":"@userid:kaer.morhen","type":"m.room.member"}`), + []byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}],["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"content":{"join_rule":"public"},"depth":2,"event_id":"$SMHlqUrNhhBBRLeN:kaer.morhen","hashes":{"sha256":"vIuJQvmMjrGxshAkj1SXe0C4RqvMbv4ZADDw9pFCWqQ"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"prev_state":[],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"hBMsb3Qppo3RaqqAl4JyTgaiWEbW5hlckATky6PrHun+F3YM203TzG7w9clwuQU5F5pZoB1a6nw+to0hN90FAw"}},"state_key":"","type":"m.room.join_rules"}`), + []byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}],["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"content":{"history_visibility":"shared"},"depth":3,"event_id":"$6F1yGIbO0J7TM93h:kaer.morhen","hashes":{"sha256":"Mr23GKSlZW7UCCYLgOWawI2Sg6KIoMjUWO2TDenuOgw"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$SMHlqUrNhhBBRLeN:kaer.morhen",{"sha256":"SylzE8U02I+6eyEHgL+FlU0L5YdqrVp8OOlxKS9VQW0"}]],"prev_state":[],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"sHLKrFI3hKGrEJfpMVZSDS3LvLasQsy50CTsOwru9XTVxgRsPo6wozNtRVjxo1J3Rk18RC9JppovmQ5VR5EcDw"}},"state_key":"","type":"m.room.history_visibility"}`), + []byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}],["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"content":{"ban":50,"events":null,"events_default":0,"invite":0,"kick":50,"redact":50,"state_default":50,"users":null,"users_default":0},"depth":4,"event_id":"$UKNe10XzYzG0TeA9:kaer.morhen","hashes":{"sha256":"ngbP3yja9U5dlckKerUs/fSOhtKxZMCVvsfhPURSS28"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$6F1yGIbO0J7TM93h:kaer.morhen",{"sha256":"A4CucrKSoWX4IaJXhq02mBg1sxIyZEftbC+5p3fZAvk"}]],"prev_state":[],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"zOmwlP01QL3yFchzuR9WHvogOoBZA3oVtNIF3lM0ZfDnqlSYZB9sns27G/4HVq0k7alaK7ZE3oGoCrVnMkPNCw"}},"state_key":"","type":"m.room.power_levels"}`), + // messages + []byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}],["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"content":{"body":"Test Message"},"depth":5,"event_id":"$gl2T9l3qm0kUbiIJ:kaer.morhen","hashes":{"sha256":"Qx3nRMHLDPSL5hBAzuX84FiSSP0K0Kju2iFoBWH4Za8"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$UKNe10XzYzG0TeA9:kaer.morhen",{"sha256":"KtSRyMjt0ZSjsv2koixTRCxIRCGoOp6QrKscsW97XRo"}]],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"sqDgv3EG7ml5VREzmT9aZeBpS4gAPNIaIeJOwqjDhY0GPU/BcpX5wY4R7hYLrNe5cChgV+eFy/GWm1Zfg5FfDg"}},"type":"m.room.message"}`), + []byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}],["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"content":{"body":"Test Message"},"depth":6,"event_id":"$MYSbs8m4rEbsCWXD:kaer.morhen","hashes":{"sha256":"kgbYM7v4Ud2YaBsjBTolM4ySg6rHcJNYI6nWhMSdFUA"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$gl2T9l3qm0kUbiIJ:kaer.morhen",{"sha256":"C/rD04h9wGxRdN2G/IBfrgoE1UovzLZ+uskwaKZ37/Q"}]],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"x0UoKh968jj/F5l1/R7Ew0T6CTKuew3PLNHASNxqck/bkNe8yYQiDHXRr+kZxObeqPZZTpaF1+EI+bLU9W8GDQ"}},"type":"m.room.message"}`), + []byte(`{"auth_events":[["$0ok8ynDp7kjc95e3:kaer.morhen",{"sha256":"sWCi6Ckp9rDimQON+MrUlNRkyfZ2tjbPbWfg2NMB18Q"}],["$LEwEu0kxrtu5fOiS:kaer.morhen",{"sha256":"1aKajq6DWHru1R1HJjvdWMEavkJJHGaTmPvfuERUXaA"}]],"content":{"body":"Test Message"},"depth":7,"event_id":"$N5x9WJkl9ClPrAEg:kaer.morhen","hashes":{"sha256":"FWM8oz4yquTunRZ67qlW2gzPDzdWfBP6RPHXhK1I/x8"},"origin":"kaer.morhen","origin_server_ts":0,"prev_events":[["$MYSbs8m4rEbsCWXD:kaer.morhen",{"sha256":"fatqgW+SE8mb2wFn3UN+drmluoD4UJ/EcSrL6Ur9q1M"}]],"room_id":"!roomid:kaer.morhen","sender":"@userid:kaer.morhen","signatures":{"kaer.morhen":{"ed25519:auto":"Y+LX/xcyufoXMOIoqQBNOzy6lZfUGB1ffgXIrSugk6obMiyAsiRejHQN/pciZXsHKxMJLYRFAz4zSJoS/LGPAA"}},"type":"m.room.message"}`), + } + testEvents = []gomatrixserverlib.HeaderedEvent{} + testStateEvents = make(map[gomatrixserverlib.StateKeyTuple]gomatrixserverlib.HeaderedEvent) +) + +func init() { + for _, j := range testData { + e, err := gomatrixserverlib.NewEventFromTrustedJSON(j, false, testRoomVersion) + if err != nil { + panic("cannot load test data: " + err.Error()) + } + h := e.Headered(testRoomVersion) + testEvents = append(testEvents, h) + if e.StateKey() != nil { + testStateEvents[gomatrixserverlib.StateKeyTuple{ + EventType: e.Type(), + StateKey: *e.StateKey(), + }] = h + } + } +} + +type testNopJSONVerifier struct { + // this verifier verifies nothing +} + +func (t *testNopJSONVerifier) VerifyJSONs(ctx context.Context, requests []gomatrixserverlib.VerifyJSONRequest) ([]gomatrixserverlib.VerifyJSONResult, error) { + result := make([]gomatrixserverlib.VerifyJSONResult, len(requests)) + return result, nil +} + +type testEDUProducer struct { + // this producer keeps track of calls to InputTypingEvent + invocations []eduAPI.InputTypingEventRequest +} + +func (p *testEDUProducer) InputTypingEvent( + ctx context.Context, + request *eduAPI.InputTypingEventRequest, + response *eduAPI.InputTypingEventResponse, +) error { + p.invocations = append(p.invocations, *request) + return nil +} + +type testRoomserverAPI struct { + inputRoomEvents []api.InputRoomEvent + queryStateAfterEvents func(*api.QueryStateAfterEventsRequest) api.QueryStateAfterEventsResponse +} + +func (t *testRoomserverAPI) SetFederationSenderAPI(fsAPI fsAPI.FederationSenderInternalAPI) {} + +func (t *testRoomserverAPI) InputRoomEvents( + ctx context.Context, + request *api.InputRoomEventsRequest, + response *api.InputRoomEventsResponse, +) error { + t.inputRoomEvents = append(t.inputRoomEvents, request.InputRoomEvents...) + return nil +} + +func (t *testRoomserverAPI) PerformJoin( + ctx context.Context, + req *api.PerformJoinRequest, + res *api.PerformJoinResponse, +) error { + return nil +} + +func (t *testRoomserverAPI) PerformLeave( + ctx context.Context, + req *api.PerformLeaveRequest, + res *api.PerformLeaveResponse, +) error { + return nil +} + +// Query the latest events and state for a room from the room server. +func (t *testRoomserverAPI) QueryLatestEventsAndState( + ctx context.Context, + request *api.QueryLatestEventsAndStateRequest, + response *api.QueryLatestEventsAndStateResponse, +) error { + return nil +} + +// Query the state after a list of events in a room from the room server. +func (t *testRoomserverAPI) QueryStateAfterEvents( + ctx context.Context, + request *api.QueryStateAfterEventsRequest, + response *api.QueryStateAfterEventsResponse, +) error { + response.RoomVersion = testRoomVersion + response.QueryStateAfterEventsRequest = *request + res := t.queryStateAfterEvents(request) + response.PrevEventsExist = res.PrevEventsExist + response.RoomExists = res.RoomExists + response.StateEvents = res.StateEvents + return nil +} + +// Query a list of events by event ID. +func (t *testRoomserverAPI) QueryEventsByID( + ctx context.Context, + request *api.QueryEventsByIDRequest, + response *api.QueryEventsByIDResponse, +) error { + return nil +} + +// Query the membership event for an user for a room. +func (t *testRoomserverAPI) QueryMembershipForUser( + ctx context.Context, + request *api.QueryMembershipForUserRequest, + response *api.QueryMembershipForUserResponse, +) error { + return nil +} + +// Query a list of membership events for a room +func (t *testRoomserverAPI) QueryMembershipsForRoom( + ctx context.Context, + request *api.QueryMembershipsForRoomRequest, + response *api.QueryMembershipsForRoomResponse, +) error { + return nil +} + +// Query a list of invite event senders for a user in a room. +func (t *testRoomserverAPI) QueryInvitesForUser( + ctx context.Context, + request *api.QueryInvitesForUserRequest, + response *api.QueryInvitesForUserResponse, +) error { + return nil +} + +// Query whether a server is allowed to see an event +func (t *testRoomserverAPI) QueryServerAllowedToSeeEvent( + ctx context.Context, + request *api.QueryServerAllowedToSeeEventRequest, + response *api.QueryServerAllowedToSeeEventResponse, +) error { + return nil +} + +// Query missing events for a room from roomserver +func (t *testRoomserverAPI) QueryMissingEvents( + ctx context.Context, + request *api.QueryMissingEventsRequest, + response *api.QueryMissingEventsResponse, +) error { + return nil +} + +// Query to get state and auth chain for a (potentially hypothetical) event. +// Takes lists of PrevEventIDs and AuthEventsIDs and uses them to calculate +// the state and auth chain to return. +func (t *testRoomserverAPI) QueryStateAndAuthChain( + ctx context.Context, + request *api.QueryStateAndAuthChainRequest, + response *api.QueryStateAndAuthChainResponse, +) error { + return nil +} + +// Query a given amount (or less) of events prior to a given set of events. +func (t *testRoomserverAPI) QueryBackfill( + ctx context.Context, + request *api.QueryBackfillRequest, + response *api.QueryBackfillResponse, +) error { + return nil +} + +// Asks for the default room version as preferred by the server. +func (t *testRoomserverAPI) QueryRoomVersionCapabilities( + ctx context.Context, + request *api.QueryRoomVersionCapabilitiesRequest, + response *api.QueryRoomVersionCapabilitiesResponse, +) error { + return nil +} + +// Asks for the room version for a given room. +func (t *testRoomserverAPI) QueryRoomVersionForRoom( + ctx context.Context, + request *api.QueryRoomVersionForRoomRequest, + response *api.QueryRoomVersionForRoomResponse, +) error { + response.RoomVersion = testRoomVersion + return nil +} + +// Set a room alias +func (t *testRoomserverAPI) SetRoomAlias( + ctx context.Context, + req *api.SetRoomAliasRequest, + response *api.SetRoomAliasResponse, +) error { + return nil +} + +// Get the room ID for an alias +func (t *testRoomserverAPI) GetRoomIDForAlias( + ctx context.Context, + req *api.GetRoomIDForAliasRequest, + response *api.GetRoomIDForAliasResponse, +) error { + return nil +} + +// Get all known aliases for a room ID +func (t *testRoomserverAPI) GetAliasesForRoomID( + ctx context.Context, + req *api.GetAliasesForRoomIDRequest, + response *api.GetAliasesForRoomIDResponse, +) error { + return nil +} + +// Get the user ID of the creator of an alias +func (t *testRoomserverAPI) GetCreatorIDForAlias( + ctx context.Context, + req *api.GetCreatorIDForAliasRequest, + response *api.GetCreatorIDForAliasResponse, +) error { + return nil +} + +// Remove a room alias +func (t *testRoomserverAPI) RemoveRoomAlias( + ctx context.Context, + req *api.RemoveRoomAliasRequest, + response *api.RemoveRoomAliasResponse, +) error { + return nil +} + +type txnFedClient struct{} + +func (c *txnFedClient) LookupState(ctx context.Context, s gomatrixserverlib.ServerName, roomID string, eventID string, roomVersion gomatrixserverlib.RoomVersion) ( + res gomatrixserverlib.RespState, err error, +) { + return +} +func (c *txnFedClient) LookupStateIDs(ctx context.Context, s gomatrixserverlib.ServerName, roomID string, eventID string) (res gomatrixserverlib.RespStateIDs, err error) { + return +} +func (c *txnFedClient) GetEvent(ctx context.Context, s gomatrixserverlib.ServerName, eventID string) (res gomatrixserverlib.Transaction, err error) { + return +} + +func mustCreateTransaction(rsAPI api.RoomserverInternalAPI, fedClient txnFederationClient, pdus []json.RawMessage, edus []gomatrixserverlib.EDU) *txnReq { + t := &txnReq{ + context: context.Background(), + rsAPI: rsAPI, + producer: producers.NewRoomserverProducer(rsAPI), + eduProducer: producers.NewEDUServerProducer(&testEDUProducer{}), + keys: &testNopJSONVerifier{}, + federation: fedClient, + } + t.PDUs = pdus + t.EDUs = edus + t.Origin = testOrigin + t.TransactionID = gomatrixserverlib.TransactionID(fmt.Sprintf("%d", time.Now().UnixNano())) + t.Destination = testDestination + return t +} + +func mustProcessTransaction(t *testing.T, txn *txnReq, pdusWithErrors []string) { + res, err := txn.processTransaction() + if err != nil { + t.Errorf("txn.processTransaction returned an error: %s", err) + return + } + if len(res.PDUs) != len(txn.PDUs) { + t.Errorf("txn.processTransaction did not return results for all PDUs, got %d want %d", len(res.PDUs), len(txn.PDUs)) + return + } +NextPDU: + for eventID, result := range res.PDUs { + if result.Error == "" { + continue + } + for _, eventIDWantError := range pdusWithErrors { + if eventID == eventIDWantError { + break NextPDU + } + } + t.Errorf("txn.processTransaction PDU %s returned an error %s", eventID, result.Error) + } +} + +func fromStateTuples(tuples []gomatrixserverlib.StateKeyTuple, omitTuples []gomatrixserverlib.StateKeyTuple) (result []gomatrixserverlib.HeaderedEvent) { +NextTuple: + for _, t := range tuples { + for _, o := range omitTuples { + if t == o { + break NextTuple + } + } + h, ok := testStateEvents[t] + if ok { + result = append(result, h) + } + } + return +} + +func assertInputRoomEvents(t *testing.T, got []api.InputRoomEvent, want []gomatrixserverlib.HeaderedEvent) { + if len(got) != len(want) { + t.Errorf("wrong number of InputRoomEvents: got %d want %d", len(got), len(want)) + return + } + for i := range got { + if got[i].Event.EventID() != want[i].EventID() { + t.Errorf("InputRoomEvents[%d] got %s want %s", i, got[i].Event.EventID(), want[i].EventID()) + } + } +} + +// The purpose of this test is to check that receiving an event over federation for which we have the prev_events works correctly, and passes it on +// to the roomserver. It's the most basic test possible. +func TestBasicTransaction(t *testing.T) { + rsAPI := &testRoomserverAPI{ + queryStateAfterEvents: func(req *api.QueryStateAfterEventsRequest) api.QueryStateAfterEventsResponse { + return api.QueryStateAfterEventsResponse{ + PrevEventsExist: true, + RoomExists: true, + StateEvents: fromStateTuples(req.StateToFetch, nil), + } + }, + } + pdus := []json.RawMessage{ + testData[len(testData)-1], // a message event + } + txn := mustCreateTransaction(rsAPI, &txnFedClient{}, pdus, nil) + mustProcessTransaction(t, txn, nil) + assertInputRoomEvents(t, rsAPI.inputRoomEvents, []gomatrixserverlib.HeaderedEvent{testEvents[len(testEvents)-1]}) +} + +// The purpose of this test is to check that if the event received fails auth checks the transaction is failed. +func TestTransactionFailAuthChecks(t *testing.T) { + rsAPI := &testRoomserverAPI{ + queryStateAfterEvents: func(req *api.QueryStateAfterEventsRequest) api.QueryStateAfterEventsResponse { + return api.QueryStateAfterEventsResponse{ + PrevEventsExist: true, + RoomExists: true, + // omit the create event so auth checks fail + StateEvents: fromStateTuples(req.StateToFetch, []gomatrixserverlib.StateKeyTuple{ + {EventType: gomatrixserverlib.MRoomCreate, StateKey: ""}, + }), + } + }, + } + pdus := []json.RawMessage{ + testData[len(testData)-1], // a message event + } + txn := mustCreateTransaction(rsAPI, &txnFedClient{}, pdus, nil) + mustProcessTransaction(t, txn, []string{ + // expect the event to have an error + testEvents[len(testEvents)-1].EventID(), + }) + assertInputRoomEvents(t, rsAPI.inputRoomEvents, nil) // expect no messages to be sent to the roomserver +} From 3b98535dc5d85a36168c857da55b17d4ab030f9d Mon Sep 17 00:00:00 2001 From: Kegsay Date: Wed, 6 May 2020 18:03:25 +0100 Subject: [PATCH 035/200] only send new events to RS; add tests for /state_ids and /event (#1011) * only send new events to RS; add tests for /state_ids and /event * Review comments: send in auth event order * Ignore order of state events for this test as RespState.Events is non-deterministic --- clientapi/producers/roomserver.go | 8 +- federationapi/routing/send.go | 36 ++++--- federationapi/routing/send_test.go | 162 ++++++++++++++++++++++++++++- 3 files changed, 183 insertions(+), 23 deletions(-) diff --git a/clientapi/producers/roomserver.go b/clientapi/producers/roomserver.go index 7eee83f5e..f0733db9c 100644 --- a/clientapi/producers/roomserver.go +++ b/clientapi/producers/roomserver.go @@ -52,9 +52,10 @@ func (c *RoomserverProducer) SendEvents( } // SendEventWithState writes an event with KindNew to the roomserver input log -// with the state at the event as KindOutlier before it. +// with the state at the event as KindOutlier before it. Will not send any event that is +// marked as `true` in haveEventIDs func (c *RoomserverProducer) SendEventWithState( - ctx context.Context, state *gomatrixserverlib.RespState, event gomatrixserverlib.HeaderedEvent, + ctx context.Context, state *gomatrixserverlib.RespState, event gomatrixserverlib.HeaderedEvent, haveEventIDs map[string]bool, ) error { outliers, err := state.Events() if err != nil { @@ -63,6 +64,9 @@ func (c *RoomserverProducer) SendEventWithState( var ires []api.InputRoomEvent for _, outlier := range outliers { + if haveEventIDs[outlier.EventID()] { + continue + } ires = append(ires, api.InputRoomEvent{ Kind: api.KindOutlier, Event: outlier.Headered(event.RoomVersion), diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index 10210db64..e6f91d94a 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -309,9 +309,7 @@ func (t *txnReq) processEventWithMissingState(e gomatrixserverlib.Event, roomVer // TODO: Attempt to fill in the gap using /get_missing_events // Attempt to fetch the missing state using /state_ids and /events - var respState *gomatrixserverlib.RespState - var err error - respState, err = t.lookupMissingStateViaStateIDs(e, roomVersion) + respState, haveEventIDs, err := t.lookupMissingStateViaStateIDs(e, roomVersion) if err != nil { // Fallback to /state util.GetLogger(t.context).WithError(err).Warn("processEventWithMissingState failed to /state_ids, falling back to /state") @@ -343,8 +341,9 @@ retryAllowedState: return err } - // pass the event along with the state to the roomserver - return t.producer.SendEventWithState(t.context, respState, e.Headered(roomVersion)) + // pass the event along with the state to the roomserver using a background context so we don't + // needlessly expire + return t.producer.SendEventWithState(context.Background(), respState, e.Headered(roomVersion), haveEventIDs) } func (t *txnReq) lookupMissingStateViaState(e gomatrixserverlib.Event, roomVersion gomatrixserverlib.RoomVersion) ( @@ -361,28 +360,30 @@ func (t *txnReq) lookupMissingStateViaState(e gomatrixserverlib.Event, roomVersi } func (t *txnReq) lookupMissingStateViaStateIDs(e gomatrixserverlib.Event, roomVersion gomatrixserverlib.RoomVersion) ( - *gomatrixserverlib.RespState, error) { + *gomatrixserverlib.RespState, map[string]bool, error) { // fetch the state event IDs at the time of the event stateIDs, err := t.federation.LookupStateIDs(t.context, t.Origin, e.RoomID(), e.EventID()) if err != nil { - return nil, err + return nil, nil, err } // fetch as many as we can from the roomserver, do them as 2 calls rather than // 1 to try to reduce the number of parameters in the bulk query this will use haveEventMap := make(map[string]*gomatrixserverlib.HeaderedEvent, len(stateIDs.StateEventIDs)) + haveEventIDs := make(map[string]bool) for _, eventList := range [][]string{stateIDs.StateEventIDs, stateIDs.AuthEventIDs} { queryReq := api.QueryEventsByIDRequest{ EventIDs: eventList, } var queryRes api.QueryEventsByIDResponse - if err := t.rsAPI.QueryEventsByID(t.context, &queryReq, &queryRes); err != nil { - return nil, err + if err = t.rsAPI.QueryEventsByID(t.context, &queryReq, &queryRes); err != nil { + return nil, nil, err } // allow indexing of current state by event ID for i := range queryRes.Events { haveEventMap[queryRes.Events[i].EventID()] = &queryRes.Events[i] + haveEventIDs[queryRes.Events[i].EventID()] = true } } @@ -404,26 +405,29 @@ func (t *txnReq) lookupMissingStateViaStateIDs(e gomatrixserverlib.Event, roomVe }).Info("Fetching missing state at event") for missingEventID := range missing { - txn, err := t.federation.GetEvent(t.context, t.Origin, missingEventID) + var txn gomatrixserverlib.Transaction + txn, err = t.federation.GetEvent(t.context, t.Origin, missingEventID) if err != nil { util.GetLogger(t.context).WithError(err).WithField("event_id", missingEventID).Warn("failed to get missing /event for event ID") - return nil, err + return nil, nil, err } for _, pdu := range txn.PDUs { - event, err := gomatrixserverlib.NewEventFromUntrustedJSON(pdu, roomVersion) + var event gomatrixserverlib.Event + event, err = gomatrixserverlib.NewEventFromUntrustedJSON(pdu, roomVersion) if err != nil { util.GetLogger(t.context).WithError(err).Warnf("Transaction: Failed to parse event JSON of event %q", event.EventID()) - return nil, unmarshalError{err} + return nil, nil, unmarshalError{err} } - if err := gomatrixserverlib.VerifyAllEventSignatures(t.context, []gomatrixserverlib.Event{event}, t.keys); err != nil { + if err = gomatrixserverlib.VerifyAllEventSignatures(t.context, []gomatrixserverlib.Event{event}, t.keys); err != nil { util.GetLogger(t.context).WithError(err).Warnf("Transaction: Couldn't validate signature of event %q", event.EventID()) - return nil, verifySigError{event.EventID(), err} + return nil, nil, verifySigError{event.EventID(), err} } h := event.Headered(roomVersion) haveEventMap[event.EventID()] = &h } } - return t.createRespStateFromStateIDs(stateIDs, haveEventMap) + resp, err := t.createRespStateFromStateIDs(stateIDs, haveEventMap) + return resp, haveEventIDs, err } func (t *txnReq) createRespStateFromStateIDs(stateIDs gomatrixserverlib.RespStateIDs, haveEventMap map[string]*gomatrixserverlib.HeaderedEvent) ( diff --git a/federationapi/routing/send_test.go b/federationapi/routing/send_test.go index 5e8e503a8..89d28aa1c 100644 --- a/federationapi/routing/send_test.go +++ b/federationapi/routing/send_test.go @@ -4,6 +4,8 @@ import ( "context" "encoding/json" "fmt" + "reflect" + "sort" "testing" "time" @@ -79,6 +81,7 @@ func (p *testEDUProducer) InputTypingEvent( type testRoomserverAPI struct { inputRoomEvents []api.InputRoomEvent queryStateAfterEvents func(*api.QueryStateAfterEventsRequest) api.QueryStateAfterEventsResponse + queryEventsByID func(req *api.QueryEventsByIDRequest) api.QueryEventsByIDResponse } func (t *testRoomserverAPI) SetFederationSenderAPI(fsAPI fsAPI.FederationSenderInternalAPI) {} @@ -138,6 +141,8 @@ func (t *testRoomserverAPI) QueryEventsByID( request *api.QueryEventsByIDRequest, response *api.QueryEventsByIDResponse, ) error { + res := t.queryEventsByID(request) + response.Events = res.Events return nil } @@ -270,21 +275,43 @@ func (t *testRoomserverAPI) RemoveRoomAlias( return nil } -type txnFedClient struct{} +type txnFedClient struct { + state map[string]gomatrixserverlib.RespState // event_id to response + stateIDs map[string]gomatrixserverlib.RespStateIDs // event_id to response + getEvent map[string]gomatrixserverlib.Transaction // event_id to response +} func (c *txnFedClient) LookupState(ctx context.Context, s gomatrixserverlib.ServerName, roomID string, eventID string, roomVersion gomatrixserverlib.RoomVersion) ( res gomatrixserverlib.RespState, err error, ) { + r, ok := c.state[eventID] + if !ok { + err = fmt.Errorf("txnFedClient: no /state for event %s", eventID) + return + } + res = r return } func (c *txnFedClient) LookupStateIDs(ctx context.Context, s gomatrixserverlib.ServerName, roomID string, eventID string) (res gomatrixserverlib.RespStateIDs, err error) { + r, ok := c.stateIDs[eventID] + if !ok { + err = fmt.Errorf("txnFedClient: no /state_ids for event %s", eventID) + return + } + res = r return } func (c *txnFedClient) GetEvent(ctx context.Context, s gomatrixserverlib.ServerName, eventID string) (res gomatrixserverlib.Transaction, err error) { + r, ok := c.getEvent[eventID] + if !ok { + err = fmt.Errorf("txnFedClient: no /event for event ID %s", eventID) + return + } + res = r return } -func mustCreateTransaction(rsAPI api.RoomserverInternalAPI, fedClient txnFederationClient, pdus []json.RawMessage, edus []gomatrixserverlib.EDU) *txnReq { +func mustCreateTransaction(rsAPI api.RoomserverInternalAPI, fedClient txnFederationClient, pdus []json.RawMessage) *txnReq { t := &txnReq{ context: context.Background(), rsAPI: rsAPI, @@ -294,7 +321,6 @@ func mustCreateTransaction(rsAPI api.RoomserverInternalAPI, fedClient txnFederat federation: fedClient, } t.PDUs = pdus - t.EDUs = edus t.Origin = testOrigin t.TransactionID = gomatrixserverlib.TransactionID(fmt.Sprintf("%d", time.Now().UnixNano())) t.Destination = testDestination @@ -368,7 +394,7 @@ func TestBasicTransaction(t *testing.T) { pdus := []json.RawMessage{ testData[len(testData)-1], // a message event } - txn := mustCreateTransaction(rsAPI, &txnFedClient{}, pdus, nil) + txn := mustCreateTransaction(rsAPI, &txnFedClient{}, pdus) mustProcessTransaction(t, txn, nil) assertInputRoomEvents(t, rsAPI.inputRoomEvents, []gomatrixserverlib.HeaderedEvent{testEvents[len(testEvents)-1]}) } @@ -390,10 +416,136 @@ func TestTransactionFailAuthChecks(t *testing.T) { pdus := []json.RawMessage{ testData[len(testData)-1], // a message event } - txn := mustCreateTransaction(rsAPI, &txnFedClient{}, pdus, nil) + txn := mustCreateTransaction(rsAPI, &txnFedClient{}, pdus) mustProcessTransaction(t, txn, []string{ // expect the event to have an error testEvents[len(testEvents)-1].EventID(), }) assertInputRoomEvents(t, rsAPI.inputRoomEvents, nil) // expect no messages to be sent to the roomserver } + +// The purpose of this test is to check that when there are missing prev_events that state is fetched via /state_ids +// and /event and not /state. It works by setting PrevEventsExist=false in the roomserver query response, resulting in +// a call to /state_ids which returns the whole room state. It should attempt to fetch as many of these events from the +// roomserver FIRST, resulting in a call to QueryEventsByID. However, this will be missing the m.room.power_levels event which +// should then be requested via /event. The net result is that the transaction should succeed and there should be 2 +// new events, first the m.room.power_levels event we were missing, then the transaction PDU. +func TestTransactionFetchMissingStateByStateIDs(t *testing.T) { + missingStateEvent := testStateEvents[gomatrixserverlib.StateKeyTuple{ + EventType: gomatrixserverlib.MRoomPowerLevels, + StateKey: "", + }] + rsAPI := &testRoomserverAPI{ + queryStateAfterEvents: func(req *api.QueryStateAfterEventsRequest) api.QueryStateAfterEventsResponse { + return api.QueryStateAfterEventsResponse{ + // setting this to false should trigger a call to /state_ids + PrevEventsExist: false, + RoomExists: true, + StateEvents: nil, + } + }, + queryEventsByID: func(req *api.QueryEventsByIDRequest) api.QueryEventsByIDResponse { + var res api.QueryEventsByIDResponse + for _, wantEventID := range req.EventIDs { + for _, ev := range testStateEvents { + // roomserver is missing the power levels event + if wantEventID == missingStateEvent.EventID() { + continue + } + if ev.EventID() == wantEventID { + res.Events = append(res.Events, ev) + } + } + } + res.QueryEventsByIDRequest = *req + return res + }, + } + inputEvent := testEvents[len(testEvents)-1] + var stateEventIDs []string + for _, ev := range testStateEvents { + stateEventIDs = append(stateEventIDs, ev.EventID()) + } + cli := &txnFedClient{ + // /state_ids returns all the state events + stateIDs: map[string]gomatrixserverlib.RespStateIDs{ + inputEvent.EventID(): gomatrixserverlib.RespStateIDs{ + StateEventIDs: stateEventIDs, + AuthEventIDs: stateEventIDs, + }, + }, + // /event for the missing state event returns it + getEvent: map[string]gomatrixserverlib.Transaction{ + missingStateEvent.EventID(): gomatrixserverlib.Transaction{ + PDUs: []json.RawMessage{ + missingStateEvent.JSON(), + }, + }, + }, + } + + pdus := []json.RawMessage{ + testData[len(testData)-1], // a message event + } + txn := mustCreateTransaction(rsAPI, cli, pdus) + mustProcessTransaction(t, txn, nil) + assertInputRoomEvents(t, rsAPI.inputRoomEvents, []gomatrixserverlib.HeaderedEvent{missingStateEvent, inputEvent}) +} + +// The purpose of this test is to check that when there are missing prev_events and /state_ids fails, that we fallback to +// calling /state which returns the entire room state at that event. It works by setting PrevEventsExist=false in the +// roomserver query response, resulting in a call to /state_ids which fails (unset). It should then fetch via /state. +func TestTransactionFetchMissingStateByFallbackState(t *testing.T) { + rsAPI := &testRoomserverAPI{ + queryStateAfterEvents: func(req *api.QueryStateAfterEventsRequest) api.QueryStateAfterEventsResponse { + return api.QueryStateAfterEventsResponse{ + // setting this to false should trigger a call to /state_ids + PrevEventsExist: false, + RoomExists: true, + StateEvents: nil, + } + }, + } + inputEvent := testEvents[len(testEvents)-1] + // first 5 events are the state events, in auth event order. + stateEvents := testEvents[:5] + + cli := &txnFedClient{ + // /state_ids purposefully unset + stateIDs: nil, + // /state returns the state at that event (which is the current state) + state: map[string]gomatrixserverlib.RespState{ + inputEvent.EventID(): gomatrixserverlib.RespState{ + AuthEvents: gomatrixserverlib.UnwrapEventHeaders(stateEvents), + StateEvents: gomatrixserverlib.UnwrapEventHeaders(stateEvents), + }, + }, + } + + pdus := []json.RawMessage{ + testData[len(testData)-1], // a message event + } + txn := mustCreateTransaction(rsAPI, cli, pdus) + mustProcessTransaction(t, txn, nil) + // the roomserver should get all state events and the new input event + // TODO: it should really be only giving the missing ones + got := rsAPI.inputRoomEvents + if len(got) != len(stateEvents)+1 { + t.Fatalf("wrong number of InputRoomEvents: got %d want %d", len(got), len(stateEvents)+1) + } + last := got[len(got)-1] + if last.Event.EventID() != inputEvent.EventID() { + t.Errorf("last event should be the input event but it wasn't. got %s want %s", last.Event.EventID(), inputEvent.EventID()) + } + gots := make([]string, len(stateEvents)) + wants := make([]string, len(stateEvents)) + for i := range stateEvents { + gots[i] = got[i].Event.EventID() + wants[i] = stateEvents[i].EventID() + } + sort.Strings(gots) + sort.Strings(wants) + if !reflect.DeepEqual(gots, wants) { + t.Errorf("state events returned mismatch, got (sorted): %+v want %+v", gots, wants) + } +} From a16db1c4085c0079f72615f0c077fa5016c4fe0f Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 7 May 2020 12:42:06 +0100 Subject: [PATCH 036/200] Improve federation sender performance, implement backoff and blacklisting, fix up invites a bit (#1007) * Improve federation sender performance and behaviour, add backoff * Tweaks * Tweaks * Tweaks * Take copies of events before passing to destination queues * Don't accidentally drop queued messages * Don't take copies again * Tidy up a bit * Break out statistics (tracked component-wide), report success and failures from Perform actions * Fix comment, use atomic add * Improve logic a bit, don't block on wakeup, move idle check * Don't retry sucessful invites, don't dispatch sendEvent, sendInvite etc * Dedupe destinations, fix other bug hopefully * Dispatch sends again * Federation sender to ignore invites that are destined locally * Loopback invite events * Remodel a bit with channels * Linter * Only loopback invite event if we know the room * We should tell other resident servers about the invite if we know about the room * Correct invite signing * Fix invite loopback * Check HTTP response codes, push new invites to front of queue * Review comments --- federationsender/consumers/roomserver.go | 21 +- federationsender/federationsender.go | 11 +- federationsender/internal/api.go | 4 + federationsender/internal/perform.go | 8 + federationsender/producers/roomserver.go | 8 +- federationsender/queue/destinationqueue.go | 257 +++++++++++++++------ federationsender/queue/queue.go | 105 ++++----- federationsender/types/statistics.go | 122 ++++++++++ roomserver/internal/input.go | 17 +- roomserver/internal/input_events.go | 63 ++++- 10 files changed, 474 insertions(+), 142 deletions(-) create mode 100644 federationsender/types/statistics.go diff --git a/federationsender/consumers/roomserver.go b/federationsender/consumers/roomserver.go index 67d08b339..901239472 100644 --- a/federationsender/consumers/roomserver.go +++ b/federationsender/consumers/roomserver.go @@ -188,11 +188,30 @@ func (s *OutputRoomEventConsumer) processInvite(oie api.OutputNewInviteEvent) er return nil } + // Ignore invites that don't have state keys - they are invalid. + if oie.Event.StateKey() == nil { + return fmt.Errorf("event %q doesn't have state key", oie.Event.EventID()) + } + + // Don't try to handle events that are actually destined for us. + stateKey := *oie.Event.StateKey() + _, destination, err := gomatrixserverlib.SplitID('@', stateKey) + if err != nil { + log.WithFields(log.Fields{ + "event_id": oie.Event.EventID(), + "state_key": stateKey, + }).Info("failed to split destination from state key") + return nil + } + if s.cfg.Matrix.ServerName == destination { + return nil + } + // Try to extract the room invite state. The roomserver will have stashed // this for us in invite_room_state if it didn't already exist. strippedState := []gomatrixserverlib.InviteV2StrippedState{} if inviteRoomState := gjson.GetBytes(oie.Event.Unsigned(), "invite_room_state"); inviteRoomState.Exists() { - if err := json.Unmarshal([]byte(inviteRoomState.Raw), &strippedState); err != nil { + if err = json.Unmarshal([]byte(inviteRoomState.Raw), &strippedState); err != nil { log.WithError(err).Warn("failed to extract invite_room_state from event unsigned") } } diff --git a/federationsender/federationsender.go b/federationsender/federationsender.go index cf4395527..8e2f256dc 100644 --- a/federationsender/federationsender.go +++ b/federationsender/federationsender.go @@ -24,6 +24,7 @@ import ( "github.com/matrix-org/dendrite/federationsender/producers" "github.com/matrix-org/dendrite/federationsender/queue" "github.com/matrix-org/dendrite/federationsender/storage" + "github.com/matrix-org/dendrite/federationsender/types" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" @@ -42,9 +43,14 @@ func SetupFederationSenderComponent( logrus.WithError(err).Panic("failed to connect to federation sender db") } - roomserverProducer := producers.NewRoomserverProducer(rsAPI, base.Cfg.Matrix.ServerName) + roomserverProducer := producers.NewRoomserverProducer( + rsAPI, base.Cfg.Matrix.ServerName, base.Cfg.Matrix.KeyID, base.Cfg.Matrix.PrivateKey, + ) - queues := queue.NewOutgoingQueues(base.Cfg.Matrix.ServerName, federation, roomserverProducer) + statistics := &types.Statistics{} + queues := queue.NewOutgoingQueues( + base.Cfg.Matrix.ServerName, federation, roomserverProducer, statistics, + ) rsConsumer := consumers.NewOutputRoomEventConsumer( base.Cfg, base.KafkaConsumer, queues, @@ -63,6 +69,7 @@ func SetupFederationSenderComponent( queryAPI := internal.NewFederationSenderInternalAPI( federationSenderDB, base.Cfg, roomserverProducer, federation, keyRing, + statistics, ) queryAPI.SetupHTTP(http.DefaultServeMux) diff --git a/federationsender/internal/api.go b/federationsender/internal/api.go index 89a1fda40..481795220 100644 --- a/federationsender/internal/api.go +++ b/federationsender/internal/api.go @@ -9,6 +9,7 @@ import ( "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/federationsender/producers" "github.com/matrix-org/dendrite/federationsender/storage" + "github.com/matrix-org/dendrite/federationsender/types" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -18,6 +19,7 @@ type FederationSenderInternalAPI struct { api.FederationSenderInternalAPI db storage.Database cfg *config.Dendrite + statistics *types.Statistics producer *producers.RoomserverProducer federation *gomatrixserverlib.FederationClient keyRing *gomatrixserverlib.KeyRing @@ -28,6 +30,7 @@ func NewFederationSenderInternalAPI( producer *producers.RoomserverProducer, federation *gomatrixserverlib.FederationClient, keyRing *gomatrixserverlib.KeyRing, + statistics *types.Statistics, ) *FederationSenderInternalAPI { return &FederationSenderInternalAPI{ db: db, @@ -35,6 +38,7 @@ func NewFederationSenderInternalAPI( producer: producer, federation: federation, keyRing: keyRing, + statistics: statistics, } } diff --git a/federationsender/internal/perform.go b/federationsender/internal/perform.go index ff7f821c1..431b2a2d3 100644 --- a/federationsender/internal/perform.go +++ b/federationsender/internal/perform.go @@ -25,10 +25,12 @@ func (r *FederationSenderInternalAPI) PerformDirectoryLookup( request.RoomAlias, ) if err != nil { + r.statistics.ForServer(request.ServerName).Failure() return err } response.RoomID = dir.RoomID response.ServerNames = dir.Servers + r.statistics.ForServer(request.ServerName).Success() return nil } @@ -61,6 +63,7 @@ func (r *FederationSenderInternalAPI) PerformJoin( ) if err != nil { // TODO: Check if the user was not allowed to join the room. + r.statistics.ForServer(serverName).Failure() return fmt.Errorf("r.federation.MakeJoin: %w", err) } @@ -112,6 +115,7 @@ func (r *FederationSenderInternalAPI) PerformJoin( ) if err != nil { logrus.WithError(err).Warnf("r.federation.SendJoin failed") + r.statistics.ForServer(serverName).Failure() continue } @@ -137,6 +141,7 @@ func (r *FederationSenderInternalAPI) PerformJoin( } // We're all good. + r.statistics.ForServer(serverName).Success() return nil } @@ -170,6 +175,7 @@ func (r *FederationSenderInternalAPI) PerformLeave( if err != nil { // TODO: Check if the user was not allowed to leave the room. logrus.WithError(err).Warnf("r.federation.MakeLeave failed") + r.statistics.ForServer(serverName).Failure() continue } @@ -221,9 +227,11 @@ func (r *FederationSenderInternalAPI) PerformLeave( ) if err != nil { logrus.WithError(err).Warnf("r.federation.SendLeave failed") + r.statistics.ForServer(serverName).Failure() continue } + r.statistics.ForServer(serverName).Success() return nil } diff --git a/federationsender/producers/roomserver.go b/federationsender/producers/roomserver.go index 48aeed8cc..76fedf537 100644 --- a/federationsender/producers/roomserver.go +++ b/federationsender/producers/roomserver.go @@ -16,6 +16,7 @@ package producers import ( "context" + "crypto/ed25519" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" @@ -25,15 +26,20 @@ import ( type RoomserverProducer struct { InputAPI api.RoomserverInternalAPI serverName gomatrixserverlib.ServerName + keyID gomatrixserverlib.KeyID + privateKey ed25519.PrivateKey } // NewRoomserverProducer creates a new RoomserverProducer func NewRoomserverProducer( rsAPI api.RoomserverInternalAPI, serverName gomatrixserverlib.ServerName, + keyID gomatrixserverlib.KeyID, privateKey ed25519.PrivateKey, ) *RoomserverProducer { return &RoomserverProducer{ InputAPI: rsAPI, serverName: serverName, + keyID: keyID, + privateKey: privateKey, } } @@ -43,7 +49,7 @@ func NewRoomserverProducer( func (c *RoomserverProducer) SendInviteResponse( ctx context.Context, res gomatrixserverlib.RespInviteV2, roomVersion gomatrixserverlib.RoomVersion, ) (string, error) { - ev := res.Event.Headered(roomVersion) + ev := res.Event.Sign(string(c.serverName), c.keyID, c.privateKey).Headered(roomVersion) ire := api.InputRoomEvent{ Kind: api.KindNew, Event: ev, diff --git a/federationsender/queue/destinationqueue.go b/federationsender/queue/destinationqueue.go index 89526fcfd..45faa287c 100644 --- a/federationsender/queue/destinationqueue.go +++ b/federationsender/queue/destinationqueue.go @@ -18,12 +18,13 @@ import ( "context" "encoding/json" "fmt" - "sync" "time" "github.com/matrix-org/dendrite/federationsender/producers" + "github.com/matrix-org/dendrite/federationsender/types" + "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" + "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus" "go.uber.org/atomic" ) @@ -33,92 +34,190 @@ import ( // ensures that only one request is in flight to a given destination // at a time. type destinationQueue struct { - rsProducer *producers.RoomserverProducer - client *gomatrixserverlib.FederationClient - origin gomatrixserverlib.ServerName - destination gomatrixserverlib.ServerName - running atomic.Bool - // The running mutex protects sentCounter, lastTransactionIDs and - // pendingEvents, pendingEDUs. - runningMutex sync.Mutex - sentCounter int - lastTransactionIDs []gomatrixserverlib.TransactionID - pendingEvents []*gomatrixserverlib.HeaderedEvent - pendingEDUs []*gomatrixserverlib.EDU - pendingInvites []*gomatrixserverlib.InviteV2Request + rsProducer *producers.RoomserverProducer // roomserver producer + client *gomatrixserverlib.FederationClient // federation client + origin gomatrixserverlib.ServerName // origin of requests + destination gomatrixserverlib.ServerName // destination of requests + running atomic.Bool // is the queue worker running? + statistics *types.ServerStatistics // statistics about this remote server + incomingPDUs chan *gomatrixserverlib.HeaderedEvent // PDUs to send + incomingEDUs chan *gomatrixserverlib.EDU // EDUs to send + incomingInvites chan *gomatrixserverlib.InviteV2Request // invites to send + lastTransactionIDs []gomatrixserverlib.TransactionID // last transaction ID + pendingPDUs []*gomatrixserverlib.HeaderedEvent // owned by backgroundSend + pendingEDUs []*gomatrixserverlib.EDU // owned by backgroundSend + pendingInvites []*gomatrixserverlib.InviteV2Request // owned by backgroundSend } // Send event adds the event to the pending queue for the destination. // If the queue is empty then it starts a background goroutine to // start sending events to that destination. func (oq *destinationQueue) sendEvent(ev *gomatrixserverlib.HeaderedEvent) { - oq.runningMutex.Lock() - defer oq.runningMutex.Unlock() - oq.pendingEvents = append(oq.pendingEvents, ev) + if oq.statistics.Blacklisted() { + // If the destination is blacklisted then drop the event. + return + } if !oq.running.Load() { go oq.backgroundSend() } + oq.incomingPDUs <- ev } // sendEDU adds the EDU event to the pending queue for the destination. // If the queue is empty then it starts a background goroutine to // start sending events to that destination. -func (oq *destinationQueue) sendEDU(e *gomatrixserverlib.EDU) { - oq.runningMutex.Lock() - defer oq.runningMutex.Unlock() - oq.pendingEDUs = append(oq.pendingEDUs, e) +func (oq *destinationQueue) sendEDU(ev *gomatrixserverlib.EDU) { + if oq.statistics.Blacklisted() { + // If the destination is blacklisted then drop the event. + return + } if !oq.running.Load() { go oq.backgroundSend() } + oq.incomingEDUs <- ev } // sendInvite adds the invite event to the pending queue for the // destination. If the queue is empty then it starts a background // goroutine to start sending events to that destination. func (oq *destinationQueue) sendInvite(ev *gomatrixserverlib.InviteV2Request) { - oq.runningMutex.Lock() - defer oq.runningMutex.Unlock() - oq.pendingInvites = append(oq.pendingInvites, ev) + if oq.statistics.Blacklisted() { + // If the destination is blacklisted then drop the event. + return + } if !oq.running.Load() { go oq.backgroundSend() } + oq.incomingInvites <- ev } // backgroundSend is the worker goroutine for sending events. +// nolint:gocyclo func (oq *destinationQueue) backgroundSend() { - oq.running.Store(true) + // Check if a worker is already running, and if it isn't, then + // mark it as started. + if !oq.running.CAS(false, true) { + return + } defer oq.running.Store(false) for { - transaction, invites := oq.nextTransaction(), oq.nextInvites() - if !transaction && !invites { - // If the queue is empty then stop processing for this destination. - // TODO: Remove this destination from the queue map. + // Wait either for incoming events, or until we hit an + // idle timeout. + select { + case pdu := <-oq.incomingPDUs: + // Ordering of PDUs is important so we add them to the end + // of the queue and they will all be added to transactions + // in order. + oq.pendingPDUs = append(oq.pendingPDUs, pdu) + case edu := <-oq.incomingEDUs: + // Likewise for EDUs, although we should probably not try + // too hard with some EDUs (like typing notifications) after + // a certain amount of time has passed. + // TODO: think about EDU expiry some more + oq.pendingEDUs = append(oq.pendingEDUs, edu) + case invite := <-oq.incomingInvites: + // There's no strict ordering requirement for invites like + // there is for transactions, so we put the invite onto the + // front of the queue. This means that if an invite that is + // stuck failing already, that it won't block our new invite + // from being sent. + oq.pendingInvites = append( + []*gomatrixserverlib.InviteV2Request{invite}, + oq.pendingInvites..., + ) + case <-time.After(time.Second * 30): + // The worker is idle so stop the goroutine. It'll + // get restarted automatically the next time we + // get an event. return } - // TODO: handle retries. - // TODO: blacklist uncooperative servers. + // If we are backing off this server then wait for the + // backoff duration to complete first. + if backoff, duration := oq.statistics.BackoffDuration(); backoff { + <-time.After(duration) + } + + // How many things do we have waiting? + numPDUs := len(oq.pendingPDUs) + numEDUs := len(oq.pendingEDUs) + numInvites := len(oq.pendingInvites) + + // If we have pending PDUs or EDUs then construct a transaction. + if numPDUs > 0 || numEDUs > 0 { + // Try sending the next transaction and see what happens. + transaction, terr := oq.nextTransaction(oq.pendingPDUs, oq.pendingEDUs, oq.statistics.SuccessCount()) + if terr != nil { + // We failed to send the transaction. + if giveUp := oq.statistics.Failure(); giveUp { + // It's been suggested that we should give up because + // the backoff has exceeded a maximum allowable value. + return + } + } else if transaction { + // If we successfully sent the transaction then clear out + // the pending events and EDUs. + oq.statistics.Success() + // Reallocate so that the underlying arrays can be GC'd, as + // opposed to growing forever. + for i := 0; i < numPDUs; i++ { + oq.pendingPDUs[i] = nil + } + for i := 0; i < numEDUs; i++ { + oq.pendingEDUs[i] = nil + } + oq.pendingPDUs = append( + []*gomatrixserverlib.HeaderedEvent{}, + oq.pendingPDUs[numPDUs:]..., + ) + oq.pendingEDUs = append( + []*gomatrixserverlib.EDU{}, + oq.pendingEDUs[numEDUs:]..., + ) + } + } + + // Try sending the next invite and see what happens. + if numInvites > 0 { + sent, ierr := oq.nextInvites(oq.pendingInvites) + if ierr != nil { + // We failed to send the transaction so increase the + // backoff and give it another go shortly. + if giveUp := oq.statistics.Failure(); giveUp { + // It's been suggested that we should give up because + // the backoff has exceeded a maximum allowable value. + return + } + } else if sent > 0 { + // If we successfully sent the invites then clear out + // the pending invites. + oq.statistics.Success() + // Reallocate so that the underlying array can be GC'd, as + // opposed to growing forever. + oq.pendingInvites = append( + []*gomatrixserverlib.InviteV2Request{}, + oq.pendingInvites[sent:]..., + ) + } + } } } // nextTransaction creates a new transaction from the pending event // queue and sends it. Returns true if a transaction was sent or // false otherwise. -func (oq *destinationQueue) nextTransaction() bool { - oq.runningMutex.Lock() - defer oq.runningMutex.Unlock() - - if len(oq.pendingEvents) == 0 && len(oq.pendingEDUs) == 0 { - return false - } - +func (oq *destinationQueue) nextTransaction( + pendingPDUs []*gomatrixserverlib.HeaderedEvent, + pendingEDUs []*gomatrixserverlib.EDU, + sentCounter uint32, +) (bool, error) { t := gomatrixserverlib.Transaction{ PDUs: []json.RawMessage{}, EDUs: []gomatrixserverlib.EDU{}, } now := gomatrixserverlib.AsTimestamp(time.Now()) - t.TransactionID = gomatrixserverlib.TransactionID(fmt.Sprintf("%d-%d", now, oq.sentCounter)) + t.TransactionID = gomatrixserverlib.TransactionID(fmt.Sprintf("%d-%d", now, sentCounter)) t.Origin = oq.origin t.Destination = oq.destination t.OriginServerTS = now @@ -129,44 +228,54 @@ func (oq *destinationQueue) nextTransaction() bool { oq.lastTransactionIDs = []gomatrixserverlib.TransactionID{t.TransactionID} - for _, pdu := range oq.pendingEvents { + for _, pdu := range pendingPDUs { // Append the JSON of the event, since this is a json.RawMessage type in the // gomatrixserverlib.Transaction struct t.PDUs = append(t.PDUs, (*pdu).JSON()) } - oq.pendingEvents = nil - oq.sentCounter += len(t.PDUs) - for _, edu := range oq.pendingEDUs { + for _, edu := range pendingEDUs { t.EDUs = append(t.EDUs, *edu) } - oq.pendingEDUs = nil - oq.sentCounter += len(t.EDUs) - util.GetLogger(context.TODO()).Infof("Sending transaction %q containing %d PDUs, %d EDUs", t.TransactionID, len(t.PDUs), len(t.EDUs)) + logrus.WithField("server_name", oq.destination).Infof("Sending transaction %q containing %d PDUs, %d EDUs", t.TransactionID, len(t.PDUs), len(t.EDUs)) + // TODO: we should check for 500-ish fails vs 400-ish here, + // since we shouldn't queue things indefinitely in response + // to a 400-ish error _, err := oq.client.SendTransaction(context.TODO(), t) - if err != nil { + switch e := err.(type) { + case nil: + // No error was returned so the transaction looks to have + // been successfully sent. + return true, nil + case gomatrix.HTTPError: + // We received a HTTP error back. In this instance we only + // should report an error if + if e.Code >= 400 && e.Code <= 499 { + // We tried but the remote side has sent back a client error. + // It's no use retrying because it will happen again. + return true, nil + } + // Otherwise, report that we failed to send the transaction + // and we will retry again. + return false, err + default: log.WithFields(log.Fields{ "destination": oq.destination, log.ErrorKey: err, }).Info("problem sending transaction") + return false, err } - - return true } // nextInvite takes pending invite events from the queue and sends // them. Returns true if a transaction was sent or false otherwise. -func (oq *destinationQueue) nextInvites() bool { - oq.runningMutex.Lock() - defer oq.runningMutex.Unlock() - - if len(oq.pendingInvites) == 0 { - return false - } - - for _, inviteReq := range oq.pendingInvites { +func (oq *destinationQueue) nextInvites( + pendingInvites []*gomatrixserverlib.InviteV2Request, +) (int, error) { + done := 0 + for _, inviteReq := range pendingInvites { ev, roomVersion := inviteReq.Event(), inviteReq.RoomVersion() log.WithFields(log.Fields{ @@ -180,13 +289,32 @@ func (oq *destinationQueue) nextInvites() bool { oq.destination, *inviteReq, ) - if err != nil { + switch e := err.(type) { + case nil: + done++ + case gomatrix.HTTPError: + log.WithFields(log.Fields{ + "event_id": ev.EventID(), + "state_key": ev.StateKey(), + "destination": oq.destination, + "status_code": e.Code, + }).WithError(err).Error("failed to send invite due to HTTP error") + // Check whether we should do something about the error or + // just accept it as unavoidable. + if e.Code >= 400 && e.Code <= 499 { + // We tried but the remote side has sent back a client error. + // It's no use retrying because it will happen again. + done++ + continue + } + return done, err + default: log.WithFields(log.Fields{ "event_id": ev.EventID(), "state_key": ev.StateKey(), "destination": oq.destination, }).WithError(err).Error("failed to send invite") - continue + return done, err } if _, err = oq.rsProducer.SendInviteResponse( @@ -199,10 +327,9 @@ func (oq *destinationQueue) nextInvites() bool { "state_key": ev.StateKey(), "destination": oq.destination, }).WithError(err).Error("failed to return signed invite to roomserver") + return done, err } } - oq.pendingInvites = nil - - return true + return done, nil } diff --git a/federationsender/queue/queue.go b/federationsender/queue/queue.go index 33abc8fdd..aae6c53a0 100644 --- a/federationsender/queue/queue.go +++ b/federationsender/queue/queue.go @@ -19,18 +19,20 @@ import ( "sync" "github.com/matrix-org/dendrite/federationsender/producers" + "github.com/matrix-org/dendrite/federationsender/types" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" log "github.com/sirupsen/logrus" ) // OutgoingQueues is a collection of queues for sending transactions to other // matrix servers type OutgoingQueues struct { - rsProducer *producers.RoomserverProducer - origin gomatrixserverlib.ServerName - client *gomatrixserverlib.FederationClient - // The queuesMutex protects queues - queuesMutex sync.Mutex + rsProducer *producers.RoomserverProducer + origin gomatrixserverlib.ServerName + client *gomatrixserverlib.FederationClient + statistics *types.Statistics + queuesMutex sync.Mutex // protects the below queues map[gomatrixserverlib.ServerName]*destinationQueue } @@ -39,15 +41,37 @@ func NewOutgoingQueues( origin gomatrixserverlib.ServerName, client *gomatrixserverlib.FederationClient, rsProducer *producers.RoomserverProducer, + statistics *types.Statistics, ) *OutgoingQueues { return &OutgoingQueues{ rsProducer: rsProducer, origin: origin, client: client, + statistics: statistics, queues: map[gomatrixserverlib.ServerName]*destinationQueue{}, } } +func (oqs *OutgoingQueues) getQueue(destination gomatrixserverlib.ServerName) *destinationQueue { + oqs.queuesMutex.Lock() + defer oqs.queuesMutex.Unlock() + oq := oqs.queues[destination] + if oq == nil { + oq = &destinationQueue{ + rsProducer: oqs.rsProducer, + origin: oqs.origin, + destination: destination, + client: oqs.client, + statistics: oqs.statistics.ForServer(destination), + incomingPDUs: make(chan *gomatrixserverlib.HeaderedEvent, 128), + incomingEDUs: make(chan *gomatrixserverlib.EDU, 128), + incomingInvites: make(chan *gomatrixserverlib.InviteV2Request, 128), + } + oqs.queues[destination] = oq + } + return oq +} + // SendEvent sends an event to the destinations func (oqs *OutgoingQueues) SendEvent( ev *gomatrixserverlib.HeaderedEvent, origin gomatrixserverlib.ServerName, @@ -62,27 +86,14 @@ func (oqs *OutgoingQueues) SendEvent( } // Remove our own server from the list of destinations. - destinations = filterDestinations(oqs.origin, destinations) + destinations = filterAndDedupeDests(oqs.origin, destinations) log.WithFields(log.Fields{ "destinations": destinations, "event": ev.EventID(), }).Info("Sending event") - oqs.queuesMutex.Lock() - defer oqs.queuesMutex.Unlock() for _, destination := range destinations { - oq := oqs.queues[destination] - if oq == nil { - oq = &destinationQueue{ - rsProducer: oqs.rsProducer, - origin: oqs.origin, - destination: destination, - client: oqs.client, - } - oqs.queues[destination] = oq - } - - oq.sendEvent(ev) + oqs.getQueue(destination).sendEvent(ev) } return nil @@ -111,23 +122,11 @@ func (oqs *OutgoingQueues) SendInvite( } log.WithFields(log.Fields{ - "event_id": ev.EventID(), + "event_id": ev.EventID(), + "server_name": destination, }).Info("Sending invite") - oqs.queuesMutex.Lock() - defer oqs.queuesMutex.Unlock() - oq := oqs.queues[destination] - if oq == nil { - oq = &destinationQueue{ - rsProducer: oqs.rsProducer, - origin: oqs.origin, - destination: destination, - client: oqs.client, - } - oqs.queues[destination] = oq - } - - oq.sendInvite(inviteReq) + oqs.getQueue(destination).sendInvite(inviteReq) return nil } @@ -146,7 +145,7 @@ func (oqs *OutgoingQueues) SendEDU( } // Remove our own server from the list of destinations. - destinations = filterDestinations(oqs.origin, destinations) + destinations = filterAndDedupeDests(oqs.origin, destinations) if len(destinations) > 0 { log.WithFields(log.Fields{ @@ -154,35 +153,27 @@ func (oqs *OutgoingQueues) SendEDU( }).Info("Sending EDU event") } - oqs.queuesMutex.Lock() - defer oqs.queuesMutex.Unlock() for _, destination := range destinations { - oq := oqs.queues[destination] - if oq == nil { - oq = &destinationQueue{ - rsProducer: oqs.rsProducer, - origin: oqs.origin, - destination: destination, - client: oqs.client, - } - oqs.queues[destination] = oq - } - - oq.sendEDU(e) + oqs.getQueue(destination).sendEDU(e) } return nil } -// filterDestinations removes our own server from the list of destinations. -// Otherwise we could end up trying to talk to ourselves. -func filterDestinations(origin gomatrixserverlib.ServerName, destinations []gomatrixserverlib.ServerName) []gomatrixserverlib.ServerName { - var result []gomatrixserverlib.ServerName - for _, destination := range destinations { - if destination == origin { +// filterAndDedupeDests removes our own server from the list of destinations +// and deduplicates any servers in the list that may appear more than once. +func filterAndDedupeDests(origin gomatrixserverlib.ServerName, destinations []gomatrixserverlib.ServerName) ( + result []gomatrixserverlib.ServerName, +) { + strs := make([]string, len(destinations)) + for i, d := range destinations { + strs[i] = string(d) + } + for _, destination := range util.UniqueStrings(strs) { + if gomatrixserverlib.ServerName(destination) == origin { continue } - result = append(result, destination) + result = append(result, gomatrixserverlib.ServerName(destination)) } return result } diff --git a/federationsender/types/statistics.go b/federationsender/types/statistics.go new file mode 100644 index 000000000..63f82756e --- /dev/null +++ b/federationsender/types/statistics.go @@ -0,0 +1,122 @@ +package types + +import ( + "math" + "sync" + "time" + + "github.com/matrix-org/gomatrixserverlib" + "go.uber.org/atomic" +) + +const ( + // How many times should we tolerate consecutive failures before we + // just blacklist the host altogether? Bear in mind that the backoff + // is exponential, so the max time here to attempt is 2**failures. + FailuresUntilBlacklist = 16 // 16 equates to roughly 18 hours. +) + +// Statistics contains information about all of the remote federated +// hosts that we have interacted with. It is basically a threadsafe +// wrapper. +type Statistics struct { + servers map[gomatrixserverlib.ServerName]*ServerStatistics + mutex sync.RWMutex +} + +// ForServer returns server statistics for the given server name. If it +// does not exist, it will create empty statistics and return those. +func (s *Statistics) ForServer(serverName gomatrixserverlib.ServerName) *ServerStatistics { + // If the map hasn't been initialised yet then do that. + if s.servers == nil { + s.mutex.Lock() + s.servers = make(map[gomatrixserverlib.ServerName]*ServerStatistics) + s.mutex.Unlock() + } + // Look up if we have statistics for this server already. + s.mutex.RLock() + server, found := s.servers[serverName] + s.mutex.RUnlock() + // If we don't, then make one. + if !found { + s.mutex.Lock() + server = &ServerStatistics{} + s.servers[serverName] = server + s.mutex.Unlock() + } + return server +} + +// ServerStatistics contains information about our interactions with a +// remote federated host, e.g. how many times we were successful, how +// many times we failed etc. It also manages the backoff time and black- +// listing a remote host if it remains uncooperative. +type ServerStatistics struct { + blacklisted atomic.Bool // is the remote side dead? + backoffUntil atomic.Value // time.Time to wait until before sending requests + failCounter atomic.Uint32 // how many times have we failed? + successCounter atomic.Uint32 // how many times have we succeeded? +} + +// Success updates the server statistics with a new successful +// attempt, which increases the sent counter and resets the idle and +// failure counters. If a host was blacklisted at this point then +// we will unblacklist it. +func (s *ServerStatistics) Success() { + s.successCounter.Add(1) + s.failCounter.Store(0) + s.blacklisted.Store(false) +} + +// Failure marks a failure and works out when to backoff until. It +// returns true if the worker should give up altogether because of +// too many consecutive failures. At this point the host is marked +// as blacklisted. +func (s *ServerStatistics) Failure() bool { + // Increase the fail counter. + failCounter := s.failCounter.Add(1) + + // Check that we haven't failed more times than is acceptable. + if failCounter >= FailuresUntilBlacklist { + // We've exceeded the maximum amount of times we're willing + // to back off, which is probably in the region of hours by + // now. Mark the host as blacklisted and tell the caller to + // give up. + s.blacklisted.Store(true) + return true + } + + // We're still under the threshold so work out the exponential + // backoff based on how many times we have failed already. The + // worker goroutine will wait until this time before processing + // anything from the queue. + backoffSeconds := time.Second * time.Duration(math.Exp2(float64(failCounter))) + s.backoffUntil.Store( + time.Now().Add(backoffSeconds), + ) + return false +} + +// BackoffDuration returns both a bool stating whether to wait, +// and then if true, a duration to wait for. +func (s *ServerStatistics) BackoffDuration() (bool, time.Duration) { + backoff, until := false, time.Second + if b, ok := s.backoffUntil.Load().(time.Time); ok { + if b.After(time.Now()) { + backoff, until = true, time.Until(b) + } + } + return backoff, until +} + +// Blacklisted returns true if the server is blacklisted and false +// otherwise. +func (s *ServerStatistics) Blacklisted() bool { + return s.blacklisted.Load() +} + +// SuccessCount returns the number of successful requests. This is +// usually useful in constructing transaction IDs. +func (s *ServerStatistics) SuccessCount() uint32 { + return s.successCounter.Load() +} diff --git a/roomserver/internal/input.go b/roomserver/internal/input.go index 19ebea660..ab3d7516b 100644 --- a/roomserver/internal/input.go +++ b/roomserver/internal/input.go @@ -58,15 +58,22 @@ func (r *RoomserverInternalAPI) InputRoomEvents( // We lock as processRoomEvent can only be called once at a time r.mutex.Lock() defer r.mutex.Unlock() + for i := range request.InputInviteEvents { + var loopback *api.InputRoomEvent + if loopback, err = processInviteEvent(ctx, r.DB, r, request.InputInviteEvents[i]); err != nil { + return err + } + // The processInviteEvent function can optionally return a + // loopback room event containing the invite, for local invites. + // If it does, we should process it with the room events below. + if loopback != nil { + request.InputRoomEvents = append(request.InputRoomEvents, *loopback) + } + } for i := range request.InputRoomEvents { if response.EventID, err = processRoomEvent(ctx, r.DB, r, request.InputRoomEvents[i]); err != nil { return err } } - for i := range request.InputInviteEvents { - if err = processInviteEvent(ctx, r.DB, r, request.InputInviteEvents[i]); err != nil { - return err - } - } return nil } diff --git a/roomserver/internal/input_events.go b/roomserver/internal/input_events.go index 6da63716c..b17076efe 100644 --- a/roomserver/internal/input_events.go +++ b/roomserver/internal/input_events.go @@ -18,6 +18,7 @@ package internal import ( "context" + "errors" "fmt" "github.com/matrix-org/dendrite/common" @@ -132,11 +133,11 @@ func calculateAndSetState( func processInviteEvent( ctx context.Context, db storage.Database, - ow OutputRoomEventWriter, + ow *RoomserverInternalAPI, input api.InputInviteEvent, -) (err error) { +) (*api.InputRoomEvent, error) { if input.Event.StateKey() == nil { - return fmt.Errorf("invite must be a state event") + return nil, fmt.Errorf("invite must be a state event") } roomID := input.Event.RoomID() @@ -151,7 +152,7 @@ func processInviteEvent( updater, err := db.MembershipUpdater(ctx, roomID, targetUserID, input.RoomVersion) if err != nil { - return err + return nil, err } succeeded := false defer func() { @@ -189,17 +190,27 @@ func processInviteEvent( // For now we will implement option 2. Since in the abesence of a retry // mechanism it will be equivalent to option 1, and we don't have a // signalling mechanism to implement option 3. - return nil + return nil, nil + } + + // Normally, with a federated invite, the federation sender would do + // the /v2/invite request (in which the remote server signs the invite) + // and then the signed event gets sent back to the roomserver as an input + // event. When the invite is local, we don't interact with the federation + // sender therefore we need to generate the loopback invite event for + // the room ourselves. + loopback, err := localInviteLoopback(ow, input) + if err != nil { + return nil, err } event := input.Event.Unwrap() - if len(input.InviteRoomState) > 0 { // If we were supplied with some invite room state already (which is // most likely to be if the event came in over federation) then use // that. if err = event.SetUnsignedField("invite_room_state", input.InviteRoomState); err != nil { - return err + return nil, err } } else { // There's no invite room state, so let's have a go at building it @@ -208,22 +219,52 @@ func processInviteEvent( // the invite room state, if we don't then we just fail quietly. if irs, ierr := buildInviteStrippedState(ctx, db, input); ierr == nil { if err = event.SetUnsignedField("invite_room_state", irs); err != nil { - return err + return nil, err } } } outputUpdates, err := updateToInviteMembership(updater, &event, nil, input.Event.RoomVersion) if err != nil { - return err + return nil, err } if err = ow.WriteOutputEvents(roomID, outputUpdates); err != nil { - return err + return nil, err } succeeded = true - return nil + return loopback, nil +} + +func localInviteLoopback( + ow *RoomserverInternalAPI, + input api.InputInviteEvent, +) (ire *api.InputRoomEvent, err error) { + if input.Event.StateKey() == nil { + return nil, errors.New("no state key on invite event") + } + ourServerName := string(ow.Cfg.Matrix.ServerName) + _, theirServerName, err := gomatrixserverlib.SplitID('@', *input.Event.StateKey()) + if err != nil { + return nil, err + } + // Check if the invite originated locally and is destined locally. + if input.Event.Origin() == ow.Cfg.Matrix.ServerName && string(theirServerName) == ourServerName { + rsEvent := input.Event.Sign( + ourServerName, + ow.Cfg.Matrix.KeyID, + ow.Cfg.Matrix.PrivateKey, + ).Headered(input.RoomVersion) + ire = &api.InputRoomEvent{ + Kind: api.KindNew, + Event: rsEvent, + AuthEventIDs: rsEvent.AuthEventIDs(), + SendAsServer: ourServerName, + TransactionID: nil, + } + } + return ire, nil } func buildInviteStrippedState( From c8e11dfe53a97ac2e207c893f3f21f1216d86343 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 7 May 2020 16:46:11 +0100 Subject: [PATCH 037/200] Direct messages (#1012) * Initial DM support, include invite event in stripped state for regular invites * Update go.mod, go.sum, test list --- are-we-synapse-yet.list | 1 + clientapi/routing/createroom.go | 45 +++++++++++++++++++++++++++++ clientapi/routing/membership.go | 6 ++-- go.mod | 2 +- go.sum | 4 +-- roomserver/internal/input_events.go | 1 + 6 files changed, 54 insertions(+), 5 deletions(-) diff --git a/are-we-synapse-yet.list b/are-we-synapse-yet.list index 71b05d1c4..cb90d6282 100644 --- a/are-we-synapse-yet.list +++ b/are-we-synapse-yet.list @@ -833,3 +833,4 @@ gst Guest user can call /events on another world_readable room (SYN-606) gst Real user can call /events on another world_readable room (SYN-606) gst Events come down the correct room pub Asking for a remote rooms list, but supplying the local server's name, returns the local rooms list +std Can send a to-device message to two users which both receive it using /sync \ No newline at end of file diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index 28e2b1514..43a16945d 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -30,6 +30,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" + "github.com/matrix-org/dendrite/clientapi/threepid" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/gomatrixserverlib" @@ -351,6 +352,50 @@ func createRoom( } } + // If this is a direct message then we should invite the participants. + for _, invitee := range r.Invite { + // Build the membership request. + body := threepid.MembershipRequest{ + UserID: invitee, + } + // Build the invite event. + inviteEvent, err := buildMembershipEvent( + req.Context(), body, accountDB, device, gomatrixserverlib.Invite, + roomID, true, cfg, evTime, rsAPI, asAPI, + ) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("buildMembershipEvent failed") + continue + } + // Build some stripped state for the invite. + candidates := append(gomatrixserverlib.UnwrapEventHeaders(builtEvents), *inviteEvent) + var strippedState []gomatrixserverlib.InviteV2StrippedState + for _, event := range candidates { + switch event.Type() { + // TODO: case gomatrixserverlib.MRoomEncryption: + // fallthrough + case gomatrixserverlib.MRoomMember: + fallthrough + case gomatrixserverlib.MRoomJoinRules: + strippedState = append( + strippedState, + gomatrixserverlib.NewInviteV2StrippedState(&event), + ) + } + } + // Send the invite event to the roomserver. + if err = producer.SendInvite( + req.Context(), + inviteEvent.Headered(roomVersion), + strippedState, // invite room state + cfg.Matrix.ServerName, // send as server + nil, // transaction ID + ); err != nil { + util.GetLogger(req.Context()).WithError(err).Error("producer.SendEvents failed") + return jsonerror.InternalServerError() + } + } + response := createRoomResponse{ RoomID: roomID, RoomAlias: roomAlias, diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index 9030f9f7e..0a56eec57 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -89,7 +89,8 @@ func SendMembership( } event, err := buildMembershipEvent( - req.Context(), body, accountDB, device, membership, roomID, cfg, evTime, rsAPI, asAPI, + req.Context(), body, accountDB, device, membership, + roomID, false, cfg, evTime, rsAPI, asAPI, ) if err == errMissingUserID { return util.JSONResponse{ @@ -151,7 +152,7 @@ func buildMembershipEvent( ctx context.Context, body threepid.MembershipRequest, accountDB accounts.Database, device *authtypes.Device, - membership, roomID string, + membership, roomID string, isDirect bool, cfg *config.Dendrite, evTime time.Time, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, ) (*gomatrixserverlib.Event, error) { @@ -182,6 +183,7 @@ func buildMembershipEvent( DisplayName: profile.DisplayName, AvatarURL: profile.AvatarURL, Reason: reason, + IsDirect: isDirect, } if err = builder.SetContent(content); err != nil { diff --git a/go.mod b/go.mod index fd1bb3d70..3da557325 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f github.com/matrix-org/go-sqlite3-js v0.0.0-20200325174927-327088cdef10 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200505092542-ef8abbde3f6b + github.com/matrix-org/gomatrixserverlib v0.0.0-20200507150553-025991c971ea github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible diff --git a/go.sum b/go.sum index 7ab035364..871648069 100644 --- a/go.sum +++ b/go.sum @@ -367,8 +367,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:kmRjpmFOenVpOaV/DRlo9p6z/IbOKlUC+hhKsAAh8Qg= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200505092542-ef8abbde3f6b h1:gxLun/noFJ7DplX7rqT8E4v4NkeDJ45tqW7LXC6k4C4= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200505092542-ef8abbde3f6b/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200507150553-025991c971ea h1:1qfbSjg3PwULY68AVRdZ3QIJoccNMbre0mSR7m7mqI4= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200507150553-025991c971ea/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y= diff --git a/roomserver/internal/input_events.go b/roomserver/internal/input_events.go index b17076efe..a0bfaa2e3 100644 --- a/roomserver/internal/input_events.go +++ b/roomserver/internal/input_events.go @@ -308,6 +308,7 @@ func buildInviteStrippedState( inviteState := []gomatrixserverlib.InviteV2StrippedState{ gomatrixserverlib.NewInviteV2StrippedState(&input.Event.Event), } + stateEvents = append(stateEvents, types.Event{Event: input.Event.Unwrap()}) for _, event := range stateEvents { inviteState = append(inviteState, gomatrixserverlib.NewInviteV2StrippedState(&event.Event)) } From 17d27331a3329c10f1aa38a409f451402fb770aa Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 7 May 2020 17:14:32 +0100 Subject: [PATCH 038/200] Fix 'input to Unique() must be sorted' panic --- federationsender/internal/perform.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/federationsender/internal/perform.go b/federationsender/internal/perform.go index 431b2a2d3..7c5fa73fc 100644 --- a/federationsender/internal/perform.go +++ b/federationsender/internal/perform.go @@ -47,7 +47,7 @@ func (r *FederationSenderInternalAPI) PerformJoin( } // Deduplicate the server names we were provided. - util.Unique(request.ServerNames) + util.SortAndUnique(request.ServerNames) // Try each server that we were provided until we land on one that // successfully completes the make-join send-join dance. @@ -159,7 +159,7 @@ func (r *FederationSenderInternalAPI) PerformLeave( response *api.PerformLeaveResponse, ) (err error) { // Deduplicate the server names we were provided. - util.Unique(request.ServerNames) + util.SortAndUnique(request.ServerNames) // Try each server that we were provided until we land on one that // successfully completes the make-leave send-leave dance. From d6e18a33ce1dd85c922165b89aaf680fd5de8535 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Fri, 8 May 2020 12:00:32 +0100 Subject: [PATCH 039/200] Add registration_disabled to dendrite-config.yaml (#1013) Missed this when first setting up my instance, and it feels important enough that it should be part of the sample config --- dendrite-config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dendrite-config.yaml b/dendrite-config.yaml index 8c8fba390..536b0f42b 100644 --- a/dendrite-config.yaml +++ b/dendrite-config.yaml @@ -25,6 +25,8 @@ matrix: # public_key: Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw # - key_id: ed25519:a_RXGa # public_key: l8Hft5qXKn1vfHrg3p4+W8gELQVo8N13JkluMfmn2sQ + # Disables new users from registering (except via shared secrets) + registration_disabled: false # The media repository config media: From 4fd97df2c5ac86bf908432d29259bdda5912b89b Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 11 May 2020 11:01:24 +0100 Subject: [PATCH 040/200] Don't return 500s from media API download requests --- mediaapi/routing/download.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mediaapi/routing/download.go b/mediaapi/routing/download.go index 8544bd64f..9feca90e9 100644 --- a/mediaapi/routing/download.go +++ b/mediaapi/routing/download.go @@ -118,7 +118,10 @@ func Download( ) if err != nil { // TODO: Handle the fact we might have started writing the response - dReq.jsonErrorResponse(w, util.ErrorResponse(err)) + dReq.jsonErrorResponse(w, util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound("Failed to download: " + err.Error()), + }) return } @@ -138,7 +141,7 @@ func (r *downloadRequest) jsonErrorResponse(w http.ResponseWriter, res util.JSON if err != nil { r.Logger.WithError(err).Error("Failed to marshal JSONResponse") // this should never fail to be marshalled so drop err to the floor - res = util.MessageResponse(http.StatusInternalServerError, "Internal Server Error") + res = util.MessageResponse(http.StatusNotFound, "Download request failed: "+err.Error()) resBytes, _ = json.Marshal(res.JSON) } From 6e643860b12dbcf910fb6c7ba6d27f220c9fa594 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 11 May 2020 11:50:29 +0100 Subject: [PATCH 041/200] Update sytest-whitelist --- sytest-whitelist | 1 + 1 file changed, 1 insertion(+) diff --git a/sytest-whitelist b/sytest-whitelist index c957021cb..6e1b4f631 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -264,3 +264,4 @@ User can invite local user to room with version 5 remote user can join room with version 5 User can invite remote user to room with version 5 Remote user can backfill in a room with version 5 +Alternative server names do not cause a routing loop From 615de25347bdaf5bfa1079c004757e5e1558d9cf Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 11 May 2020 16:02:23 +0100 Subject: [PATCH 042/200] Update gomatrixserverlib for more memory-efficient state res v2 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 3da557325..cb912be3b 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f github.com/matrix-org/go-sqlite3-js v0.0.0-20200325174927-327088cdef10 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200507150553-025991c971ea + github.com/matrix-org/gomatrixserverlib v0.0.0-20200511150133-f4a8869b5366 github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible diff --git a/go.sum b/go.sum index 871648069..a6f160052 100644 --- a/go.sum +++ b/go.sum @@ -367,8 +367,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:kmRjpmFOenVpOaV/DRlo9p6z/IbOKlUC+hhKsAAh8Qg= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200507150553-025991c971ea h1:1qfbSjg3PwULY68AVRdZ3QIJoccNMbre0mSR7m7mqI4= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200507150553-025991c971ea/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200511150133-f4a8869b5366 h1:LbtCldIuE/kEgGnzKm6xebnZDfDie1vQA0mDq7ReyM4= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200511150133-f4a8869b5366/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y= From 99e0a7dff27501df482ba929f53637f4a96c78d6 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 11 May 2020 16:43:50 +0100 Subject: [PATCH 043/200] Update gomatrixserverlib for even more memory-efficient state res v2 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index cb912be3b..8e6d63cd5 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f github.com/matrix-org/go-sqlite3-js v0.0.0-20200325174927-327088cdef10 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200511150133-f4a8869b5366 + github.com/matrix-org/gomatrixserverlib v0.0.0-20200511154227-5cc71d36632b github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible diff --git a/go.sum b/go.sum index a6f160052..53aefed37 100644 --- a/go.sum +++ b/go.sum @@ -367,8 +367,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:kmRjpmFOenVpOaV/DRlo9p6z/IbOKlUC+hhKsAAh8Qg= github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200511150133-f4a8869b5366 h1:LbtCldIuE/kEgGnzKm6xebnZDfDie1vQA0mDq7ReyM4= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200511150133-f4a8869b5366/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200511154227-5cc71d36632b h1:nAmSc1KvQOumoRTz/LD68KyrB6Q5/6q7CmQ5Bswc2nM= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200511154227-5cc71d36632b/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y= From 0c892d59fa5846097647d08244059de4f73e39a6 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 11 May 2020 18:21:25 +0100 Subject: [PATCH 044/200] Prevent panic in membership updater (#1021) --- roomserver/internal/input_membership.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/roomserver/internal/input_membership.go b/roomserver/internal/input_membership.go index cba75b4fc..19b7d8055 100644 --- a/roomserver/internal/input_membership.go +++ b/roomserver/internal/input_membership.go @@ -16,6 +16,7 @@ package internal import ( "context" + "errors" "fmt" "github.com/matrix-org/dendrite/roomserver/api" @@ -106,6 +107,13 @@ func updateMembership( return updates, nil } + if add == nil { + // This shouldn't happen. Returning an error here is better than panicking + // in the membership updater functions later on. + // TODO: Why does this happen to begin with? + return updates, errors.New("add should not be nil") + } + mu, err := updater.MembershipUpdater(targetUserNID) if err != nil { return nil, err From 32624697fd2d74d4a6e23549b647d323b210fb2a Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 11 May 2020 18:21:39 +0100 Subject: [PATCH 045/200] Add PPROFLISTEN (#1019) * Add PPROFLISTEN env var * Direct logging to more useful places * Space --- cmd/dendrite-room-server/main.go | 2 -- common/basecomponent/base.go | 3 +++ common/log.go | 12 ++++++++++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/cmd/dendrite-room-server/main.go b/cmd/dendrite-room-server/main.go index 41149ad90..172468448 100644 --- a/cmd/dendrite-room-server/main.go +++ b/cmd/dendrite-room-server/main.go @@ -15,8 +15,6 @@ package main import ( - _ "net/http/pprof" - "github.com/matrix-org/dendrite/common/basecomponent" "github.com/matrix-org/dendrite/common/keydb" "github.com/matrix-org/dendrite/roomserver" diff --git a/common/basecomponent/base.go b/common/basecomponent/base.go index a7e6736a6..cb04a308e 100644 --- a/common/basecomponent/base.go +++ b/common/basecomponent/base.go @@ -42,6 +42,8 @@ import ( federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/sirupsen/logrus" + + _ "net/http/pprof" ) // BaseDendrite is a base for creating new instances of dendrite. It parses @@ -71,6 +73,7 @@ const HTTPClientTimeout = time.Second * 30 func NewBaseDendrite(cfg *config.Dendrite, componentName string) *BaseDendrite { common.SetupStdLogging() common.SetupHookLogging(cfg.Logging, componentName) + common.SetupPprof() closer, err := cfg.SetupTracing("Dendrite" + componentName) if err != nil { diff --git a/common/log.go b/common/log.go index 11339ada4..60e969650 100644 --- a/common/log.go +++ b/common/log.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "io" + "net/http" "os" "path" "path/filepath" @@ -79,6 +80,17 @@ func callerPrettyfier(f *runtime.Frame) (string, string) { return funcname, filename } +// SetupPprof starts a pprof listener. We use the DefaultServeMux here because it is +// simplest, and it gives us the freedom to run pprof on a separate port. +func SetupPprof() { + if hostPort := os.Getenv("PPROFLISTEN"); hostPort != "" { + logrus.Warn("Starting pprof on ", hostPort) + go func() { + logrus.WithError(http.ListenAndServe(hostPort, nil)).Error("Failed to setup pprof listener") + }() + } +} + // SetupStdLogging configures the logging format to standard output. Typically, it is called when the config is not yet loaded. func SetupStdLogging() { logrus.SetReportCaller(true) From ce5dfbebf98a55c0efe4f86ba30956916bf0b3ad Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 12 May 2020 16:24:28 +0100 Subject: [PATCH 046/200] Implement /get_missing_events (#1022) * WIP get_missing_events work * More WIP get_missing_events work * First working /get_missing_events implementation Flakey currently due to racing between /sync and /send * Final tweaks * Remove log lines * Linting * go mod tidy * Clamp min depth to 0 * sort events by depth because sytest makes me sad Specifically I think it's https://github.com/matrix-org/sytest/blob/4172585c2521ec6d640b4b580080276da1ab5353/lib/SyTest/Federation/Client.pm#L265 to blame here. --- federationapi/routing/join.go | 19 ++ federationapi/routing/routing.go | 22 ++ federationapi/routing/send.go | 505 ++++++++++++++++++++++------ federationapi/routing/send_test.go | 318 ++++++++++++------ roomserver/internal/input_events.go | 2 +- sytest-whitelist | 7 + 6 files changed, 675 insertions(+), 198 deletions(-) diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go index ada89e4a6..3533bbba8 100644 --- a/federationapi/routing/join.go +++ b/federationapi/routing/join.go @@ -17,6 +17,7 @@ package routing import ( "fmt" "net/http" + "sort" "time" "github.com/matrix-org/dendrite/clientapi/jsonerror" @@ -275,6 +276,12 @@ func SendJoin( } } + // sort events deterministically by depth (lower is earlier) + // We also do this because sytest's basic federation server isn't good at using the correct + // state if these lists are randomised, resulting in flakey tests. :( + sort.Sort(eventsByDepth(stateAndAuthChainResponse.StateEvents)) + sort.Sort(eventsByDepth(stateAndAuthChainResponse.AuthChainEvents)) + // https://matrix.org/docs/spec/server_server/latest#put-matrix-federation-v1-send-join-roomid-eventid return util.JSONResponse{ Code: http.StatusOK, @@ -285,3 +292,15 @@ func SendJoin( }, } } + +type eventsByDepth []gomatrixserverlib.HeaderedEvent + +func (e eventsByDepth) Len() int { + return len(e) +} +func (e eventsByDepth) Swap(i, j int) { + e[i], e[j] = e[j], e[i] +} +func (e eventsByDepth) Less(i, j int) bool { + return e[i].Depth() < e[j].Depth() +} diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index a5b8ce24e..67f3a957c 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -226,6 +226,28 @@ func Setup( }, )).Methods(http.MethodGet) + v1fedmux.Handle("/send_join/{roomID}/{eventID}", common.MakeFedAPI( + "federation_send_join", cfg.Matrix.ServerName, keys, + func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { + vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) + if err != nil { + return util.ErrorResponse(err) + } + roomID := vars["roomID"] + eventID := vars["eventID"] + res := SendJoin( + httpReq, request, cfg, rsAPI, producer, keys, roomID, eventID, + ) + return util.JSONResponse{ + Headers: res.Headers, + Code: res.Code, + JSON: []interface{}{ + res.Code, res.JSON, + }, + } + }, + )).Methods(http.MethodPut) + v2fedmux.Handle("/send_join/{roomID}/{eventID}", common.MakeFedAPI( "federation_send_join", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index e6f91d94a..990220741 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -48,6 +48,8 @@ func Send( eduProducer: eduProducer, keys: keys, federation: federation, + haveEvents: make(map[string]*gomatrixserverlib.HeaderedEvent), + newEvents: make(map[string]bool), } var txnEvents struct { @@ -105,6 +107,11 @@ type txnReq struct { eduProducer *producers.EDUServerProducer keys gomatrixserverlib.JSONVerifier federation txnFederationClient + // local cache of events for auth checks, etc - this may include events + // which the roomserver is unaware of. + haveEvents map[string]*gomatrixserverlib.HeaderedEvent + // new events which the roomserver does not know about + newEvents map[string]bool } // A subset of FederationClient functionality that txn requires. Useful for testing. @@ -114,6 +121,8 @@ type txnFederationClient interface { ) LookupStateIDs(ctx context.Context, s gomatrixserverlib.ServerName, roomID string, eventID string) (res gomatrixserverlib.RespStateIDs, err error) GetEvent(ctx context.Context, s gomatrixserverlib.ServerName, eventID string) (res gomatrixserverlib.Transaction, err error) + LookupMissingEvents(ctx context.Context, s gomatrixserverlib.ServerName, roomID string, missing gomatrixserverlib.MissingEvents, + roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMissingEvents, err error) } func (t *txnReq) processTransaction() (*gomatrixserverlib.RespSend, error) { @@ -148,7 +157,7 @@ func (t *txnReq) processTransaction() (*gomatrixserverlib.RespSend, error) { // Process the events. for _, e := range pdus { - err := t.processEvent(e.Unwrap()) + err := t.processEvent(e.Unwrap(), true) if err != nil { // If the error is due to the event itself being bad then we skip // it and move onto the next event. We report an error so that the @@ -168,7 +177,9 @@ func (t *txnReq) processTransaction() (*gomatrixserverlib.RespSend, error) { switch err.(type) { case roomNotFoundError: case *gomatrixserverlib.NotAllowed: + case missingPrevEventsError: default: + util.GetLogger(t.context).Warnf("Processing %s failed: %s", e.EventID(), err) // Any other error should be the result of a temporary error in // our server so we should bail processing the transaction entirely. return nil, err @@ -197,12 +208,30 @@ type verifySigError struct { eventID string err error } +type missingPrevEventsError struct { + eventID string + err error +} func (e roomNotFoundError) Error() string { return fmt.Sprintf("room %q not found", e.roomID) } func (e unmarshalError) Error() string { return fmt.Sprintf("unable to parse event: %s", e.err) } func (e verifySigError) Error() string { return fmt.Sprintf("unable to verify signature of event %q: %s", e.eventID, e.err) } +func (e missingPrevEventsError) Error() string { + return fmt.Sprintf("unable to get prev_events for event %q: %s", e.eventID, e.err) +} + +func (t *txnReq) haveEventIDs() map[string]bool { + result := make(map[string]bool, len(t.haveEvents)) + for eventID := range t.haveEvents { + if t.newEvents[eventID] { + continue + } + result[eventID] = true + } + return result +} func (t *txnReq) processEDUs(edus []gomatrixserverlib.EDU) { for _, e := range edus { @@ -227,7 +256,7 @@ func (t *txnReq) processEDUs(edus []gomatrixserverlib.EDU) { } } -func (t *txnReq) processEvent(e gomatrixserverlib.Event) error { +func (t *txnReq) processEvent(e gomatrixserverlib.Event, isInboundTxn bool) error { prevEventIDs := e.PrevEventIDs() // Fetch the state needed to authenticate the event. @@ -253,21 +282,14 @@ func (t *txnReq) processEvent(e gomatrixserverlib.Event) error { } if !stateResp.PrevEventsExist { - return t.processEventWithMissingState(e, stateResp.RoomVersion) + return t.processEventWithMissingState(e, stateResp.RoomVersion, isInboundTxn) } // Check that the event is allowed by the state at the event. - var events []gomatrixserverlib.Event - for _, headeredEvent := range stateResp.StateEvents { - events = append(events, headeredEvent.Unwrap()) - } - if err := checkAllowedByState(e, events); err != nil { + if err := checkAllowedByState(e, gomatrixserverlib.UnwrapEventHeaders(stateResp.StateEvents)); err != nil { return err } - // TODO: Check that the roomserver has a copy of all of the auth_events. - // TODO: Check that the event is allowed by its auth_events. - // pass the event to the roomserver _, err := t.producer.SendEvents( t.context, @@ -291,7 +313,7 @@ func checkAllowedByState(e gomatrixserverlib.Event, stateEvents []gomatrixserver return gomatrixserverlib.Allowed(e, &authUsingState) } -func (t *txnReq) processEventWithMissingState(e gomatrixserverlib.Event, roomVersion gomatrixserverlib.RoomVersion) error { +func (t *txnReq) processEventWithMissingState(e gomatrixserverlib.Event, roomVersion gomatrixserverlib.RoomVersion, isInboundTxn bool) error { // We are missing the previous events for this events. // This means that there is a gap in our view of the history of the // room. There two ways that we can handle such a gap: @@ -306,49 +328,315 @@ func (t *txnReq) processEventWithMissingState(e gomatrixserverlib.Event, roomVer // event ids and then use /event to fetch the individual events. // However not all version of synapse support /state_ids so you may // need to fallback to /state. - // TODO: Attempt to fill in the gap using /get_missing_events - // Attempt to fetch the missing state using /state_ids and /events - respState, haveEventIDs, err := t.lookupMissingStateViaStateIDs(e, roomVersion) + // Attempt to fill in the gap using /get_missing_events + // This will either: + // - fill in the gap completely then process event `e` returning no backwards extremity + // - fail to fill in the gap and tell us to terminate the transaction err=not nil + // - fail to fill in the gap and tell us to fetch state at the new backwards extremity, and to not terminate the transaction + backwardsExtremity, err := t.getMissingEvents(e, roomVersion, isInboundTxn) if err != nil { - // Fallback to /state - util.GetLogger(t.context).WithError(err).Warn("processEventWithMissingState failed to /state_ids, falling back to /state") - respState, err = t.lookupMissingStateViaState(e, roomVersion) - if err != nil { - return err - } + return err + } + if backwardsExtremity == nil { + // we filled in the gap! + return nil } - // Check that the event is allowed by the state. -retryAllowedState: - if err := checkAllowedByState(e, respState.StateEvents); err != nil { - switch missing := err.(type) { - case gomatrixserverlib.MissingAuthEventError: - // An auth event was missing so let's look up that event over federation - for _, s := range respState.StateEvents { - if s.EventID() != missing.AuthEventID { - continue - } - err = t.processEventWithMissingState(s, roomVersion) - // If there was no error retrieving the event from federation then - // we assume that it succeeded, so retry the original state check - if err == nil { - goto retryAllowedState - } - } - default: + // at this point we know we're going to have a gap: we need to work out the room state at the new backwards extremity. + // security: we have to do state resolution on the new backwards extremity (TODO: WHY) + // Therefore, we cannot just query /state_ids with this event to get the state before. Instead, we need to query + // the state AFTER all the prev_events for this event, then mix in our current room state and apply state resolution + // to that to get the state before the event. + var states []*gomatrixserverlib.RespState + needed := gomatrixserverlib.StateNeededForAuth([]gomatrixserverlib.Event{*backwardsExtremity}).Tuples() + for _, prevEventID := range backwardsExtremity.PrevEventIDs() { + var prevState *gomatrixserverlib.RespState + prevState, err = t.lookupStateAfterEvent(roomVersion, backwardsExtremity.RoomID(), prevEventID, needed) + if err != nil { + util.GetLogger(t.context).WithError(err).Errorf("Failed to lookup state after prev_event: %s", prevEventID) + return err } + states = append(states, prevState) + } + // mix in the current room state + currState, err := t.lookupCurrentState(backwardsExtremity) + if err != nil { + util.GetLogger(t.context).WithError(err).Errorf("Failed to lookup current room state") + return err + } + states = append(states, currState) + resolvedState, err := t.resolveStatesAndCheck(roomVersion, states, backwardsExtremity) + if err != nil { + util.GetLogger(t.context).WithError(err).Errorf("Failed to resolve state conflicts for event %s", backwardsExtremity.EventID()) return err } // pass the event along with the state to the roomserver using a background context so we don't // needlessly expire - return t.producer.SendEventWithState(context.Background(), respState, e.Headered(roomVersion), haveEventIDs) + return t.producer.SendEventWithState(context.Background(), resolvedState, e.Headered(roomVersion), t.haveEventIDs()) } -func (t *txnReq) lookupMissingStateViaState(e gomatrixserverlib.Event, roomVersion gomatrixserverlib.RoomVersion) ( +// lookupStateAfterEvent returns the room state after `eventID`, which is the state before eventID with the state of `eventID` (if it's a state event) +// added into the mix. +func (t *txnReq) lookupStateAfterEvent(roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string, needed []gomatrixserverlib.StateKeyTuple) (*gomatrixserverlib.RespState, error) { + // try doing all this locally before we resort to querying federation + respState := t.lookupStateAfterEventLocally(roomID, eventID, needed) + if respState != nil { + return respState, nil + } + + respState, err := t.lookupStateBeforeEvent(roomVersion, roomID, eventID) + if err != nil { + return nil, err + } + + // fetch the event we're missing and add it to the pile + h, err := t.lookupEvent(roomVersion, eventID, false) + if err != nil { + return nil, err + } + t.haveEvents[h.EventID()] = h + if h.StateKey() != nil { + addedToState := false + for i := range respState.StateEvents { + se := respState.StateEvents[i] + if se.Type() == h.Type() && se.StateKeyEquals(*h.StateKey()) { + respState.StateEvents[i] = h.Unwrap() + addedToState = true + break + } + } + if !addedToState { + respState.StateEvents = append(respState.StateEvents, h.Unwrap()) + } + } + + return respState, nil +} + +func (t *txnReq) lookupStateAfterEventLocally(roomID, eventID string, needed []gomatrixserverlib.StateKeyTuple) *gomatrixserverlib.RespState { + var res api.QueryStateAfterEventsResponse + err := t.rsAPI.QueryStateAfterEvents(t.context, &api.QueryStateAfterEventsRequest{ + RoomID: roomID, + PrevEventIDs: []string{eventID}, + StateToFetch: needed, + }, &res) + if err != nil || !res.PrevEventsExist { + util.GetLogger(t.context).WithError(err).Warnf("failed to query state after %s locally", eventID) + return nil + } + for i, ev := range res.StateEvents { + t.haveEvents[ev.EventID()] = &res.StateEvents[i] + } + var authEvents []gomatrixserverlib.Event + missingAuthEvents := make(map[string]bool) + for _, ev := range res.StateEvents { + for _, ae := range ev.AuthEventIDs() { + aev, ok := t.haveEvents[ae] + if ok { + authEvents = append(authEvents, aev.Unwrap()) + } else { + missingAuthEvents[ae] = true + } + } + } + // QueryStateAfterEvents does not return the auth events, so fetch them now. We know the roomserver has them else it wouldn't + // have stored the event. + var missingEventList []string + for evID := range missingAuthEvents { + missingEventList = append(missingEventList, evID) + } + queryReq := api.QueryEventsByIDRequest{ + EventIDs: missingEventList, + } + util.GetLogger(t.context).Infof("Fetching missing auth events: %v", missingEventList) + var queryRes api.QueryEventsByIDResponse + if err = t.rsAPI.QueryEventsByID(t.context, &queryReq, &queryRes); err != nil { + return nil + } + for i := range queryRes.Events { + evID := queryRes.Events[i].EventID() + t.haveEvents[evID] = &queryRes.Events[i] + authEvents = append(authEvents, queryRes.Events[i].Unwrap()) + } + + evs := gomatrixserverlib.UnwrapEventHeaders(res.StateEvents) + return &gomatrixserverlib.RespState{ + StateEvents: evs, + AuthEvents: authEvents, + } +} + +func (t *txnReq) lookupCurrentState(newEvent *gomatrixserverlib.Event) (*gomatrixserverlib.RespState, error) { + // Ask the roomserver for information about this room + queryReq := api.QueryLatestEventsAndStateRequest{ + RoomID: newEvent.RoomID(), + StateToFetch: gomatrixserverlib.StateNeededForAuth([]gomatrixserverlib.Event{*newEvent}).Tuples(), + } + var queryRes api.QueryLatestEventsAndStateResponse + if err := t.rsAPI.QueryLatestEventsAndState(t.context, &queryReq, &queryRes); err != nil { + return nil, fmt.Errorf("lookupCurrentState rsAPI.QueryLatestEventsAndState: %w", err) + } + evs := gomatrixserverlib.UnwrapEventHeaders(queryRes.StateEvents) + return &gomatrixserverlib.RespState{ + StateEvents: evs, + AuthEvents: evs, + }, nil +} + +// lookuptStateBeforeEvent returns the room state before the event e, which is just /state_ids and/or /state depending on what +// the server supports. +func (t *txnReq) lookupStateBeforeEvent(roomVersion gomatrixserverlib.RoomVersion, roomID, eventID string) ( respState *gomatrixserverlib.RespState, err error) { - state, err := t.federation.LookupState(t.context, t.Origin, e.RoomID(), e.EventID(), roomVersion) + + util.GetLogger(t.context).Infof("lookupStateBeforeEvent %s", eventID) + + // Attempt to fetch the missing state using /state_ids and /events + respState, err = t.lookupMissingStateViaStateIDs(roomID, eventID, roomVersion) + if err != nil { + // Fallback to /state + util.GetLogger(t.context).WithError(err).Warn("lookupStateBeforeEvent failed to /state_ids, falling back to /state") + respState, err = t.lookupMissingStateViaState(roomID, eventID, roomVersion) + } + return +} + +func (t *txnReq) resolveStatesAndCheck(roomVersion gomatrixserverlib.RoomVersion, states []*gomatrixserverlib.RespState, backwardsExtremity *gomatrixserverlib.Event) (*gomatrixserverlib.RespState, error) { + var authEventList []gomatrixserverlib.Event + var stateEventList []gomatrixserverlib.Event + for _, state := range states { + authEventList = append(authEventList, state.AuthEvents...) + stateEventList = append(stateEventList, state.StateEvents...) + } + resolvedStateEvents, err := gomatrixserverlib.ResolveConflicts(roomVersion, stateEventList, authEventList) + if err != nil { + return nil, err + } + // apply the current event +retryAllowedState: + if err = checkAllowedByState(*backwardsExtremity, resolvedStateEvents); err != nil { + switch missing := err.(type) { + case gomatrixserverlib.MissingAuthEventError: + h, err2 := t.lookupEvent(roomVersion, missing.AuthEventID, true) + if err2 != nil { + return nil, fmt.Errorf("missing auth event %s and failed to look it up: %w", missing.AuthEventID, err2) + } + util.GetLogger(t.context).Infof("fetched event %s", missing.AuthEventID) + resolvedStateEvents = append(resolvedStateEvents, h.Unwrap()) + goto retryAllowedState + default: + } + return nil, err + } + return &gomatrixserverlib.RespState{ + AuthEvents: authEventList, + StateEvents: resolvedStateEvents, + }, nil +} + +// getMissingEvents returns a nil backwardsExtremity if missing events were fetched and handled, else returns the new backwards extremity which we should +// begin from. Returns an error only if we should terminate the transaction which initiated /get_missing_events +// This function recursively calls txnReq.processEvent with the missing events, which will be processed before this function returns. +// This means that we may recursively call this function, as we spider back up prev_events to the min depth. +func (t *txnReq) getMissingEvents(e gomatrixserverlib.Event, roomVersion gomatrixserverlib.RoomVersion, isInboundTxn bool) (backwardsExtremity *gomatrixserverlib.Event, err error) { + if !isInboundTxn { + // we've recursed here, so just take a state snapshot please! + return &e, nil + } + logger := util.GetLogger(t.context).WithField("event_id", e.EventID()).WithField("room_id", e.RoomID()) + needed := gomatrixserverlib.StateNeededForAuth([]gomatrixserverlib.Event{e}) + // query latest events (our trusted forward extremities) + req := api.QueryLatestEventsAndStateRequest{ + RoomID: e.RoomID(), + StateToFetch: needed.Tuples(), + } + var res api.QueryLatestEventsAndStateResponse + if err = t.rsAPI.QueryLatestEventsAndState(t.context, &req, &res); err != nil { + logger.WithError(err).Warn("Failed to query latest events") + return &e, nil + } + latestEvents := make([]string, len(res.LatestEvents)) + for i := range res.LatestEvents { + latestEvents[i] = res.LatestEvents[i].EventID + } + // this server just sent us an event for which we do not know its prev_events - ask that server for those prev_events. + minDepth := int(res.Depth) - 20 + if minDepth < 0 { + minDepth = 0 + } + missingResp, err := t.federation.LookupMissingEvents(t.context, t.Origin, e.RoomID(), gomatrixserverlib.MissingEvents{ + Limit: 20, + // synapse uses the min depth they've ever seen in that room + MinDepth: minDepth, + // The latest event IDs that the sender already has. These are skipped when retrieving the previous events of latest_events. + EarliestEvents: latestEvents, + // The event IDs to retrieve the previous events for. + LatestEvents: []string{e.EventID()}, + }, roomVersion) + + // security: how we handle failures depends on whether or not this event will become the new forward extremity for the room. + // There's 2 scenarios to consider: + // - Case A: We got pushed an event and are now fetching missing prev_events. (isInboundTxn=true) + // - Case B: We are fetching missing prev_events already and now fetching some more (isInboundTxn=false) + // In Case B, we know for sure that the event we are currently processing will not become the new forward extremity for the room, + // as it was called in response to an inbound txn which had it as a prev_event. + // In Case A, the event is a forward extremity, and could eventually become the _only_ forward extremity in the room. This is bad + // because it means we would trust the state at that event to be the state for the entire room, and allows rooms to be hijacked. + // https://github.com/matrix-org/synapse/pull/3456 + // https://github.com/matrix-org/synapse/blob/229eb81498b0fe1da81e9b5b333a0285acde9446/synapse/handlers/federation.py#L335 + // For now, we do not allow Case B, so reject the event. + if err != nil { + logger.WithError(err).Errorf( + "%s pushed us an event but couldn't give us details about prev_events via /get_missing_events - dropping this event until it can", + t.Origin, + ) + return nil, missingPrevEventsError{ + eventID: e.EventID(), + err: err, + } + } + logger.Infof("get_missing_events returned %d events", len(missingResp.Events)) + + // topologically sort and sanity check that we are making forward progress + newEvents := gomatrixserverlib.ReverseTopologicalOrdering(missingResp.Events, gomatrixserverlib.TopologicalOrderByPrevEvents) + shouldHaveSomeEventIDs := e.PrevEventIDs() + hasPrevEvent := false +Event: + for _, pe := range shouldHaveSomeEventIDs { + for _, ev := range newEvents { + if ev.EventID() == pe { + hasPrevEvent = true + break Event + } + } + } + if !hasPrevEvent { + err = fmt.Errorf("called /get_missing_events but server %s didn't return any prev_events with IDs %v", t.Origin, shouldHaveSomeEventIDs) + logger.WithError(err).Errorf( + "%s pushed us an event but couldn't give us details about prev_events via /get_missing_events - dropping this event until it can", + t.Origin, + ) + return nil, missingPrevEventsError{ + eventID: e.EventID(), + err: err, + } + } + // process the missing events then the event which started this whole thing + for _, ev := range append(newEvents, e) { + err := t.processEvent(ev, false) + if err != nil { + return nil, err + } + } + + // we processed everything! + return nil, nil +} + +func (t *txnReq) lookupMissingStateViaState(roomID, eventID string, roomVersion gomatrixserverlib.RoomVersion) ( + respState *gomatrixserverlib.RespState, err error) { + state, err := t.federation.LookupState(t.context, t.Origin, roomID, eventID, roomVersion) if err != nil { return nil, err } @@ -359,78 +647,64 @@ func (t *txnReq) lookupMissingStateViaState(e gomatrixserverlib.Event, roomVersi return &state, nil } -func (t *txnReq) lookupMissingStateViaStateIDs(e gomatrixserverlib.Event, roomVersion gomatrixserverlib.RoomVersion) ( - *gomatrixserverlib.RespState, map[string]bool, error) { - +func (t *txnReq) lookupMissingStateViaStateIDs(roomID, eventID string, roomVersion gomatrixserverlib.RoomVersion) ( + *gomatrixserverlib.RespState, error) { + util.GetLogger(t.context).Infof("lookupMissingStateViaStateIDs %s", eventID) // fetch the state event IDs at the time of the event - stateIDs, err := t.federation.LookupStateIDs(t.context, t.Origin, e.RoomID(), e.EventID()) + stateIDs, err := t.federation.LookupStateIDs(t.context, t.Origin, roomID, eventID) if err != nil { - return nil, nil, err + return nil, err } - - // fetch as many as we can from the roomserver, do them as 2 calls rather than - // 1 to try to reduce the number of parameters in the bulk query this will use - haveEventMap := make(map[string]*gomatrixserverlib.HeaderedEvent, len(stateIDs.StateEventIDs)) - haveEventIDs := make(map[string]bool) - for _, eventList := range [][]string{stateIDs.StateEventIDs, stateIDs.AuthEventIDs} { - queryReq := api.QueryEventsByIDRequest{ - EventIDs: eventList, - } - var queryRes api.QueryEventsByIDResponse - if err = t.rsAPI.QueryEventsByID(t.context, &queryReq, &queryRes); err != nil { - return nil, nil, err - } - // allow indexing of current state by event ID - for i := range queryRes.Events { - haveEventMap[queryRes.Events[i].EventID()] = &queryRes.Events[i] - haveEventIDs[queryRes.Events[i].EventID()] = true - } - } - // work out which auth/state IDs are missing wantIDs := append(stateIDs.StateEventIDs, stateIDs.AuthEventIDs...) missing := make(map[string]bool) + var missingEventList []string for _, sid := range wantIDs { - if _, ok := haveEventMap[sid]; !ok { - missing[sid] = true + if _, ok := t.haveEvents[sid]; !ok { + if !missing[sid] { + missing[sid] = true + missingEventList = append(missingEventList, sid) + } } } + + // fetch as many as we can from the roomserver + queryReq := api.QueryEventsByIDRequest{ + EventIDs: missingEventList, + } + var queryRes api.QueryEventsByIDResponse + if err = t.rsAPI.QueryEventsByID(t.context, &queryReq, &queryRes); err != nil { + return nil, err + } + for i := range queryRes.Events { + evID := queryRes.Events[i].EventID() + t.haveEvents[evID] = &queryRes.Events[i] + if missing[evID] { + delete(missing, evID) + } + } + util.GetLogger(t.context).WithFields(logrus.Fields{ "missing": len(missing), - "event_id": e.EventID(), - "room_id": e.RoomID(), - "already_have": len(haveEventMap), + "event_id": eventID, + "room_id": roomID, "total_state": len(stateIDs.StateEventIDs), "total_auth_events": len(stateIDs.AuthEventIDs), }).Info("Fetching missing state at event") for missingEventID := range missing { - var txn gomatrixserverlib.Transaction - txn, err = t.federation.GetEvent(t.context, t.Origin, missingEventID) + var h *gomatrixserverlib.HeaderedEvent + h, err = t.lookupEvent(roomVersion, missingEventID, false) if err != nil { - util.GetLogger(t.context).WithError(err).WithField("event_id", missingEventID).Warn("failed to get missing /event for event ID") - return nil, nil, err - } - for _, pdu := range txn.PDUs { - var event gomatrixserverlib.Event - event, err = gomatrixserverlib.NewEventFromUntrustedJSON(pdu, roomVersion) - if err != nil { - util.GetLogger(t.context).WithError(err).Warnf("Transaction: Failed to parse event JSON of event %q", event.EventID()) - return nil, nil, unmarshalError{err} - } - if err = gomatrixserverlib.VerifyAllEventSignatures(t.context, []gomatrixserverlib.Event{event}, t.keys); err != nil { - util.GetLogger(t.context).WithError(err).Warnf("Transaction: Couldn't validate signature of event %q", event.EventID()) - return nil, nil, verifySigError{event.EventID(), err} - } - h := event.Headered(roomVersion) - haveEventMap[event.EventID()] = &h + return nil, err } + t.haveEvents[h.EventID()] = h } - resp, err := t.createRespStateFromStateIDs(stateIDs, haveEventMap) - return resp, haveEventIDs, err + resp, err := t.createRespStateFromStateIDs(stateIDs) + return resp, err } -func (t *txnReq) createRespStateFromStateIDs(stateIDs gomatrixserverlib.RespStateIDs, haveEventMap map[string]*gomatrixserverlib.HeaderedEvent) ( +func (t *txnReq) createRespStateFromStateIDs(stateIDs gomatrixserverlib.RespStateIDs) ( *gomatrixserverlib.RespState, error) { // create a RespState response using the response to /state_ids as a guide respState := gomatrixserverlib.RespState{ @@ -439,22 +713,55 @@ func (t *txnReq) createRespStateFromStateIDs(stateIDs gomatrixserverlib.RespStat } for i := range stateIDs.StateEventIDs { - ev, ok := haveEventMap[stateIDs.StateEventIDs[i]] + ev, ok := t.haveEvents[stateIDs.StateEventIDs[i]] if !ok { return nil, fmt.Errorf("missing state event %s", stateIDs.StateEventIDs[i]) } respState.StateEvents[i] = ev.Unwrap() } for i := range stateIDs.AuthEventIDs { - ev, ok := haveEventMap[stateIDs.AuthEventIDs[i]] + ev, ok := t.haveEvents[stateIDs.AuthEventIDs[i]] if !ok { return nil, fmt.Errorf("missing auth event %s", stateIDs.AuthEventIDs[i]) } respState.AuthEvents[i] = ev.Unwrap() } - // Check that the returned state is valid. - if err := respState.Check(t.context, t.keys); err != nil { - return nil, err - } + // We purposefully do not do auth checks on the returned events, as they will still + // be processed in the exact same way, just as a 'rejected' event + // TODO: Add a field to HeaderedEvent to indicate if the event is rejected. return &respState, nil } + +func (t *txnReq) lookupEvent(roomVersion gomatrixserverlib.RoomVersion, missingEventID string, localFirst bool) (*gomatrixserverlib.HeaderedEvent, error) { + if localFirst { + // fetch from the roomserver + queryReq := api.QueryEventsByIDRequest{ + EventIDs: []string{missingEventID}, + } + var queryRes api.QueryEventsByIDResponse + if err := t.rsAPI.QueryEventsByID(t.context, &queryReq, &queryRes); err != nil { + util.GetLogger(t.context).Warnf("Failed to query roomserver for missing event %s: %s - falling back to remote", missingEventID, err) + } else if len(queryRes.Events) == 1 { + return &queryRes.Events[0], nil + } + } + txn, err := t.federation.GetEvent(t.context, t.Origin, missingEventID) + if err != nil || len(txn.PDUs) == 0 { + util.GetLogger(t.context).WithError(err).WithField("event_id", missingEventID).Warn("failed to get missing /event for event ID") + return nil, err + } + pdu := txn.PDUs[0] + var event gomatrixserverlib.Event + event, err = gomatrixserverlib.NewEventFromUntrustedJSON(pdu, roomVersion) + if err != nil { + util.GetLogger(t.context).WithError(err).Warnf("Transaction: Failed to parse event JSON of event %q", event.EventID()) + return nil, unmarshalError{err} + } + if err = gomatrixserverlib.VerifyAllEventSignatures(t.context, []gomatrixserverlib.Event{event}, t.keys); err != nil { + util.GetLogger(t.context).WithError(err).Warnf("Transaction: Couldn't validate signature of event %q", event.EventID()) + return nil, verifySigError{event.EventID(), err} + } + h := event.Headered(roomVersion) + t.newEvents[h.EventID()] = true + return &h, nil +} diff --git a/federationapi/routing/send_test.go b/federationapi/routing/send_test.go index 89d28aa1c..cb8aec6f5 100644 --- a/federationapi/routing/send_test.go +++ b/federationapi/routing/send_test.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "reflect" - "sort" "testing" "time" @@ -79,9 +78,10 @@ func (p *testEDUProducer) InputTypingEvent( } type testRoomserverAPI struct { - inputRoomEvents []api.InputRoomEvent - queryStateAfterEvents func(*api.QueryStateAfterEventsRequest) api.QueryStateAfterEventsResponse - queryEventsByID func(req *api.QueryEventsByIDRequest) api.QueryEventsByIDResponse + inputRoomEvents []api.InputRoomEvent + queryStateAfterEvents func(*api.QueryStateAfterEventsRequest) api.QueryStateAfterEventsResponse + queryEventsByID func(req *api.QueryEventsByIDRequest) api.QueryEventsByIDResponse + queryLatestEventsAndState func(*api.QueryLatestEventsAndStateRequest) api.QueryLatestEventsAndStateResponse } func (t *testRoomserverAPI) SetFederationSenderAPI(fsAPI fsAPI.FederationSenderInternalAPI) {} @@ -92,6 +92,9 @@ func (t *testRoomserverAPI) InputRoomEvents( response *api.InputRoomEventsResponse, ) error { t.inputRoomEvents = append(t.inputRoomEvents, request.InputRoomEvents...) + for _, ire := range request.InputRoomEvents { + fmt.Println("InputRoomEvents: ", ire.Event.EventID()) + } return nil } @@ -117,6 +120,13 @@ func (t *testRoomserverAPI) QueryLatestEventsAndState( request *api.QueryLatestEventsAndStateRequest, response *api.QueryLatestEventsAndStateResponse, ) error { + r := t.queryLatestEventsAndState(request) + response.QueryLatestEventsAndStateRequest = *request + response.RoomExists = r.RoomExists + response.RoomVersion = testRoomVersion + response.LatestEvents = r.LatestEvents + response.StateEvents = r.StateEvents + response.Depth = r.Depth return nil } @@ -152,7 +162,7 @@ func (t *testRoomserverAPI) QueryMembershipForUser( request *api.QueryMembershipForUserRequest, response *api.QueryMembershipForUserResponse, ) error { - return nil + return fmt.Errorf("not implemented") } // Query a list of membership events for a room @@ -161,7 +171,7 @@ func (t *testRoomserverAPI) QueryMembershipsForRoom( request *api.QueryMembershipsForRoomRequest, response *api.QueryMembershipsForRoomResponse, ) error { - return nil + return fmt.Errorf("not implemented") } // Query a list of invite event senders for a user in a room. @@ -170,7 +180,7 @@ func (t *testRoomserverAPI) QueryInvitesForUser( request *api.QueryInvitesForUserRequest, response *api.QueryInvitesForUserResponse, ) error { - return nil + return fmt.Errorf("not implemented") } // Query whether a server is allowed to see an event @@ -179,7 +189,7 @@ func (t *testRoomserverAPI) QueryServerAllowedToSeeEvent( request *api.QueryServerAllowedToSeeEventRequest, response *api.QueryServerAllowedToSeeEventResponse, ) error { - return nil + return fmt.Errorf("not implemented") } // Query missing events for a room from roomserver @@ -188,7 +198,7 @@ func (t *testRoomserverAPI) QueryMissingEvents( request *api.QueryMissingEventsRequest, response *api.QueryMissingEventsResponse, ) error { - return nil + return fmt.Errorf("not implemented") } // Query to get state and auth chain for a (potentially hypothetical) event. @@ -199,7 +209,7 @@ func (t *testRoomserverAPI) QueryStateAndAuthChain( request *api.QueryStateAndAuthChainRequest, response *api.QueryStateAndAuthChainResponse, ) error { - return nil + return fmt.Errorf("not implemented") } // Query a given amount (or less) of events prior to a given set of events. @@ -208,7 +218,7 @@ func (t *testRoomserverAPI) QueryBackfill( request *api.QueryBackfillRequest, response *api.QueryBackfillResponse, ) error { - return nil + return fmt.Errorf("not implemented") } // Asks for the default room version as preferred by the server. @@ -217,7 +227,7 @@ func (t *testRoomserverAPI) QueryRoomVersionCapabilities( request *api.QueryRoomVersionCapabilitiesRequest, response *api.QueryRoomVersionCapabilitiesResponse, ) error { - return nil + return fmt.Errorf("not implemented") } // Asks for the room version for a given room. @@ -236,7 +246,7 @@ func (t *testRoomserverAPI) SetRoomAlias( req *api.SetRoomAliasRequest, response *api.SetRoomAliasResponse, ) error { - return nil + return fmt.Errorf("not implemented") } // Get the room ID for an alias @@ -245,7 +255,7 @@ func (t *testRoomserverAPI) GetRoomIDForAlias( req *api.GetRoomIDForAliasRequest, response *api.GetRoomIDForAliasResponse, ) error { - return nil + return fmt.Errorf("not implemented") } // Get all known aliases for a room ID @@ -254,7 +264,7 @@ func (t *testRoomserverAPI) GetAliasesForRoomID( req *api.GetAliasesForRoomIDRequest, response *api.GetAliasesForRoomIDResponse, ) error { - return nil + return fmt.Errorf("not implemented") } // Get the user ID of the creator of an alias @@ -263,7 +273,7 @@ func (t *testRoomserverAPI) GetCreatorIDForAlias( req *api.GetCreatorIDForAliasRequest, response *api.GetCreatorIDForAliasResponse, ) error { - return nil + return fmt.Errorf("not implemented") } // Remove a room alias @@ -272,18 +282,20 @@ func (t *testRoomserverAPI) RemoveRoomAlias( req *api.RemoveRoomAliasRequest, response *api.RemoveRoomAliasResponse, ) error { - return nil + return fmt.Errorf("not implemented") } type txnFedClient struct { - state map[string]gomatrixserverlib.RespState // event_id to response - stateIDs map[string]gomatrixserverlib.RespStateIDs // event_id to response - getEvent map[string]gomatrixserverlib.Transaction // event_id to response + state map[string]gomatrixserverlib.RespState // event_id to response + stateIDs map[string]gomatrixserverlib.RespStateIDs // event_id to response + getEvent map[string]gomatrixserverlib.Transaction // event_id to response + getMissingEvents func(gomatrixserverlib.MissingEvents) (res gomatrixserverlib.RespMissingEvents, err error) } func (c *txnFedClient) LookupState(ctx context.Context, s gomatrixserverlib.ServerName, roomID string, eventID string, roomVersion gomatrixserverlib.RoomVersion) ( res gomatrixserverlib.RespState, err error, ) { + fmt.Println("testFederationClient.LookupState", eventID) r, ok := c.state[eventID] if !ok { err = fmt.Errorf("txnFedClient: no /state for event %s", eventID) @@ -293,6 +305,7 @@ func (c *txnFedClient) LookupState(ctx context.Context, s gomatrixserverlib.Serv return } func (c *txnFedClient) LookupStateIDs(ctx context.Context, s gomatrixserverlib.ServerName, roomID string, eventID string) (res gomatrixserverlib.RespStateIDs, err error) { + fmt.Println("testFederationClient.LookupStateIDs", eventID) r, ok := c.stateIDs[eventID] if !ok { err = fmt.Errorf("txnFedClient: no /state_ids for event %s", eventID) @@ -302,6 +315,7 @@ func (c *txnFedClient) LookupStateIDs(ctx context.Context, s gomatrixserverlib.S return } func (c *txnFedClient) GetEvent(ctx context.Context, s gomatrixserverlib.ServerName, eventID string) (res gomatrixserverlib.Transaction, err error) { + fmt.Println("testFederationClient.GetEvent", eventID) r, ok := c.getEvent[eventID] if !ok { err = fmt.Errorf("txnFedClient: no /event for event ID %s", eventID) @@ -310,6 +324,10 @@ func (c *txnFedClient) GetEvent(ctx context.Context, s gomatrixserverlib.ServerN res = r return } +func (c *txnFedClient) LookupMissingEvents(ctx context.Context, s gomatrixserverlib.ServerName, roomID string, missing gomatrixserverlib.MissingEvents, + roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMissingEvents, err error) { + return c.getMissingEvents(missing) +} func mustCreateTransaction(rsAPI api.RoomserverInternalAPI, fedClient txnFederationClient, pdus []json.RawMessage) *txnReq { t := &txnReq{ @@ -319,6 +337,8 @@ func mustCreateTransaction(rsAPI api.RoomserverInternalAPI, fedClient txnFederat eduProducer: producers.NewEDUServerProducer(&testEDUProducer{}), keys: &testNopJSONVerifier{}, federation: fedClient, + haveEvents: make(map[string]*gomatrixserverlib.HeaderedEvent), + newEvents: make(map[string]bool), } t.PDUs = pdus t.Origin = testOrigin @@ -368,6 +388,9 @@ NextTuple: } func assertInputRoomEvents(t *testing.T, got []api.InputRoomEvent, want []gomatrixserverlib.HeaderedEvent) { + for _, g := range got { + fmt.Println("GOT ", g.Event.EventID()) + } if len(got) != len(want) { t.Errorf("wrong number of InputRoomEvents: got %d want %d", len(got), len(want)) return @@ -424,32 +447,167 @@ func TestTransactionFailAuthChecks(t *testing.T) { assertInputRoomEvents(t, rsAPI.inputRoomEvents, nil) // expect no messages to be sent to the roomserver } -// The purpose of this test is to check that when there are missing prev_events that state is fetched via /state_ids -// and /event and not /state. It works by setting PrevEventsExist=false in the roomserver query response, resulting in -// a call to /state_ids which returns the whole room state. It should attempt to fetch as many of these events from the -// roomserver FIRST, resulting in a call to QueryEventsByID. However, this will be missing the m.room.power_levels event which -// should then be requested via /event. The net result is that the transaction should succeed and there should be 2 -// new events, first the m.room.power_levels event we were missing, then the transaction PDU. +// The purpose of this test is to make sure that when an event is received for which we do not know the prev_events, +// we request them from /get_missing_events. It works by setting PrevEventsExist=false in the roomserver query response, +// resulting in a call to /get_missing_events which returns the missing prev event. Both events should be processed in +// topological order and sent to the roomserver. +func TestTransactionFetchMissingPrevEvents(t *testing.T) { + haveEvent := testEvents[len(testEvents)-3] + prevEvent := testEvents[len(testEvents)-2] + inputEvent := testEvents[len(testEvents)-1] + + var rsAPI *testRoomserverAPI // ref here so we can refer to inputRoomEvents inside these functions + rsAPI = &testRoomserverAPI{ + queryStateAfterEvents: func(req *api.QueryStateAfterEventsRequest) api.QueryStateAfterEventsResponse { + // we expect this to be called three times: + // - first with input event to realise there's a gap + // - second with the prevEvent to realise there is no gap + // - third with the input event to realise there is no longer a gap + prevEventsExist := false + if len(req.PrevEventIDs) == 1 { + switch req.PrevEventIDs[0] { + case haveEvent.EventID(): + prevEventsExist = true + case prevEvent.EventID(): + // we only have this event if we've been send prevEvent + if len(rsAPI.inputRoomEvents) == 1 && rsAPI.inputRoomEvents[0].Event.EventID() == prevEvent.EventID() { + prevEventsExist = true + } + } + } + + return api.QueryStateAfterEventsResponse{ + PrevEventsExist: prevEventsExist, + RoomExists: true, + StateEvents: fromStateTuples(req.StateToFetch, nil), + } + }, + queryLatestEventsAndState: func(req *api.QueryLatestEventsAndStateRequest) api.QueryLatestEventsAndStateResponse { + return api.QueryLatestEventsAndStateResponse{ + RoomExists: true, + Depth: haveEvent.Depth(), + LatestEvents: []gomatrixserverlib.EventReference{ + haveEvent.EventReference(), + }, + StateEvents: fromStateTuples(req.StateToFetch, nil), + } + }, + } + + cli := &txnFedClient{ + getMissingEvents: func(missing gomatrixserverlib.MissingEvents) (res gomatrixserverlib.RespMissingEvents, err error) { + if !reflect.DeepEqual(missing.EarliestEvents, []string{haveEvent.EventID()}) { + t.Errorf("call to /get_missing_events wrong earliest events: got %v want %v", missing.EarliestEvents, haveEvent.EventID()) + } + if !reflect.DeepEqual(missing.LatestEvents, []string{inputEvent.EventID()}) { + t.Errorf("call to /get_missing_events wrong latest events: got %v want %v", missing.LatestEvents, inputEvent.EventID()) + } + return gomatrixserverlib.RespMissingEvents{ + Events: []gomatrixserverlib.Event{ + prevEvent.Unwrap(), + }, + }, nil + }, + } + + pdus := []json.RawMessage{ + inputEvent.JSON(), + } + txn := mustCreateTransaction(rsAPI, cli, pdus) + mustProcessTransaction(t, txn, nil) + assertInputRoomEvents(t, rsAPI.inputRoomEvents, []gomatrixserverlib.HeaderedEvent{prevEvent, inputEvent}) +} + +// The purpose of this test is to check that when there are missing prev_events and we still haven't been able to fill +// in the hole with /get_missing_events that the state BEFORE the events we want to persist is fetched via /state_ids +// and /event. It works by setting PrevEventsExist=false in the roomserver query response, resulting in +// a call to /get_missing_events which returns 1 out of the 2 events it needs to fill in the gap. Synapse and Dendrite +// both give up after 1x /get_missing_events call, relying on requesting the state AFTER the missing event in order to +// continue. The DAG looks something like: +// FE GME TXN +// A ---> B ---> C ---> D +// TXN=event in the txn, GME=response to /get_missing_events, FE=roomserver's forward extremity. Should result in: +// - /state_ids?event=B is requested, then /event/B to get the state AFTER B. B is a state event. +// - state resolution is done to check C is allowed. +// This results in B being sent as an outlier FIRST, then C,D. func TestTransactionFetchMissingStateByStateIDs(t *testing.T) { - missingStateEvent := testStateEvents[gomatrixserverlib.StateKeyTuple{ + eventA := testEvents[len(testEvents)-5] + // this is also len(testEvents)-4 + eventB := testStateEvents[gomatrixserverlib.StateKeyTuple{ EventType: gomatrixserverlib.MRoomPowerLevels, StateKey: "", }] - rsAPI := &testRoomserverAPI{ + eventC := testEvents[len(testEvents)-3] + eventD := testEvents[len(testEvents)-2] + fmt.Println("a:", eventA.EventID()) + fmt.Println("b:", eventB.EventID()) + fmt.Println("c:", eventC.EventID()) + fmt.Println("d:", eventD.EventID()) + var rsAPI *testRoomserverAPI + rsAPI = &testRoomserverAPI{ queryStateAfterEvents: func(req *api.QueryStateAfterEventsRequest) api.QueryStateAfterEventsResponse { + omitTuples := []gomatrixserverlib.StateKeyTuple{ + { + EventType: gomatrixserverlib.MRoomPowerLevels, + StateKey: "", + }, + } + askingForEvent := req.PrevEventIDs[0] + haveEventB := false + haveEventC := false + for _, ev := range rsAPI.inputRoomEvents { + switch ev.Event.EventID() { + case eventB.EventID(): + haveEventB = true + omitTuples = nil // include event B now + case eventC.EventID(): + haveEventC = true + } + } + prevEventExists := false + if askingForEvent == eventC.EventID() { + prevEventExists = haveEventC + } else if askingForEvent == eventB.EventID() { + prevEventExists = haveEventB + } + var stateEvents []gomatrixserverlib.HeaderedEvent + if prevEventExists { + stateEvents = fromStateTuples(req.StateToFetch, omitTuples) + } return api.QueryStateAfterEventsResponse{ - // setting this to false should trigger a call to /state_ids - PrevEventsExist: false, + PrevEventsExist: prevEventExists, RoomExists: true, - StateEvents: nil, + StateEvents: stateEvents, + } + }, + queryLatestEventsAndState: func(req *api.QueryLatestEventsAndStateRequest) api.QueryLatestEventsAndStateResponse { + omitTuples := []gomatrixserverlib.StateKeyTuple{ + {EventType: gomatrixserverlib.MRoomPowerLevels, StateKey: ""}, + } + return api.QueryLatestEventsAndStateResponse{ + RoomExists: true, + Depth: eventA.Depth(), + LatestEvents: []gomatrixserverlib.EventReference{ + eventA.EventReference(), + }, + StateEvents: fromStateTuples(req.StateToFetch, omitTuples), } }, queryEventsByID: func(req *api.QueryEventsByIDRequest) api.QueryEventsByIDResponse { var res api.QueryEventsByIDResponse + fmt.Println("queryEventsByID ", req.EventIDs) for _, wantEventID := range req.EventIDs { for _, ev := range testStateEvents { - // roomserver is missing the power levels event - if wantEventID == missingStateEvent.EventID() { + // roomserver is missing the power levels event unless it's been sent to us recently as an outlier + if wantEventID == eventB.EventID() { + fmt.Println("Asked for pl event") + for _, inEv := range rsAPI.inputRoomEvents { + fmt.Println("recv ", inEv.Event.EventID()) + if inEv.Event.EventID() == wantEventID { + res.Events = append(res.Events, inEv.Event) + break + } + } continue } if ev.EventID() == wantEventID { @@ -461,91 +619,55 @@ func TestTransactionFetchMissingStateByStateIDs(t *testing.T) { return res }, } - inputEvent := testEvents[len(testEvents)-1] + // /state_ids for event B returns every state event but B (it's the state before) + var authEventIDs []string var stateEventIDs []string for _, ev := range testStateEvents { + if ev.EventID() == eventB.EventID() { + continue + } + // state res checks what auth events you give it, and this isn't a valid auth event + if ev.Type() != gomatrixserverlib.MRoomHistoryVisibility { + authEventIDs = append(authEventIDs, ev.EventID()) + } stateEventIDs = append(stateEventIDs, ev.EventID()) } cli := &txnFedClient{ - // /state_ids returns all the state events stateIDs: map[string]gomatrixserverlib.RespStateIDs{ - inputEvent.EventID(): gomatrixserverlib.RespStateIDs{ + eventB.EventID(): { StateEventIDs: stateEventIDs, - AuthEventIDs: stateEventIDs, + AuthEventIDs: authEventIDs, }, }, - // /event for the missing state event returns it + // /event for event B returns it getEvent: map[string]gomatrixserverlib.Transaction{ - missingStateEvent.EventID(): gomatrixserverlib.Transaction{ + eventB.EventID(): { PDUs: []json.RawMessage{ - missingStateEvent.JSON(), + eventB.JSON(), }, }, }, - } - - pdus := []json.RawMessage{ - testData[len(testData)-1], // a message event - } - txn := mustCreateTransaction(rsAPI, cli, pdus) - mustProcessTransaction(t, txn, nil) - assertInputRoomEvents(t, rsAPI.inputRoomEvents, []gomatrixserverlib.HeaderedEvent{missingStateEvent, inputEvent}) -} - -// The purpose of this test is to check that when there are missing prev_events and /state_ids fails, that we fallback to -// calling /state which returns the entire room state at that event. It works by setting PrevEventsExist=false in the -// roomserver query response, resulting in a call to /state_ids which fails (unset). It should then fetch via /state. -func TestTransactionFetchMissingStateByFallbackState(t *testing.T) { - rsAPI := &testRoomserverAPI{ - queryStateAfterEvents: func(req *api.QueryStateAfterEventsRequest) api.QueryStateAfterEventsResponse { - return api.QueryStateAfterEventsResponse{ - // setting this to false should trigger a call to /state_ids - PrevEventsExist: false, - RoomExists: true, - StateEvents: nil, + // /get_missing_events should be done exactly once + getMissingEvents: func(missing gomatrixserverlib.MissingEvents) (res gomatrixserverlib.RespMissingEvents, err error) { + if !reflect.DeepEqual(missing.EarliestEvents, []string{eventA.EventID()}) { + t.Errorf("call to /get_missing_events wrong earliest events: got %v want %v", missing.EarliestEvents, eventA.EventID()) } - }, - } - inputEvent := testEvents[len(testEvents)-1] - // first 5 events are the state events, in auth event order. - stateEvents := testEvents[:5] - - cli := &txnFedClient{ - // /state_ids purposefully unset - stateIDs: nil, - // /state returns the state at that event (which is the current state) - state: map[string]gomatrixserverlib.RespState{ - inputEvent.EventID(): gomatrixserverlib.RespState{ - AuthEvents: gomatrixserverlib.UnwrapEventHeaders(stateEvents), - StateEvents: gomatrixserverlib.UnwrapEventHeaders(stateEvents), - }, + if !reflect.DeepEqual(missing.LatestEvents, []string{eventD.EventID()}) { + t.Errorf("call to /get_missing_events wrong latest events: got %v want %v", missing.LatestEvents, eventD.EventID()) + } + // just return event C, not event B so /state_ids logic kicks in as there will STILL be missing prev_events + return gomatrixserverlib.RespMissingEvents{ + Events: []gomatrixserverlib.Event{ + eventC.Unwrap(), + }, + }, nil }, } pdus := []json.RawMessage{ - testData[len(testData)-1], // a message event + eventD.JSON(), } txn := mustCreateTransaction(rsAPI, cli, pdus) mustProcessTransaction(t, txn, nil) - // the roomserver should get all state events and the new input event - // TODO: it should really be only giving the missing ones - got := rsAPI.inputRoomEvents - if len(got) != len(stateEvents)+1 { - t.Fatalf("wrong number of InputRoomEvents: got %d want %d", len(got), len(stateEvents)+1) - } - last := got[len(got)-1] - if last.Event.EventID() != inputEvent.EventID() { - t.Errorf("last event should be the input event but it wasn't. got %s want %s", last.Event.EventID(), inputEvent.EventID()) - } - gots := make([]string, len(stateEvents)) - wants := make([]string, len(stateEvents)) - for i := range stateEvents { - gots[i] = got[i].Event.EventID() - wants[i] = stateEvents[i].EventID() - } - sort.Strings(gots) - sort.Strings(wants) - if !reflect.DeepEqual(gots, wants) { - t.Errorf("state events returned mismatch, got (sorted): %+v want %+v", gots, wants) - } + assertInputRoomEvents(t, rsAPI.inputRoomEvents, []gomatrixserverlib.HeaderedEvent{eventB, eventC, eventD}) } diff --git a/roomserver/internal/input_events.go b/roomserver/internal/input_events.go index a0bfaa2e3..f4d3bb8f5 100644 --- a/roomserver/internal/input_events.go +++ b/roomserver/internal/input_events.go @@ -55,7 +55,7 @@ func processRoomEvent( // Check that the event passes authentication checks and work out the numeric IDs for the auth events. authEventNIDs, err := checkAuthEvents(ctx, db, headered, input.AuthEventIDs) if err != nil { - logrus.WithError(err).WithField("event_id", event.EventID()).Error("processRoomEvent.checkAuthEvents failed for event") + logrus.WithError(err).WithField("event_id", event.EventID()).WithField("auth_event_ids", input.AuthEventIDs).Error("processRoomEvent.checkAuthEvents failed for event") return } diff --git a/sytest-whitelist b/sytest-whitelist index 6e1b4f631..6dbc7ab2b 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -264,4 +264,11 @@ User can invite local user to room with version 5 remote user can join room with version 5 User can invite remote user to room with version 5 Remote user can backfill in a room with version 5 +Inbound federation can receive v1 /send_join +Inbound federation can get state for a room +Inbound federation of state requires event_id as a mandatory paramater +Inbound federation can get state_ids for a room +Inbound federation of state_ids requires event_id as a mandatory paramater +Federation rejects inbound events where the prev_events cannot be found +Outbound federation requests missing prev_events and then asks for /state_ids and resolves the state Alternative server names do not cause a routing loop From 106a335b7d4fcef8d204dfc15c7cf19f927e75d9 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 12 May 2020 17:49:04 +0100 Subject: [PATCH 047/200] Update sytest-whitelist --- sytest-whitelist | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sytest-whitelist b/sytest-whitelist index 6dbc7ab2b..035b9b36e 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -272,3 +272,10 @@ Inbound federation of state_ids requires event_id as a mandatory paramater Federation rejects inbound events where the prev_events cannot be found Outbound federation requests missing prev_events and then asks for /state_ids and resolves the state Alternative server names do not cause a routing loop +Events whose auth_events are in the wrong room do not mess up the room state +Inbound federation can return events +Inbound federation can return missing events for world_readable visibility +Inbound federation can return missing events for invite visibility +Inbound federation can get public room list +An event which redacts itself should be ignored +A pair of events which redact each other should be ignored From 31e6a7f1932c11d9b5b682ad06a5b8db9d74a44f Mon Sep 17 00:00:00 2001 From: Sid Karunaratne Date: Wed, 13 May 2020 19:04:54 +0800 Subject: [PATCH 048/200] Enforce `mediaIDRegex` to be only valid `mediaIDCharacters` (#1020) Error messages indicate that: > mediaId must be a non-empty string using only characters in `mediaIDCharacters` However the regex used only required that some characters in the filename match the restriction, not that the entire filename does. This commit ensures that the filename must entirely fullfill the `mediaIDCharacters` restriction Signed-off-by: Sid Karunaratne Co-authored-by: Kegsay --- mediaapi/routing/download.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediaapi/routing/download.go b/mediaapi/routing/download.go index 9feca90e9..75df313f6 100644 --- a/mediaapi/routing/download.go +++ b/mediaapi/routing/download.go @@ -43,7 +43,7 @@ import ( const mediaIDCharacters = "A-Za-z0-9_=-" // Note: unfortunately regex.MustCompile() cannot be assigned to a const -var mediaIDRegex = regexp.MustCompile("[" + mediaIDCharacters + "]+") +var mediaIDRegex = regexp.MustCompile("^[" + mediaIDCharacters + "]+$") // downloadRequest metadata included in or derivable from a download or thumbnail request // https://matrix.org/docs/spec/client_server/r0.2.0.html#get-matrix-media-r0-download-servername-mediaid From 5e9dce1c0c66736937eeddd5c33c92700d9a65a7 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Wed, 13 May 2020 12:14:50 +0100 Subject: [PATCH 049/200] syncapi: Rename and split out tokens (#1025) * syncapi: Rename and split out tokens Previously we used the badly named `PaginationToken` which was used for both `/sync` and `/messages` requests. This quickly became confusing because named fields like `PDUPosition` meant different things depending on the token type. Instead, we now have two token types: `TopologyToken` and `StreamingToken`, both of which have fields which make more sense for their specific situations. Updated the codebase to use one or the other. `PaginationToken` still lives on as `syncToken`, an unexported type which both tokens rely on. This allows us to guarantee that the specific mappings of positions to a string remain solely under the control of the `types` package. This enables us to move high-level conceptual things like "decrement this topological token" to function calls e.g `TopologicalToken.Decrement()`. Currently broken because `/messages` seemingly used both stream and topological tokens, though I need to confirm this. * final tweaks/hacks * spurious logging * Review comments and linting --- syncapi/consumers/clientapi.go | 2 +- syncapi/consumers/eduserver.go | 6 +- syncapi/consumers/roomserver.go | 4 +- syncapi/routing/messages.go | 77 +++---- syncapi/storage/interface.go | 11 +- syncapi/storage/postgres/syncserver.go | 127 +++++------ syncapi/storage/sqlite3/syncserver.go | 154 ++++++------- syncapi/storage/storage_test.go | 64 +++--- syncapi/sync/notifier.go | 10 +- syncapi/sync/notifier_test.go | 70 ++---- syncapi/sync/request.go | 26 +-- syncapi/sync/requestpool.go | 6 +- syncapi/sync/userstream.go | 12 +- syncapi/types/types.go | 293 ++++++++++++++++--------- syncapi/types/types_test.go | 34 +-- 15 files changed, 457 insertions(+), 439 deletions(-) diff --git a/syncapi/consumers/clientapi.go b/syncapi/consumers/clientapi.go index f5b8c43ec..b65d01a04 100644 --- a/syncapi/consumers/clientapi.go +++ b/syncapi/consumers/clientapi.go @@ -90,7 +90,7 @@ func (s *OutputClientDataConsumer) onMessage(msg *sarama.ConsumerMessage) error }).Panicf("could not save account data") } - s.notifier.OnNewEvent(nil, "", []string{string(msg.Key)}, types.PaginationToken{PDUPosition: pduPos}) + s.notifier.OnNewEvent(nil, "", []string{string(msg.Key)}, types.NewStreamToken(pduPos, 0)) return nil } diff --git a/syncapi/consumers/eduserver.go b/syncapi/consumers/eduserver.go index 249452af5..ece999d59 100644 --- a/syncapi/consumers/eduserver.go +++ b/syncapi/consumers/eduserver.go @@ -65,9 +65,7 @@ func (s *OutputTypingEventConsumer) Start() error { s.db.SetTypingTimeoutCallback(func(userID, roomID string, latestSyncPosition int64) { s.notifier.OnNewEvent( nil, roomID, nil, - types.PaginationToken{ - EDUTypingPosition: types.StreamPosition(latestSyncPosition), - }, + types.NewStreamToken(0, types.StreamPosition(latestSyncPosition)), ) }) @@ -96,6 +94,6 @@ func (s *OutputTypingEventConsumer) onMessage(msg *sarama.ConsumerMessage) error typingPos = s.db.RemoveTypingUser(typingEvent.UserID, typingEvent.RoomID) } - s.notifier.OnNewEvent(nil, output.Event.RoomID, nil, types.PaginationToken{EDUTypingPosition: typingPos}) + s.notifier.OnNewEvent(nil, output.Event.RoomID, nil, types.NewStreamToken(0, typingPos)) return nil } diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index 987cc5df6..368420a6c 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -146,7 +146,7 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent( }).Panicf("roomserver output log: write event failure") return nil } - s.notifier.OnNewEvent(&ev, "", nil, types.PaginationToken{PDUPosition: pduPos}) + s.notifier.OnNewEvent(&ev, "", nil, types.NewStreamToken(pduPos, 0)) return nil } @@ -164,7 +164,7 @@ func (s *OutputRoomEventConsumer) onNewInviteEvent( }).Panicf("roomserver output log: write invite failure") return nil } - s.notifier.OnNewEvent(&msg.Event, "", nil, types.PaginationToken{PDUPosition: pduPos}) + s.notifier.OnNewEvent(&msg.Event, "", nil, types.NewStreamToken(pduPos, 0)) return nil } diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index 270b0ee95..72c306d4f 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -38,8 +38,9 @@ type messagesReq struct { federation *gomatrixserverlib.FederationClient cfg *config.Dendrite roomID string - from *types.PaginationToken - to *types.PaginationToken + from *types.TopologyToken + to *types.TopologyToken + fromStream *types.StreamingToken wasToProvided bool limit int backwardOrdering bool @@ -66,11 +67,16 @@ func OnIncomingMessagesRequest( // Extract parameters from the request's URL. // Pagination tokens. - from, err := types.NewPaginationTokenFromString(req.URL.Query().Get("from")) + var fromStream *types.StreamingToken + from, err := types.NewTopologyTokenFromString(req.URL.Query().Get("from")) if err != nil { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.InvalidArgumentValue("Invalid from parameter: " + err.Error()), + fs, err2 := types.NewStreamTokenFromString(req.URL.Query().Get("from")) + fromStream = &fs + if err2 != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.InvalidArgumentValue("Invalid from parameter: " + err2.Error()), + } } } @@ -88,10 +94,10 @@ func OnIncomingMessagesRequest( // Pagination tokens. To is optional, and its default value depends on the // direction ("b" or "f"). - var to *types.PaginationToken + var to types.TopologyToken wasToProvided := true if s := req.URL.Query().Get("to"); len(s) > 0 { - to, err = types.NewPaginationTokenFromString(s) + to, err = types.NewTopologyTokenFromString(s) if err != nil { return util.JSONResponse{ Code: http.StatusBadRequest, @@ -139,8 +145,9 @@ func OnIncomingMessagesRequest( federation: federation, cfg: cfg, roomID: roomID, - from: from, - to: to, + from: &from, + to: &to, + fromStream: fromStream, wasToProvided: wasToProvided, limit: limit, backwardOrdering: backwardOrdering, @@ -178,12 +185,20 @@ func OnIncomingMessagesRequest( // remote homeserver. func (r *messagesReq) retrieveEvents() ( clientEvents []gomatrixserverlib.ClientEvent, start, - end *types.PaginationToken, err error, + end types.TopologyToken, err error, ) { // Retrieve the events from the local database. - streamEvents, err := r.db.GetEventsInRange( - r.ctx, r.from, r.to, r.roomID, r.limit, r.backwardOrdering, - ) + var streamEvents []types.StreamEvent + if r.fromStream != nil { + toStream := r.to.StreamToken() + streamEvents, err = r.db.GetEventsInStreamingRange( + r.ctx, r.fromStream, &toStream, r.roomID, r.limit, r.backwardOrdering, + ) + } else { + streamEvents, err = r.db.GetEventsInTopologicalRange( + r.ctx, r.from, r.to, r.roomID, r.limit, r.backwardOrdering, + ) + } if err != nil { err = fmt.Errorf("GetEventsInRange: %w", err) return @@ -206,7 +221,7 @@ func (r *messagesReq) retrieveEvents() ( // If we didn't get any event, we don't need to proceed any further. if len(events) == 0 { - return []gomatrixserverlib.ClientEvent{}, r.from, r.to, nil + return []gomatrixserverlib.ClientEvent{}, *r.from, *r.to, nil } // Sort the events to ensure we send them in the right order. @@ -246,12 +261,8 @@ func (r *messagesReq) retrieveEvents() ( } // Generate pagination tokens to send to the client using the positions // retrieved previously. - start = types.NewPaginationTokenFromTypeAndPosition( - types.PaginationTokenTypeTopology, startPos, startStreamPos, - ) - end = types.NewPaginationTokenFromTypeAndPosition( - types.PaginationTokenTypeTopology, endPos, endStreamPos, - ) + start = types.NewTopologyToken(startPos, startStreamPos) + end = types.NewTopologyToken(endPos, endStreamPos) if r.backwardOrdering { // A stream/topological position is a cursor located between two events. @@ -259,14 +270,7 @@ func (r *messagesReq) retrieveEvents() ( // we consider a left to right chronological order), tokens need to refer // to them by the event on their left, therefore we need to decrement the // end position we send in the response if we're going backward. - end.PDUPosition-- - end.EDUTypingPosition += 1000 - } - - // The lowest token value is 1, therefore we need to manually set it to that - // value if we're below it. - if end.PDUPosition < types.StreamPosition(1) { - end.PDUPosition = types.StreamPosition(1) + end.Decrement() } return clientEvents, start, end, err @@ -317,11 +321,11 @@ func (r *messagesReq) handleNonEmptyEventsSlice(streamEvents []types.StreamEvent // The condition in the SQL query is a strict "greater than" so // we need to check against to-1. streamPos := types.StreamPosition(streamEvents[len(streamEvents)-1].StreamPosition) - isSetLargeEnough = (r.to.PDUPosition-1 == streamPos) + isSetLargeEnough = (r.to.PDUPosition()-1 == streamPos) } } else { streamPos := types.StreamPosition(streamEvents[0].StreamPosition) - isSetLargeEnough = (r.from.PDUPosition-1 == streamPos) + isSetLargeEnough = (r.from.PDUPosition()-1 == streamPos) } } @@ -424,18 +428,17 @@ func (r *messagesReq) backfill(roomID string, fromEventIDs []string, limit int) func setToDefault( ctx context.Context, db storage.Database, backwardOrdering bool, roomID string, -) (to *types.PaginationToken, err error) { +) (to types.TopologyToken, err error) { if backwardOrdering { // go 1 earlier than the first event so we correctly fetch the earliest event - to = types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, 0, 0) + to = types.NewTopologyToken(0, 0) } else { - var pos, stream types.StreamPosition - pos, stream, err = db.MaxTopologicalPosition(ctx, roomID) + var depth, stream types.StreamPosition + depth, stream, err = db.MaxTopologicalPosition(ctx, roomID) if err != nil { return } - - to = types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, pos, stream) + to = types.NewTopologyToken(depth, stream) } return diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index 7d6376438..63af11365 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -50,13 +50,13 @@ type Database interface { // Returns an error if there was an issue with the retrieval. GetStateEventsForRoom(ctx context.Context, roomID string, stateFilterPart *gomatrixserverlib.StateFilter) (stateEvents []gomatrixserverlib.HeaderedEvent, err error) // SyncPosition returns the latest positions for syncing. - SyncPosition(ctx context.Context) (types.PaginationToken, error) + SyncPosition(ctx context.Context) (types.StreamingToken, error) // IncrementalSync returns all the data needed in order to create an incremental // sync response for the given user. Events returned will include any client // transaction IDs associated with the given device. These transaction IDs come // from when the device sent the event via an API that included a transaction // ID. - IncrementalSync(ctx context.Context, device authtypes.Device, fromPos, toPos types.PaginationToken, numRecentEventsPerRoom int, wantFullState bool) (*types.Response, error) + IncrementalSync(ctx context.Context, device authtypes.Device, fromPos, toPos types.StreamingToken, numRecentEventsPerRoom int, wantFullState bool) (*types.Response, error) // CompleteSync returns a complete /sync API response for the given user. CompleteSync(ctx context.Context, userID string, numRecentEventsPerRoom int) (*types.Response, error) // GetAccountDataInRange returns all account data for a given user inserted or @@ -88,9 +88,10 @@ type Database interface { // RemoveTypingUser removes a typing user from the typing cache. // Returns the newly calculated sync position for typing notifications. RemoveTypingUser(userID, roomID string) types.StreamPosition - // GetEventsInRange retrieves all of the events on a given ordering using the - // given extremities and limit. - GetEventsInRange(ctx context.Context, from, to *types.PaginationToken, roomID string, limit int, backwardOrdering bool) (events []types.StreamEvent, err error) + // GetEventsInStreamingRange retrieves all of the events on a given ordering using the given extremities and limit. + GetEventsInStreamingRange(ctx context.Context, from, to *types.StreamingToken, roomID string, limit int, backwardOrdering bool) (events []types.StreamEvent, err error) + // GetEventsInTopologicalRange retrieves all of the events on a given ordering using the given extremities and limit. + GetEventsInTopologicalRange(ctx context.Context, from, to *types.TopologyToken, roomID string, limit int, backwardOrdering bool) (events []types.StreamEvent, err error) // EventPositionInTopology returns the depth and stream position of the given event. EventPositionInTopology(ctx context.Context, eventID string) (depth types.StreamPosition, stream types.StreamPosition, err error) // EventsAtTopologicalPosition returns all of the events matching a given diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go index 1845ac386..d45bc09e5 100644 --- a/syncapi/storage/postgres/syncserver.go +++ b/syncapi/storage/postgres/syncserver.go @@ -228,69 +228,68 @@ func (d *SyncServerDatasource) GetStateEventsForRoom( return } -func (d *SyncServerDatasource) GetEventsInRange( +func (d *SyncServerDatasource) GetEventsInTopologicalRange( ctx context.Context, - from, to *types.PaginationToken, + from, to *types.TopologyToken, roomID string, limit int, backwardOrdering bool, ) (events []types.StreamEvent, err error) { - // If the pagination token's type is types.PaginationTokenTypeTopology, the - // events must be retrieved from the rooms' topology table rather than the - // table contaning the syncapi server's whole stream of events. - if from.Type == types.PaginationTokenTypeTopology { - // Determine the backward and forward limit, i.e. the upper and lower - // limits to the selection in the room's topology, from the direction. - var backwardLimit, forwardLimit, forwardMicroLimit types.StreamPosition - if backwardOrdering { - // Backward ordering is antichronological (latest event to oldest - // one). - backwardLimit = to.PDUPosition - forwardLimit = from.PDUPosition - forwardMicroLimit = from.EDUTypingPosition - } else { - // Forward ordering is chronological (oldest event to latest one). - backwardLimit = from.PDUPosition - forwardLimit = to.PDUPosition - } + // Determine the backward and forward limit, i.e. the upper and lower + // limits to the selection in the room's topology, from the direction. + var backwardLimit, forwardLimit, forwardMicroLimit types.StreamPosition + if backwardOrdering { + // Backward ordering is antichronological (latest event to oldest + // one). + backwardLimit = to.Depth() + forwardLimit = from.Depth() + forwardMicroLimit = from.PDUPosition() + } else { + // Forward ordering is chronological (oldest event to latest one). + backwardLimit = from.Depth() + forwardLimit = to.Depth() + } - // Select the event IDs from the defined range. - var eIDs []string - eIDs, err = d.topology.selectEventIDsInRange( - ctx, roomID, backwardLimit, forwardLimit, forwardMicroLimit, limit, !backwardOrdering, - ) - if err != nil { - return - } - - // Retrieve the events' contents using their IDs. - events, err = d.events.selectEvents(ctx, nil, eIDs) + // Select the event IDs from the defined range. + var eIDs []string + eIDs, err = d.topology.selectEventIDsInRange( + ctx, roomID, backwardLimit, forwardLimit, forwardMicroLimit, limit, !backwardOrdering, + ) + if err != nil { return } - // If the pagination token's type is types.PaginationTokenTypeStream, the - // events must be retrieved from the table contaning the syncapi server's - // whole stream of events. + // Retrieve the events' contents using their IDs. + events, err = d.events.selectEvents(ctx, nil, eIDs) + return +} +// GetEventsInStreamingRange retrieves all of the events on a given ordering using the +// given extremities and limit. +func (d *SyncServerDatasource) GetEventsInStreamingRange( + ctx context.Context, + from, to *types.StreamingToken, + roomID string, limit int, + backwardOrdering bool, +) (events []types.StreamEvent, err error) { if backwardOrdering { // When using backward ordering, we want the most recent events first. if events, err = d.events.selectRecentEvents( - ctx, nil, roomID, to.PDUPosition, from.PDUPosition, limit, false, false, + ctx, nil, roomID, to.PDUPosition(), from.PDUPosition(), limit, false, false, ); err != nil { return } } else { // When using forward ordering, we want the least recent events first. if events, err = d.events.selectEarlyEvents( - ctx, nil, roomID, from.PDUPosition, to.PDUPosition, limit, + ctx, nil, roomID, from.PDUPosition(), to.PDUPosition(), limit, ); err != nil { return } } - - return + return events, err } -func (d *SyncServerDatasource) SyncPosition(ctx context.Context) (types.PaginationToken, error) { +func (d *SyncServerDatasource) SyncPosition(ctx context.Context) (types.StreamingToken, error) { return d.syncPositionTx(ctx, nil) } @@ -353,7 +352,7 @@ func (d *SyncServerDatasource) syncStreamPositionTx( func (d *SyncServerDatasource) syncPositionTx( ctx context.Context, txn *sql.Tx, -) (sp types.PaginationToken, err error) { +) (sp types.StreamingToken, err error) { maxEventID, err := d.events.selectMaxEventID(ctx, txn) if err != nil { @@ -373,8 +372,7 @@ func (d *SyncServerDatasource) syncPositionTx( if maxInviteID > maxEventID { maxEventID = maxInviteID } - sp.PDUPosition = types.StreamPosition(maxEventID) - sp.EDUTypingPosition = types.StreamPosition(d.eduCache.GetLatestSyncPosition()) + sp = types.NewStreamToken(types.StreamPosition(maxEventID), types.StreamPosition(d.eduCache.GetLatestSyncPosition())) return } @@ -439,7 +437,7 @@ func (d *SyncServerDatasource) addPDUDeltaToResponse( // addTypingDeltaToResponse adds all typing notifications to a sync response // since the specified position. func (d *SyncServerDatasource) addTypingDeltaToResponse( - since types.PaginationToken, + since types.StreamingToken, joinedRoomIDs []string, res *types.Response, ) error { @@ -448,7 +446,7 @@ func (d *SyncServerDatasource) addTypingDeltaToResponse( var err error for _, roomID := range joinedRoomIDs { if typingUsers, updated := d.eduCache.GetTypingUsersIfUpdatedAfter( - roomID, int64(since.EDUTypingPosition), + roomID, int64(since.EDUPosition()), ); updated { ev := gomatrixserverlib.ClientEvent{ Type: gomatrixserverlib.MTyping, @@ -473,12 +471,12 @@ func (d *SyncServerDatasource) addTypingDeltaToResponse( // addEDUDeltaToResponse adds updates for EDUs of each type since fromPos if // the positions of that type are not equal in fromPos and toPos. func (d *SyncServerDatasource) addEDUDeltaToResponse( - fromPos, toPos types.PaginationToken, + fromPos, toPos types.StreamingToken, joinedRoomIDs []string, res *types.Response, ) (err error) { - if fromPos.EDUTypingPosition != toPos.EDUTypingPosition { + if fromPos.EDUPosition() != toPos.EDUPosition() { err = d.addTypingDeltaToResponse( fromPos, joinedRoomIDs, res, ) @@ -490,7 +488,7 @@ func (d *SyncServerDatasource) addEDUDeltaToResponse( func (d *SyncServerDatasource) IncrementalSync( ctx context.Context, device authtypes.Device, - fromPos, toPos types.PaginationToken, + fromPos, toPos types.StreamingToken, numRecentEventsPerRoom int, wantFullState bool, ) (*types.Response, error) { @@ -499,9 +497,9 @@ func (d *SyncServerDatasource) IncrementalSync( var joinedRoomIDs []string var err error - if fromPos.PDUPosition != toPos.PDUPosition || wantFullState { + if fromPos.PDUPosition() != toPos.PDUPosition() || wantFullState { joinedRoomIDs, err = d.addPDUDeltaToResponse( - ctx, device, fromPos.PDUPosition, toPos.PDUPosition, numRecentEventsPerRoom, wantFullState, res, + ctx, device, fromPos.PDUPosition(), toPos.PDUPosition(), numRecentEventsPerRoom, wantFullState, res, ) } else { joinedRoomIDs, err = d.roomstate.selectRoomIDsWithMembership( @@ -530,7 +528,7 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( numRecentEventsPerRoom int, ) ( res *types.Response, - toPos types.PaginationToken, + toPos types.StreamingToken, joinedRoomIDs []string, err error, ) { @@ -577,7 +575,7 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( // See: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L316 var recentStreamEvents []types.StreamEvent recentStreamEvents, err = d.events.selectRecentEvents( - ctx, txn, roomID, types.StreamPosition(0), toPos.PDUPosition, + ctx, txn, roomID, types.StreamPosition(0), toPos.PDUPosition(), numRecentEventsPerRoom, true, true, ) if err != nil { @@ -588,27 +586,25 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( // oldest event in the room's topology. var backwardTopologyPos, backwardStreamPos types.StreamPosition backwardTopologyPos, backwardStreamPos, err = d.topology.selectPositionInTopology(ctx, recentStreamEvents[0].EventID()) - if backwardTopologyPos-1 <= 0 { - backwardTopologyPos = types.StreamPosition(1) - } else { - backwardTopologyPos-- + if err != nil { + return } + prevBatch := types.NewTopologyToken(backwardTopologyPos, backwardStreamPos) + prevBatch.Decrement() // We don't include a device here as we don't need to send down // transaction IDs for complete syncs recentEvents := d.StreamEventsToEvents(nil, recentStreamEvents) stateEvents = removeDuplicates(stateEvents, recentEvents) jr := types.NewJoinResponse() - jr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition( - types.PaginationTokenTypeTopology, backwardTopologyPos, backwardStreamPos, - ).String() + jr.Timeline.PrevBatch = prevBatch.String() jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = true jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(stateEvents, gomatrixserverlib.FormatSync) res.Rooms.Join[roomID] = *jr } - if err = d.addInvitesToResponse(ctx, txn, userID, 0, toPos.PDUPosition, res); err != nil { + if err = d.addInvitesToResponse(ctx, txn, userID, 0, toPos.PDUPosition(), res); err != nil { return } @@ -628,7 +624,7 @@ func (d *SyncServerDatasource) CompleteSync( // Use a zero value SyncPosition for fromPos so all EDU states are added. err = d.addEDUDeltaToResponse( - types.PaginationToken{}, toPos, joinedRoomIDs, res, + types.NewStreamToken(0, 0), toPos, joinedRoomIDs, res, ) if err != nil { return nil, err @@ -757,14 +753,15 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( recentEvents := d.StreamEventsToEvents(device, recentStreamEvents) delta.stateEvents = removeDuplicates(delta.stateEvents, recentEvents) // roll back backwardTopologyPos, backwardStreamPos := d.getBackwardTopologyPos(ctx, recentStreamEvents) + prevBatch := types.NewTopologyToken( + backwardTopologyPos, backwardStreamPos, + ) switch delta.membership { case gomatrixserverlib.Join: jr := types.NewJoinResponse() - jr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition( - types.PaginationTokenTypeTopology, backwardTopologyPos, backwardStreamPos, - ).String() + jr.Timeline.PrevBatch = prevBatch.String() jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) @@ -775,9 +772,7 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( // TODO: recentEvents may contain events that this user is not allowed to see because they are // no longer in the room. lr := types.NewLeaveResponse() - lr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition( - types.PaginationTokenTypeTopology, backwardTopologyPos, backwardStreamPos, - ).String() + lr.Timeline.PrevBatch = prevBatch.String() lr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) lr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true lr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index 314ea2aa3..212f882b1 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -269,63 +269,63 @@ func (d *SyncServerDatasource) GetStateEventsForRoom( return } -// GetEventsInRange retrieves all of the events on a given ordering using the +// GetEventsInTopologicalRange retrieves all of the events on a given ordering using the // given extremities and limit. -func (d *SyncServerDatasource) GetEventsInRange( +func (d *SyncServerDatasource) GetEventsInTopologicalRange( ctx context.Context, - from, to *types.PaginationToken, + from, to *types.TopologyToken, roomID string, limit int, backwardOrdering bool, ) (events []types.StreamEvent, err error) { - // If the pagination token's type is types.PaginationTokenTypeTopology, the - // events must be retrieved from the rooms' topology table rather than the - // table contaning the syncapi server's whole stream of events. - if from.Type == types.PaginationTokenTypeTopology { - // TODO: ARGH CONFUSING - // Determine the backward and forward limit, i.e. the upper and lower - // limits to the selection in the room's topology, from the direction. - var backwardLimit, forwardLimit, forwardMicroLimit types.StreamPosition - if backwardOrdering { - // Backward ordering is antichronological (latest event to oldest - // one). - backwardLimit = to.PDUPosition - forwardLimit = from.PDUPosition - forwardMicroLimit = from.EDUTypingPosition - } else { - // Forward ordering is chronological (oldest event to latest one). - backwardLimit = from.PDUPosition - forwardLimit = to.PDUPosition - } + // TODO: ARGH CONFUSING + // Determine the backward and forward limit, i.e. the upper and lower + // limits to the selection in the room's topology, from the direction. + var backwardLimit, forwardLimit, forwardMicroLimit types.StreamPosition + if backwardOrdering { + // Backward ordering is antichronological (latest event to oldest + // one). + backwardLimit = to.Depth() + forwardLimit = from.Depth() + forwardMicroLimit = from.PDUPosition() + } else { + // Forward ordering is chronological (oldest event to latest one). + backwardLimit = from.Depth() + forwardLimit = to.Depth() + } - // Select the event IDs from the defined range. - var eIDs []string - eIDs, err = d.topology.selectEventIDsInRange( - ctx, nil, roomID, backwardLimit, forwardLimit, forwardMicroLimit, limit, !backwardOrdering, - ) - if err != nil { - return - } - - // Retrieve the events' contents using their IDs. - events, err = d.events.selectEvents(ctx, nil, eIDs) + // Select the event IDs from the defined range. + var eIDs []string + eIDs, err = d.topology.selectEventIDsInRange( + ctx, nil, roomID, backwardLimit, forwardLimit, forwardMicroLimit, limit, !backwardOrdering, + ) + if err != nil { return } - // If the pagination token's type is types.PaginationTokenTypeStream, the - // events must be retrieved from the table contaning the syncapi server's - // whole stream of events. + // Retrieve the events' contents using their IDs. + events, err = d.events.selectEvents(ctx, nil, eIDs) + return +} +// GetEventsInStreamingRange retrieves all of the events on a given ordering using the +// given extremities and limit. +func (d *SyncServerDatasource) GetEventsInStreamingRange( + ctx context.Context, + from, to *types.StreamingToken, + roomID string, limit int, + backwardOrdering bool, +) (events []types.StreamEvent, err error) { if backwardOrdering { // When using backward ordering, we want the most recent events first. if events, err = d.events.selectRecentEvents( - ctx, nil, roomID, to.PDUPosition, from.PDUPosition, limit, false, false, + ctx, nil, roomID, to.PDUPosition(), from.PDUPosition(), limit, false, false, ); err != nil { return } } else { // When using forward ordering, we want the least recent events first. if events, err = d.events.selectEarlyEvents( - ctx, nil, roomID, from.PDUPosition, to.PDUPosition, limit, + ctx, nil, roomID, from.PDUPosition(), to.PDUPosition(), limit, ); err != nil { return } @@ -334,10 +334,14 @@ func (d *SyncServerDatasource) GetEventsInRange( } // SyncPosition returns the latest positions for syncing. -func (d *SyncServerDatasource) SyncPosition(ctx context.Context) (tok types.PaginationToken, err error) { +func (d *SyncServerDatasource) SyncPosition(ctx context.Context) (tok types.StreamingToken, err error) { err = common.WithTransaction(d.db, func(txn *sql.Tx) error { - tok, err = d.syncPositionTx(ctx, txn) - return err + pos, err := d.syncPositionTx(ctx, txn) + if err != nil { + return err + } + tok = *pos + return nil }) return } @@ -412,30 +416,31 @@ func (d *SyncServerDatasource) syncStreamPositionTx( func (d *SyncServerDatasource) syncPositionTx( ctx context.Context, txn *sql.Tx, -) (sp types.PaginationToken, err error) { +) (*types.StreamingToken, error) { maxEventID, err := d.events.selectMaxEventID(ctx, txn) if err != nil { - return sp, err + return nil, err } maxAccountDataID, err := d.accountData.selectMaxAccountDataID(ctx, txn) if err != nil { - return sp, err + return nil, err } if maxAccountDataID > maxEventID { maxEventID = maxAccountDataID } maxInviteID, err := d.invites.selectMaxInviteID(ctx, txn) if err != nil { - return sp, err + return nil, err } if maxInviteID > maxEventID { maxEventID = maxInviteID } - sp.PDUPosition = types.StreamPosition(maxEventID) - sp.EDUTypingPosition = types.StreamPosition(d.eduCache.GetLatestSyncPosition()) - sp.Type = types.PaginationTokenTypeStream - return + sp := types.NewStreamToken( + types.StreamPosition(maxEventID), + types.StreamPosition(d.eduCache.GetLatestSyncPosition()), + ) + return &sp, nil } // addPDUDeltaToResponse adds all PDU deltas to a sync response. @@ -499,7 +504,7 @@ func (d *SyncServerDatasource) addPDUDeltaToResponse( // addTypingDeltaToResponse adds all typing notifications to a sync response // since the specified position. func (d *SyncServerDatasource) addTypingDeltaToResponse( - since types.PaginationToken, + since types.StreamingToken, joinedRoomIDs []string, res *types.Response, ) error { @@ -508,7 +513,7 @@ func (d *SyncServerDatasource) addTypingDeltaToResponse( var err error for _, roomID := range joinedRoomIDs { if typingUsers, updated := d.eduCache.GetTypingUsersIfUpdatedAfter( - roomID, int64(since.EDUTypingPosition), + roomID, int64(since.EDUPosition()), ); updated { ev := gomatrixserverlib.ClientEvent{ Type: gomatrixserverlib.MTyping, @@ -533,12 +538,12 @@ func (d *SyncServerDatasource) addTypingDeltaToResponse( // addEDUDeltaToResponse adds updates for EDUs of each type since fromPos if // the positions of that type are not equal in fromPos and toPos. func (d *SyncServerDatasource) addEDUDeltaToResponse( - fromPos, toPos types.PaginationToken, + fromPos, toPos types.StreamingToken, joinedRoomIDs []string, res *types.Response, ) (err error) { - if fromPos.EDUTypingPosition != toPos.EDUTypingPosition { + if fromPos.EDUPosition() != toPos.EDUPosition() { err = d.addTypingDeltaToResponse( fromPos, joinedRoomIDs, res, ) @@ -555,18 +560,21 @@ func (d *SyncServerDatasource) addEDUDeltaToResponse( func (d *SyncServerDatasource) IncrementalSync( ctx context.Context, device authtypes.Device, - fromPos, toPos types.PaginationToken, + fromPos, toPos types.StreamingToken, numRecentEventsPerRoom int, wantFullState bool, ) (*types.Response, error) { + fmt.Println("from ", fromPos, "to", toPos) nextBatchPos := fromPos.WithUpdates(toPos) res := types.NewResponse(nextBatchPos) + fmt.Println("from ", fromPos, "to", toPos, "next", nextBatchPos) var joinedRoomIDs []string var err error - if fromPos.PDUPosition != toPos.PDUPosition || wantFullState { + fmt.Println("from", fromPos.PDUPosition(), "to", toPos.PDUPosition()) + if fromPos.PDUPosition() != toPos.PDUPosition() || wantFullState { joinedRoomIDs, err = d.addPDUDeltaToResponse( - ctx, device, fromPos.PDUPosition, toPos.PDUPosition, numRecentEventsPerRoom, wantFullState, res, + ctx, device, fromPos.PDUPosition(), toPos.PDUPosition(), numRecentEventsPerRoom, wantFullState, res, ) } else { joinedRoomIDs, err = d.roomstate.selectRoomIDsWithMembership( @@ -595,7 +603,7 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( numRecentEventsPerRoom int, ) ( res *types.Response, - toPos types.PaginationToken, + toPos *types.StreamingToken, joinedRoomIDs []string, err error, ) { @@ -621,7 +629,7 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( return } - res = types.NewResponse(toPos) + res = types.NewResponse(*toPos) // Extract room state and recent events for all rooms the user is joined to. joinedRoomIDs, err = d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, gomatrixserverlib.Join) @@ -643,7 +651,7 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( // See: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L316 var recentStreamEvents []types.StreamEvent recentStreamEvents, err = d.events.selectRecentEvents( - ctx, txn, roomID, types.StreamPosition(0), toPos.PDUPosition, + ctx, txn, roomID, types.StreamPosition(0), toPos.PDUPosition(), numRecentEventsPerRoom, true, true, ) if err != nil { @@ -655,28 +663,22 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( // oldest event in the room's topology. var backwardTopologyPos, backwardTopologyStreamPos types.StreamPosition backwardTopologyPos, backwardTopologyStreamPos, err = d.topology.selectPositionInTopology(ctx, txn, recentStreamEvents[0].EventID()) - if backwardTopologyPos-1 <= 0 { - backwardTopologyPos = types.StreamPosition(1) - } else { - backwardTopologyPos-- - backwardTopologyStreamPos += 1000 // this has to be bigger than the number of events we backfill per request - } + prevBatch := types.NewTopologyToken(backwardTopologyPos, backwardTopologyStreamPos) + prevBatch.Decrement() // We don't include a device here as we don't need to send down // transaction IDs for complete syncs recentEvents := d.StreamEventsToEvents(nil, recentStreamEvents) stateEvents = removeDuplicates(stateEvents, recentEvents) jr := types.NewJoinResponse() - jr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition( - types.PaginationTokenTypeTopology, backwardTopologyPos, backwardTopologyStreamPos, - ).String() + jr.Timeline.PrevBatch = prevBatch.String() jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = true jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(stateEvents, gomatrixserverlib.FormatSync) res.Rooms.Join[roomID] = *jr } - if err = d.addInvitesToResponse(ctx, txn, userID, 0, toPos.PDUPosition, res); err != nil { + if err = d.addInvitesToResponse(ctx, txn, userID, 0, toPos.PDUPosition(), res); err != nil { return } @@ -697,7 +699,7 @@ func (d *SyncServerDatasource) CompleteSync( // Use a zero value SyncPosition for fromPos so all EDU states are added. err = d.addEDUDeltaToResponse( - types.PaginationToken{}, toPos, joinedRoomIDs, res, + types.NewStreamToken(0, 0), *toPos, joinedRoomIDs, res, ) if err != nil { return nil, err @@ -860,14 +862,14 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( recentEvents := d.StreamEventsToEvents(device, recentStreamEvents) delta.stateEvents = removeDuplicates(delta.stateEvents, recentEvents) backwardTopologyPos, backwardStreamPos := d.getBackwardTopologyPos(ctx, txn, recentStreamEvents) + prevBatch := types.NewTopologyToken( + backwardTopologyPos, backwardStreamPos, + ) switch delta.membership { case gomatrixserverlib.Join: jr := types.NewJoinResponse() - - jr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition( - types.PaginationTokenTypeTopology, backwardTopologyPos, backwardStreamPos, - ).String() + jr.Timeline.PrevBatch = prevBatch.String() jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) @@ -878,9 +880,7 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( // TODO: recentEvents may contain events that this user is not allowed to see because they are // no longer in the room. lr := types.NewLeaveResponse() - lr.Timeline.PrevBatch = types.NewPaginationTokenFromTypeAndPosition( - types.PaginationTokenTypeTopology, backwardTopologyPos, backwardStreamPos, - ).String() + lr.Timeline.PrevBatch = prevBatch.String() lr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) lr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true lr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) diff --git a/syncapi/storage/storage_test.go b/syncapi/storage/storage_test.go index b951efa45..f7fa1a870 100644 --- a/syncapi/storage/storage_test.go +++ b/syncapi/storage/storage_test.go @@ -154,10 +154,10 @@ func TestSyncResponse(t *testing.T) { { Name: "IncrementalSync penultimate", DoSync: func() (*types.Response, error) { - from := types.NewPaginationTokenFromTypeAndPosition( // pretend we are at the penultimate event - types.PaginationTokenTypeStream, positions[len(positions)-2], types.StreamPosition(0), + from := types.NewStreamToken( // pretend we are at the penultimate event + positions[len(positions)-2], types.StreamPosition(0), ) - return db.IncrementalSync(ctx, testUserDeviceA, *from, latest, 5, false) + return db.IncrementalSync(ctx, testUserDeviceA, from, latest, 5, false) }, WantTimeline: events[len(events)-1:], }, @@ -166,11 +166,11 @@ func TestSyncResponse(t *testing.T) { { Name: "IncrementalSync limited", DoSync: func() (*types.Response, error) { - from := types.NewPaginationTokenFromTypeAndPosition( // pretend we are 10 events behind - types.PaginationTokenTypeStream, positions[len(positions)-11], types.StreamPosition(0), + from := types.NewStreamToken( // pretend we are 10 events behind + positions[len(positions)-11], types.StreamPosition(0), ) // limit is set to 5 - return db.IncrementalSync(ctx, testUserDeviceA, *from, latest, 5, false) + return db.IncrementalSync(ctx, testUserDeviceA, from, latest, 5, false) }, // want the last 5 events, NOT the last 10. WantTimeline: events[len(events)-5:], @@ -207,7 +207,7 @@ func TestSyncResponse(t *testing.T) { if err != nil { st.Fatalf("failed to do sync: %s", err) } - next := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeStream, latest.PDUPosition, latest.EDUTypingPosition) + next := types.NewStreamToken(latest.PDUPosition(), latest.EDUPosition()) if res.NextBatch != next.String() { st.Errorf("NextBatch got %s want %s", res.NextBatch, next.String()) } @@ -230,11 +230,11 @@ func TestGetEventsInRangeWithPrevBatch(t *testing.T) { if err != nil { t.Fatalf("failed to get SyncPosition: %s", err) } - from := types.NewPaginationTokenFromTypeAndPosition( - types.PaginationTokenTypeStream, positions[len(positions)-2], types.StreamPosition(0), + from := types.NewStreamToken( + positions[len(positions)-2], types.StreamPosition(0), ) - res, err := db.IncrementalSync(ctx, testUserDeviceA, *from, latest, 5, false) + res, err := db.IncrementalSync(ctx, testUserDeviceA, from, latest, 5, false) if err != nil { t.Fatalf("failed to IncrementalSync with latest token") } @@ -249,14 +249,14 @@ func TestGetEventsInRangeWithPrevBatch(t *testing.T) { if prev == "" { t.Fatalf("IncrementalSync expected prev_batch token") } - prevBatchToken, err := types.NewPaginationTokenFromString(prev) + prevBatchToken, err := types.NewTopologyTokenFromString(prev) if err != nil { - t.Fatalf("failed to NewPaginationTokenFromString : %s", err) + t.Fatalf("failed to NewTopologyTokenFromString : %s", err) } // backpaginate 5 messages starting at the latest position. // head towards the beginning of time - to := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, 0, 0) - paginatedEvents, err := db.GetEventsInRange(ctx, prevBatchToken, to, testRoomID, 5, true) + to := types.NewTopologyToken(0, 0) + paginatedEvents, err := db.GetEventsInTopologicalRange(ctx, &prevBatchToken, &to, testRoomID, 5, true) if err != nil { t.Fatalf("GetEventsInRange returned an error: %s", err) } @@ -275,10 +275,10 @@ func TestGetEventsInRangeWithStreamToken(t *testing.T) { t.Fatalf("failed to get SyncPosition: %s", err) } // head towards the beginning of time - to := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, 0, 0) + to := types.NewStreamToken(0, 0) // backpaginate 5 messages starting at the latest position. - paginatedEvents, err := db.GetEventsInRange(ctx, &latest, to, testRoomID, 5, true) + paginatedEvents, err := db.GetEventsInStreamingRange(ctx, &latest, &to, testRoomID, 5, true) if err != nil { t.Fatalf("GetEventsInRange returned an error: %s", err) } @@ -296,12 +296,12 @@ func TestGetEventsInRangeWithTopologyToken(t *testing.T) { if err != nil { t.Fatalf("failed to get MaxTopologicalPosition: %s", err) } - from := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, latest, latestStream) + from := types.NewTopologyToken(latest, latestStream) // head towards the beginning of time - to := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, 0, 0) + to := types.NewTopologyToken(0, 0) // backpaginate 5 messages starting at the latest position. - paginatedEvents, err := db.GetEventsInRange(ctx, from, to, testRoomID, 5, true) + paginatedEvents, err := db.GetEventsInTopologicalRange(ctx, &from, &to, testRoomID, 5, true) if err != nil { t.Fatalf("GetEventsInRange returned an error: %s", err) } @@ -366,14 +366,14 @@ func TestGetEventsInRangeWithEventsSameDepth(t *testing.T) { if err != nil { t.Fatalf("failed to get EventPositionInTopology for event: %s", err) } - fromLatest := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, latestPos, latestStreamPos) - fromFork := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, topoPos, streamPos) + fromLatest := types.NewTopologyToken(latestPos, latestStreamPos) + fromFork := types.NewTopologyToken(topoPos, streamPos) // head towards the beginning of time - to := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, 0, 0) + to := types.NewTopologyToken(0, 0) testCases := []struct { Name string - From *types.PaginationToken + From types.TopologyToken Limit int Wants []gomatrixserverlib.HeaderedEvent }{ @@ -399,7 +399,7 @@ func TestGetEventsInRangeWithEventsSameDepth(t *testing.T) { for _, tc := range testCases { // backpaginate messages starting at the latest position. - paginatedEvents, err := db.GetEventsInRange(ctx, tc.From, to, testRoomID, tc.Limit, true) + paginatedEvents, err := db.GetEventsInTopologicalRange(ctx, &tc.From, &to, testRoomID, tc.Limit, true) if err != nil { t.Fatalf("%s GetEventsInRange returned an error: %s", tc.Name, err) } @@ -446,13 +446,13 @@ func TestGetEventsInRangeWithEventsInsertedLikeBackfill(t *testing.T) { } // head towards the beginning of time - to := types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, 0, 0) + to := types.NewTopologyToken(0, 0) // starting at `from`, backpaginate to the beginning of time, asserting as we go. chunkSize = 3 events = reversed(events) for i := 0; i < len(events); i += chunkSize { - paginatedEvents, err := db.GetEventsInRange(ctx, from, to, testRoomID, chunkSize, true) + paginatedEvents, err := db.GetEventsInTopologicalRange(ctx, from, &to, testRoomID, chunkSize, true) if err != nil { t.Fatalf("GetEventsInRange returned an error: %s", err) } @@ -506,19 +506,15 @@ func assertEventsEqual(t *testing.T, msg string, checkRoomID bool, gots []gomatr } } -func topologyTokenBefore(t *testing.T, db storage.Database, eventID string) *types.PaginationToken { +func topologyTokenBefore(t *testing.T, db storage.Database, eventID string) *types.TopologyToken { pos, spos, err := db.EventPositionInTopology(ctx, eventID) if err != nil { t.Fatalf("failed to get EventPositionInTopology: %s", err) } - if pos-1 <= 0 { - pos = types.StreamPosition(1) - } else { - pos = pos - 1 - spos += 1000 // this has to be bigger than the chunk limit - } - return types.NewPaginationTokenFromTypeAndPosition(types.PaginationTokenTypeTopology, pos, spos) + tok := types.NewTopologyToken(pos, spos) + tok.Decrement() + return &tok } func reversed(in []gomatrixserverlib.HeaderedEvent) []gomatrixserverlib.HeaderedEvent { diff --git a/syncapi/sync/notifier.go b/syncapi/sync/notifier.go index 0d8050112..b3ed5cd03 100644 --- a/syncapi/sync/notifier.go +++ b/syncapi/sync/notifier.go @@ -36,7 +36,7 @@ type Notifier struct { // Protects currPos and userStreams. streamLock *sync.Mutex // The latest sync position - currPos types.PaginationToken + currPos types.StreamingToken // A map of user_id => UserStream which can be used to wake a given user's /sync request. userStreams map[string]*UserStream // The last time we cleaned out stale entries from the userStreams map @@ -46,7 +46,7 @@ type Notifier struct { // NewNotifier creates a new notifier set to the given sync position. // In order for this to be of any use, the Notifier needs to be told all rooms and // the joined users within each of them by calling Notifier.Load(*storage.SyncServerDatabase). -func NewNotifier(pos types.PaginationToken) *Notifier { +func NewNotifier(pos types.StreamingToken) *Notifier { return &Notifier{ currPos: pos, roomIDToJoinedUsers: make(map[string]userIDSet), @@ -68,7 +68,7 @@ func NewNotifier(pos types.PaginationToken) *Notifier { // event type it handles, leaving other fields as 0. func (n *Notifier) OnNewEvent( ev *gomatrixserverlib.HeaderedEvent, roomID string, userIDs []string, - posUpdate types.PaginationToken, + posUpdate types.StreamingToken, ) { // update the current position then notify relevant /sync streams. // This needs to be done PRIOR to waking up users as they will read this value. @@ -151,7 +151,7 @@ func (n *Notifier) Load(ctx context.Context, db storage.Database) error { } // CurrentPosition returns the current sync position -func (n *Notifier) CurrentPosition() types.PaginationToken { +func (n *Notifier) CurrentPosition() types.StreamingToken { n.streamLock.Lock() defer n.streamLock.Unlock() @@ -173,7 +173,7 @@ func (n *Notifier) setUsersJoinedToRooms(roomIDToUserIDs map[string][]string) { } } -func (n *Notifier) wakeupUsers(userIDs []string, newPos types.PaginationToken) { +func (n *Notifier) wakeupUsers(userIDs []string, newPos types.StreamingToken) { for _, userID := range userIDs { stream := n.fetchUserStream(userID, false) if stream != nil { diff --git a/syncapi/sync/notifier_test.go b/syncapi/sync/notifier_test.go index 350d757c6..7d979fcc9 100644 --- a/syncapi/sync/notifier_test.go +++ b/syncapi/sync/notifier_test.go @@ -33,11 +33,11 @@ var ( randomMessageEvent gomatrixserverlib.HeaderedEvent aliceInviteBobEvent gomatrixserverlib.HeaderedEvent bobLeaveEvent gomatrixserverlib.HeaderedEvent - syncPositionVeryOld types.PaginationToken - syncPositionBefore types.PaginationToken - syncPositionAfter types.PaginationToken - syncPositionNewEDU types.PaginationToken - syncPositionAfter2 types.PaginationToken + syncPositionVeryOld = types.NewStreamToken(5, 0) + syncPositionBefore = types.NewStreamToken(11, 0) + syncPositionAfter = types.NewStreamToken(12, 0) + syncPositionNewEDU = types.NewStreamToken(syncPositionAfter.PDUPosition(), 1) + syncPositionAfter2 = types.NewStreamToken(13, 0) ) var ( @@ -47,26 +47,6 @@ var ( ) func init() { - baseSyncPos := types.PaginationToken{ - PDUPosition: 0, - EDUTypingPosition: 0, - } - - syncPositionVeryOld = baseSyncPos - syncPositionVeryOld.PDUPosition = 5 - - syncPositionBefore = baseSyncPos - syncPositionBefore.PDUPosition = 11 - - syncPositionAfter = baseSyncPos - syncPositionAfter.PDUPosition = 12 - - syncPositionNewEDU = syncPositionAfter - syncPositionNewEDU.EDUTypingPosition = 1 - - syncPositionAfter2 = baseSyncPos - syncPositionAfter2.PDUPosition = 13 - var err error err = json.Unmarshal([]byte(`{ "_room_version": "1", @@ -118,6 +98,12 @@ func init() { } } +func mustEqualPositions(t *testing.T, got, want types.StreamingToken) { + if got.String() != want.String() { + t.Fatalf("mustEqualPositions got %s want %s", got.String(), want.String()) + } +} + // Test that the current position is returned if a request is already behind. func TestImmediateNotification(t *testing.T) { n := NewNotifier(syncPositionBefore) @@ -125,9 +111,7 @@ func TestImmediateNotification(t *testing.T) { if err != nil { t.Fatalf("TestImmediateNotification error: %s", err) } - if pos != syncPositionBefore { - t.Fatalf("TestImmediateNotification want %v, got %v", syncPositionBefore, pos) - } + mustEqualPositions(t, pos, syncPositionBefore) } // Test that new events to a joined room unblocks the request. @@ -144,9 +128,7 @@ func TestNewEventAndJoinedToRoom(t *testing.T) { if err != nil { t.Errorf("TestNewEventAndJoinedToRoom error: %w", err) } - if pos != syncPositionAfter { - t.Errorf("TestNewEventAndJoinedToRoom want %v, got %v", syncPositionAfter, pos) - } + mustEqualPositions(t, pos, syncPositionAfter) wg.Done() }() @@ -172,9 +154,7 @@ func TestNewInviteEventForUser(t *testing.T) { if err != nil { t.Errorf("TestNewInviteEventForUser error: %w", err) } - if pos != syncPositionAfter { - t.Errorf("TestNewInviteEventForUser want %v, got %v", syncPositionAfter, pos) - } + mustEqualPositions(t, pos, syncPositionAfter) wg.Done() }() @@ -200,9 +180,7 @@ func TestEDUWakeup(t *testing.T) { if err != nil { t.Errorf("TestNewInviteEventForUser error: %w", err) } - if pos != syncPositionNewEDU { - t.Errorf("TestNewInviteEventForUser want %v, got %v", syncPositionNewEDU, pos) - } + mustEqualPositions(t, pos, syncPositionNewEDU) wg.Done() }() @@ -228,9 +206,7 @@ func TestMultipleRequestWakeup(t *testing.T) { if err != nil { t.Errorf("TestMultipleRequestWakeup error: %w", err) } - if pos != syncPositionAfter { - t.Errorf("TestMultipleRequestWakeup want %v, got %v", syncPositionAfter, pos) - } + mustEqualPositions(t, pos, syncPositionAfter) wg.Done() } go poll() @@ -268,9 +244,7 @@ func TestNewEventAndWasPreviouslyJoinedToRoom(t *testing.T) { if err != nil { t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom error: %w", err) } - if pos != syncPositionAfter { - t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom want %v, got %v", syncPositionAfter, pos) - } + mustEqualPositions(t, pos, syncPositionAfter) leaveWG.Done() }() bobStream := lockedFetchUserStream(n, bob) @@ -287,9 +261,7 @@ func TestNewEventAndWasPreviouslyJoinedToRoom(t *testing.T) { if err != nil { t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom error: %w", err) } - if pos != syncPositionAfter2 { - t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom want %v, got %v", syncPositionAfter2, pos) - } + mustEqualPositions(t, pos, syncPositionAfter2) aliceWG.Done() }() @@ -312,13 +284,13 @@ func TestNewEventAndWasPreviouslyJoinedToRoom(t *testing.T) { time.Sleep(1 * time.Millisecond) } -func waitForEvents(n *Notifier, req syncRequest) (types.PaginationToken, error) { +func waitForEvents(n *Notifier, req syncRequest) (types.StreamingToken, error) { listener := n.GetListener(req) defer listener.Close() select { case <-time.After(5 * time.Second): - return types.PaginationToken{}, fmt.Errorf( + return types.StreamingToken{}, fmt.Errorf( "waitForEvents timed out waiting for %s (pos=%v)", req.device.UserID, req.since, ) case <-listener.GetNotifyChannel(*req.since): @@ -344,7 +316,7 @@ func lockedFetchUserStream(n *Notifier, userID string) *UserStream { return n.fetchUserStream(userID, true) } -func newTestSyncRequest(userID string, since types.PaginationToken) syncRequest { +func newTestSyncRequest(userID string, since types.StreamingToken) syncRequest { return syncRequest{ device: authtypes.Device{UserID: userID}, timeout: 1 * time.Minute, diff --git a/syncapi/sync/request.go b/syncapi/sync/request.go index f2e199d23..66663cf0a 100644 --- a/syncapi/sync/request.go +++ b/syncapi/sync/request.go @@ -36,7 +36,7 @@ type syncRequest struct { device authtypes.Device limit int timeout time.Duration - since *types.PaginationToken // nil means that no since token was supplied + since *types.StreamingToken // nil means that no since token was supplied wantFullState bool log *log.Entry } @@ -45,9 +45,14 @@ func newSyncRequest(req *http.Request, device authtypes.Device) (*syncRequest, e timeout := getTimeout(req.URL.Query().Get("timeout")) fullState := req.URL.Query().Get("full_state") wantFullState := fullState != "" && fullState != "false" - since, err := getPaginationToken(req.URL.Query().Get("since")) - if err != nil { - return nil, err + var since *types.StreamingToken + sinceStr := req.URL.Query().Get("since") + if sinceStr != "" { + tok, err := types.NewStreamTokenFromString(sinceStr) + if err != nil { + return nil, err + } + since = &tok } // TODO: Additional query params: set_presence, filter return &syncRequest{ @@ -71,16 +76,3 @@ func getTimeout(timeoutMS string) time.Duration { } return time.Duration(i) * time.Millisecond } - -// getSyncStreamPosition tries to parse a 'since' token taken from the API to a -// types.PaginationToken. If the string is empty then (nil, nil) is returned. -// There are two forms of tokens: The full length form containing all PDU and EDU -// positions separated by "_", and the short form containing only the PDU -// position. Short form can be used for, e.g., `prev_batch` tokens. -func getPaginationToken(since string) (*types.PaginationToken, error) { - if since == "" { - return nil, nil - } - - return types.NewPaginationTokenFromString(since) -} diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index 69efd8aa8..126e76f5b 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -132,7 +132,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *authtype } } -func (rp *RequestPool) currentSyncForUser(req syncRequest, latestPos types.PaginationToken) (res *types.Response, err error) { +func (rp *RequestPool) currentSyncForUser(req syncRequest, latestPos types.StreamingToken) (res *types.Response, err error) { // TODO: handle ignored users if req.since == nil { res, err = rp.db.CompleteSync(req.ctx, req.device.UserID, req.limit) @@ -145,7 +145,7 @@ func (rp *RequestPool) currentSyncForUser(req syncRequest, latestPos types.Pagin } accountDataFilter := gomatrixserverlib.DefaultEventFilter() // TODO: use filter provided in req instead - res, err = rp.appendAccountData(res, req.device.UserID, req, latestPos.PDUPosition, &accountDataFilter) + res, err = rp.appendAccountData(res, req.device.UserID, req, latestPos.PDUPosition(), &accountDataFilter) return } @@ -187,7 +187,7 @@ func (rp *RequestPool) appendAccountData( // Sync is not initial, get all account data since the latest sync dataTypes, err := rp.db.GetAccountDataInRange( req.ctx, userID, - types.StreamPosition(req.since.PDUPosition), types.StreamPosition(currentPos), + types.StreamPosition(req.since.PDUPosition()), types.StreamPosition(currentPos), accountDataFilter, ) if err != nil { diff --git a/syncapi/sync/userstream.go b/syncapi/sync/userstream.go index 88867005e..b2eafa3dc 100644 --- a/syncapi/sync/userstream.go +++ b/syncapi/sync/userstream.go @@ -34,7 +34,7 @@ type UserStream struct { // Closed when there is an update. signalChannel chan struct{} // The last sync position that there may have been an update for the user - pos types.PaginationToken + pos types.StreamingToken // The last time when we had some listeners waiting timeOfLastChannel time.Time // The number of listeners waiting @@ -50,7 +50,7 @@ type UserStreamListener struct { } // NewUserStream creates a new user stream -func NewUserStream(userID string, currPos types.PaginationToken) *UserStream { +func NewUserStream(userID string, currPos types.StreamingToken) *UserStream { return &UserStream{ UserID: userID, timeOfLastChannel: time.Now(), @@ -83,7 +83,7 @@ func (s *UserStream) GetListener(ctx context.Context) UserStreamListener { } // Broadcast a new sync position for this user. -func (s *UserStream) Broadcast(pos types.PaginationToken) { +func (s *UserStream) Broadcast(pos types.StreamingToken) { s.lock.Lock() defer s.lock.Unlock() @@ -116,9 +116,9 @@ func (s *UserStream) TimeOfLastNonEmpty() time.Time { return s.timeOfLastChannel } -// GetStreamPosition returns last sync position which the UserStream was +// GetSyncPosition returns last sync position which the UserStream was // notified about -func (s *UserStreamListener) GetSyncPosition() types.PaginationToken { +func (s *UserStreamListener) GetSyncPosition() types.StreamingToken { s.userStream.lock.Lock() defer s.userStream.lock.Unlock() @@ -130,7 +130,7 @@ func (s *UserStreamListener) GetSyncPosition() types.PaginationToken { // sincePos specifies from which point we want to be notified about. If there // has already been an update after sincePos we'll return a closed channel // immediately. -func (s *UserStreamListener) GetNotifyChannel(sincePos types.PaginationToken) <-chan struct{} { +func (s *UserStreamListener) GetNotifyChannel(sincePos types.StreamingToken) <-chan struct{} { s.userStream.lock.Lock() defer s.userStream.lock.Unlock() diff --git a/syncapi/types/types.go b/syncapi/types/types.go index c04fe5219..c1b6d7dd5 100644 --- a/syncapi/types/types.go +++ b/syncapi/types/types.go @@ -27,19 +27,19 @@ import ( ) var ( - // ErrInvalidPaginationTokenType is returned when an attempt at creating a - // new instance of PaginationToken with an invalid type (i.e. neither "s" + // ErrInvalidSyncTokenType is returned when an attempt at creating a + // new instance of SyncToken with an invalid type (i.e. neither "s" // nor "t"). - ErrInvalidPaginationTokenType = fmt.Errorf("Pagination token has an unknown prefix (should be either s or t)") - // ErrInvalidPaginationTokenLen is returned when the pagination token is an + ErrInvalidSyncTokenType = fmt.Errorf("Sync token has an unknown prefix (should be either s or t)") + // ErrInvalidSyncTokenLen is returned when the pagination token is an // invalid length - ErrInvalidPaginationTokenLen = fmt.Errorf("Pagination token has an invalid length") + ErrInvalidSyncTokenLen = fmt.Errorf("Sync token has an invalid length") ) // StreamPosition represents the offset in the sync stream a client is at. type StreamPosition int64 -// Same as gomatrixserverlib.Event but also has the PDU stream position for this event. +// StreamEvent is the same as gomatrixserverlib.Event but also has the PDU stream position for this event. type StreamEvent struct { gomatrixserverlib.HeaderedEvent StreamPosition StreamPosition @@ -47,118 +47,201 @@ type StreamEvent struct { ExcludeFromSync bool } -// PaginationTokenType represents the type of a pagination token. +// SyncTokenType represents the type of a sync token. // It can be either "s" (representing a position in the whole stream of events) // or "t" (representing a position in a room's topology/depth). -type PaginationTokenType string +type SyncTokenType string const ( - // PaginationTokenTypeStream represents a position in the server's whole + // SyncTokenTypeStream represents a position in the server's whole // stream of events - PaginationTokenTypeStream PaginationTokenType = "s" - // PaginationTokenTypeTopology represents a position in a room's topology. - PaginationTokenTypeTopology PaginationTokenType = "t" + SyncTokenTypeStream SyncTokenType = "s" + // SyncTokenTypeTopology represents a position in a room's topology. + SyncTokenTypeTopology SyncTokenType = "t" ) -// PaginationToken represents a pagination token, used for interactions with -// /sync or /messages, for example. -type PaginationToken struct { - //Position StreamPosition - Type PaginationTokenType - // For /sync, this is the PDU position. For /messages, this is the topological position (depth). - // TODO: Given how different the positions are depending on the token type, they should probably be renamed - // or use different structs altogether. - PDUPosition StreamPosition - // For /sync, this is the EDU position. For /messages, this is the stream (PDU) position. - // TODO: Given how different the positions are depending on the token type, they should probably be renamed - // or use different structs altogether. - EDUTypingPosition StreamPosition +type StreamingToken struct { + syncToken } -// NewPaginationTokenFromString takes a string of the form "xyyyy..." where "x" -// represents the type of a pagination token and "yyyy..." the token itself, and -// parses it in order to create a new instance of PaginationToken. Returns an -// error if the token couldn't be parsed into an int64, or if the token type -// isn't a known type (returns ErrInvalidPaginationTokenType in the latter -// case). -func NewPaginationTokenFromString(s string) (token *PaginationToken, err error) { - if len(s) == 0 { - return nil, ErrInvalidPaginationTokenLen - } +func (t *StreamingToken) PDUPosition() StreamPosition { + return t.Positions[0] +} +func (t *StreamingToken) EDUPosition() StreamPosition { + return t.Positions[1] +} - token = new(PaginationToken) - var positions []string - - switch t := PaginationTokenType(s[:1]); t { - case PaginationTokenTypeStream, PaginationTokenTypeTopology: - token.Type = t - positions = strings.Split(s[1:], "_") - default: - token.Type = PaginationTokenTypeStream - positions = strings.Split(s, "_") - } - - // Try to get the PDU position. - if len(positions) >= 1 { - if pduPos, err := strconv.ParseInt(positions[0], 10, 64); err != nil { - return nil, err - } else if pduPos < 0 { - return nil, errors.New("negative PDU position not allowed") - } else { - token.PDUPosition = StreamPosition(pduPos) +// IsAfter returns true if ANY position in this token is greater than `other`. +func (t *StreamingToken) IsAfter(other StreamingToken) bool { + for i := range other.Positions { + if t.Positions[i] > other.Positions[i] { + return true } } + return false +} - // Try to get the typing position. - if len(positions) >= 2 { - if typPos, err := strconv.ParseInt(positions[1], 10, 64); err != nil { - return nil, err - } else if typPos < 0 { - return nil, errors.New("negative EDU typing position not allowed") - } else { - token.EDUTypingPosition = StreamPosition(typPos) +// WithUpdates returns a copy of the StreamingToken with updates applied from another StreamingToken. +// If the latter StreamingToken contains a field that is not 0, it is considered an update, +// and its value will replace the corresponding value in the StreamingToken on which WithUpdates is called. +func (t *StreamingToken) WithUpdates(other StreamingToken) (ret StreamingToken) { + ret.Type = t.Type + ret.Positions = make([]StreamPosition, len(t.Positions)) + for i := range t.Positions { + ret.Positions[i] = t.Positions[i] + if other.Positions[i] == 0 { + continue } - } - - return -} - -// NewPaginationTokenFromTypeAndPosition takes a PaginationTokenType and a -// StreamPosition and returns an instance of PaginationToken. -func NewPaginationTokenFromTypeAndPosition( - t PaginationTokenType, pdupos StreamPosition, typpos StreamPosition, -) (p *PaginationToken) { - return &PaginationToken{ - Type: t, - PDUPosition: pdupos, - EDUTypingPosition: typpos, - } -} - -// String translates a PaginationToken to a string of the "xyyyy..." (see -// NewPaginationToken to know what it represents). -func (p *PaginationToken) String() string { - return fmt.Sprintf("%s%d_%d", p.Type, p.PDUPosition, p.EDUTypingPosition) -} - -// WithUpdates returns a copy of the PaginationToken with updates applied from another PaginationToken. -// If the latter PaginationToken contains a field that is not 0, it is considered an update, -// and its value will replace the corresponding value in the PaginationToken on which WithUpdates is called. -func (pt *PaginationToken) WithUpdates(other PaginationToken) PaginationToken { - ret := *pt - if other.PDUPosition != 0 { - ret.PDUPosition = other.PDUPosition - } - if other.EDUTypingPosition != 0 { - ret.EDUTypingPosition = other.EDUTypingPosition + ret.Positions[i] = other.Positions[i] } return ret } -// IsAfter returns whether one PaginationToken refers to states newer than another PaginationToken. -func (sp *PaginationToken) IsAfter(other PaginationToken) bool { - return sp.PDUPosition > other.PDUPosition || - sp.EDUTypingPosition > other.EDUTypingPosition +type TopologyToken struct { + syncToken +} + +func (t *TopologyToken) Depth() StreamPosition { + return t.Positions[0] +} +func (t *TopologyToken) PDUPosition() StreamPosition { + return t.Positions[1] +} +func (t *TopologyToken) StreamToken() StreamingToken { + return NewStreamToken(t.PDUPosition(), 0) +} +func (t *TopologyToken) String() string { + return t.syncToken.String() +} + +// Decrement the topology token to one event earlier. +func (t *TopologyToken) Decrement() { + depth := t.Positions[0] + pduPos := t.Positions[1] + if depth-1 <= 0 { + depth = 1 + } else { + depth-- + pduPos += 1000 + } + // The lowest token value is 1, therefore we need to manually set it to that + // value if we're below it. + if depth < 1 { + depth = 1 + } + t.Positions = []StreamPosition{ + depth, pduPos, + } +} + +// NewSyncTokenFromString takes a string of the form "xyyyy..." where "x" +// represents the type of a pagination token and "yyyy..." the token itself, and +// parses it in order to create a new instance of SyncToken. Returns an +// error if the token couldn't be parsed into an int64, or if the token type +// isn't a known type (returns ErrInvalidSyncTokenType in the latter +// case). +func newSyncTokenFromString(s string) (token *syncToken, err error) { + if len(s) == 0 { + return nil, ErrInvalidSyncTokenLen + } + + token = new(syncToken) + var positions []string + + switch t := SyncTokenType(s[:1]); t { + case SyncTokenTypeStream, SyncTokenTypeTopology: + token.Type = t + positions = strings.Split(s[1:], "_") + default: + return nil, ErrInvalidSyncTokenType + } + + for _, pos := range positions { + if posInt, err := strconv.ParseInt(pos, 10, 64); err != nil { + return nil, err + } else if posInt < 0 { + return nil, errors.New("negative position not allowed") + } else { + token.Positions = append(token.Positions, StreamPosition(posInt)) + } + } + return +} + +// NewTopologyToken creates a new sync token for /messages +func NewTopologyToken(depth, streamPos StreamPosition) TopologyToken { + if depth < 0 { + depth = 1 + } + return TopologyToken{ + syncToken: syncToken{ + Type: SyncTokenTypeTopology, + Positions: []StreamPosition{depth, streamPos}, + }, + } +} +func NewTopologyTokenFromString(tok string) (token TopologyToken, err error) { + t, err := newSyncTokenFromString(tok) + if err != nil { + return + } + if t.Type != SyncTokenTypeTopology { + err = fmt.Errorf("token %s is not a topology token", tok) + return + } + if len(t.Positions) != 2 { + err = fmt.Errorf("token %s wrong number of values, got %d want 2", tok, len(t.Positions)) + return + } + return TopologyToken{ + syncToken: *t, + }, nil +} + +// NewStreamToken creates a new sync token for /sync +func NewStreamToken(pduPos, eduPos StreamPosition) StreamingToken { + return StreamingToken{ + syncToken: syncToken{ + Type: SyncTokenTypeStream, + Positions: []StreamPosition{pduPos, eduPos}, + }, + } +} +func NewStreamTokenFromString(tok string) (token StreamingToken, err error) { + t, err := newSyncTokenFromString(tok) + if err != nil { + return + } + if t.Type != SyncTokenTypeStream { + err = fmt.Errorf("token %s is not a streaming token", tok) + return + } + if len(t.Positions) != 2 { + err = fmt.Errorf("token %s wrong number of values, got %d want 2", tok, len(t.Positions)) + return + } + return StreamingToken{ + syncToken: *t, + }, nil +} + +// syncToken represents a syncapi token, used for interactions with +// /sync or /messages, for example. +type syncToken struct { + Type SyncTokenType + // A list of stream positions, their meanings vary depending on the token type. + Positions []StreamPosition +} + +// String translates a SyncToken to a string of the "xyyyy..." (see +// NewSyncToken to know what it represents). +func (p *syncToken) String() string { + posStr := make([]string, len(p.Positions)) + for i := range p.Positions { + posStr[i] = strconv.FormatInt(int64(p.Positions[i]), 10) + } + + return fmt.Sprintf("%s%s", p.Type, strings.Join(posStr, "_")) } // PrevEventRef represents a reference to a previous event in a state event upgrade @@ -185,7 +268,7 @@ type Response struct { } // NewResponse creates an empty response with initialised maps. -func NewResponse(token PaginationToken) *Response { +func NewResponse(token StreamingToken) *Response { res := Response{ NextBatch: token.String(), } @@ -202,14 +285,6 @@ func NewResponse(token PaginationToken) *Response { res.AccountData.Events = make([]gomatrixserverlib.ClientEvent, 0) res.Presence.Events = make([]gomatrixserverlib.ClientEvent, 0) - // Fill next_batch with a pagination token. Since this is a response to a sync request, we can assume - // we'll always return a stream token. - res.NextBatch = NewPaginationTokenFromTypeAndPosition( - PaginationTokenTypeStream, - StreamPosition(token.PDUPosition), - StreamPosition(token.EDUTypingPosition), - ).String() - return &res } diff --git a/syncapi/types/types_test.go b/syncapi/types/types_test.go index f4c84e0d1..1e27a8e32 100644 --- a/syncapi/types/types_test.go +++ b/syncapi/types/types_test.go @@ -2,26 +2,11 @@ package types import "testing" -func TestNewPaginationTokenFromString(t *testing.T) { - shouldPass := map[string]PaginationToken{ - "2": PaginationToken{ - Type: PaginationTokenTypeStream, - PDUPosition: 2, - }, - "s4": PaginationToken{ - Type: PaginationTokenTypeStream, - PDUPosition: 4, - }, - "s3_1": PaginationToken{ - Type: PaginationTokenTypeStream, - PDUPosition: 3, - EDUTypingPosition: 1, - }, - "t3_1_4": PaginationToken{ - Type: PaginationTokenTypeTopology, - PDUPosition: 3, - EDUTypingPosition: 1, - }, +func TestNewSyncTokenFromString(t *testing.T) { + shouldPass := map[string]syncToken{ + "s4_0": NewStreamToken(4, 0).syncToken, + "s3_1": NewStreamToken(3, 1).syncToken, + "t3_1": NewTopologyToken(3, 1).syncToken, } shouldFail := []string{ @@ -32,20 +17,21 @@ func TestNewPaginationTokenFromString(t *testing.T) { "b", "b-1", "-4", + "2", } for test, expected := range shouldPass { - result, err := NewPaginationTokenFromString(test) + result, err := newSyncTokenFromString(test) if err != nil { t.Error(err) } - if *result != expected { - t.Errorf("expected %v but got %v", expected.String(), result.String()) + if result.String() != expected.String() { + t.Errorf("%s expected %v but got %v", test, expected.String(), result.String()) } } for _, test := range shouldFail { - if _, err := NewPaginationTokenFromString(test); err == nil { + if _, err := newSyncTokenFromString(test); err == nil { t.Errorf("input '%v' should have errored but didn't", test) } } From ee140c9d6a18646f1bf9765362a6f3b7ffeb7075 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 13 May 2020 13:01:45 +0100 Subject: [PATCH 050/200] Reduce 500s (#1017) * Try to avoid returning 500s on /send * Don't return 500s from media API download requests * Don't 500 on context errors * Update sytest-whitelist * Fix lint, add comments --- federationapi/routing/send.go | 86 ++++++++++++++++++++--------------- 1 file changed, 50 insertions(+), 36 deletions(-) diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index 990220741..e62855abb 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -74,28 +74,17 @@ func Send( util.GetLogger(httpReq.Context()).Infof("Received transaction %q containing %d PDUs, %d EDUs", txnID, len(t.PDUs), len(t.EDUs)) resp, err := t.processTransaction() - switch err.(type) { - // No error? Great! Send back a 200. - case nil: - return util.JSONResponse{ - Code: http.StatusOK, - JSON: resp, - } - // Handle known error cases as we will return a 400 error for these. - case roomNotFoundError: - case unmarshalError: - case verifySigError: - // Handle unknown error cases. Sending 500 errors back should be a last - // resort as this can make other homeservers back off sending federation - // events. - default: + if err != nil { util.GetLogger(httpReq.Context()).WithError(err).Error("t.processTransaction failed") - return jsonerror.InternalServerError() } - // Return a 400 error for bad requests as fallen through from above. + + // https://matrix.org/docs/spec/server_server/r0.1.3#put-matrix-federation-v1-send-txnid + // Status code 200: + // The result of processing the transaction. The server is to use this response + // even in the event of one or more PDUs failing to be processed. return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.BadJSON(err.Error()), + Code: http.StatusOK, + JSON: resp, } } @@ -135,30 +124,39 @@ func (t *txnReq) processTransaction() (*gomatrixserverlib.RespSend, error) { } if err := json.Unmarshal(pdu, &header); err != nil { util.GetLogger(t.context).WithError(err).Warn("Transaction: Failed to extract room ID from event") - return nil, unmarshalError{err} + // We don't know the event ID at this point so we can't return the + // failure in the PDU results + continue } verReq := api.QueryRoomVersionForRoomRequest{RoomID: header.RoomID} verRes := api.QueryRoomVersionForRoomResponse{} if err := t.rsAPI.QueryRoomVersionForRoom(t.context, &verReq, &verRes); err != nil { util.GetLogger(t.context).WithError(err).Warn("Transaction: Failed to query room version for room", verReq.RoomID) - return nil, roomNotFoundError{verReq.RoomID} + // We don't know the event ID at this point so we can't return the + // failure in the PDU results + continue } event, err := gomatrixserverlib.NewEventFromUntrustedJSON(pdu, verRes.RoomVersion) if err != nil { util.GetLogger(t.context).WithError(err).Warnf("Transaction: Failed to parse event JSON of event %q", event.EventID()) - return nil, unmarshalError{err} + results[event.EventID()] = gomatrixserverlib.PDUResult{ + Error: err.Error(), + } + continue } - if err := gomatrixserverlib.VerifyAllEventSignatures(t.context, []gomatrixserverlib.Event{event}, t.keys); err != nil { + if err = gomatrixserverlib.VerifyAllEventSignatures(t.context, []gomatrixserverlib.Event{event}, t.keys); err != nil { util.GetLogger(t.context).WithError(err).Warnf("Transaction: Couldn't validate signature of event %q", event.EventID()) - return nil, verifySigError{event.EventID(), err} + results[event.EventID()] = gomatrixserverlib.PDUResult{ + Error: err.Error(), + } + continue } pdus = append(pdus, event.Headered(verRes.RoomVersion)) } // Process the events. for _, e := range pdus { - err := t.processEvent(e.Unwrap(), true) - if err != nil { + if err := t.processEvent(e.Unwrap(), true); err != nil { // If the error is due to the event itself being bad then we skip // it and move onto the next event. We report an error so that the // sender knows that we have skipped processing it. @@ -174,20 +172,17 @@ func (t *txnReq) processTransaction() (*gomatrixserverlib.RespSend, error) { // receive another event referencing it. // If we bail and stop processing then we risk wedging incoming // transactions from that server forever. - switch err.(type) { - case roomNotFoundError: - case *gomatrixserverlib.NotAllowed: - case missingPrevEventsError: - default: - util.GetLogger(t.context).Warnf("Processing %s failed: %s", e.EventID(), err) + if isProcessingErrorFatal(err) { // Any other error should be the result of a temporary error in // our server so we should bail processing the transaction entirely. + util.GetLogger(t.context).Warnf("Processing %s failed fatally: %s", e.EventID(), err) return nil, err + } else { + util.GetLogger(t.context).WithError(err).WithField("event_id", e.EventID()).Warn("Failed to process incoming federation event, skipping") + results[e.EventID()] = gomatrixserverlib.PDUResult{ + Error: err.Error(), + } } - results[e.EventID()] = gomatrixserverlib.PDUResult{ - Error: err.Error(), - } - util.GetLogger(t.context).WithError(err).WithField("event_id", e.EventID()).Warn("Failed to process incoming federation event, skipping it.") } else { results[e.EventID()] = gomatrixserverlib.PDUResult{} } @@ -198,6 +193,25 @@ func (t *txnReq) processTransaction() (*gomatrixserverlib.RespSend, error) { return &gomatrixserverlib.RespSend{PDUs: results}, nil } +// isProcessingErrorFatal returns true if the error is really bad and +// we should stop processing the transaction, and returns false if it +// is just some less serious error about a specific event. +func isProcessingErrorFatal(err error) bool { + switch err.(type) { + case roomNotFoundError: + case *gomatrixserverlib.NotAllowed: + case missingPrevEventsError: + default: + switch err { + case context.Canceled: + case context.DeadlineExceeded: + default: + return true + } + } + return false +} + type roomNotFoundError struct { roomID string } From 37b2299b71875bafbbf58c4d539e3688f4aeda05 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Wed, 13 May 2020 13:02:34 +0100 Subject: [PATCH 051/200] Fix #1015 (#1026) Co-authored-by: Neil Alexander --- syncapi/storage/postgres/syncserver.go | 18 +++++++++++------- syncapi/storage/sqlite3/syncserver.go | 17 ++++++++++++----- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go index d45bc09e5..909761688 100644 --- a/syncapi/storage/postgres/syncserver.go +++ b/syncapi/storage/postgres/syncserver.go @@ -584,20 +584,24 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( // Retrieve the backward topology position, i.e. the position of the // oldest event in the room's topology. - var backwardTopologyPos, backwardStreamPos types.StreamPosition - backwardTopologyPos, backwardStreamPos, err = d.topology.selectPositionInTopology(ctx, recentStreamEvents[0].EventID()) - if err != nil { - return + var prevBatchStr string + if len(recentStreamEvents) > 0 { + var backwardTopologyPos, backwardStreamPos types.StreamPosition + backwardTopologyPos, backwardStreamPos, err = d.topology.selectPositionInTopology(ctx, recentStreamEvents[0].EventID()) + if err != nil { + return + } + prevBatch := types.NewTopologyToken(backwardTopologyPos, backwardStreamPos) + prevBatch.Decrement() + prevBatchStr = prevBatch.String() } - prevBatch := types.NewTopologyToken(backwardTopologyPos, backwardStreamPos) - prevBatch.Decrement() // We don't include a device here as we don't need to send down // transaction IDs for complete syncs recentEvents := d.StreamEventsToEvents(nil, recentStreamEvents) stateEvents = removeDuplicates(stateEvents, recentEvents) jr := types.NewJoinResponse() - jr.Timeline.PrevBatch = prevBatch.String() + jr.Timeline.PrevBatch = prevBatchStr jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = true jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(stateEvents, gomatrixserverlib.FormatSync) diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index 212f882b1..a807b0c17 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -661,17 +661,24 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( // Retrieve the backward topology position, i.e. the position of the // oldest event in the room's topology. - var backwardTopologyPos, backwardTopologyStreamPos types.StreamPosition - backwardTopologyPos, backwardTopologyStreamPos, err = d.topology.selectPositionInTopology(ctx, txn, recentStreamEvents[0].EventID()) - prevBatch := types.NewTopologyToken(backwardTopologyPos, backwardTopologyStreamPos) - prevBatch.Decrement() + var prevBatchStr string + if len(recentStreamEvents) > 0 { + var backwardTopologyPos, backwardStreamPos types.StreamPosition + backwardTopologyPos, backwardStreamPos, err = d.topology.selectPositionInTopology(ctx, txn, recentStreamEvents[0].EventID()) + if err != nil { + return + } + prevBatch := types.NewTopologyToken(backwardTopologyPos, backwardStreamPos) + prevBatch.Decrement() + prevBatchStr = prevBatch.String() + } // We don't include a device here as we don't need to send down // transaction IDs for complete syncs recentEvents := d.StreamEventsToEvents(nil, recentStreamEvents) stateEvents = removeDuplicates(stateEvents, recentEvents) jr := types.NewJoinResponse() - jr.Timeline.PrevBatch = prevBatch.String() + jr.Timeline.PrevBatch = prevBatchStr jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = true jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(stateEvents, gomatrixserverlib.FormatSync) From 334680a216b98389efbcc758648a6385f5ac00d8 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 13 May 2020 14:53:25 +0100 Subject: [PATCH 052/200] Get profile content for join request again (#1027) --- clientapi/routing/joinroom.go | 18 ++++++++++++++++++ clientapi/routing/routing.go | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/clientapi/routing/joinroom.go b/clientapi/routing/joinroom.go index 48e42214d..2a2fce185 100644 --- a/clientapi/routing/joinroom.go +++ b/clientapi/routing/joinroom.go @@ -18,9 +18,11 @@ import ( "net/http" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -28,6 +30,7 @@ func JoinRoomByIDOrAlias( req *http.Request, device *authtypes.Device, rsAPI roomserverAPI.RoomserverInternalAPI, + accountDB accounts.Database, roomIDOrAlias string, ) util.JSONResponse { // Prepare to ask the roomserver to perform the room join. @@ -44,6 +47,21 @@ func JoinRoomByIDOrAlias( return *err } + // Work out our localpart for the client profile request. + localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") + } else { + // Request our profile content to populate the request content with. + profile, err := accountDB.GetProfileByLocalpart(req.Context(), localpart) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("accountDB.GetProfileByLocalpart failed") + } else { + joinReq.Content["displayname"] = profile.DisplayName + joinReq.Content["avatar_url"] = profile.AvatarURL + } + } + // Ask the roomserver to perform the join. if err := rsAPI.PerformJoin(req.Context(), &joinReq, &joinRes); err != nil { return util.JSONResponse{ diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index ead8a4c85..f8cdcf5b8 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -100,7 +100,7 @@ func Setup( return util.ErrorResponse(err) } return JoinRoomByIDOrAlias( - req, device, rsAPI, vars["roomIDOrAlias"], + req, device, rsAPI, accountDB, vars["roomIDOrAlias"], ) }), ).Methods(http.MethodPost, http.MethodOptions) From bdddd83753e31c69e99a695b01cf9009ba7cb6c8 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 13 May 2020 15:00:47 +0100 Subject: [PATCH 053/200] Spurious logging --- syncapi/storage/sqlite3/syncserver.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index a807b0c17..84c562bd8 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -564,10 +564,8 @@ func (d *SyncServerDatasource) IncrementalSync( numRecentEventsPerRoom int, wantFullState bool, ) (*types.Response, error) { - fmt.Println("from ", fromPos, "to", toPos) nextBatchPos := fromPos.WithUpdates(toPos) res := types.NewResponse(nextBatchPos) - fmt.Println("from ", fromPos, "to", toPos, "next", nextBatchPos) var joinedRoomIDs []string var err error From a25d477cdb8f1ba49b3b5e9d931f808ae45b4853 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Wed, 13 May 2020 17:28:42 +0100 Subject: [PATCH 054/200] Initial syncapi storage refactor to share pq/sqlite code (#1030) * Initial syncapi storage refactor to share pq/sqlite code This goes down a different route than https://github.com/matrix-org/dendrite/pull/985 which tried to even reduce the boilerplate of `ExecContext` etc. The previous pattern fails badly when there are subtle differences in parameters and hence the shared boilerplate to read from `QueryContext` breaks. Rather than attacking it at that level, the main place where we want to reuse code is for the `syncserver.go` itself - the database implementation which has lots of complex logic. So instead, this commit: - Makes `invites_table.go` an interface. - Makes `SyncServerDatasource` use that interface - This means some functions are now identical for pq/sqlite, so factor them out to a temporary `shared.Database` struct which will grow until it replaces all of `SyncServerDatasource`. * Missing files --- syncapi/storage/postgres/invites_table.go | 28 ++++++------ syncapi/storage/postgres/syncserver.go | 25 ++++++----- syncapi/storage/shared/syncserver.go | 37 ++++++++++++++++ syncapi/storage/sqlite3/invites_table.go | 54 +++++++++++++---------- syncapi/storage/sqlite3/syncserver.go | 32 ++++++-------- syncapi/storage/tables/interface.go | 16 +++++++ 6 files changed, 127 insertions(+), 65 deletions(-) create mode 100644 syncapi/storage/shared/syncserver.go create mode 100644 syncapi/storage/tables/interface.go diff --git a/syncapi/storage/postgres/invites_table.go b/syncapi/storage/postgres/invites_table.go index ca0c64fb9..78ca4d6d8 100644 --- a/syncapi/storage/postgres/invites_table.go +++ b/syncapi/storage/postgres/invites_table.go @@ -21,6 +21,7 @@ import ( "encoding/json" "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -66,28 +67,29 @@ type inviteEventsStatements struct { selectMaxInviteIDStmt *sql.Stmt } -func (s *inviteEventsStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(inviteEventsSchema) +func NewPostgresInvitesTable(db *sql.DB) (tables.Invites, error) { + s := &inviteEventsStatements{} + _, err := db.Exec(inviteEventsSchema) if err != nil { - return + return nil, err } if s.insertInviteEventStmt, err = db.Prepare(insertInviteEventSQL); err != nil { - return + return nil, err } if s.selectInviteEventsInRangeStmt, err = db.Prepare(selectInviteEventsInRangeSQL); err != nil { - return + return nil, err } if s.deleteInviteEventStmt, err = db.Prepare(deleteInviteEventSQL); err != nil { - return + return nil, err } if s.selectMaxInviteIDStmt, err = db.Prepare(selectMaxInviteIDSQL); err != nil { - return + return nil, err } - return + return s, nil } -func (s *inviteEventsStatements) insertInviteEvent( - ctx context.Context, inviteEvent gomatrixserverlib.HeaderedEvent, +func (s *inviteEventsStatements) InsertInviteEvent( + ctx context.Context, txn *sql.Tx, inviteEvent gomatrixserverlib.HeaderedEvent, ) (streamPos types.StreamPosition, err error) { var headeredJSON []byte headeredJSON, err = json.Marshal(inviteEvent) @@ -105,7 +107,7 @@ func (s *inviteEventsStatements) insertInviteEvent( return } -func (s *inviteEventsStatements) deleteInviteEvent( +func (s *inviteEventsStatements) DeleteInviteEvent( ctx context.Context, inviteEventID string, ) error { _, err := s.deleteInviteEventStmt.ExecContext(ctx, inviteEventID) @@ -114,7 +116,7 @@ func (s *inviteEventsStatements) deleteInviteEvent( // selectInviteEventsInRange returns a map of room ID to invite event for the // active invites for the target user ID in the supplied range. -func (s *inviteEventsStatements) selectInviteEventsInRange( +func (s *inviteEventsStatements) SelectInviteEventsInRange( ctx context.Context, txn *sql.Tx, targetUserID string, startPos, endPos types.StreamPosition, ) (map[string]gomatrixserverlib.HeaderedEvent, error) { stmt := common.TxStmt(txn, s.selectInviteEventsInRangeStmt) @@ -143,7 +145,7 @@ func (s *inviteEventsStatements) selectInviteEventsInRange( return result, rows.Err() } -func (s *inviteEventsStatements) selectMaxInviteID( +func (s *inviteEventsStatements) SelectMaxInviteID( ctx context.Context, txn *sql.Tx, ) (id int64, err error) { var nullableID sql.NullInt64 diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go index 909761688..9883c3629 100644 --- a/syncapi/storage/postgres/syncserver.go +++ b/syncapi/storage/postgres/syncserver.go @@ -32,6 +32,7 @@ import ( _ "github.com/lib/pq" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/eduserver/cache" + "github.com/matrix-org/dendrite/syncapi/storage/shared" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -54,10 +55,10 @@ type SyncServerDatasource struct { accountData accountDataStatements events outputRoomEventsStatements roomstate currentRoomStateStatements - invites inviteEventsStatements eduCache *cache.EDUCache topology outputRoomEventsTopologyStatements backwardExtremities tables.BackwardsExtremities + shared *shared.Database } // NewSyncServerDatasource creates a new sync server database @@ -79,7 +80,8 @@ func NewSyncServerDatasource(dbDataSourceName string, dbProperties common.DbProp if err = d.roomstate.prepare(d.db); err != nil { return nil, err } - if err = d.invites.prepare(d.db); err != nil { + invites, err := NewPostgresInvitesTable(d.db) + if err != nil { return nil, err } if err = d.topology.prepare(d.db); err != nil { @@ -90,6 +92,10 @@ func NewSyncServerDatasource(dbDataSourceName string, dbProperties common.DbProp return nil, err } d.eduCache = cache.New() + d.shared = &shared.Database{ + DB: d.db, + Invites: invites, + } return &d, nil } @@ -340,7 +346,7 @@ func (d *SyncServerDatasource) syncStreamPositionTx( if maxAccountDataID > maxID { maxID = maxAccountDataID } - maxInviteID, err := d.invites.selectMaxInviteID(ctx, txn) + maxInviteID, err := d.shared.Invites.SelectMaxInviteID(ctx, txn) if err != nil { return 0, err } @@ -365,7 +371,7 @@ func (d *SyncServerDatasource) syncPositionTx( if maxAccountDataID > maxEventID { maxEventID = maxAccountDataID } - maxInviteID, err := d.invites.selectMaxInviteID(ctx, txn) + maxInviteID, err := d.shared.Invites.SelectMaxInviteID(ctx, txn) if err != nil { return sp, err } @@ -662,17 +668,14 @@ func (d *SyncServerDatasource) UpsertAccountData( func (d *SyncServerDatasource) AddInviteEvent( ctx context.Context, inviteEvent gomatrixserverlib.HeaderedEvent, -) (types.StreamPosition, error) { - return d.invites.insertInviteEvent(ctx, inviteEvent) +) (sp types.StreamPosition, err error) { + return d.shared.AddInviteEvent(ctx, inviteEvent) } func (d *SyncServerDatasource) RetireInviteEvent( ctx context.Context, inviteEventID string, ) error { - // TODO: Record that invite has been retired in a stream so that we can - // notify the user in an incremental sync. - err := d.invites.deleteInviteEvent(ctx, inviteEventID) - return err + return d.shared.RetireInviteEvent(ctx, inviteEventID) } func (d *SyncServerDatasource) SetTypingTimeoutCallback(fn cache.TimeoutCallbackFn) { @@ -697,7 +700,7 @@ func (d *SyncServerDatasource) addInvitesToResponse( fromPos, toPos types.StreamPosition, res *types.Response, ) error { - invites, err := d.invites.selectInviteEventsInRange( + invites, err := d.shared.Invites.SelectInviteEventsInRange( ctx, txn, userID, fromPos, toPos, ) if err != nil { diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go new file mode 100644 index 000000000..e89976df6 --- /dev/null +++ b/syncapi/storage/shared/syncserver.go @@ -0,0 +1,37 @@ +package shared + +import ( + "context" + "database/sql" + + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/syncapi/storage/tables" + "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/gomatrixserverlib" +) + +// Database is a temporary struct until we have made syncserver.go the same for both pq/sqlite +// For now this contains the shared functions +type Database struct { + DB *sql.DB + Invites tables.Invites +} + +func (d *Database) AddInviteEvent( + ctx context.Context, inviteEvent gomatrixserverlib.HeaderedEvent, +) (sp types.StreamPosition, err error) { + err = common.WithTransaction(d.DB, func(txn *sql.Tx) error { + sp, err = d.Invites.InsertInviteEvent(ctx, txn, inviteEvent) + return err + }) + return +} + +func (d *Database) RetireInviteEvent( + ctx context.Context, inviteEventID string, +) error { + // TODO: Record that invite has been retired in a stream so that we can + // notify the user in an incremental sync. + err := d.Invites.DeleteInviteEvent(ctx, inviteEventID) + return err +} diff --git a/syncapi/storage/sqlite3/invites_table.go b/syncapi/storage/sqlite3/invites_table.go index 22efeaeb0..26b3a3169 100644 --- a/syncapi/storage/sqlite3/invites_table.go +++ b/syncapi/storage/sqlite3/invites_table.go @@ -21,6 +21,7 @@ import ( "encoding/json" "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -62,30 +63,37 @@ type inviteEventsStatements struct { selectMaxInviteIDStmt *sql.Stmt } -func (s *inviteEventsStatements) prepare(db *sql.DB, streamID *streamIDStatements) (err error) { - s.streamIDStatements = streamID - _, err = db.Exec(inviteEventsSchema) +func NewSqliteInvitesTable(db *sql.DB, streamID *streamIDStatements) (tables.Invites, error) { + s := &inviteEventsStatements{ + streamIDStatements: streamID, + } + _, err := db.Exec(inviteEventsSchema) + if err != nil { + return nil, err + } + if s.insertInviteEventStmt, err = db.Prepare(insertInviteEventSQL); err != nil { + return nil, err + } + if s.selectInviteEventsInRangeStmt, err = db.Prepare(selectInviteEventsInRangeSQL); err != nil { + return nil, err + } + if s.deleteInviteEventStmt, err = db.Prepare(deleteInviteEventSQL); err != nil { + return nil, err + } + if s.selectMaxInviteIDStmt, err = db.Prepare(selectMaxInviteIDSQL); err != nil { + return nil, err + } + return s, nil +} + +func (s *inviteEventsStatements) InsertInviteEvent( + ctx context.Context, txn *sql.Tx, inviteEvent gomatrixserverlib.HeaderedEvent, +) (streamPos types.StreamPosition, err error) { + streamPos, err = s.streamIDStatements.nextStreamID(ctx, txn) if err != nil { return } - if s.insertInviteEventStmt, err = db.Prepare(insertInviteEventSQL); err != nil { - return - } - if s.selectInviteEventsInRangeStmt, err = db.Prepare(selectInviteEventsInRangeSQL); err != nil { - return - } - if s.deleteInviteEventStmt, err = db.Prepare(deleteInviteEventSQL); err != nil { - return - } - if s.selectMaxInviteIDStmt, err = db.Prepare(selectMaxInviteIDSQL); err != nil { - return - } - return -} -func (s *inviteEventsStatements) insertInviteEvent( - ctx context.Context, txn *sql.Tx, inviteEvent gomatrixserverlib.HeaderedEvent, streamPos types.StreamPosition, -) (err error) { var headeredJSON []byte headeredJSON, err = json.Marshal(inviteEvent) if err != nil { @@ -103,7 +111,7 @@ func (s *inviteEventsStatements) insertInviteEvent( return } -func (s *inviteEventsStatements) deleteInviteEvent( +func (s *inviteEventsStatements) DeleteInviteEvent( ctx context.Context, inviteEventID string, ) error { _, err := s.deleteInviteEventStmt.ExecContext(ctx, inviteEventID) @@ -112,7 +120,7 @@ func (s *inviteEventsStatements) deleteInviteEvent( // selectInviteEventsInRange returns a map of room ID to invite event for the // active invites for the target user ID in the supplied range. -func (s *inviteEventsStatements) selectInviteEventsInRange( +func (s *inviteEventsStatements) SelectInviteEventsInRange( ctx context.Context, txn *sql.Tx, targetUserID string, startPos, endPos types.StreamPosition, ) (map[string]gomatrixserverlib.HeaderedEvent, error) { stmt := common.TxStmt(txn, s.selectInviteEventsInRangeStmt) @@ -141,7 +149,7 @@ func (s *inviteEventsStatements) selectInviteEventsInRange( return result, nil } -func (s *inviteEventsStatements) selectMaxInviteID( +func (s *inviteEventsStatements) SelectMaxInviteID( ctx context.Context, txn *sql.Tx, ) (id int64, err error) { var nullableID sql.NullInt64 diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index 84c562bd8..a2253dcd1 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -35,6 +35,7 @@ import ( "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/eduserver/cache" + "github.com/matrix-org/dendrite/syncapi/storage/shared" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -58,10 +59,10 @@ type SyncServerDatasource struct { accountData accountDataStatements events outputRoomEventsStatements roomstate currentRoomStateStatements - invites inviteEventsStatements eduCache *cache.EDUCache topology outputRoomEventsTopologyStatements backwardExtremities tables.BackwardsExtremities + shared *shared.Database } // NewSyncServerDatasource creates a new sync server database @@ -106,7 +107,8 @@ func (d *SyncServerDatasource) prepare() (err error) { if err = d.roomstate.prepare(d.db, &d.streamID); err != nil { return err } - if err = d.invites.prepare(d.db, &d.streamID); err != nil { + invites, err := NewSqliteInvitesTable(d.db, &d.streamID) + if err != nil { return err } if err = d.topology.prepare(d.db); err != nil { @@ -116,6 +118,10 @@ func (d *SyncServerDatasource) prepare() (err error) { if err != nil { return err } + d.shared = &shared.Database{ + DB: d.db, + Invites: invites, + } return nil } @@ -404,7 +410,7 @@ func (d *SyncServerDatasource) syncStreamPositionTx( if maxAccountDataID > maxID { maxID = maxAccountDataID } - maxInviteID, err := d.invites.selectMaxInviteID(ctx, txn) + maxInviteID, err := d.shared.Invites.SelectMaxInviteID(ctx, txn) if err != nil { return 0, err } @@ -429,7 +435,7 @@ func (d *SyncServerDatasource) syncPositionTx( if maxAccountDataID > maxEventID { maxEventID = maxAccountDataID } - maxInviteID, err := d.invites.selectMaxInviteID(ctx, txn) + maxInviteID, err := d.shared.Invites.SelectMaxInviteID(ctx, txn) if err != nil { return nil, err } @@ -756,15 +762,8 @@ func (d *SyncServerDatasource) UpsertAccountData( // Returns an error if there was a problem communicating with the database. func (d *SyncServerDatasource) AddInviteEvent( ctx context.Context, inviteEvent gomatrixserverlib.HeaderedEvent, -) (streamPos types.StreamPosition, err error) { - err = common.WithTransaction(d.db, func(txn *sql.Tx) error { - streamPos, err = d.streamID.nextStreamID(ctx, txn) - if err != nil { - return err - } - return d.invites.insertInviteEvent(ctx, txn, inviteEvent, streamPos) - }) - return +) (sp types.StreamPosition, err error) { + return d.shared.AddInviteEvent(ctx, inviteEvent) } // RetireInviteEvent removes an old invite event from the database. @@ -772,10 +771,7 @@ func (d *SyncServerDatasource) AddInviteEvent( func (d *SyncServerDatasource) RetireInviteEvent( ctx context.Context, inviteEventID string, ) error { - // TODO: Record that invite has been retired in a stream so that we can - // notify the user in an incremental sync. - err := d.invites.deleteInviteEvent(ctx, inviteEventID) - return err + return d.shared.RetireInviteEvent(ctx, inviteEventID) } func (d *SyncServerDatasource) SetTypingTimeoutCallback(fn cache.TimeoutCallbackFn) { @@ -804,7 +800,7 @@ func (d *SyncServerDatasource) addInvitesToResponse( fromPos, toPos types.StreamPosition, res *types.Response, ) error { - invites, err := d.invites.selectInviteEventsInRange( + invites, err := d.shared.Invites.SelectInviteEventsInRange( ctx, txn, userID, fromPos, toPos, ) if err != nil { diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go new file mode 100644 index 000000000..1a9940524 --- /dev/null +++ b/syncapi/storage/tables/interface.go @@ -0,0 +1,16 @@ +package tables + +import ( + "context" + "database/sql" + + "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/gomatrixserverlib" +) + +type Invites interface { + InsertInviteEvent(ctx context.Context, txn *sql.Tx, inviteEvent gomatrixserverlib.HeaderedEvent) (streamPos types.StreamPosition, err error) + DeleteInviteEvent(ctx context.Context, inviteEventID string) error + SelectInviteEventsInRange(ctx context.Context, txn *sql.Tx, targetUserID string, startPos, endPos types.StreamPosition) (map[string]gomatrixserverlib.HeaderedEvent, error) + SelectMaxInviteID(ctx context.Context, txn *sql.Tx) (id int64, err error) +} From 9ed68a3125d9024f52bf89810abf3b203f4b25b7 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Thu, 14 May 2020 09:53:55 +0100 Subject: [PATCH 055/200] Factor out account data and events table (#1031) * Factor out account data * Factor out events table and EDU cache * linting * fix npe --- .../storage/postgres/account_data_table.go | 24 ++- .../postgres/output_room_events_table.go | 36 ++-- syncapi/storage/postgres/syncserver.go | 180 +++-------------- syncapi/storage/shared/syncserver.go | 151 +++++++++++++- syncapi/storage/sqlite3/account_data_table.go | 25 ++- .../sqlite3/output_room_events_table.go | 39 ++-- syncapi/storage/sqlite3/syncserver.go | 191 ++++-------------- syncapi/storage/tables/interface.go | 16 ++ 8 files changed, 300 insertions(+), 362 deletions(-) diff --git a/syncapi/storage/postgres/account_data_table.go b/syncapi/storage/postgres/account_data_table.go index d1e3b527f..58fb21983 100644 --- a/syncapi/storage/postgres/account_data_table.go +++ b/syncapi/storage/postgres/account_data_table.go @@ -21,6 +21,7 @@ import ( "github.com/lib/pq" "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -70,32 +71,33 @@ type accountDataStatements struct { selectMaxAccountDataIDStmt *sql.Stmt } -func (s *accountDataStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(accountDataSchema) +func NewPostgresAccountDataTable(db *sql.DB) (tables.AccountData, error) { + s := &accountDataStatements{} + _, err := db.Exec(accountDataSchema) if err != nil { - return + return nil, err } if s.insertAccountDataStmt, err = db.Prepare(insertAccountDataSQL); err != nil { - return + return nil, err } if s.selectAccountDataInRangeStmt, err = db.Prepare(selectAccountDataInRangeSQL); err != nil { - return + return nil, err } if s.selectMaxAccountDataIDStmt, err = db.Prepare(selectMaxAccountDataIDSQL); err != nil { - return + return nil, err } - return + return s, nil } -func (s *accountDataStatements) insertAccountData( - ctx context.Context, +func (s *accountDataStatements) InsertAccountData( + ctx context.Context, txn *sql.Tx, userID, roomID, dataType string, ) (pos types.StreamPosition, err error) { err = s.insertAccountDataStmt.QueryRowContext(ctx, userID, roomID, dataType).Scan(&pos) return } -func (s *accountDataStatements) selectAccountDataInRange( +func (s *accountDataStatements) SelectAccountDataInRange( ctx context.Context, userID string, oldPos, newPos types.StreamPosition, @@ -137,7 +139,7 @@ func (s *accountDataStatements) selectAccountDataInRange( return data, rows.Err() } -func (s *accountDataStatements) selectMaxAccountDataID( +func (s *accountDataStatements) SelectMaxAccountDataID( ctx context.Context, txn *sql.Tx, ) (id int64, err error) { var nullableID sql.NullInt64 diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go index 0b53dfa9e..5870bfd52 100644 --- a/syncapi/storage/postgres/output_room_events_table.go +++ b/syncapi/storage/postgres/output_room_events_table.go @@ -22,6 +22,7 @@ import ( "sort" "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/lib/pq" @@ -120,39 +121,40 @@ type outputRoomEventsStatements struct { selectStateInRangeStmt *sql.Stmt } -func (s *outputRoomEventsStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(outputRoomEventsSchema) +func NewPostgresEventsTable(db *sql.DB) (tables.Events, error) { + s := &outputRoomEventsStatements{} + _, err := db.Exec(outputRoomEventsSchema) if err != nil { - return + return nil, err } if s.insertEventStmt, err = db.Prepare(insertEventSQL); err != nil { - return + return nil, err } if s.selectEventsStmt, err = db.Prepare(selectEventsSQL); err != nil { - return + return nil, err } if s.selectMaxEventIDStmt, err = db.Prepare(selectMaxEventIDSQL); err != nil { - return + return nil, err } if s.selectRecentEventsStmt, err = db.Prepare(selectRecentEventsSQL); err != nil { - return + return nil, err } if s.selectRecentEventsForSyncStmt, err = db.Prepare(selectRecentEventsForSyncSQL); err != nil { - return + return nil, err } if s.selectEarlyEventsStmt, err = db.Prepare(selectEarlyEventsSQL); err != nil { - return + return nil, err } if s.selectStateInRangeStmt, err = db.Prepare(selectStateInRangeSQL); err != nil { - return + return nil, err } - return + return s, nil } // selectStateInRange returns the state events between the two given PDU stream positions, exclusive of oldPos, inclusive of newPos. // Results are bucketed based on the room ID. If the same state is overwritten multiple times between the // two positions, only the most recent state is returned. -func (s *outputRoomEventsStatements) selectStateInRange( +func (s *outputRoomEventsStatements) SelectStateInRange( ctx context.Context, txn *sql.Tx, oldPos, newPos types.StreamPosition, stateFilter *gomatrixserverlib.StateFilter, ) (map[string]map[string]bool, map[string]types.StreamEvent, error) { @@ -233,7 +235,7 @@ func (s *outputRoomEventsStatements) selectStateInRange( // MaxID returns the ID of the last inserted event in this table. 'txn' is optional. If it is not supplied, // then this function should only ever be used at startup, as it will race with inserting events if it is // done afterwards. If there are no inserted events, 0 is returned. -func (s *outputRoomEventsStatements) selectMaxEventID( +func (s *outputRoomEventsStatements) SelectMaxEventID( ctx context.Context, txn *sql.Tx, ) (id int64, err error) { var nullableID sql.NullInt64 @@ -247,7 +249,7 @@ func (s *outputRoomEventsStatements) selectMaxEventID( // InsertEvent into the output_room_events table. addState and removeState are an optional list of state event IDs. Returns the position // of the inserted event. -func (s *outputRoomEventsStatements) insertEvent( +func (s *outputRoomEventsStatements) InsertEvent( ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.HeaderedEvent, addState, removeState []string, transactionID *api.TransactionID, excludeFromSync bool, @@ -294,7 +296,7 @@ func (s *outputRoomEventsStatements) insertEvent( // selectRecentEvents returns the most recent events in the given room, up to a maximum of 'limit'. // If onlySyncEvents has a value of true, only returns the events that aren't marked as to exclude // from sync. -func (s *outputRoomEventsStatements) selectRecentEvents( +func (s *outputRoomEventsStatements) SelectRecentEvents( ctx context.Context, txn *sql.Tx, roomID string, fromPos, toPos types.StreamPosition, limit int, chronologicalOrder bool, onlySyncEvents bool, @@ -327,7 +329,7 @@ func (s *outputRoomEventsStatements) selectRecentEvents( // selectEarlyEvents returns the earliest events in the given room, starting // from a given position, up to a maximum of 'limit'. -func (s *outputRoomEventsStatements) selectEarlyEvents( +func (s *outputRoomEventsStatements) SelectEarlyEvents( ctx context.Context, txn *sql.Tx, roomID string, fromPos, toPos types.StreamPosition, limit int, ) ([]types.StreamEvent, error) { @@ -352,7 +354,7 @@ func (s *outputRoomEventsStatements) selectEarlyEvents( // selectEvents returns the events for the given event IDs. If an event is // missing from the database, it will be omitted. -func (s *outputRoomEventsStatements) selectEvents( +func (s *outputRoomEventsStatements) SelectEvents( ctx context.Context, txn *sql.Tx, eventIDs []string, ) ([]types.StreamEvent, error) { stmt := common.TxStmt(txn, s.selectEventsStmt) diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go index 9883c3629..4fa08ce5b 100644 --- a/syncapi/storage/postgres/syncserver.go +++ b/syncapi/storage/postgres/syncserver.go @@ -20,9 +20,6 @@ import ( "database/sql" "encoding/json" "fmt" - "time" - - "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/internal/sqlutil" @@ -50,15 +47,12 @@ type stateDelta struct { // SyncServerDatasource represents a sync server datasource which manages // both the database for PDUs and caches for EDUs. type SyncServerDatasource struct { + shared.Database db *sql.DB common.PartitionOffsetStatements - accountData accountDataStatements - events outputRoomEventsStatements roomstate currentRoomStateStatements - eduCache *cache.EDUCache topology outputRoomEventsTopologyStatements backwardExtremities tables.BackwardsExtremities - shared *shared.Database } // NewSyncServerDatasource creates a new sync server database @@ -71,10 +65,12 @@ func NewSyncServerDatasource(dbDataSourceName string, dbProperties common.DbProp if err = d.PartitionOffsetStatements.Prepare(d.db, "syncapi"); err != nil { return nil, err } - if err = d.accountData.prepare(d.db); err != nil { + accountData, err := NewPostgresAccountDataTable(d.db) + if err != nil { return nil, err } - if err = d.events.prepare(d.db); err != nil { + events, err := NewPostgresEventsTable(d.db) + if err != nil { return nil, err } if err = d.roomstate.prepare(d.db); err != nil { @@ -91,10 +87,12 @@ func NewSyncServerDatasource(dbDataSourceName string, dbProperties common.DbProp if err != nil { return nil, err } - d.eduCache = cache.New() - d.shared = &shared.Database{ - DB: d.db, - Invites: invites, + d.Database = shared.Database{ + DB: d.db, + Invites: invites, + AccountData: accountData, + OutputEvents: events, + EDUCache: cache.New(), } return &d, nil } @@ -103,17 +101,6 @@ func (d *SyncServerDatasource) AllJoinedUsersInRooms(ctx context.Context) (map[s return d.roomstate.selectJoinedUsers(ctx) } -func (d *SyncServerDatasource) Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.HeaderedEvent, error) { - streamEvents, err := d.events.selectEvents(ctx, nil, eventIDs) - if err != nil { - return nil, err - } - - // We don't include a device here as we only include transaction IDs in - // incremental syncs. - return d.StreamEventsToEvents(nil, streamEvents), nil -} - // handleBackwardExtremities adds this event as a backwards extremity if and only if we do not have all of // the events listed in the event's 'prev_events'. This function also updates the backwards extremities table // to account for the fact that the given event is no longer a backwards extremity, but may be marked as such. @@ -124,7 +111,7 @@ func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, tx // Check if we have all of the event's previous events. If an event is // missing, add it to the room's backward extremities. - prevEvents, err := d.events.selectEvents(ctx, txn, ev.PrevEventIDs()) + prevEvents, err := d.Database.OutputEvents.SelectEvents(ctx, txn, ev.PrevEventIDs()) if err != nil { return err } @@ -157,7 +144,7 @@ func (d *SyncServerDatasource) WriteEvent( ) (pduPosition types.StreamPosition, returnErr error) { returnErr = common.WithTransaction(d.db, func(txn *sql.Tx) error { var err error - pos, err := d.events.insertEvent( + pos, err := d.Database.OutputEvents.InsertEvent( ctx, txn, ev, addStateEventIDs, removeStateEventIDs, transactionID, excludeFromSync, ) if err != nil { @@ -265,36 +252,10 @@ func (d *SyncServerDatasource) GetEventsInTopologicalRange( } // Retrieve the events' contents using their IDs. - events, err = d.events.selectEvents(ctx, nil, eIDs) + events, err = d.Database.OutputEvents.SelectEvents(ctx, nil, eIDs) return } -// GetEventsInStreamingRange retrieves all of the events on a given ordering using the -// given extremities and limit. -func (d *SyncServerDatasource) GetEventsInStreamingRange( - ctx context.Context, - from, to *types.StreamingToken, - roomID string, limit int, - backwardOrdering bool, -) (events []types.StreamEvent, err error) { - if backwardOrdering { - // When using backward ordering, we want the most recent events first. - if events, err = d.events.selectRecentEvents( - ctx, nil, roomID, to.PDUPosition(), from.PDUPosition(), limit, false, false, - ); err != nil { - return - } - } else { - // When using forward ordering, we want the least recent events first. - if events, err = d.events.selectEarlyEvents( - ctx, nil, roomID, from.PDUPosition(), to.PDUPosition(), limit, - ); err != nil { - return - } - } - return events, err -} - func (d *SyncServerDatasource) SyncPosition(ctx context.Context) (types.StreamingToken, error) { return d.syncPositionTx(ctx, nil) } @@ -319,7 +280,7 @@ func (d *SyncServerDatasource) EventsAtTopologicalPosition( return nil, err } - return d.events.selectEvents(ctx, nil, eIDs) + return d.Database.OutputEvents.SelectEvents(ctx, nil, eIDs) } func (d *SyncServerDatasource) EventPositionInTopology( @@ -328,57 +289,29 @@ func (d *SyncServerDatasource) EventPositionInTopology( return d.topology.selectPositionInTopology(ctx, eventID) } -func (d *SyncServerDatasource) SyncStreamPosition(ctx context.Context) (types.StreamPosition, error) { - return d.syncStreamPositionTx(ctx, nil) -} - -func (d *SyncServerDatasource) syncStreamPositionTx( - ctx context.Context, txn *sql.Tx, -) (types.StreamPosition, error) { - maxID, err := d.events.selectMaxEventID(ctx, txn) - if err != nil { - return 0, err - } - maxAccountDataID, err := d.accountData.selectMaxAccountDataID(ctx, txn) - if err != nil { - return 0, err - } - if maxAccountDataID > maxID { - maxID = maxAccountDataID - } - maxInviteID, err := d.shared.Invites.SelectMaxInviteID(ctx, txn) - if err != nil { - return 0, err - } - if maxInviteID > maxID { - maxID = maxInviteID - } - return types.StreamPosition(maxID), nil -} - func (d *SyncServerDatasource) syncPositionTx( ctx context.Context, txn *sql.Tx, ) (sp types.StreamingToken, err error) { - maxEventID, err := d.events.selectMaxEventID(ctx, txn) + maxEventID, err := d.Database.OutputEvents.SelectMaxEventID(ctx, txn) if err != nil { return sp, err } - maxAccountDataID, err := d.accountData.selectMaxAccountDataID(ctx, txn) + maxAccountDataID, err := d.Database.AccountData.SelectMaxAccountDataID(ctx, txn) if err != nil { return sp, err } if maxAccountDataID > maxEventID { maxEventID = maxAccountDataID } - maxInviteID, err := d.shared.Invites.SelectMaxInviteID(ctx, txn) + maxInviteID, err := d.Database.Invites.SelectMaxInviteID(ctx, txn) if err != nil { return sp, err } if maxInviteID > maxEventID { maxEventID = maxInviteID } - sp = types.NewStreamToken(types.StreamPosition(maxEventID), types.StreamPosition(d.eduCache.GetLatestSyncPosition())) + sp = types.NewStreamToken(types.StreamPosition(maxEventID), types.StreamPosition(d.Database.EDUCache.GetLatestSyncPosition())) return } @@ -451,7 +384,7 @@ func (d *SyncServerDatasource) addTypingDeltaToResponse( var ok bool var err error for _, roomID := range joinedRoomIDs { - if typingUsers, updated := d.eduCache.GetTypingUsersIfUpdatedAfter( + if typingUsers, updated := d.Database.EDUCache.GetTypingUsersIfUpdatedAfter( roomID, int64(since.EDUPosition()), ); updated { ev := gomatrixserverlib.ClientEvent{ @@ -580,7 +513,7 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( // TODO: When filters are added, we may need to call this multiple times to get enough events. // See: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L316 var recentStreamEvents []types.StreamEvent - recentStreamEvents, err = d.events.selectRecentEvents( + recentStreamEvents, err = d.Database.OutputEvents.SelectRecentEvents( ctx, txn, roomID, types.StreamPosition(0), toPos.PDUPosition(), numRecentEventsPerRoom, true, true, ) @@ -653,54 +586,13 @@ var txReadOnlySnapshot = sql.TxOptions{ ReadOnly: true, } -func (d *SyncServerDatasource) GetAccountDataInRange( - ctx context.Context, userID string, oldPos, newPos types.StreamPosition, - accountDataFilterPart *gomatrixserverlib.EventFilter, -) (map[string][]string, error) { - return d.accountData.selectAccountDataInRange(ctx, userID, oldPos, newPos, accountDataFilterPart) -} - -func (d *SyncServerDatasource) UpsertAccountData( - ctx context.Context, userID, roomID, dataType string, -) (types.StreamPosition, error) { - return d.accountData.insertAccountData(ctx, userID, roomID, dataType) -} - -func (d *SyncServerDatasource) AddInviteEvent( - ctx context.Context, inviteEvent gomatrixserverlib.HeaderedEvent, -) (sp types.StreamPosition, err error) { - return d.shared.AddInviteEvent(ctx, inviteEvent) -} - -func (d *SyncServerDatasource) RetireInviteEvent( - ctx context.Context, inviteEventID string, -) error { - return d.shared.RetireInviteEvent(ctx, inviteEventID) -} - -func (d *SyncServerDatasource) SetTypingTimeoutCallback(fn cache.TimeoutCallbackFn) { - d.eduCache.SetTimeoutCallback(fn) -} - -func (d *SyncServerDatasource) AddTypingUser( - userID, roomID string, expireTime *time.Time, -) types.StreamPosition { - return types.StreamPosition(d.eduCache.AddTypingUser(userID, roomID, expireTime)) -} - -func (d *SyncServerDatasource) RemoveTypingUser( - userID, roomID string, -) types.StreamPosition { - return types.StreamPosition(d.eduCache.RemoveUser(userID, roomID)) -} - func (d *SyncServerDatasource) addInvitesToResponse( ctx context.Context, txn *sql.Tx, userID string, fromPos, toPos types.StreamPosition, res *types.Response, ) error { - invites, err := d.shared.Invites.SelectInviteEventsInRange( + invites, err := d.Database.Invites.SelectInviteEventsInRange( ctx, txn, userID, fromPos, toPos, ) if err != nil { @@ -750,7 +642,7 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( // This is all "okay" assuming history_visibility == "shared" which it is by default. endPos = delta.membershipPos } - recentStreamEvents, err := d.events.selectRecentEvents( + recentStreamEvents, err := d.Database.OutputEvents.SelectRecentEvents( ctx, txn, delta.roomID, types.StreamPosition(fromPos), types.StreamPosition(endPos), numRecentEventsPerRoom, true, true, ) @@ -841,7 +733,7 @@ func (d *SyncServerDatasource) fetchMissingStateEvents( ) ([]types.StreamEvent, error) { // Fetch from the events table first so we pick up the stream ID for the // event. - events, err := d.events.selectEvents(ctx, txn, eventIDs) + events, err := d.Database.OutputEvents.SelectEvents(ctx, txn, eventIDs) if err != nil { return nil, err } @@ -895,7 +787,7 @@ func (d *SyncServerDatasource) getStateDeltas( var deltas []stateDelta // get all the state events ever between these two positions - stateNeeded, eventMap, err := d.events.selectStateInRange(ctx, txn, fromPos, toPos, stateFilter) + stateNeeded, eventMap, err := d.Database.OutputEvents.SelectStateInRange(ctx, txn, fromPos, toPos, stateFilter) if err != nil { return nil, nil, err } @@ -981,7 +873,7 @@ func (d *SyncServerDatasource) getStateDeltasForFullStateSync( } // Get all the state events ever between these two positions - stateNeeded, eventMap, err := d.events.selectStateInRange(ctx, txn, fromPos, toPos, stateFilter) + stateNeeded, eventMap, err := d.Database.OutputEvents.SelectStateInRange(ctx, txn, fromPos, toPos, stateFilter) if err != nil { return nil, nil, err } @@ -1025,26 +917,6 @@ func (d *SyncServerDatasource) currentStateStreamEventsForRoom( return s, nil } -func (d *SyncServerDatasource) StreamEventsToEvents(device *authtypes.Device, in []types.StreamEvent) []gomatrixserverlib.HeaderedEvent { - out := make([]gomatrixserverlib.HeaderedEvent, len(in)) - for i := 0; i < len(in); i++ { - out[i] = in[i].HeaderedEvent - if device != nil && in[i].TransactionID != nil { - if device.UserID == in[i].Sender() && device.SessionID == in[i].TransactionID.SessionID { - err := out[i].SetUnsignedField( - "transaction_id", in[i].TransactionID.TransactionID, - ) - if err != nil { - logrus.WithFields(logrus.Fields{ - "event_id": out[i].EventID(), - }).WithError(err).Warnf("Failed to add transaction ID to event") - } - } - } - } - return out -} - // There may be some overlap where events in stateEvents are already in recentEvents, so filter // them out so we don't include them twice in the /sync response. They should be in recentEvents // only, so clients get to the correct state once they have rolled forward. diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go index e89976df6..e9fed758b 100644 --- a/syncapi/storage/shared/syncserver.go +++ b/syncapi/storage/shared/syncserver.go @@ -3,20 +3,117 @@ package shared import ( "context" "database/sql" + "time" + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" + "github.com/sirupsen/logrus" ) // Database is a temporary struct until we have made syncserver.go the same for both pq/sqlite // For now this contains the shared functions type Database struct { - DB *sql.DB - Invites tables.Invites + DB *sql.DB + Invites tables.Invites + AccountData tables.AccountData + OutputEvents tables.Events + EDUCache *cache.EDUCache } +// Events lookups a list of event by their event ID. +// Returns a list of events matching the requested IDs found in the database. +// If an event is not found in the database then it will be omitted from the list. +// Returns an error if there was a problem talking with the database. +// Does not include any transaction IDs in the returned events. +func (d *Database) Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.HeaderedEvent, error) { + streamEvents, err := d.OutputEvents.SelectEvents(ctx, nil, eventIDs) + if err != nil { + return nil, err + } + + // We don't include a device here as we only include transaction IDs in + // incremental syncs. + return d.StreamEventsToEvents(nil, streamEvents), nil +} + +// GetEventsInStreamingRange retrieves all of the events on a given ordering using the +// given extremities and limit. +func (d *Database) GetEventsInStreamingRange( + ctx context.Context, + from, to *types.StreamingToken, + roomID string, limit int, + backwardOrdering bool, +) (events []types.StreamEvent, err error) { + if backwardOrdering { + // When using backward ordering, we want the most recent events first. + if events, err = d.OutputEvents.SelectRecentEvents( + ctx, nil, roomID, to.PDUPosition(), from.PDUPosition(), limit, false, false, + ); err != nil { + return + } + } else { + // When using forward ordering, we want the least recent events first. + if events, err = d.OutputEvents.SelectEarlyEvents( + ctx, nil, roomID, from.PDUPosition(), to.PDUPosition(), limit, + ); err != nil { + return + } + } + return events, err +} + +func (d *Database) AddTypingUser( + userID, roomID string, expireTime *time.Time, +) types.StreamPosition { + return types.StreamPosition(d.EDUCache.AddTypingUser(userID, roomID, expireTime)) +} + +func (d *Database) RemoveTypingUser( + userID, roomID string, +) types.StreamPosition { + return types.StreamPosition(d.EDUCache.RemoveUser(userID, roomID)) +} + +func (d *Database) SetTypingTimeoutCallback(fn cache.TimeoutCallbackFn) { + d.EDUCache.SetTimeoutCallback(fn) +} + +func (d *Database) SyncStreamPosition(ctx context.Context) (types.StreamPosition, error) { + var maxID int64 + var err error + err = common.WithTransaction(d.DB, func(txn *sql.Tx) error { + maxID, err = d.OutputEvents.SelectMaxEventID(ctx, txn) + if err != nil { + return err + } + var maxAccountDataID int64 + maxAccountDataID, err = d.AccountData.SelectMaxAccountDataID(ctx, txn) + if err != nil { + return err + } + if maxAccountDataID > maxID { + maxID = maxAccountDataID + } + var maxInviteID int64 + maxInviteID, err = d.Invites.SelectMaxInviteID(ctx, txn) + if err != nil { + return err + } + if maxInviteID > maxID { + maxID = maxInviteID + } + return nil + }) + return types.StreamPosition(maxID), err +} + +// AddInviteEvent stores a new invite event for a user. +// If the invite was successfully stored this returns the stream ID it was stored at. +// Returns an error if there was a problem communicating with the database. func (d *Database) AddInviteEvent( ctx context.Context, inviteEvent gomatrixserverlib.HeaderedEvent, ) (sp types.StreamPosition, err error) { @@ -27,6 +124,8 @@ func (d *Database) AddInviteEvent( return } +// RetireInviteEvent removes an old invite event from the database. +// Returns an error if there was a problem communicating with the database. func (d *Database) RetireInviteEvent( ctx context.Context, inviteEventID string, ) error { @@ -35,3 +134,51 @@ func (d *Database) RetireInviteEvent( err := d.Invites.DeleteInviteEvent(ctx, inviteEventID) return err } + +// GetAccountDataInRange returns all account data for a given user inserted or +// updated between two given positions +// Returns a map following the format data[roomID] = []dataTypes +// If no data is retrieved, returns an empty map +// If there was an issue with the retrieval, returns an error +func (d *Database) GetAccountDataInRange( + ctx context.Context, userID string, oldPos, newPos types.StreamPosition, + accountDataFilterPart *gomatrixserverlib.EventFilter, +) (map[string][]string, error) { + return d.AccountData.SelectAccountDataInRange(ctx, userID, oldPos, newPos, accountDataFilterPart) +} + +// UpsertAccountData keeps track of new or updated account data, by saving the type +// of the new/updated data, and the user ID and room ID the data is related to (empty) +// room ID means the data isn't specific to any room) +// If no data with the given type, user ID and room ID exists in the database, +// creates a new row, else update the existing one +// Returns an error if there was an issue with the upsert +func (d *Database) UpsertAccountData( + ctx context.Context, userID, roomID, dataType string, +) (sp types.StreamPosition, err error) { + err = common.WithTransaction(d.DB, func(txn *sql.Tx) error { + sp, err = d.AccountData.InsertAccountData(ctx, txn, userID, roomID, dataType) + return err + }) + return +} + +func (d *Database) StreamEventsToEvents(device *authtypes.Device, in []types.StreamEvent) []gomatrixserverlib.HeaderedEvent { + out := make([]gomatrixserverlib.HeaderedEvent, len(in)) + for i := 0; i < len(in); i++ { + out[i] = in[i].HeaderedEvent + if device != nil && in[i].TransactionID != nil { + if device.UserID == in[i].Sender() && device.SessionID == in[i].TransactionID.SessionID { + err := out[i].SetUnsignedField( + "transaction_id", in[i].TransactionID.TransactionID, + ) + if err != nil { + logrus.WithFields(logrus.Fields{ + "event_id": out[i].EventID(), + }).WithError(err).Warnf("Failed to add transaction ID to event") + } + } + } + } + return out +} diff --git a/syncapi/storage/sqlite3/account_data_table.go b/syncapi/storage/sqlite3/account_data_table.go index 3dbf961b4..e5f2417b8 100644 --- a/syncapi/storage/sqlite3/account_data_table.go +++ b/syncapi/storage/sqlite3/account_data_table.go @@ -21,6 +21,7 @@ import ( "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -55,25 +56,27 @@ type accountDataStatements struct { selectAccountDataInRangeStmt *sql.Stmt } -func (s *accountDataStatements) prepare(db *sql.DB, streamID *streamIDStatements) (err error) { - s.streamIDStatements = streamID - _, err = db.Exec(accountDataSchema) +func NewSqliteAccountDataTable(db *sql.DB, streamID *streamIDStatements) (tables.AccountData, error) { + s := &accountDataStatements{ + streamIDStatements: streamID, + } + _, err := db.Exec(accountDataSchema) if err != nil { - return + return nil, err } if s.insertAccountDataStmt, err = db.Prepare(insertAccountDataSQL); err != nil { - return + return nil, err } if s.selectMaxAccountDataIDStmt, err = db.Prepare(selectMaxAccountDataIDSQL); err != nil { - return + return nil, err } if s.selectAccountDataInRangeStmt, err = db.Prepare(selectAccountDataInRangeSQL); err != nil { - return + return nil, err } - return + return s, nil } -func (s *accountDataStatements) insertAccountData( +func (s *accountDataStatements) InsertAccountData( ctx context.Context, txn *sql.Tx, userID, roomID, dataType string, ) (pos types.StreamPosition, err error) { @@ -85,7 +88,7 @@ func (s *accountDataStatements) insertAccountData( return } -func (s *accountDataStatements) selectAccountDataInRange( +func (s *accountDataStatements) SelectAccountDataInRange( ctx context.Context, userID string, oldPos, newPos types.StreamPosition, @@ -146,7 +149,7 @@ func (s *accountDataStatements) selectAccountDataInRange( return data, nil } -func (s *accountDataStatements) selectMaxAccountDataID( +func (s *accountDataStatements) SelectMaxAccountDataID( ctx context.Context, txn *sql.Tx, ) (id int64, err error) { var nullableID sql.NullInt64 diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index 08299f64b..d3e88a549 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -22,6 +22,7 @@ import ( "sort" "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/dendrite/common" @@ -109,40 +110,42 @@ type outputRoomEventsStatements struct { selectStateInRangeStmt *sql.Stmt } -func (s *outputRoomEventsStatements) prepare(db *sql.DB, streamID *streamIDStatements) (err error) { - s.streamIDStatements = streamID - _, err = db.Exec(outputRoomEventsSchema) +func NewSqliteEventsTable(db *sql.DB, streamID *streamIDStatements) (tables.Events, error) { + s := &outputRoomEventsStatements{ + streamIDStatements: streamID, + } + _, err := db.Exec(outputRoomEventsSchema) if err != nil { - return + return nil, err } if s.insertEventStmt, err = db.Prepare(insertEventSQL); err != nil { - return + return nil, err } if s.selectEventsStmt, err = db.Prepare(selectEventsSQL); err != nil { - return + return nil, err } if s.selectMaxEventIDStmt, err = db.Prepare(selectMaxEventIDSQL); err != nil { - return + return nil, err } if s.selectRecentEventsStmt, err = db.Prepare(selectRecentEventsSQL); err != nil { - return + return nil, err } if s.selectRecentEventsForSyncStmt, err = db.Prepare(selectRecentEventsForSyncSQL); err != nil { - return + return nil, err } if s.selectEarlyEventsStmt, err = db.Prepare(selectEarlyEventsSQL); err != nil { - return + return nil, err } if s.selectStateInRangeStmt, err = db.Prepare(selectStateInRangeSQL); err != nil { - return + return nil, err } - return + return s, nil } // selectStateInRange returns the state events between the two given PDU stream positions, exclusive of oldPos, inclusive of newPos. // Results are bucketed based on the room ID. If the same state is overwritten multiple times between the // two positions, only the most recent state is returned. -func (s *outputRoomEventsStatements) selectStateInRange( +func (s *outputRoomEventsStatements) SelectStateInRange( ctx context.Context, txn *sql.Tx, oldPos, newPos types.StreamPosition, stateFilterPart *gomatrixserverlib.StateFilter, ) (map[string]map[string]bool, map[string]types.StreamEvent, error) { @@ -229,7 +232,7 @@ func (s *outputRoomEventsStatements) selectStateInRange( // MaxID returns the ID of the last inserted event in this table. 'txn' is optional. If it is not supplied, // then this function should only ever be used at startup, as it will race with inserting events if it is // done afterwards. If there are no inserted events, 0 is returned. -func (s *outputRoomEventsStatements) selectMaxEventID( +func (s *outputRoomEventsStatements) SelectMaxEventID( ctx context.Context, txn *sql.Tx, ) (id int64, err error) { var nullableID sql.NullInt64 @@ -243,7 +246,7 @@ func (s *outputRoomEventsStatements) selectMaxEventID( // InsertEvent into the output_room_events table. addState and removeState are an optional list of state event IDs. Returns the position // of the inserted event. -func (s *outputRoomEventsStatements) insertEvent( +func (s *outputRoomEventsStatements) InsertEvent( ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.HeaderedEvent, addState, removeState []string, transactionID *api.TransactionID, excludeFromSync bool, @@ -306,7 +309,7 @@ func (s *outputRoomEventsStatements) insertEvent( // selectRecentEvents returns the most recent events in the given room, up to a maximum of 'limit'. // If onlySyncEvents has a value of true, only returns the events that aren't marked as to exclude // from sync. -func (s *outputRoomEventsStatements) selectRecentEvents( +func (s *outputRoomEventsStatements) SelectRecentEvents( ctx context.Context, txn *sql.Tx, roomID string, fromPos, toPos types.StreamPosition, limit int, chronologicalOrder bool, onlySyncEvents bool, @@ -340,7 +343,7 @@ func (s *outputRoomEventsStatements) selectRecentEvents( // selectEarlyEvents returns the earliest events in the given room, starting // from a given position, up to a maximum of 'limit'. -func (s *outputRoomEventsStatements) selectEarlyEvents( +func (s *outputRoomEventsStatements) SelectEarlyEvents( ctx context.Context, txn *sql.Tx, roomID string, fromPos, toPos types.StreamPosition, limit int, ) ([]types.StreamEvent, error) { @@ -365,7 +368,7 @@ func (s *outputRoomEventsStatements) selectEarlyEvents( // selectEvents returns the events for the given event IDs. If an event is // missing from the database, it will be omitted. -func (s *outputRoomEventsStatements) selectEvents( +func (s *outputRoomEventsStatements) SelectEvents( ctx context.Context, txn *sql.Tx, eventIDs []string, ) ([]types.StreamEvent, error) { var returnEvents []types.StreamEvent diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index a2253dcd1..0da05eab3 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -22,7 +22,6 @@ import ( "errors" "fmt" "net/url" - "time" "github.com/sirupsen/logrus" @@ -53,16 +52,13 @@ type stateDelta struct { // SyncServerDatasource represents a sync server datasource which manages // both the database for PDUs and caches for EDUs. type SyncServerDatasource struct { + shared.Database db *sql.DB common.PartitionOffsetStatements streamID streamIDStatements - accountData accountDataStatements - events outputRoomEventsStatements roomstate currentRoomStateStatements - eduCache *cache.EDUCache topology outputRoomEventsTopologyStatements backwardExtremities tables.BackwardsExtremities - shared *shared.Database } // NewSyncServerDatasource creates a new sync server database @@ -87,7 +83,6 @@ func NewSyncServerDatasource(dataSourceName string) (*SyncServerDatasource, erro if err = d.prepare(); err != nil { return nil, err } - d.eduCache = cache.New() return &d, nil } @@ -98,10 +93,12 @@ func (d *SyncServerDatasource) prepare() (err error) { if err = d.streamID.prepare(d.db); err != nil { return err } - if err = d.accountData.prepare(d.db, &d.streamID); err != nil { + accountData, err := NewSqliteAccountDataTable(d.db, &d.streamID) + if err != nil { return err } - if err = d.events.prepare(d.db, &d.streamID); err != nil { + events, err := NewSqliteEventsTable(d.db, &d.streamID) + if err != nil { return err } if err = d.roomstate.prepare(d.db, &d.streamID); err != nil { @@ -118,9 +115,12 @@ func (d *SyncServerDatasource) prepare() (err error) { if err != nil { return err } - d.shared = &shared.Database{ - DB: d.db, - Invites: invites, + d.Database = shared.Database{ + DB: d.db, + Invites: invites, + AccountData: accountData, + OutputEvents: events, + EDUCache: cache.New(), } return nil } @@ -130,22 +130,6 @@ func (d *SyncServerDatasource) AllJoinedUsersInRooms(ctx context.Context) (map[s return d.roomstate.selectJoinedUsers(ctx) } -// Events lookups a list of event by their event ID. -// Returns a list of events matching the requested IDs found in the database. -// If an event is not found in the database then it will be omitted from the list. -// Returns an error if there was a problem talking with the database. -// Does not include any transaction IDs in the returned events. -func (d *SyncServerDatasource) Events(ctx context.Context, eventIDs []string) ([]gomatrixserverlib.HeaderedEvent, error) { - streamEvents, err := d.events.selectEvents(ctx, nil, eventIDs) - if err != nil { - return nil, err - } - - // We don't include a device here as we only include transaction IDs in - // incremental syncs. - return d.StreamEventsToEvents(nil, streamEvents), nil -} - // handleBackwardExtremities adds this event as a backwards extremity if and only if we do not have all of // the events listed in the event's 'prev_events'. This function also updates the backwards extremities table // to account for the fact that the given event is no longer a backwards extremity, but may be marked as such. @@ -156,7 +140,7 @@ func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, tx // Check if we have all of the event's previous events. If an event is // missing, add it to the room's backward extremities. - prevEvents, err := d.events.selectEvents(ctx, txn, ev.PrevEventIDs()) + prevEvents, err := d.Database.OutputEvents.SelectEvents(ctx, txn, ev.PrevEventIDs()) if err != nil { return err } @@ -192,7 +176,7 @@ func (d *SyncServerDatasource) WriteEvent( ) (pduPosition types.StreamPosition, returnErr error) { returnErr = common.WithTransaction(d.db, func(txn *sql.Tx) error { var err error - pos, err := d.events.insertEvent( + pos, err := d.Database.OutputEvents.InsertEvent( ctx, txn, ev, addStateEventIDs, removeStateEventIDs, transactionID, excludeFromSync, ) if err != nil { @@ -253,6 +237,19 @@ func (d *SyncServerDatasource) updateRoomState( return nil } +// SyncPosition returns the latest positions for syncing. +func (d *SyncServerDatasource) SyncPosition(ctx context.Context) (tok types.StreamingToken, err error) { + err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + pos, err := d.syncPositionTx(ctx, txn) + if err != nil { + return err + } + tok = *pos + return nil + }) + return +} + // GetStateEvent returns the Matrix state event of a given type for a given room with a given state key // If no event could be found, returns nil // If there was an issue during the retrieval, returns an error @@ -309,46 +306,7 @@ func (d *SyncServerDatasource) GetEventsInTopologicalRange( } // Retrieve the events' contents using their IDs. - events, err = d.events.selectEvents(ctx, nil, eIDs) - return -} - -// GetEventsInStreamingRange retrieves all of the events on a given ordering using the -// given extremities and limit. -func (d *SyncServerDatasource) GetEventsInStreamingRange( - ctx context.Context, - from, to *types.StreamingToken, - roomID string, limit int, - backwardOrdering bool, -) (events []types.StreamEvent, err error) { - if backwardOrdering { - // When using backward ordering, we want the most recent events first. - if events, err = d.events.selectRecentEvents( - ctx, nil, roomID, to.PDUPosition(), from.PDUPosition(), limit, false, false, - ); err != nil { - return - } - } else { - // When using forward ordering, we want the least recent events first. - if events, err = d.events.selectEarlyEvents( - ctx, nil, roomID, from.PDUPosition(), to.PDUPosition(), limit, - ); err != nil { - return - } - } - return events, err -} - -// SyncPosition returns the latest positions for syncing. -func (d *SyncServerDatasource) SyncPosition(ctx context.Context) (tok types.StreamingToken, err error) { - err = common.WithTransaction(d.db, func(txn *sql.Tx) error { - pos, err := d.syncPositionTx(ctx, txn) - if err != nil { - return err - } - tok = *pos - return nil - }) + events, err = d.Database.OutputEvents.SelectEvents(ctx, nil, eIDs) return } @@ -378,7 +336,7 @@ func (d *SyncServerDatasource) EventsAtTopologicalPosition( return nil, err } - return d.events.selectEvents(ctx, nil, eIDs) + return d.Database.OutputEvents.SelectEvents(ctx, nil, eIDs) } func (d *SyncServerDatasource) EventPositionInTopology( @@ -399,18 +357,18 @@ func (d *SyncServerDatasource) SyncStreamPosition(ctx context.Context) (pos type func (d *SyncServerDatasource) syncStreamPositionTx( ctx context.Context, txn *sql.Tx, ) (types.StreamPosition, error) { - maxID, err := d.events.selectMaxEventID(ctx, txn) + maxID, err := d.Database.OutputEvents.SelectMaxEventID(ctx, txn) if err != nil { return 0, err } - maxAccountDataID, err := d.accountData.selectMaxAccountDataID(ctx, txn) + maxAccountDataID, err := d.Database.AccountData.SelectMaxAccountDataID(ctx, txn) if err != nil { return 0, err } if maxAccountDataID > maxID { maxID = maxAccountDataID } - maxInviteID, err := d.shared.Invites.SelectMaxInviteID(ctx, txn) + maxInviteID, err := d.Database.Invites.SelectMaxInviteID(ctx, txn) if err != nil { return 0, err } @@ -424,18 +382,18 @@ func (d *SyncServerDatasource) syncPositionTx( ctx context.Context, txn *sql.Tx, ) (*types.StreamingToken, error) { - maxEventID, err := d.events.selectMaxEventID(ctx, txn) + maxEventID, err := d.Database.OutputEvents.SelectMaxEventID(ctx, txn) if err != nil { return nil, err } - maxAccountDataID, err := d.accountData.selectMaxAccountDataID(ctx, txn) + maxAccountDataID, err := d.Database.AccountData.SelectMaxAccountDataID(ctx, txn) if err != nil { return nil, err } if maxAccountDataID > maxEventID { maxEventID = maxAccountDataID } - maxInviteID, err := d.shared.Invites.SelectMaxInviteID(ctx, txn) + maxInviteID, err := d.Database.Invites.SelectMaxInviteID(ctx, txn) if err != nil { return nil, err } @@ -444,7 +402,7 @@ func (d *SyncServerDatasource) syncPositionTx( } sp := types.NewStreamToken( types.StreamPosition(maxEventID), - types.StreamPosition(d.eduCache.GetLatestSyncPosition()), + types.StreamPosition(d.Database.EDUCache.GetLatestSyncPosition()), ) return &sp, nil } @@ -518,7 +476,7 @@ func (d *SyncServerDatasource) addTypingDeltaToResponse( var ok bool var err error for _, roomID := range joinedRoomIDs { - if typingUsers, updated := d.eduCache.GetTypingUsersIfUpdatedAfter( + if typingUsers, updated := d.Database.EDUCache.GetTypingUsersIfUpdatedAfter( roomID, int64(since.EDUPosition()), ); updated { ev := gomatrixserverlib.ClientEvent{ @@ -654,7 +612,7 @@ func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( // TODO: When filters are added, we may need to call this multiple times to get enough events. // See: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L316 var recentStreamEvents []types.StreamEvent - recentStreamEvents, err = d.events.selectRecentEvents( + recentStreamEvents, err = d.Database.OutputEvents.SelectRecentEvents( ctx, txn, roomID, types.StreamPosition(0), toPos.PDUPosition(), numRecentEventsPerRoom, true, true, ) @@ -729,78 +687,13 @@ var txReadOnlySnapshot = sql.TxOptions{ ReadOnly: true, } -// GetAccountDataInRange returns all account data for a given user inserted or -// updated between two given positions -// Returns a map following the format data[roomID] = []dataTypes -// If no data is retrieved, returns an empty map -// If there was an issue with the retrieval, returns an error -func (d *SyncServerDatasource) GetAccountDataInRange( - ctx context.Context, userID string, oldPos, newPos types.StreamPosition, - accountDataFilterPart *gomatrixserverlib.EventFilter, -) (map[string][]string, error) { - return d.accountData.selectAccountDataInRange(ctx, userID, oldPos, newPos, accountDataFilterPart) -} - -// UpsertAccountData keeps track of new or updated account data, by saving the type -// of the new/updated data, and the user ID and room ID the data is related to (empty) -// room ID means the data isn't specific to any room) -// If no data with the given type, user ID and room ID exists in the database, -// creates a new row, else update the existing one -// Returns an error if there was an issue with the upsert -func (d *SyncServerDatasource) UpsertAccountData( - ctx context.Context, userID, roomID, dataType string, -) (sp types.StreamPosition, err error) { - err = common.WithTransaction(d.db, func(txn *sql.Tx) error { - sp, err = d.accountData.insertAccountData(ctx, txn, userID, roomID, dataType) - return err - }) - return -} - -// AddInviteEvent stores a new invite event for a user. -// If the invite was successfully stored this returns the stream ID it was stored at. -// Returns an error if there was a problem communicating with the database. -func (d *SyncServerDatasource) AddInviteEvent( - ctx context.Context, inviteEvent gomatrixserverlib.HeaderedEvent, -) (sp types.StreamPosition, err error) { - return d.shared.AddInviteEvent(ctx, inviteEvent) -} - -// RetireInviteEvent removes an old invite event from the database. -// Returns an error if there was a problem communicating with the database. -func (d *SyncServerDatasource) RetireInviteEvent( - ctx context.Context, inviteEventID string, -) error { - return d.shared.RetireInviteEvent(ctx, inviteEventID) -} - -func (d *SyncServerDatasource) SetTypingTimeoutCallback(fn cache.TimeoutCallbackFn) { - d.eduCache.SetTimeoutCallback(fn) -} - -// AddTypingUser adds a typing user to the typing cache. -// Returns the newly calculated sync position for typing notifications. -func (d *SyncServerDatasource) AddTypingUser( - userID, roomID string, expireTime *time.Time, -) types.StreamPosition { - return types.StreamPosition(d.eduCache.AddTypingUser(userID, roomID, expireTime)) -} - -// RemoveTypingUser removes a typing user from the typing cache. -// Returns the newly calculated sync position for typing notifications. -func (d *SyncServerDatasource) RemoveTypingUser( - userID, roomID string, -) types.StreamPosition { - return types.StreamPosition(d.eduCache.RemoveUser(userID, roomID)) -} - func (d *SyncServerDatasource) addInvitesToResponse( ctx context.Context, txn *sql.Tx, userID string, fromPos, toPos types.StreamPosition, res *types.Response, ) error { - invites, err := d.shared.Invites.SelectInviteEventsInRange( + invites, err := d.Database.Invites.SelectInviteEventsInRange( ctx, txn, userID, fromPos, toPos, ) if err != nil { @@ -853,7 +746,7 @@ func (d *SyncServerDatasource) addRoomDeltaToResponse( // This is all "okay" assuming history_visibility == "shared" which it is by default. endPos = delta.membershipPos } - recentStreamEvents, err := d.events.selectRecentEvents( + recentStreamEvents, err := d.Database.OutputEvents.SelectRecentEvents( ctx, txn, delta.roomID, types.StreamPosition(fromPos), types.StreamPosition(endPos), numRecentEventsPerRoom, true, true, ) @@ -943,7 +836,7 @@ func (d *SyncServerDatasource) fetchMissingStateEvents( ) ([]types.StreamEvent, error) { // Fetch from the events table first so we pick up the stream ID for the // event. - events, err := d.events.selectEvents(ctx, txn, eventIDs) + events, err := d.Database.OutputEvents.SelectEvents(ctx, txn, eventIDs) if err != nil { return nil, err } @@ -997,7 +890,7 @@ func (d *SyncServerDatasource) getStateDeltas( var deltas []stateDelta // get all the state events ever between these two positions - stateNeeded, eventMap, err := d.events.selectStateInRange(ctx, txn, fromPos, toPos, stateFilterPart) + stateNeeded, eventMap, err := d.Database.OutputEvents.SelectStateInRange(ctx, txn, fromPos, toPos, stateFilterPart) if err != nil { return nil, nil, err } @@ -1083,7 +976,7 @@ func (d *SyncServerDatasource) getStateDeltasForFullStateSync( } // Get all the state events ever between these two positions - stateNeeded, eventMap, err := d.events.selectStateInRange(ctx, txn, fromPos, toPos, stateFilterPart) + stateNeeded, eventMap, err := d.Database.OutputEvents.SelectStateInRange(ctx, txn, fromPos, toPos, stateFilterPart) if err != nil { return nil, nil, err } diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go index 1a9940524..ffd502165 100644 --- a/syncapi/storage/tables/interface.go +++ b/syncapi/storage/tables/interface.go @@ -4,13 +4,29 @@ import ( "context" "database/sql" + "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" ) +type AccountData interface { + InsertAccountData(ctx context.Context, txn *sql.Tx, userID, roomID, dataType string) (pos types.StreamPosition, err error) + SelectAccountDataInRange(ctx context.Context, userID string, oldPos, newPos types.StreamPosition, accountDataEventFilter *gomatrixserverlib.EventFilter) (data map[string][]string, err error) + SelectMaxAccountDataID(ctx context.Context, txn *sql.Tx) (id int64, err error) +} + type Invites interface { InsertInviteEvent(ctx context.Context, txn *sql.Tx, inviteEvent gomatrixserverlib.HeaderedEvent) (streamPos types.StreamPosition, err error) DeleteInviteEvent(ctx context.Context, inviteEventID string) error SelectInviteEventsInRange(ctx context.Context, txn *sql.Tx, targetUserID string, startPos, endPos types.StreamPosition) (map[string]gomatrixserverlib.HeaderedEvent, error) SelectMaxInviteID(ctx context.Context, txn *sql.Tx) (id int64, err error) } + +type Events interface { + SelectStateInRange(ctx context.Context, txn *sql.Tx, oldPos, newPos types.StreamPosition, stateFilter *gomatrixserverlib.StateFilter) (map[string]map[string]bool, map[string]types.StreamEvent, error) + SelectMaxEventID(ctx context.Context, txn *sql.Tx) (id int64, err error) + InsertEvent(ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.HeaderedEvent, addState, removeState []string, transactionID *api.TransactionID, excludeFromSync bool) (streamPos types.StreamPosition, err error) + SelectRecentEvents(ctx context.Context, txn *sql.Tx, roomID string, fromPos, toPos types.StreamPosition, limit int, chronologicalOrder bool, onlySyncEvents bool) ([]types.StreamEvent, error) + SelectEarlyEvents(ctx context.Context, txn *sql.Tx, roomID string, fromPos, toPos types.StreamPosition, limit int) ([]types.StreamEvent, error) + SelectEvents(ctx context.Context, txn *sql.Tx, eventIDs []string) ([]types.StreamEvent, error) +} From 8adc128225127f6e600e12987dd4767cbb681703 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 14 May 2020 14:05:14 +0100 Subject: [PATCH 056/200] Keyserver skeleton (#1032) * Keyserver skeleton * Indentation --- cmd/dendrite-key-server/main.go | 34 +++++++++++++++++ cmd/dendrite-monolith-server/main.go | 4 ++ common/config/config.go | 2 + dendrite-config.yaml | 1 + docker/docker-compose.yml | 10 +++++ docker/services/key-server.sh | 5 +++ keyserver/keyserver.go | 32 ++++++++++++++++ keyserver/routing/keys.go | 33 ++++++++++++++++ keyserver/routing/routing.go | 56 ++++++++++++++++++++++++++++ 9 files changed, 177 insertions(+) create mode 100644 cmd/dendrite-key-server/main.go create mode 100644 docker/services/key-server.sh create mode 100644 keyserver/keyserver.go create mode 100644 keyserver/routing/keys.go create mode 100644 keyserver/routing/routing.go diff --git a/cmd/dendrite-key-server/main.go b/cmd/dendrite-key-server/main.go new file mode 100644 index 000000000..5b2166d9a --- /dev/null +++ b/cmd/dendrite-key-server/main.go @@ -0,0 +1,34 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "github.com/matrix-org/dendrite/common/basecomponent" + "github.com/matrix-org/dendrite/keyserver" +) + +func main() { + cfg := basecomponent.ParseFlags() + base := basecomponent.NewBaseDendrite(cfg, "KeyServer") + defer base.Close() // nolint: errcheck + + accountDB := base.CreateAccountsDB() + deviceDB := base.CreateDeviceDB() + + keyserver.SetupKeyServerComponent(base, deviceDB, accountDB) + + base.SetupAndServeHTTP(string(base.Cfg.Bind.KeyServer), string(base.Cfg.Listen.KeyServer)) + +} diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index e004bc12e..f22610616 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -29,6 +29,7 @@ import ( "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/federationsender" + "github.com/matrix-org/dendrite/keyserver" "github.com/matrix-org/dendrite/mediaapi" "github.com/matrix-org/dendrite/publicroomsapi" "github.com/matrix-org/dendrite/publicroomsapi/storage" @@ -76,6 +77,9 @@ func main() { federation, &keyRing, rsAPI, eduInputAPI, asAPI, transactions.New(), fsAPI, ) + keyserver.SetupKeyServerComponent( + base, deviceDB, accountDB, + ) eduProducer := producers.NewEDUServerProducer(eduInputAPI) federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, rsAPI, asAPI, fsAPI, eduProducer) mediaapi.SetupMediaAPIComponent(base, deviceDB) diff --git a/common/config/config.go b/common/config/config.go index 9a29186a6..e1e96f9d5 100644 --- a/common/config/config.go +++ b/common/config/config.go @@ -229,6 +229,7 @@ type Dendrite struct { FederationSender Address `yaml:"federation_sender"` PublicRoomsAPI Address `yaml:"public_rooms_api"` EDUServer Address `yaml:"edu_server"` + KeyServer Address `yaml:"key_server"` } `yaml:"bind"` // The addresses for talking to other microservices. @@ -242,6 +243,7 @@ type Dendrite struct { FederationSender Address `yaml:"federation_sender"` PublicRoomsAPI Address `yaml:"public_rooms_api"` EDUServer Address `yaml:"edu_server"` + KeyServer Address `yaml:"key_server"` } `yaml:"listen"` // The config for tracing the dendrite servers. diff --git a/dendrite-config.yaml b/dendrite-config.yaml index 536b0f42b..2616d74db 100644 --- a/dendrite-config.yaml +++ b/dendrite-config.yaml @@ -137,6 +137,7 @@ listen: federation_sender: "localhost:7776" appservice_api: "localhost:7777" edu_server: "localhost:7778" + key_server: "localhost:7779" # The configuration for tracing the dendrite components. tracing: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 957c3bf3f..c6bb45813 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -153,6 +153,16 @@ services: - postgres networks: - internal + + key_server: + container_name: dendrite_key_server + hostname: key_server + entrypoint: ["bash", "./docker/services/key-server.sh"] + build: ./ + volumes: + - ..:/build + networks: + - internal postgres: container_name: dendrite_postgres diff --git a/docker/services/key-server.sh b/docker/services/key-server.sh new file mode 100644 index 000000000..965fa8543 --- /dev/null +++ b/docker/services/key-server.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +bash ./docker/build.sh + +./bin/dendrite-key-server --config dendrite.yaml diff --git a/keyserver/keyserver.go b/keyserver/keyserver.go new file mode 100644 index 000000000..1e0d5cb42 --- /dev/null +++ b/keyserver/keyserver.go @@ -0,0 +1,32 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package keyserver + +import ( + "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" + "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" + "github.com/matrix-org/dendrite/common/basecomponent" + "github.com/matrix-org/dendrite/keyserver/routing" +) + +// SetupFederationSenderComponent sets up and registers HTTP handlers for the +// FederationSender component. +func SetupKeyServerComponent( + base *basecomponent.BaseDendrite, + deviceDB devices.Database, + accountsDB accounts.Database, +) { + routing.Setup(base.APIMux, base.Cfg, accountsDB, deviceDB) +} diff --git a/keyserver/routing/keys.go b/keyserver/routing/keys.go new file mode 100644 index 000000000..a279a747c --- /dev/null +++ b/keyserver/routing/keys.go @@ -0,0 +1,33 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routing + +import ( + "net/http" + + "github.com/matrix-org/util" +) + +func QueryKeys( + req *http.Request, +) util.JSONResponse { + return util.JSONResponse{ + Code: http.StatusOK, + JSON: map[string]interface{}{ + "failures": map[string]interface{}{}, + "device_keys": map[string]interface{}{}, + }, + } +} diff --git a/keyserver/routing/routing.go b/keyserver/routing/routing.go new file mode 100644 index 000000000..d79ce6d40 --- /dev/null +++ b/keyserver/routing/routing.go @@ -0,0 +1,56 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routing + +import ( + "net/http" + + "github.com/gorilla/mux" + "github.com/matrix-org/dendrite/clientapi/auth" + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" + "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/util" +) + +const pathPrefixR0 = "/_matrix/client/r0" + +// Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client +// to clients which need to make outbound HTTP requests. +// +// Due to Setup being used to call many other functions, a gocyclo nolint is +// applied: +// nolint: gocyclo +func Setup( + apiMux *mux.Router, cfg *config.Dendrite, + accountDB accounts.Database, + deviceDB devices.Database, +) { + r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter() + + authData := auth.Data{ + AccountDB: accountDB, + DeviceDB: deviceDB, + AppServices: cfg.Derived.ApplicationServices, + } + + r0mux.Handle("/keys/query", + common.MakeAuthAPI("queryKeys", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + return QueryKeys(req) + }), + ).Methods(http.MethodPost, http.MethodOptions) +} From 640a0265df922a5aa3731f3bf61fe99c6acd0bff Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 14 May 2020 14:58:47 +0100 Subject: [PATCH 057/200] Fix some cases where accepting invites over federation doesn't work (#1028) * Handle cases where accepting invites doesn't work for historic rooms * Rewrite pairUpChanges * Review comments --- roomserver/internal/input_membership.go | 92 +++++++++---------------- roomserver/internal/perform_join.go | 53 ++++++++++---- roomserver/internal/perform_leave.go | 13 ++-- 3 files changed, 79 insertions(+), 79 deletions(-) diff --git a/roomserver/internal/input_membership.go b/roomserver/internal/input_membership.go index 19b7d8055..a0029a289 100644 --- a/roomserver/internal/input_membership.go +++ b/roomserver/internal/input_membership.go @@ -231,8 +231,7 @@ func updateToLeaveMembership( return updates, nil } -// membershipChanges pairs up the membership state changes from a sorted list -// of state removed and a sorted list of state added. +// membershipChanges pairs up the membership state changes. func membershipChanges(removed, added []types.StateEntry) []stateChange { changes := pairUpChanges(removed, added) var result []stateChange @@ -251,64 +250,39 @@ type stateChange struct { } // pairUpChanges pairs up the state events added and removed for each type, -// state key tuple. Assumes that removed and added are sorted. +// state key tuple. func pairUpChanges(removed, added []types.StateEntry) []stateChange { - var ai int - var ri int - var result []stateChange - for { - switch { - case ai == len(added): - // We've reached the end of the added entries. - // The rest of the removed list are events that were removed without - // an event with the same state key being added. - for _, s := range removed[ri:] { - result = append(result, stateChange{ - StateKeyTuple: s.StateKeyTuple, - removedEventNID: s.EventNID, - }) - } - return result - case ri == len(removed): - // We've reached the end of the removed entries. - // The rest of the added list are events that were added without - // an event with the same state key being removed. - for _, s := range added[ai:] { - result = append(result, stateChange{ - StateKeyTuple: s.StateKeyTuple, - addedEventNID: s.EventNID, - }) - } - return result - case added[ai].StateKeyTuple == removed[ri].StateKeyTuple: - // The tuple is in both lists so an event with that key is being - // removed and another event with the same key is being added. - result = append(result, stateChange{ - StateKeyTuple: added[ai].StateKeyTuple, - removedEventNID: removed[ri].EventNID, - addedEventNID: added[ai].EventNID, - }) - ai++ - ri++ - case added[ai].StateKeyTuple.LessThan(removed[ri].StateKeyTuple): - // The lists are sorted so the added entry being less than the - // removed entry means that the added event was added without an - // event with the same key being removed. - result = append(result, stateChange{ - StateKeyTuple: added[ai].StateKeyTuple, - addedEventNID: added[ai].EventNID, - }) - ai++ - default: - // Reaching the default case implies that the removed entry is less - // than the added entry. Since the lists are sorted this means that - // the removed event was removed without an event with the same - // key being added. - result = append(result, stateChange{ - StateKeyTuple: removed[ai].StateKeyTuple, - removedEventNID: removed[ri].EventNID, - }) - ri++ + tuples := make(map[types.StateKeyTuple]stateChange) + changes := []stateChange{} + + // First, go through the newly added state entries. + for _, add := range added { + if change, ok := tuples[add.StateKeyTuple]; ok { + // If we already have an entry, update it. + change.addedEventNID = add.EventNID + tuples[add.StateKeyTuple] = change + } else { + // Otherwise, create a new entry. + tuples[add.StateKeyTuple] = stateChange{add.StateKeyTuple, 0, add.EventNID} } } + + // Now go through the removed state entries. + for _, remove := range removed { + if change, ok := tuples[remove.StateKeyTuple]; ok { + // If we already have an entry, update it. + change.removedEventNID = remove.EventNID + tuples[remove.StateKeyTuple] = change + } else { + // Otherwise, create a new entry. + tuples[remove.StateKeyTuple] = stateChange{remove.StateKeyTuple, remove.EventNID, 0} + } + } + + // Now return the changes as an array. + for _, change := range tuples { + changes = append(changes, change) + } + + return changes } diff --git a/roomserver/internal/perform_join.go b/roomserver/internal/perform_join.go index 99e10d974..8f2f84e0f 100644 --- a/roomserver/internal/perform_join.go +++ b/roomserver/internal/perform_join.go @@ -121,6 +121,22 @@ func (r *RoomserverInternalAPI) performJoinRoomByID( return fmt.Errorf("eb.SetContent: %w", err) } + // First work out if this is in response to an existing invite. + // If it is then we avoid the situation where we might think we + // know about a room in the following section but don't know the + // latest state as all of our users have left. + isInvitePending, inviteSender, err := r.isInvitePending(ctx, req.RoomIDOrAlias, req.UserID) + if err == nil && isInvitePending { + // Add the server of the person who invited us to the server list, + // as they should be a fairly good bet. + if _, inviterDomain, ierr := gomatrixserverlib.SplitID('@', inviteSender); ierr == nil { + req.ServerNames = append(req.ServerNames, inviterDomain) + } + + // Perform a federated room join. + return r.performFederatedJoinRoomByID(ctx, req, res) + } + // Try to construct an actual join event from the template. // If this succeeds then it is a sign that the room already exists // locally on the homeserver. @@ -178,21 +194,32 @@ func (r *RoomserverInternalAPI) performJoinRoomByID( return fmt.Errorf("Room ID %q does not exist", req.RoomIDOrAlias) } - // Try joining by all of the supplied server names. - fedReq := fsAPI.PerformJoinRequest{ - RoomID: req.RoomIDOrAlias, // the room ID to try and join - UserID: req.UserID, // the user ID joining the room - ServerNames: req.ServerNames, // the server to try joining with - Content: req.Content, // the membership event content - } - fedRes := fsAPI.PerformJoinResponse{} - err = r.fsAPI.PerformJoin(ctx, &fedReq, &fedRes) - if err != nil { - return fmt.Errorf("Error joining federated room: %q", err) - } + // Perform a federated room join. + return r.performFederatedJoinRoomByID(ctx, req, res) default: - return fmt.Errorf("Error joining room %q: %w", req.RoomIDOrAlias, err) + // Something else went wrong. + return fmt.Errorf("Error joining local room: %q", err) + } + + return nil +} + +func (r *RoomserverInternalAPI) performFederatedJoinRoomByID( + ctx context.Context, + req *api.PerformJoinRequest, + res *api.PerformJoinResponse, // nolint:unparam +) error { + // Try joining by all of the supplied server names. + fedReq := fsAPI.PerformJoinRequest{ + RoomID: req.RoomIDOrAlias, // the room ID to try and join + UserID: req.UserID, // the user ID joining the room + ServerNames: req.ServerNames, // the server to try joining with + Content: req.Content, // the membership event content + } + fedRes := fsAPI.PerformJoinResponse{} + if err := r.fsAPI.PerformJoin(ctx, &fedReq, &fedRes); err != nil { + return fmt.Errorf("Error joining federated room: %q", err) } return nil diff --git a/roomserver/internal/perform_leave.go b/roomserver/internal/perform_leave.go index 422748e6a..5d9b251c9 100644 --- a/roomserver/internal/perform_leave.go +++ b/roomserver/internal/perform_leave.go @@ -38,7 +38,7 @@ func (r *RoomserverInternalAPI) performLeaveRoomByID( ) error { // If there's an invite outstanding for the room then respond to // that. - isInvitePending, senderUser, err := r.isInvitePending(ctx, req, res) + isInvitePending, senderUser, err := r.isInvitePending(ctx, req.RoomID, req.UserID) if err == nil && isInvitePending { return r.performRejectInvite(ctx, req, res, senderUser) } @@ -160,23 +160,22 @@ func (r *RoomserverInternalAPI) performRejectInvite( func (r *RoomserverInternalAPI) isInvitePending( ctx context.Context, - req *api.PerformLeaveRequest, - res *api.PerformLeaveResponse, // nolint:unparam + roomID, userID string, ) (bool, string, error) { // Look up the room NID for the supplied room ID. - roomNID, err := r.DB.RoomNID(ctx, req.RoomID) + roomNID, err := r.DB.RoomNID(ctx, roomID) if err != nil { return false, "", fmt.Errorf("r.DB.RoomNID: %w", err) } // Look up the state key NID for the supplied user ID. - targetUserNIDs, err := r.DB.EventStateKeyNIDs(ctx, []string{req.UserID}) + targetUserNIDs, err := r.DB.EventStateKeyNIDs(ctx, []string{userID}) if err != nil { return false, "", fmt.Errorf("r.DB.EventStateKeyNIDs: %w", err) } - targetUserNID, targetUserFound := targetUserNIDs[req.UserID] + targetUserNID, targetUserFound := targetUserNIDs[userID] if !targetUserFound { - return false, "", fmt.Errorf("missing NID for user %q (%+v)", req.UserID, targetUserNIDs) + return false, "", fmt.Errorf("missing NID for user %q (%+v)", userID, targetUserNIDs) } // Let's see if we have an event active for the user in the room. If From 1b34130a5b1a47bc461fd48c8ca731eaab1a529b Mon Sep 17 00:00:00 2001 From: Kegsay Date: Thu, 14 May 2020 16:11:37 +0100 Subject: [PATCH 058/200] Finish merging syncserver.go (#1033) * Refactor all postgres tables; start work on sqlite * wip sqlite merges; database is locked errors to investigate and failing tests * Revert "wip sqlite merges; database is locked errors to investigate and failing tests" This reverts commit 26cbfc5b75ae2dc4fb31a838b917aa39d758f162. * convert current room state table * port over sqlite topology table * remove a few functions * remove more functions * Share more code * factor out completesync and a bit more * Remove remaining code --- .../postgres/backwards_extremities_table.go | 106 ++ .../postgres/current_room_state_table.go | 42 +- .../output_room_events_topology_table.go | 50 +- syncapi/storage/postgres/syncserver.go | 903 +-------------- syncapi/storage/shared/syncserver.go | 896 ++++++++++++++- .../sqlite3/backwards_extremities_table.go | 106 ++ .../sqlite3/current_room_state_table.go | 39 +- .../output_room_events_topology_table.go | 32 +- syncapi/storage/sqlite3/syncserver.go | 1005 +---------------- syncapi/storage/storage.go | 8 +- syncapi/storage/storage_test.go | 2 +- syncapi/storage/storage_wasm.go | 2 +- .../storage/tables/backward_extremities.go | 175 --- syncapi/storage/tables/interface.go | 58 + 14 files changed, 1284 insertions(+), 2140 deletions(-) create mode 100644 syncapi/storage/postgres/backwards_extremities_table.go create mode 100644 syncapi/storage/sqlite3/backwards_extremities_table.go delete mode 100644 syncapi/storage/tables/backward_extremities.go diff --git a/syncapi/storage/postgres/backwards_extremities_table.go b/syncapi/storage/postgres/backwards_extremities_table.go new file mode 100644 index 000000000..3dd1b2349 --- /dev/null +++ b/syncapi/storage/postgres/backwards_extremities_table.go @@ -0,0 +1,106 @@ +// Copyright 2018 New Vector Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package postgres + +import ( + "context" + "database/sql" + + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/syncapi/storage/tables" +) + +const backwardExtremitiesSchema = ` +-- Stores output room events received from the roomserver. +CREATE TABLE IF NOT EXISTS syncapi_backward_extremities ( + -- The 'room_id' key for the event. + room_id TEXT NOT NULL, + -- The event ID for the last known event. This is the backwards extremity. + event_id TEXT NOT NULL, + -- The prev_events for the last known event. This is used to update extremities. + prev_event_id TEXT NOT NULL, + PRIMARY KEY(room_id, event_id, prev_event_id) +); +` + +const insertBackwardExtremitySQL = "" + + "INSERT INTO syncapi_backward_extremities (room_id, event_id, prev_event_id)" + + " VALUES ($1, $2, $3)" + + " ON CONFLICT DO NOTHING" + +const selectBackwardExtremitiesForRoomSQL = "" + + "SELECT DISTINCT event_id FROM syncapi_backward_extremities WHERE room_id = $1" + +const deleteBackwardExtremitySQL = "" + + "DELETE FROM syncapi_backward_extremities WHERE room_id = $1 AND prev_event_id = $2" + +type backwardExtremitiesStatements struct { + insertBackwardExtremityStmt *sql.Stmt + selectBackwardExtremitiesForRoomStmt *sql.Stmt + deleteBackwardExtremityStmt *sql.Stmt +} + +func NewPostgresBackwardsExtremitiesTable(db *sql.DB) (tables.BackwardsExtremities, error) { + s := &backwardExtremitiesStatements{} + _, err := db.Exec(backwardExtremitiesSchema) + if err != nil { + return nil, err + } + if s.insertBackwardExtremityStmt, err = db.Prepare(insertBackwardExtremitySQL); err != nil { + return nil, err + } + if s.selectBackwardExtremitiesForRoomStmt, err = db.Prepare(selectBackwardExtremitiesForRoomSQL); err != nil { + return nil, err + } + if s.deleteBackwardExtremityStmt, err = db.Prepare(deleteBackwardExtremitySQL); err != nil { + return nil, err + } + return s, nil +} + +func (s *backwardExtremitiesStatements) InsertsBackwardExtremity( + ctx context.Context, txn *sql.Tx, roomID, eventID string, prevEventID string, +) (err error) { + _, err = txn.Stmt(s.insertBackwardExtremityStmt).ExecContext(ctx, roomID, eventID, prevEventID) + return +} + +func (s *backwardExtremitiesStatements) SelectBackwardExtremitiesForRoom( + ctx context.Context, roomID string, +) (eventIDs []string, err error) { + rows, err := s.selectBackwardExtremitiesForRoomStmt.QueryContext(ctx, roomID) + if err != nil { + return + } + defer common.CloseAndLogIfError(ctx, rows, "selectBackwardExtremitiesForRoom: rows.close() failed") + + for rows.Next() { + var eID string + if err = rows.Scan(&eID); err != nil { + return + } + + eventIDs = append(eventIDs, eID) + } + + return eventIDs, rows.Err() +} + +func (s *backwardExtremitiesStatements) DeleteBackwardExtremity( + ctx context.Context, txn *sql.Tx, roomID, knownEventID string, +) (err error) { + _, err = txn.Stmt(s.deleteBackwardExtremityStmt).ExecContext(ctx, roomID, knownEventID) + return +} diff --git a/syncapi/storage/postgres/current_room_state_table.go b/syncapi/storage/postgres/current_room_state_table.go index ab8f07b21..b6ab72215 100644 --- a/syncapi/storage/postgres/current_room_state_table.go +++ b/syncapi/storage/postgres/current_room_state_table.go @@ -22,6 +22,7 @@ import ( "github.com/lib/pq" "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -103,37 +104,38 @@ type currentRoomStateStatements struct { selectStateEventStmt *sql.Stmt } -func (s *currentRoomStateStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(currentRoomStateSchema) +func NewPostgresCurrentRoomStateTable(db *sql.DB) (tables.CurrentRoomState, error) { + s := ¤tRoomStateStatements{} + _, err := db.Exec(currentRoomStateSchema) if err != nil { - return + return nil, err } if s.upsertRoomStateStmt, err = db.Prepare(upsertRoomStateSQL); err != nil { - return + return nil, err } if s.deleteRoomStateByEventIDStmt, err = db.Prepare(deleteRoomStateByEventIDSQL); err != nil { - return + return nil, err } if s.selectRoomIDsWithMembershipStmt, err = db.Prepare(selectRoomIDsWithMembershipSQL); err != nil { - return + return nil, err } if s.selectCurrentStateStmt, err = db.Prepare(selectCurrentStateSQL); err != nil { - return + return nil, err } if s.selectJoinedUsersStmt, err = db.Prepare(selectJoinedUsersSQL); err != nil { - return + return nil, err } if s.selectEventsWithEventIDsStmt, err = db.Prepare(selectEventsWithEventIDsSQL); err != nil { - return + return nil, err } if s.selectStateEventStmt, err = db.Prepare(selectStateEventSQL); err != nil { - return + return nil, err } - return + return s, nil } -// JoinedMemberLists returns a map of room ID to a list of joined user IDs. -func (s *currentRoomStateStatements) selectJoinedUsers( +// SelectJoinedUsers returns a map of room ID to a list of joined user IDs. +func (s *currentRoomStateStatements) SelectJoinedUsers( ctx context.Context, ) (map[string][]string, error) { rows, err := s.selectJoinedUsersStmt.QueryContext(ctx) @@ -157,7 +159,7 @@ func (s *currentRoomStateStatements) selectJoinedUsers( } // SelectRoomIDsWithMembership returns the list of room IDs which have the given user in the given membership state. -func (s *currentRoomStateStatements) selectRoomIDsWithMembership( +func (s *currentRoomStateStatements) SelectRoomIDsWithMembership( ctx context.Context, txn *sql.Tx, userID string, @@ -181,8 +183,8 @@ func (s *currentRoomStateStatements) selectRoomIDsWithMembership( return result, rows.Err() } -// CurrentState returns all the current state events for the given room. -func (s *currentRoomStateStatements) selectCurrentState( +// SelectCurrentState returns all the current state events for the given room. +func (s *currentRoomStateStatements) SelectCurrentState( ctx context.Context, txn *sql.Tx, roomID string, stateFilter *gomatrixserverlib.StateFilter, ) ([]gomatrixserverlib.HeaderedEvent, error) { @@ -203,7 +205,7 @@ func (s *currentRoomStateStatements) selectCurrentState( return rowsToEvents(rows) } -func (s *currentRoomStateStatements) deleteRoomStateByEventID( +func (s *currentRoomStateStatements) DeleteRoomStateByEventID( ctx context.Context, txn *sql.Tx, eventID string, ) error { stmt := common.TxStmt(txn, s.deleteRoomStateByEventIDStmt) @@ -211,7 +213,7 @@ func (s *currentRoomStateStatements) deleteRoomStateByEventID( return err } -func (s *currentRoomStateStatements) upsertRoomState( +func (s *currentRoomStateStatements) UpsertRoomState( ctx context.Context, txn *sql.Tx, event gomatrixserverlib.HeaderedEvent, membership *string, addedAt types.StreamPosition, ) error { @@ -245,7 +247,7 @@ func (s *currentRoomStateStatements) upsertRoomState( return err } -func (s *currentRoomStateStatements) selectEventsWithEventIDs( +func (s *currentRoomStateStatements) SelectEventsWithEventIDs( ctx context.Context, txn *sql.Tx, eventIDs []string, ) ([]types.StreamEvent, error) { stmt := common.TxStmt(txn, s.selectEventsWithEventIDsStmt) @@ -274,7 +276,7 @@ func rowsToEvents(rows *sql.Rows) ([]gomatrixserverlib.HeaderedEvent, error) { return result, rows.Err() } -func (s *currentRoomStateStatements) selectStateEvent( +func (s *currentRoomStateStatements) SelectStateEvent( ctx context.Context, roomID, evType, stateKey string, ) (*gomatrixserverlib.HeaderedEvent, error) { stmt := s.selectStateEventStmt diff --git a/syncapi/storage/postgres/output_room_events_topology_table.go b/syncapi/storage/postgres/output_room_events_topology_table.go index 51cbd50d0..48ebcf251 100644 --- a/syncapi/storage/postgres/output_room_events_topology_table.go +++ b/syncapi/storage/postgres/output_room_events_topology_table.go @@ -20,6 +20,7 @@ import ( "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -84,36 +85,37 @@ type outputRoomEventsTopologyStatements struct { selectEventIDsFromPositionStmt *sql.Stmt } -func (s *outputRoomEventsTopologyStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(outputRoomEventsTopologySchema) +func NewPostgresTopologyTable(db *sql.DB) (tables.Topology, error) { + s := &outputRoomEventsTopologyStatements{} + _, err := db.Exec(outputRoomEventsTopologySchema) if err != nil { - return + return nil, err } if s.insertEventInTopologyStmt, err = db.Prepare(insertEventInTopologySQL); err != nil { - return + return nil, err } if s.selectEventIDsInRangeASCStmt, err = db.Prepare(selectEventIDsInRangeASCSQL); err != nil { - return + return nil, err } if s.selectEventIDsInRangeDESCStmt, err = db.Prepare(selectEventIDsInRangeDESCSQL); err != nil { - return + return nil, err } if s.selectPositionInTopologyStmt, err = db.Prepare(selectPositionInTopologySQL); err != nil { - return + return nil, err } if s.selectMaxPositionInTopologyStmt, err = db.Prepare(selectMaxPositionInTopologySQL); err != nil { - return + return nil, err } if s.selectEventIDsFromPositionStmt, err = db.Prepare(selectEventIDsFromPositionSQL); err != nil { - return + return nil, err } - return + return s, nil } -// insertEventInTopology inserts the given event in the room's topology, based +// InsertEventInTopology inserts the given event in the room's topology, based // on the event's depth. -func (s *outputRoomEventsTopologyStatements) insertEventInTopology( - ctx context.Context, event *gomatrixserverlib.HeaderedEvent, pos types.StreamPosition, +func (s *outputRoomEventsTopologyStatements) InsertEventInTopology( + ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.HeaderedEvent, pos types.StreamPosition, ) (err error) { _, err = s.insertEventInTopologyStmt.ExecContext( ctx, event.EventID(), event.Depth(), event.RoomID(), pos, @@ -121,11 +123,11 @@ func (s *outputRoomEventsTopologyStatements) insertEventInTopology( return } -// selectEventIDsInRange selects the IDs of events which positions are within a +// SelectEventIDsInRange selects the IDs of events which positions are within a // given range in a given room's topological order. // Returns an empty slice if no events match the given range. -func (s *outputRoomEventsTopologyStatements) selectEventIDsInRange( - ctx context.Context, roomID string, fromPos, toPos, toMicroPos types.StreamPosition, +func (s *outputRoomEventsTopologyStatements) SelectEventIDsInRange( + ctx context.Context, txn *sql.Tx, roomID string, fromPos, toPos, toMicroPos types.StreamPosition, limit int, chronologicalOrder bool, ) (eventIDs []string, err error) { // Decide on the selection's order according to whether chronological order @@ -159,26 +161,26 @@ func (s *outputRoomEventsTopologyStatements) selectEventIDsInRange( return eventIDs, rows.Err() } -// selectPositionInTopology returns the position of a given event in the +// SelectPositionInTopology returns the position of a given event in the // topology of the room it belongs to. -func (s *outputRoomEventsTopologyStatements) selectPositionInTopology( - ctx context.Context, eventID string, +func (s *outputRoomEventsTopologyStatements) SelectPositionInTopology( + ctx context.Context, txn *sql.Tx, eventID string, ) (pos, spos types.StreamPosition, err error) { err = s.selectPositionInTopologyStmt.QueryRowContext(ctx, eventID).Scan(&pos, &spos) return } -func (s *outputRoomEventsTopologyStatements) selectMaxPositionInTopology( - ctx context.Context, roomID string, +func (s *outputRoomEventsTopologyStatements) SelectMaxPositionInTopology( + ctx context.Context, txn *sql.Tx, roomID string, ) (pos types.StreamPosition, spos types.StreamPosition, err error) { err = s.selectMaxPositionInTopologyStmt.QueryRowContext(ctx, roomID).Scan(&pos, &spos) return } -// selectEventIDsFromPosition returns the IDs of all events that have a given +// SelectEventIDsFromPosition returns the IDs of all events that have a given // position in the topology of a given room. -func (s *outputRoomEventsTopologyStatements) selectEventIDsFromPosition( - ctx context.Context, roomID string, pos types.StreamPosition, +func (s *outputRoomEventsTopologyStatements) SelectEventIDsFromPosition( + ctx context.Context, txn *sql.Tx, roomID string, pos types.StreamPosition, ) (eventIDs []string, err error) { // Query the event IDs. rows, err := s.selectEventIDsFromPositionStmt.QueryContext(ctx, roomID, pos) diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go index 4fa08ce5b..d44f8b7e6 100644 --- a/syncapi/storage/postgres/syncserver.go +++ b/syncapi/storage/postgres/syncserver.go @@ -16,47 +16,27 @@ package postgres import ( - "context" "database/sql" - "encoding/json" - "fmt" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/roomserver/api" // Import the postgres database driver. _ "github.com/lib/pq" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/syncapi/storage/shared" - "github.com/matrix-org/dendrite/syncapi/storage/tables" - "github.com/matrix-org/dendrite/syncapi/types" - "github.com/matrix-org/gomatrixserverlib" ) -type stateDelta struct { - roomID string - stateEvents []gomatrixserverlib.HeaderedEvent - membership string - // The PDU stream position of the latest membership event for this user, if applicable. - // Can be 0 if there is no membership event in this delta. - membershipPos types.StreamPosition -} - // SyncServerDatasource represents a sync server datasource which manages // both the database for PDUs and caches for EDUs. type SyncServerDatasource struct { shared.Database db *sql.DB common.PartitionOffsetStatements - roomstate currentRoomStateStatements - topology outputRoomEventsTopologyStatements - backwardExtremities tables.BackwardsExtremities } -// NewSyncServerDatasource creates a new sync server database -func NewSyncServerDatasource(dbDataSourceName string, dbProperties common.DbProperties) (*SyncServerDatasource, error) { +// NewDatabase creates a new sync server database +func NewDatabase(dbDataSourceName string, dbProperties common.DbProperties) (*SyncServerDatasource, error) { var d SyncServerDatasource var err error if d.db, err = sqlutil.Open("postgres", dbDataSourceName, dbProperties); err != nil { @@ -73,884 +53,31 @@ func NewSyncServerDatasource(dbDataSourceName string, dbProperties common.DbProp if err != nil { return nil, err } - if err = d.roomstate.prepare(d.db); err != nil { + currState, err := NewPostgresCurrentRoomStateTable(d.db) + if err != nil { return nil, err } invites, err := NewPostgresInvitesTable(d.db) if err != nil { return nil, err } - if err = d.topology.prepare(d.db); err != nil { + topology, err := NewPostgresTopologyTable(d.db) + if err != nil { return nil, err } - d.backwardExtremities, err = tables.NewBackwardsExtremities(d.db, &tables.PostgresBackwardsExtremitiesStatements{}) + backwardExtremities, err := NewPostgresBackwardsExtremitiesTable(d.db) if err != nil { return nil, err } d.Database = shared.Database{ - DB: d.db, - Invites: invites, - AccountData: accountData, - OutputEvents: events, - EDUCache: cache.New(), + DB: d.db, + Invites: invites, + AccountData: accountData, + OutputEvents: events, + Topology: topology, + CurrentRoomState: currState, + BackwardExtremities: backwardExtremities, + EDUCache: cache.New(), } return &d, nil } - -func (d *SyncServerDatasource) AllJoinedUsersInRooms(ctx context.Context) (map[string][]string, error) { - return d.roomstate.selectJoinedUsers(ctx) -} - -// handleBackwardExtremities adds this event as a backwards extremity if and only if we do not have all of -// the events listed in the event's 'prev_events'. This function also updates the backwards extremities table -// to account for the fact that the given event is no longer a backwards extremity, but may be marked as such. -func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, txn *sql.Tx, ev *gomatrixserverlib.HeaderedEvent) error { - if err := d.backwardExtremities.DeleteBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID()); err != nil { - return err - } - - // Check if we have all of the event's previous events. If an event is - // missing, add it to the room's backward extremities. - prevEvents, err := d.Database.OutputEvents.SelectEvents(ctx, txn, ev.PrevEventIDs()) - if err != nil { - return err - } - var found bool - for _, eID := range ev.PrevEventIDs() { - found = false - for _, prevEv := range prevEvents { - if eID == prevEv.EventID() { - found = true - } - } - - // If the event is missing, consider it a backward extremity. - if !found { - if err = d.backwardExtremities.InsertsBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID(), eID); err != nil { - return err - } - } - } - - return nil -} - -func (d *SyncServerDatasource) WriteEvent( - ctx context.Context, - ev *gomatrixserverlib.HeaderedEvent, - addStateEvents []gomatrixserverlib.HeaderedEvent, - addStateEventIDs, removeStateEventIDs []string, - transactionID *api.TransactionID, excludeFromSync bool, -) (pduPosition types.StreamPosition, returnErr error) { - returnErr = common.WithTransaction(d.db, func(txn *sql.Tx) error { - var err error - pos, err := d.Database.OutputEvents.InsertEvent( - ctx, txn, ev, addStateEventIDs, removeStateEventIDs, transactionID, excludeFromSync, - ) - if err != nil { - return err - } - pduPosition = pos - - if err = d.topology.insertEventInTopology(ctx, ev, pos); err != nil { - return err - } - - if err = d.handleBackwardExtremities(ctx, txn, ev); err != nil { - return err - } - - if len(addStateEvents) == 0 && len(removeStateEventIDs) == 0 { - // Nothing to do, the event may have just been a message event. - return nil - } - - return d.updateRoomState(ctx, txn, removeStateEventIDs, addStateEvents, pduPosition) - }) - - return pduPosition, returnErr -} - -func (d *SyncServerDatasource) updateRoomState( - ctx context.Context, txn *sql.Tx, - removedEventIDs []string, - addedEvents []gomatrixserverlib.HeaderedEvent, - pduPosition types.StreamPosition, -) error { - // remove first, then add, as we do not ever delete state, but do replace state which is a remove followed by an add. - for _, eventID := range removedEventIDs { - if err := d.roomstate.deleteRoomStateByEventID(ctx, txn, eventID); err != nil { - return err - } - } - - for _, event := range addedEvents { - if event.StateKey() == nil { - // ignore non state events - continue - } - var membership *string - if event.Type() == "m.room.member" { - value, err := event.Membership() - if err != nil { - return err - } - membership = &value - } - if err := d.roomstate.upsertRoomState(ctx, txn, event, membership, pduPosition); err != nil { - return err - } - } - - return nil -} - -func (d *SyncServerDatasource) GetStateEvent( - ctx context.Context, roomID, evType, stateKey string, -) (*gomatrixserverlib.HeaderedEvent, error) { - return d.roomstate.selectStateEvent(ctx, roomID, evType, stateKey) -} - -func (d *SyncServerDatasource) GetStateEventsForRoom( - ctx context.Context, roomID string, stateFilter *gomatrixserverlib.StateFilter, -) (stateEvents []gomatrixserverlib.HeaderedEvent, err error) { - err = common.WithTransaction(d.db, func(txn *sql.Tx) error { - stateEvents, err = d.roomstate.selectCurrentState(ctx, txn, roomID, stateFilter) - return err - }) - return -} - -func (d *SyncServerDatasource) GetEventsInTopologicalRange( - ctx context.Context, - from, to *types.TopologyToken, - roomID string, limit int, - backwardOrdering bool, -) (events []types.StreamEvent, err error) { - // Determine the backward and forward limit, i.e. the upper and lower - // limits to the selection in the room's topology, from the direction. - var backwardLimit, forwardLimit, forwardMicroLimit types.StreamPosition - if backwardOrdering { - // Backward ordering is antichronological (latest event to oldest - // one). - backwardLimit = to.Depth() - forwardLimit = from.Depth() - forwardMicroLimit = from.PDUPosition() - } else { - // Forward ordering is chronological (oldest event to latest one). - backwardLimit = from.Depth() - forwardLimit = to.Depth() - } - - // Select the event IDs from the defined range. - var eIDs []string - eIDs, err = d.topology.selectEventIDsInRange( - ctx, roomID, backwardLimit, forwardLimit, forwardMicroLimit, limit, !backwardOrdering, - ) - if err != nil { - return - } - - // Retrieve the events' contents using their IDs. - events, err = d.Database.OutputEvents.SelectEvents(ctx, nil, eIDs) - return -} - -func (d *SyncServerDatasource) SyncPosition(ctx context.Context) (types.StreamingToken, error) { - return d.syncPositionTx(ctx, nil) -} - -func (d *SyncServerDatasource) BackwardExtremitiesForRoom( - ctx context.Context, roomID string, -) (backwardExtremities []string, err error) { - return d.backwardExtremities.SelectBackwardExtremitiesForRoom(ctx, roomID) -} - -func (d *SyncServerDatasource) MaxTopologicalPosition( - ctx context.Context, roomID string, -) (depth types.StreamPosition, stream types.StreamPosition, err error) { - return d.topology.selectMaxPositionInTopology(ctx, roomID) -} - -func (d *SyncServerDatasource) EventsAtTopologicalPosition( - ctx context.Context, roomID string, pos types.StreamPosition, -) ([]types.StreamEvent, error) { - eIDs, err := d.topology.selectEventIDsFromPosition(ctx, roomID, pos) - if err != nil { - return nil, err - } - - return d.Database.OutputEvents.SelectEvents(ctx, nil, eIDs) -} - -func (d *SyncServerDatasource) EventPositionInTopology( - ctx context.Context, eventID string, -) (depth types.StreamPosition, stream types.StreamPosition, err error) { - return d.topology.selectPositionInTopology(ctx, eventID) -} - -func (d *SyncServerDatasource) syncPositionTx( - ctx context.Context, txn *sql.Tx, -) (sp types.StreamingToken, err error) { - - maxEventID, err := d.Database.OutputEvents.SelectMaxEventID(ctx, txn) - if err != nil { - return sp, err - } - maxAccountDataID, err := d.Database.AccountData.SelectMaxAccountDataID(ctx, txn) - if err != nil { - return sp, err - } - if maxAccountDataID > maxEventID { - maxEventID = maxAccountDataID - } - maxInviteID, err := d.Database.Invites.SelectMaxInviteID(ctx, txn) - if err != nil { - return sp, err - } - if maxInviteID > maxEventID { - maxEventID = maxInviteID - } - sp = types.NewStreamToken(types.StreamPosition(maxEventID), types.StreamPosition(d.Database.EDUCache.GetLatestSyncPosition())) - return -} - -// addPDUDeltaToResponse adds all PDU deltas to a sync response. -// IDs of all rooms the user joined are returned so EDU deltas can be added for them. -func (d *SyncServerDatasource) addPDUDeltaToResponse( - ctx context.Context, - device authtypes.Device, - fromPos, toPos types.StreamPosition, - numRecentEventsPerRoom int, - wantFullState bool, - res *types.Response, -) (joinedRoomIDs []string, err error) { - txn, err := d.db.BeginTx(ctx, &txReadOnlySnapshot) - if err != nil { - return nil, err - } - var succeeded bool - defer func() { - txerr := common.EndTransaction(txn, &succeeded) - if err == nil && txerr != nil { - err = txerr - } - }() - - stateFilter := gomatrixserverlib.DefaultStateFilter() // TODO: use filter provided in request - - // Work out which rooms to return in the response. This is done by getting not only the currently - // joined rooms, but also which rooms have membership transitions for this user between the 2 PDU stream positions. - // This works out what the 'state' key should be for each room as well as which membership block - // to put the room into. - var deltas []stateDelta - if !wantFullState { - deltas, joinedRoomIDs, err = d.getStateDeltas( - ctx, &device, txn, fromPos, toPos, device.UserID, &stateFilter, - ) - } else { - deltas, joinedRoomIDs, err = d.getStateDeltasForFullStateSync( - ctx, &device, txn, fromPos, toPos, device.UserID, &stateFilter, - ) - } - if err != nil { - return nil, err - } - - for _, delta := range deltas { - err = d.addRoomDeltaToResponse(ctx, &device, txn, fromPos, toPos, delta, numRecentEventsPerRoom, res) - if err != nil { - return nil, err - } - } - - // TODO: This should be done in getStateDeltas - if err = d.addInvitesToResponse(ctx, txn, device.UserID, fromPos, toPos, res); err != nil { - return nil, err - } - - succeeded = true - return joinedRoomIDs, nil -} - -// addTypingDeltaToResponse adds all typing notifications to a sync response -// since the specified position. -func (d *SyncServerDatasource) addTypingDeltaToResponse( - since types.StreamingToken, - joinedRoomIDs []string, - res *types.Response, -) error { - var jr types.JoinResponse - var ok bool - var err error - for _, roomID := range joinedRoomIDs { - if typingUsers, updated := d.Database.EDUCache.GetTypingUsersIfUpdatedAfter( - roomID, int64(since.EDUPosition()), - ); updated { - ev := gomatrixserverlib.ClientEvent{ - Type: gomatrixserverlib.MTyping, - } - ev.Content, err = json.Marshal(map[string]interface{}{ - "user_ids": typingUsers, - }) - if err != nil { - return err - } - - if jr, ok = res.Rooms.Join[roomID]; !ok { - jr = *types.NewJoinResponse() - } - jr.Ephemeral.Events = append(jr.Ephemeral.Events, ev) - res.Rooms.Join[roomID] = jr - } - } - return nil -} - -// addEDUDeltaToResponse adds updates for EDUs of each type since fromPos if -// the positions of that type are not equal in fromPos and toPos. -func (d *SyncServerDatasource) addEDUDeltaToResponse( - fromPos, toPos types.StreamingToken, - joinedRoomIDs []string, - res *types.Response, -) (err error) { - - if fromPos.EDUPosition() != toPos.EDUPosition() { - err = d.addTypingDeltaToResponse( - fromPos, joinedRoomIDs, res, - ) - } - - return -} - -func (d *SyncServerDatasource) IncrementalSync( - ctx context.Context, - device authtypes.Device, - fromPos, toPos types.StreamingToken, - numRecentEventsPerRoom int, - wantFullState bool, -) (*types.Response, error) { - nextBatchPos := fromPos.WithUpdates(toPos) - res := types.NewResponse(nextBatchPos) - - var joinedRoomIDs []string - var err error - if fromPos.PDUPosition() != toPos.PDUPosition() || wantFullState { - joinedRoomIDs, err = d.addPDUDeltaToResponse( - ctx, device, fromPos.PDUPosition(), toPos.PDUPosition(), numRecentEventsPerRoom, wantFullState, res, - ) - } else { - joinedRoomIDs, err = d.roomstate.selectRoomIDsWithMembership( - ctx, nil, device.UserID, gomatrixserverlib.Join, - ) - } - if err != nil { - return nil, err - } - - err = d.addEDUDeltaToResponse( - fromPos, toPos, joinedRoomIDs, res, - ) - if err != nil { - return nil, err - } - - return res, nil -} - -// getResponseWithPDUsForCompleteSync creates a response and adds all PDUs needed -// to it. It returns toPos and joinedRoomIDs for use of adding EDUs. -func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( - ctx context.Context, - userID string, - numRecentEventsPerRoom int, -) ( - res *types.Response, - toPos types.StreamingToken, - joinedRoomIDs []string, - err error, -) { - // This needs to be all done in a transaction as we need to do multiple SELECTs, and we need to have - // a consistent view of the database throughout. This includes extracting the sync position. - // This does have the unfortunate side-effect that all the matrixy logic resides in this function, - // but it's better to not hide the fact that this is being done in a transaction. - txn, err := d.db.BeginTx(ctx, &txReadOnlySnapshot) - if err != nil { - return - } - var succeeded bool - defer func() { - txerr := common.EndTransaction(txn, &succeeded) - if err == nil && txerr != nil { - err = txerr - } - }() - - // Get the current sync position which we will base the sync response on. - toPos, err = d.syncPositionTx(ctx, txn) - if err != nil { - return - } - - res = types.NewResponse(toPos) - - // Extract room state and recent events for all rooms the user is joined to. - joinedRoomIDs, err = d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, gomatrixserverlib.Join) - if err != nil { - return - } - - stateFilter := gomatrixserverlib.DefaultStateFilter() // TODO: use filter provided in request - - // Build up a /sync response. Add joined rooms. - for _, roomID := range joinedRoomIDs { - var stateEvents []gomatrixserverlib.HeaderedEvent - stateEvents, err = d.roomstate.selectCurrentState(ctx, txn, roomID, &stateFilter) - if err != nil { - return - } - // TODO: When filters are added, we may need to call this multiple times to get enough events. - // See: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L316 - var recentStreamEvents []types.StreamEvent - recentStreamEvents, err = d.Database.OutputEvents.SelectRecentEvents( - ctx, txn, roomID, types.StreamPosition(0), toPos.PDUPosition(), - numRecentEventsPerRoom, true, true, - ) - if err != nil { - return - } - - // Retrieve the backward topology position, i.e. the position of the - // oldest event in the room's topology. - var prevBatchStr string - if len(recentStreamEvents) > 0 { - var backwardTopologyPos, backwardStreamPos types.StreamPosition - backwardTopologyPos, backwardStreamPos, err = d.topology.selectPositionInTopology(ctx, recentStreamEvents[0].EventID()) - if err != nil { - return - } - prevBatch := types.NewTopologyToken(backwardTopologyPos, backwardStreamPos) - prevBatch.Decrement() - prevBatchStr = prevBatch.String() - } - - // We don't include a device here as we don't need to send down - // transaction IDs for complete syncs - recentEvents := d.StreamEventsToEvents(nil, recentStreamEvents) - stateEvents = removeDuplicates(stateEvents, recentEvents) - jr := types.NewJoinResponse() - jr.Timeline.PrevBatch = prevBatchStr - jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) - jr.Timeline.Limited = true - jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(stateEvents, gomatrixserverlib.FormatSync) - res.Rooms.Join[roomID] = *jr - } - - if err = d.addInvitesToResponse(ctx, txn, userID, 0, toPos.PDUPosition(), res); err != nil { - return - } - - succeeded = true - return res, toPos, joinedRoomIDs, err -} - -func (d *SyncServerDatasource) CompleteSync( - ctx context.Context, userID string, numRecentEventsPerRoom int, -) (*types.Response, error) { - res, toPos, joinedRoomIDs, err := d.getResponseWithPDUsForCompleteSync( - ctx, userID, numRecentEventsPerRoom, - ) - if err != nil { - return nil, err - } - - // Use a zero value SyncPosition for fromPos so all EDU states are added. - err = d.addEDUDeltaToResponse( - types.NewStreamToken(0, 0), toPos, joinedRoomIDs, res, - ) - if err != nil { - return nil, err - } - - return res, nil -} - -var txReadOnlySnapshot = sql.TxOptions{ - // Set the isolation level so that we see a snapshot of the database. - // In PostgreSQL repeatable read transactions will see a snapshot taken - // at the first query, and since the transaction is read-only it can't - // run into any serialisation errors. - // https://www.postgresql.org/docs/9.5/static/transaction-iso.html#XACT-REPEATABLE-READ - Isolation: sql.LevelRepeatableRead, - ReadOnly: true, -} - -func (d *SyncServerDatasource) addInvitesToResponse( - ctx context.Context, txn *sql.Tx, - userID string, - fromPos, toPos types.StreamPosition, - res *types.Response, -) error { - invites, err := d.Database.Invites.SelectInviteEventsInRange( - ctx, txn, userID, fromPos, toPos, - ) - if err != nil { - return err - } - for roomID, inviteEvent := range invites { - ir := types.NewInviteResponse(inviteEvent) - res.Rooms.Invite[roomID] = *ir - } - return nil -} - -// Retrieve the backward topology position, i.e. the position of the -// oldest event in the room's topology. -func (d *SyncServerDatasource) getBackwardTopologyPos( - ctx context.Context, - events []types.StreamEvent, -) (pos, spos types.StreamPosition) { - if len(events) > 0 { - pos, spos, _ = d.topology.selectPositionInTopology(ctx, events[0].EventID()) - } - if pos-1 <= 0 { - pos = types.StreamPosition(1) - } else { - pos = pos - 1 - } - return -} - -// addRoomDeltaToResponse adds a room state delta to a sync response -func (d *SyncServerDatasource) addRoomDeltaToResponse( - ctx context.Context, - device *authtypes.Device, - txn *sql.Tx, - fromPos, toPos types.StreamPosition, - delta stateDelta, - numRecentEventsPerRoom int, - res *types.Response, -) error { - endPos := toPos - if delta.membershipPos > 0 && delta.membership == gomatrixserverlib.Leave { - // make sure we don't leak recent events after the leave event. - // TODO: History visibility makes this somewhat complex to handle correctly. For example: - // TODO: This doesn't work for join -> leave in a single /sync request (see events prior to join). - // TODO: This will fail on join -> leave -> sensitive msg -> join -> leave - // in a single /sync request - // This is all "okay" assuming history_visibility == "shared" which it is by default. - endPos = delta.membershipPos - } - recentStreamEvents, err := d.Database.OutputEvents.SelectRecentEvents( - ctx, txn, delta.roomID, types.StreamPosition(fromPos), types.StreamPosition(endPos), - numRecentEventsPerRoom, true, true, - ) - if err != nil { - return err - } - recentEvents := d.StreamEventsToEvents(device, recentStreamEvents) - delta.stateEvents = removeDuplicates(delta.stateEvents, recentEvents) // roll back - backwardTopologyPos, backwardStreamPos := d.getBackwardTopologyPos(ctx, recentStreamEvents) - prevBatch := types.NewTopologyToken( - backwardTopologyPos, backwardStreamPos, - ) - - switch delta.membership { - case gomatrixserverlib.Join: - jr := types.NewJoinResponse() - - jr.Timeline.PrevBatch = prevBatch.String() - jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) - jr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true - jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) - res.Rooms.Join[delta.roomID] = *jr - case gomatrixserverlib.Leave: - fallthrough // transitions to leave are the same as ban - case gomatrixserverlib.Ban: - // TODO: recentEvents may contain events that this user is not allowed to see because they are - // no longer in the room. - lr := types.NewLeaveResponse() - lr.Timeline.PrevBatch = prevBatch.String() - lr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) - lr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true - lr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) - res.Rooms.Leave[delta.roomID] = *lr - } - - return nil -} - -// fetchStateEvents converts the set of event IDs into a set of events. It will fetch any which are missing from the database. -// Returns a map of room ID to list of events. -func (d *SyncServerDatasource) fetchStateEvents( - ctx context.Context, txn *sql.Tx, - roomIDToEventIDSet map[string]map[string]bool, - eventIDToEvent map[string]types.StreamEvent, -) (map[string][]types.StreamEvent, error) { - stateBetween := make(map[string][]types.StreamEvent) - missingEvents := make(map[string][]string) - for roomID, ids := range roomIDToEventIDSet { - events := stateBetween[roomID] - for id, need := range ids { - if !need { - continue // deleted state - } - e, ok := eventIDToEvent[id] - if ok { - events = append(events, e) - } else { - m := missingEvents[roomID] - m = append(m, id) - missingEvents[roomID] = m - } - } - stateBetween[roomID] = events - } - - if len(missingEvents) > 0 { - // This happens when add_state_ids has an event ID which is not in the provided range. - // We need to explicitly fetch them. - allMissingEventIDs := []string{} - for _, missingEvIDs := range missingEvents { - allMissingEventIDs = append(allMissingEventIDs, missingEvIDs...) - } - evs, err := d.fetchMissingStateEvents(ctx, txn, allMissingEventIDs) - if err != nil { - return nil, err - } - // we know we got them all otherwise an error would've been returned, so just loop the events - for _, ev := range evs { - roomID := ev.RoomID() - stateBetween[roomID] = append(stateBetween[roomID], ev) - } - } - return stateBetween, nil -} - -func (d *SyncServerDatasource) fetchMissingStateEvents( - ctx context.Context, txn *sql.Tx, eventIDs []string, -) ([]types.StreamEvent, error) { - // Fetch from the events table first so we pick up the stream ID for the - // event. - events, err := d.Database.OutputEvents.SelectEvents(ctx, txn, eventIDs) - if err != nil { - return nil, err - } - - have := map[string]bool{} - for _, event := range events { - have[event.EventID()] = true - } - var missing []string - for _, eventID := range eventIDs { - if !have[eventID] { - missing = append(missing, eventID) - } - } - if len(missing) == 0 { - return events, nil - } - - // If they are missing from the events table then they should be state - // events that we received from outside the main event stream. - // These should be in the room state table. - stateEvents, err := d.roomstate.selectEventsWithEventIDs(ctx, txn, missing) - - if err != nil { - return nil, err - } - if len(stateEvents) != len(missing) { - return nil, fmt.Errorf("failed to map all event IDs to events: (got %d, wanted %d)", len(stateEvents), len(missing)) - } - events = append(events, stateEvents...) - return events, nil -} - -// getStateDeltas returns the state deltas between fromPos and toPos, -// exclusive of oldPos, inclusive of newPos, for the rooms in which -// the user has new membership events. -// A list of joined room IDs is also returned in case the caller needs it. -func (d *SyncServerDatasource) getStateDeltas( - ctx context.Context, device *authtypes.Device, txn *sql.Tx, - fromPos, toPos types.StreamPosition, userID string, - stateFilter *gomatrixserverlib.StateFilter, -) ([]stateDelta, []string, error) { - // Implement membership change algorithm: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L821 - // - Get membership list changes for this user in this sync response - // - For each room which has membership list changes: - // * Check if the room is 'newly joined' (insufficient to just check for a join event because we allow dupe joins TODO). - // If it is, then we need to send the full room state down (and 'limited' is always true). - // * Check if user is still CURRENTLY invited to the room. If so, add room to 'invited' block. - // * Check if the user is CURRENTLY (TODO) left/banned. If so, add room to 'archived' block. - // - Get all CURRENTLY joined rooms, and add them to 'joined' block. - var deltas []stateDelta - - // get all the state events ever between these two positions - stateNeeded, eventMap, err := d.Database.OutputEvents.SelectStateInRange(ctx, txn, fromPos, toPos, stateFilter) - if err != nil { - return nil, nil, err - } - state, err := d.fetchStateEvents(ctx, txn, stateNeeded, eventMap) - if err != nil { - return nil, nil, err - } - - for roomID, stateStreamEvents := range state { - for _, ev := range stateStreamEvents { - // TODO: Currently this will incorrectly add rooms which were ALREADY joined but they sent another no-op join event. - // We should be checking if the user was already joined at fromPos and not proceed if so. As a result of this, - // dupe join events will result in the entire room state coming down to the client again. This is added in - // the 'state' part of the response though, so is transparent modulo bandwidth concerns as it is not added to - // the timeline. - if membership := getMembershipFromEvent(&ev.Event, userID); membership != "" { - if membership == gomatrixserverlib.Join { - // send full room state down instead of a delta - var s []types.StreamEvent - s, err = d.currentStateStreamEventsForRoom(ctx, txn, roomID, stateFilter) - if err != nil { - return nil, nil, err - } - state[roomID] = s - continue // we'll add this room in when we do joined rooms - } - - deltas = append(deltas, stateDelta{ - membership: membership, - membershipPos: ev.StreamPosition, - stateEvents: d.StreamEventsToEvents(device, stateStreamEvents), - roomID: roomID, - }) - break - } - } - } - - // Add in currently joined rooms - joinedRoomIDs, err := d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, gomatrixserverlib.Join) - if err != nil { - return nil, nil, err - } - for _, joinedRoomID := range joinedRoomIDs { - deltas = append(deltas, stateDelta{ - membership: gomatrixserverlib.Join, - stateEvents: d.StreamEventsToEvents(device, state[joinedRoomID]), - roomID: joinedRoomID, - }) - } - - return deltas, joinedRoomIDs, nil -} - -// getStateDeltasForFullStateSync is a variant of getStateDeltas used for /sync -// requests with full_state=true. -// Fetches full state for all joined rooms and uses selectStateInRange to get -// updates for other rooms. -func (d *SyncServerDatasource) getStateDeltasForFullStateSync( - ctx context.Context, device *authtypes.Device, txn *sql.Tx, - fromPos, toPos types.StreamPosition, userID string, - stateFilter *gomatrixserverlib.StateFilter, -) ([]stateDelta, []string, error) { - joinedRoomIDs, err := d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, gomatrixserverlib.Join) - if err != nil { - return nil, nil, err - } - - // Use a reasonable initial capacity - deltas := make([]stateDelta, 0, len(joinedRoomIDs)) - - // Add full states for all joined rooms - for _, joinedRoomID := range joinedRoomIDs { - s, stateErr := d.currentStateStreamEventsForRoom(ctx, txn, joinedRoomID, stateFilter) - if stateErr != nil { - return nil, nil, stateErr - } - deltas = append(deltas, stateDelta{ - membership: gomatrixserverlib.Join, - stateEvents: d.StreamEventsToEvents(device, s), - roomID: joinedRoomID, - }) - } - - // Get all the state events ever between these two positions - stateNeeded, eventMap, err := d.Database.OutputEvents.SelectStateInRange(ctx, txn, fromPos, toPos, stateFilter) - if err != nil { - return nil, nil, err - } - state, err := d.fetchStateEvents(ctx, txn, stateNeeded, eventMap) - if err != nil { - return nil, nil, err - } - - for roomID, stateStreamEvents := range state { - for _, ev := range stateStreamEvents { - if membership := getMembershipFromEvent(&ev.Event, userID); membership != "" { - if membership != gomatrixserverlib.Join { // We've already added full state for all joined rooms above. - deltas = append(deltas, stateDelta{ - membership: membership, - membershipPos: ev.StreamPosition, - stateEvents: d.StreamEventsToEvents(device, stateStreamEvents), - roomID: roomID, - }) - } - - break - } - } - } - - return deltas, joinedRoomIDs, nil -} - -func (d *SyncServerDatasource) currentStateStreamEventsForRoom( - ctx context.Context, txn *sql.Tx, roomID string, - stateFilter *gomatrixserverlib.StateFilter, -) ([]types.StreamEvent, error) { - allState, err := d.roomstate.selectCurrentState(ctx, txn, roomID, stateFilter) - if err != nil { - return nil, err - } - s := make([]types.StreamEvent, len(allState)) - for i := 0; i < len(s); i++ { - s[i] = types.StreamEvent{HeaderedEvent: allState[i], StreamPosition: 0} - } - return s, nil -} - -// There may be some overlap where events in stateEvents are already in recentEvents, so filter -// them out so we don't include them twice in the /sync response. They should be in recentEvents -// only, so clients get to the correct state once they have rolled forward. -func removeDuplicates(stateEvents, recentEvents []gomatrixserverlib.HeaderedEvent) []gomatrixserverlib.HeaderedEvent { - for _, recentEv := range recentEvents { - if recentEv.StateKey() == nil { - continue // not a state event - } - // TODO: This is a linear scan over all the current state events in this room. This will - // be slow for big rooms. We should instead sort the state events by event ID (ORDER BY) - // then do a binary search to find matching events, similar to what roomserver does. - for j := 0; j < len(stateEvents); j++ { - if stateEvents[j].EventID() == recentEv.EventID() { - // overwrite the element to remove with the last element then pop the last element. - // This is orders of magnitude faster than re-slicing, but doesn't preserve ordering - // (we don't care about the order of stateEvents) - stateEvents[j] = stateEvents[len(stateEvents)-1] - stateEvents = stateEvents[:len(stateEvents)-1] - break // there shouldn't be multiple events with the same event ID - } - } - } - return stateEvents -} - -// getMembershipFromEvent returns the value of content.membership iff the event is a state event -// with type 'm.room.member' and state_key of userID. Otherwise, an empty string is returned. -func getMembershipFromEvent(ev *gomatrixserverlib.Event, userID string) string { - if ev.Type() == "m.room.member" && ev.StateKeyEquals(userID) { - membership, err := ev.Membership() - if err != nil { - return "" - } - return membership - } - return "" -} diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go index e9fed758b..96e9ff619 100644 --- a/syncapi/storage/shared/syncserver.go +++ b/syncapi/storage/shared/syncserver.go @@ -3,11 +3,14 @@ package shared import ( "context" "database/sql" + "encoding/json" + "fmt" "time" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/eduserver/cache" + "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -17,11 +20,14 @@ import ( // Database is a temporary struct until we have made syncserver.go the same for both pq/sqlite // For now this contains the shared functions type Database struct { - DB *sql.DB - Invites tables.Invites - AccountData tables.AccountData - OutputEvents tables.Events - EDUCache *cache.EDUCache + DB *sql.DB + Invites tables.Invites + AccountData tables.AccountData + OutputEvents tables.Events + Topology tables.Topology + CurrentRoomState tables.CurrentRoomState + BackwardExtremities tables.BackwardsExtremities + EDUCache *cache.EDUCache } // Events lookups a list of event by their event ID. @@ -82,6 +88,26 @@ func (d *Database) SetTypingTimeoutCallback(fn cache.TimeoutCallbackFn) { d.EDUCache.SetTimeoutCallback(fn) } +func (d *Database) AllJoinedUsersInRooms(ctx context.Context) (map[string][]string, error) { + return d.CurrentRoomState.SelectJoinedUsers(ctx) +} + +func (d *Database) GetStateEvent( + ctx context.Context, roomID, evType, stateKey string, +) (*gomatrixserverlib.HeaderedEvent, error) { + return d.CurrentRoomState.SelectStateEvent(ctx, roomID, evType, stateKey) +} + +func (d *Database) GetStateEventsForRoom( + ctx context.Context, roomID string, stateFilter *gomatrixserverlib.StateFilter, +) (stateEvents []gomatrixserverlib.HeaderedEvent, err error) { + err = common.WithTransaction(d.DB, func(txn *sql.Tx) error { + stateEvents, err = d.CurrentRoomState.SelectCurrentState(ctx, txn, roomID, stateFilter) + return err + }) + return +} + func (d *Database) SyncStreamPosition(ctx context.Context) (types.StreamPosition, error) { var maxID int64 var err error @@ -182,3 +208,863 @@ func (d *Database) StreamEventsToEvents(device *authtypes.Device, in []types.Str } return out } + +// handleBackwardExtremities adds this event as a backwards extremity if and only if we do not have all of +// the events listed in the event's 'prev_events'. This function also updates the backwards extremities table +// to account for the fact that the given event is no longer a backwards extremity, but may be marked as such. +func (d *Database) handleBackwardExtremities(ctx context.Context, txn *sql.Tx, ev *gomatrixserverlib.HeaderedEvent) error { + if err := d.BackwardExtremities.DeleteBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID()); err != nil { + return err + } + + // Check if we have all of the event's previous events. If an event is + // missing, add it to the room's backward extremities. + prevEvents, err := d.OutputEvents.SelectEvents(ctx, txn, ev.PrevEventIDs()) + if err != nil { + return err + } + var found bool + for _, eID := range ev.PrevEventIDs() { + found = false + for _, prevEv := range prevEvents { + if eID == prevEv.EventID() { + found = true + } + } + + // If the event is missing, consider it a backward extremity. + if !found { + if err = d.BackwardExtremities.InsertsBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID(), eID); err != nil { + return err + } + } + } + + return nil +} + +func (d *Database) WriteEvent( + ctx context.Context, + ev *gomatrixserverlib.HeaderedEvent, + addStateEvents []gomatrixserverlib.HeaderedEvent, + addStateEventIDs, removeStateEventIDs []string, + transactionID *api.TransactionID, excludeFromSync bool, +) (pduPosition types.StreamPosition, returnErr error) { + returnErr = common.WithTransaction(d.DB, func(txn *sql.Tx) error { + var err error + pos, err := d.OutputEvents.InsertEvent( + ctx, txn, ev, addStateEventIDs, removeStateEventIDs, transactionID, excludeFromSync, + ) + if err != nil { + return err + } + pduPosition = pos + + if err = d.Topology.InsertEventInTopology(ctx, txn, ev, pos); err != nil { + return err + } + + if err = d.handleBackwardExtremities(ctx, txn, ev); err != nil { + return err + } + + if len(addStateEvents) == 0 && len(removeStateEventIDs) == 0 { + // Nothing to do, the event may have just been a message event. + return nil + } + + return d.updateRoomState(ctx, txn, removeStateEventIDs, addStateEvents, pduPosition) + }) + + return pduPosition, returnErr +} + +func (d *Database) updateRoomState( + ctx context.Context, txn *sql.Tx, + removedEventIDs []string, + addedEvents []gomatrixserverlib.HeaderedEvent, + pduPosition types.StreamPosition, +) error { + // remove first, then add, as we do not ever delete state, but do replace state which is a remove followed by an add. + for _, eventID := range removedEventIDs { + if err := d.CurrentRoomState.DeleteRoomStateByEventID(ctx, txn, eventID); err != nil { + return err + } + } + + for _, event := range addedEvents { + if event.StateKey() == nil { + // ignore non state events + continue + } + var membership *string + if event.Type() == "m.room.member" { + value, err := event.Membership() + if err != nil { + return err + } + membership = &value + } + if err := d.CurrentRoomState.UpsertRoomState(ctx, txn, event, membership, pduPosition); err != nil { + return err + } + } + + return nil +} + +func (d *Database) GetEventsInTopologicalRange( + ctx context.Context, + from, to *types.TopologyToken, + roomID string, limit int, + backwardOrdering bool, +) (events []types.StreamEvent, err error) { + // Determine the backward and forward limit, i.e. the upper and lower + // limits to the selection in the room's topology, from the direction. + var backwardLimit, forwardLimit, forwardMicroLimit types.StreamPosition + if backwardOrdering { + // Backward ordering is antichronological (latest event to oldest + // one). + backwardLimit = to.Depth() + forwardLimit = from.Depth() + forwardMicroLimit = from.PDUPosition() + } else { + // Forward ordering is chronological (oldest event to latest one). + backwardLimit = from.Depth() + forwardLimit = to.Depth() + } + + // Select the event IDs from the defined range. + var eIDs []string + eIDs, err = d.Topology.SelectEventIDsInRange( + ctx, nil, roomID, backwardLimit, forwardLimit, forwardMicroLimit, limit, !backwardOrdering, + ) + if err != nil { + return + } + + // Retrieve the events' contents using their IDs. + events, err = d.OutputEvents.SelectEvents(ctx, nil, eIDs) + return +} + +func (d *Database) SyncPosition(ctx context.Context) (tok types.StreamingToken, err error) { + err = common.WithTransaction(d.DB, func(txn *sql.Tx) error { + pos, err := d.SyncPositionTx(ctx, txn) + if err != nil { + return err + } + tok = pos + return nil + }) + return +} + +func (d *Database) BackwardExtremitiesForRoom( + ctx context.Context, roomID string, +) (backwardExtremities []string, err error) { + return d.BackwardExtremities.SelectBackwardExtremitiesForRoom(ctx, roomID) +} + +func (d *Database) MaxTopologicalPosition( + ctx context.Context, roomID string, +) (depth types.StreamPosition, stream types.StreamPosition, err error) { + return d.Topology.SelectMaxPositionInTopology(ctx, nil, roomID) +} + +func (d *Database) EventsAtTopologicalPosition( + ctx context.Context, roomID string, pos types.StreamPosition, +) ([]types.StreamEvent, error) { + eIDs, err := d.Topology.SelectEventIDsFromPosition(ctx, nil, roomID, pos) + if err != nil { + return nil, err + } + + return d.OutputEvents.SelectEvents(ctx, nil, eIDs) +} + +func (d *Database) EventPositionInTopology( + ctx context.Context, eventID string, +) (depth types.StreamPosition, stream types.StreamPosition, err error) { + return d.Topology.SelectPositionInTopology(ctx, nil, eventID) +} + +// TODO FIXME TEMPORARY PUBLIC +func (d *Database) SyncPositionTx( + ctx context.Context, txn *sql.Tx, +) (sp types.StreamingToken, err error) { + + maxEventID, err := d.OutputEvents.SelectMaxEventID(ctx, txn) + if err != nil { + return sp, err + } + maxAccountDataID, err := d.AccountData.SelectMaxAccountDataID(ctx, txn) + if err != nil { + return sp, err + } + if maxAccountDataID > maxEventID { + maxEventID = maxAccountDataID + } + maxInviteID, err := d.Invites.SelectMaxInviteID(ctx, txn) + if err != nil { + return sp, err + } + if maxInviteID > maxEventID { + maxEventID = maxInviteID + } + sp = types.NewStreamToken(types.StreamPosition(maxEventID), types.StreamPosition(d.EDUCache.GetLatestSyncPosition())) + return +} + +// addPDUDeltaToResponse adds all PDU deltas to a sync response. +// IDs of all rooms the user joined are returned so EDU deltas can be added for them. +func (d *Database) addPDUDeltaToResponse( + ctx context.Context, + device authtypes.Device, + fromPos, toPos types.StreamPosition, + numRecentEventsPerRoom int, + wantFullState bool, + res *types.Response, +) (joinedRoomIDs []string, err error) { + txn, err := d.DB.BeginTx(ctx, &txReadOnlySnapshot) + if err != nil { + return nil, err + } + var succeeded bool + defer func() { + txerr := common.EndTransaction(txn, &succeeded) + if err == nil && txerr != nil { + err = txerr + } + }() + + stateFilter := gomatrixserverlib.DefaultStateFilter() // TODO: use filter provided in request + + // Work out which rooms to return in the response. This is done by getting not only the currently + // joined rooms, but also which rooms have membership transitions for this user between the 2 PDU stream positions. + // This works out what the 'state' key should be for each room as well as which membership block + // to put the room into. + var deltas []stateDelta + if !wantFullState { + deltas, joinedRoomIDs, err = d.getStateDeltas( + ctx, &device, txn, fromPos, toPos, device.UserID, &stateFilter, + ) + } else { + deltas, joinedRoomIDs, err = d.getStateDeltasForFullStateSync( + ctx, &device, txn, fromPos, toPos, device.UserID, &stateFilter, + ) + } + if err != nil { + return nil, err + } + + for _, delta := range deltas { + err = d.addRoomDeltaToResponse(ctx, &device, txn, fromPos, toPos, delta, numRecentEventsPerRoom, res) + if err != nil { + return nil, err + } + } + + // TODO: This should be done in getStateDeltas + if err = d.AddInvitesToResponse(ctx, txn, device.UserID, fromPos, toPos, res); err != nil { + return nil, err + } + + succeeded = true + return joinedRoomIDs, nil +} + +// addTypingDeltaToResponse adds all typing notifications to a sync response +// since the specified position. +func (d *Database) addTypingDeltaToResponse( + since types.StreamingToken, + joinedRoomIDs []string, + res *types.Response, +) error { + var jr types.JoinResponse + var ok bool + var err error + for _, roomID := range joinedRoomIDs { + if typingUsers, updated := d.EDUCache.GetTypingUsersIfUpdatedAfter( + roomID, int64(since.EDUPosition()), + ); updated { + ev := gomatrixserverlib.ClientEvent{ + Type: gomatrixserverlib.MTyping, + } + ev.Content, err = json.Marshal(map[string]interface{}{ + "user_ids": typingUsers, + }) + if err != nil { + return err + } + + if jr, ok = res.Rooms.Join[roomID]; !ok { + jr = *types.NewJoinResponse() + } + jr.Ephemeral.Events = append(jr.Ephemeral.Events, ev) + res.Rooms.Join[roomID] = jr + } + } + return nil +} + +// addEDUDeltaToResponse adds updates for EDUs of each type since fromPos if +// the positions of that type are not equal in fromPos and toPos. +// TODO FIXME TEMPORARY PUBLIC +func (d *Database) AddEDUDeltaToResponse( + fromPos, toPos types.StreamingToken, + joinedRoomIDs []string, + res *types.Response, +) (err error) { + + if fromPos.EDUPosition() != toPos.EDUPosition() { + err = d.addTypingDeltaToResponse( + fromPos, joinedRoomIDs, res, + ) + } + + return +} + +func (d *Database) IncrementalSync( + ctx context.Context, + device authtypes.Device, + fromPos, toPos types.StreamingToken, + numRecentEventsPerRoom int, + wantFullState bool, +) (*types.Response, error) { + nextBatchPos := fromPos.WithUpdates(toPos) + res := types.NewResponse(nextBatchPos) + + var joinedRoomIDs []string + var err error + if fromPos.PDUPosition() != toPos.PDUPosition() || wantFullState { + joinedRoomIDs, err = d.addPDUDeltaToResponse( + ctx, device, fromPos.PDUPosition(), toPos.PDUPosition(), numRecentEventsPerRoom, wantFullState, res, + ) + } else { + joinedRoomIDs, err = d.CurrentRoomState.SelectRoomIDsWithMembership( + ctx, nil, device.UserID, gomatrixserverlib.Join, + ) + } + if err != nil { + return nil, err + } + + err = d.AddEDUDeltaToResponse( + fromPos, toPos, joinedRoomIDs, res, + ) + if err != nil { + return nil, err + } + + return res, nil +} + +// getResponseWithPDUsForCompleteSync creates a response and adds all PDUs needed +// to it. It returns toPos and joinedRoomIDs for use of adding EDUs. +func (d *Database) getResponseWithPDUsForCompleteSync( + ctx context.Context, + userID string, + numRecentEventsPerRoom int, +) ( + res *types.Response, + toPos types.StreamingToken, + joinedRoomIDs []string, + err error, +) { + // This needs to be all done in a transaction as we need to do multiple SELECTs, and we need to have + // a consistent view of the database throughout. This includes extracting the sync position. + // This does have the unfortunate side-effect that all the matrixy logic resides in this function, + // but it's better to not hide the fact that this is being done in a transaction. + txn, err := d.DB.BeginTx(ctx, &txReadOnlySnapshot) + if err != nil { + return + } + var succeeded bool + defer func() { + txerr := common.EndTransaction(txn, &succeeded) + if err == nil && txerr != nil { + err = txerr + } + }() + + // Get the current sync position which we will base the sync response on. + toPos, err = d.SyncPositionTx(ctx, txn) + if err != nil { + return + } + + res = types.NewResponse(toPos) + + // Extract room state and recent events for all rooms the user is joined to. + joinedRoomIDs, err = d.CurrentRoomState.SelectRoomIDsWithMembership(ctx, txn, userID, gomatrixserverlib.Join) + if err != nil { + return + } + + stateFilter := gomatrixserverlib.DefaultStateFilter() // TODO: use filter provided in request + + // Build up a /sync response. Add joined rooms. + for _, roomID := range joinedRoomIDs { + var stateEvents []gomatrixserverlib.HeaderedEvent + stateEvents, err = d.CurrentRoomState.SelectCurrentState(ctx, txn, roomID, &stateFilter) + if err != nil { + return + } + // TODO: When filters are added, we may need to call this multiple times to get enough events. + // See: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L316 + var recentStreamEvents []types.StreamEvent + recentStreamEvents, err = d.OutputEvents.SelectRecentEvents( + ctx, txn, roomID, types.StreamPosition(0), toPos.PDUPosition(), + numRecentEventsPerRoom, true, true, + ) + if err != nil { + return + } + + // Retrieve the backward topology position, i.e. the position of the + // oldest event in the room's topology. + var prevBatchStr string + if len(recentStreamEvents) > 0 { + var backwardTopologyPos, backwardStreamPos types.StreamPosition + backwardTopologyPos, backwardStreamPos, err = d.Topology.SelectPositionInTopology(ctx, txn, recentStreamEvents[0].EventID()) + if err != nil { + return + } + prevBatch := types.NewTopologyToken(backwardTopologyPos, backwardStreamPos) + prevBatch.Decrement() + prevBatchStr = prevBatch.String() + } + + // We don't include a device here as we don't need to send down + // transaction IDs for complete syncs + recentEvents := d.StreamEventsToEvents(nil, recentStreamEvents) + stateEvents = removeDuplicates(stateEvents, recentEvents) + jr := types.NewJoinResponse() + jr.Timeline.PrevBatch = prevBatchStr + jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) + jr.Timeline.Limited = true + jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(stateEvents, gomatrixserverlib.FormatSync) + res.Rooms.Join[roomID] = *jr + } + + if err = d.AddInvitesToResponse(ctx, txn, userID, 0, toPos.PDUPosition(), res); err != nil { + return + } + + succeeded = true + return res, toPos, joinedRoomIDs, err +} + +func (d *Database) CompleteSync( + ctx context.Context, userID string, numRecentEventsPerRoom int, +) (*types.Response, error) { + res, toPos, joinedRoomIDs, err := d.getResponseWithPDUsForCompleteSync( + ctx, userID, numRecentEventsPerRoom, + ) + if err != nil { + return nil, err + } + + // Use a zero value SyncPosition for fromPos so all EDU states are added. + err = d.AddEDUDeltaToResponse( + types.NewStreamToken(0, 0), toPos, joinedRoomIDs, res, + ) + if err != nil { + return nil, err + } + + return res, nil +} + +var txReadOnlySnapshot = sql.TxOptions{ + // Set the isolation level so that we see a snapshot of the database. + // In PostgreSQL repeatable read transactions will see a snapshot taken + // at the first query, and since the transaction is read-only it can't + // run into any serialisation errors. + // https://www.postgresql.org/docs/9.5/static/transaction-iso.html#XACT-REPEATABLE-READ + Isolation: sql.LevelRepeatableRead, + ReadOnly: true, +} + +// TODO FIXME temporary public +func (d *Database) AddInvitesToResponse( + ctx context.Context, txn *sql.Tx, + userID string, + fromPos, toPos types.StreamPosition, + res *types.Response, +) error { + invites, err := d.Invites.SelectInviteEventsInRange( + ctx, txn, userID, fromPos, toPos, + ) + if err != nil { + return err + } + for roomID, inviteEvent := range invites { + ir := types.NewInviteResponse(inviteEvent) + res.Rooms.Invite[roomID] = *ir + } + return nil +} + +// Retrieve the backward topology position, i.e. the position of the +// oldest event in the room's topology. +func (d *Database) getBackwardTopologyPos( + ctx context.Context, txn *sql.Tx, + events []types.StreamEvent, +) (types.TopologyToken, error) { + zeroToken := types.NewTopologyToken(0, 0) + if len(events) == 0 { + return zeroToken, nil + } + pos, spos, err := d.Topology.SelectPositionInTopology(ctx, txn, events[0].EventID()) + if err != nil { + return zeroToken, err + } + tok := types.NewTopologyToken(pos, spos) + tok.Decrement() + return tok, nil +} + +// addRoomDeltaToResponse adds a room state delta to a sync response +func (d *Database) addRoomDeltaToResponse( + ctx context.Context, + device *authtypes.Device, + txn *sql.Tx, + fromPos, toPos types.StreamPosition, + delta stateDelta, + numRecentEventsPerRoom int, + res *types.Response, +) error { + endPos := toPos + if delta.membershipPos > 0 && delta.membership == gomatrixserverlib.Leave { + // make sure we don't leak recent events after the leave event. + // TODO: History visibility makes this somewhat complex to handle correctly. For example: + // TODO: This doesn't work for join -> leave in a single /sync request (see events prior to join). + // TODO: This will fail on join -> leave -> sensitive msg -> join -> leave + // in a single /sync request + // This is all "okay" assuming history_visibility == "shared" which it is by default. + endPos = delta.membershipPos + } + recentStreamEvents, err := d.OutputEvents.SelectRecentEvents( + ctx, txn, delta.roomID, types.StreamPosition(fromPos), types.StreamPosition(endPos), + numRecentEventsPerRoom, true, true, + ) + if err != nil { + return err + } + recentEvents := d.StreamEventsToEvents(device, recentStreamEvents) + delta.stateEvents = removeDuplicates(delta.stateEvents, recentEvents) // roll back + prevBatch, err := d.getBackwardTopologyPos(ctx, txn, recentStreamEvents) + if err != nil { + return err + } + + switch delta.membership { + case gomatrixserverlib.Join: + jr := types.NewJoinResponse() + + jr.Timeline.PrevBatch = prevBatch.String() + jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) + jr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true + jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) + res.Rooms.Join[delta.roomID] = *jr + case gomatrixserverlib.Leave: + fallthrough // transitions to leave are the same as ban + case gomatrixserverlib.Ban: + // TODO: recentEvents may contain events that this user is not allowed to see because they are + // no longer in the room. + lr := types.NewLeaveResponse() + lr.Timeline.PrevBatch = prevBatch.String() + lr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) + lr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true + lr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) + res.Rooms.Leave[delta.roomID] = *lr + } + + return nil +} + +// fetchStateEvents converts the set of event IDs into a set of events. It will fetch any which are missing from the database. +// Returns a map of room ID to list of events. +func (d *Database) fetchStateEvents( + ctx context.Context, txn *sql.Tx, + roomIDToEventIDSet map[string]map[string]bool, + eventIDToEvent map[string]types.StreamEvent, +) (map[string][]types.StreamEvent, error) { + stateBetween := make(map[string][]types.StreamEvent) + missingEvents := make(map[string][]string) + for roomID, ids := range roomIDToEventIDSet { + events := stateBetween[roomID] + for id, need := range ids { + if !need { + continue // deleted state + } + e, ok := eventIDToEvent[id] + if ok { + events = append(events, e) + } else { + m := missingEvents[roomID] + m = append(m, id) + missingEvents[roomID] = m + } + } + stateBetween[roomID] = events + } + + if len(missingEvents) > 0 { + // This happens when add_state_ids has an event ID which is not in the provided range. + // We need to explicitly fetch them. + allMissingEventIDs := []string{} + for _, missingEvIDs := range missingEvents { + allMissingEventIDs = append(allMissingEventIDs, missingEvIDs...) + } + evs, err := d.fetchMissingStateEvents(ctx, txn, allMissingEventIDs) + if err != nil { + return nil, err + } + // we know we got them all otherwise an error would've been returned, so just loop the events + for _, ev := range evs { + roomID := ev.RoomID() + stateBetween[roomID] = append(stateBetween[roomID], ev) + } + } + return stateBetween, nil +} + +func (d *Database) fetchMissingStateEvents( + ctx context.Context, txn *sql.Tx, eventIDs []string, +) ([]types.StreamEvent, error) { + // Fetch from the events table first so we pick up the stream ID for the + // event. + events, err := d.OutputEvents.SelectEvents(ctx, txn, eventIDs) + if err != nil { + return nil, err + } + + have := map[string]bool{} + for _, event := range events { + have[event.EventID()] = true + } + var missing []string + for _, eventID := range eventIDs { + if !have[eventID] { + missing = append(missing, eventID) + } + } + if len(missing) == 0 { + return events, nil + } + + // If they are missing from the events table then they should be state + // events that we received from outside the main event stream. + // These should be in the room state table. + stateEvents, err := d.CurrentRoomState.SelectEventsWithEventIDs(ctx, txn, missing) + + if err != nil { + return nil, err + } + if len(stateEvents) != len(missing) { + return nil, fmt.Errorf("failed to map all event IDs to events: (got %d, wanted %d)", len(stateEvents), len(missing)) + } + events = append(events, stateEvents...) + return events, nil +} + +// getStateDeltas returns the state deltas between fromPos and toPos, +// exclusive of oldPos, inclusive of newPos, for the rooms in which +// the user has new membership events. +// A list of joined room IDs is also returned in case the caller needs it. +func (d *Database) getStateDeltas( + ctx context.Context, device *authtypes.Device, txn *sql.Tx, + fromPos, toPos types.StreamPosition, userID string, + stateFilter *gomatrixserverlib.StateFilter, +) ([]stateDelta, []string, error) { + // Implement membership change algorithm: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L821 + // - Get membership list changes for this user in this sync response + // - For each room which has membership list changes: + // * Check if the room is 'newly joined' (insufficient to just check for a join event because we allow dupe joins TODO). + // If it is, then we need to send the full room state down (and 'limited' is always true). + // * Check if user is still CURRENTLY invited to the room. If so, add room to 'invited' block. + // * Check if the user is CURRENTLY (TODO) left/banned. If so, add room to 'archived' block. + // - Get all CURRENTLY joined rooms, and add them to 'joined' block. + var deltas []stateDelta + + // get all the state events ever between these two positions + stateNeeded, eventMap, err := d.OutputEvents.SelectStateInRange(ctx, txn, fromPos, toPos, stateFilter) + if err != nil { + return nil, nil, err + } + state, err := d.fetchStateEvents(ctx, txn, stateNeeded, eventMap) + if err != nil { + return nil, nil, err + } + + for roomID, stateStreamEvents := range state { + for _, ev := range stateStreamEvents { + // TODO: Currently this will incorrectly add rooms which were ALREADY joined but they sent another no-op join event. + // We should be checking if the user was already joined at fromPos and not proceed if so. As a result of this, + // dupe join events will result in the entire room state coming down to the client again. This is added in + // the 'state' part of the response though, so is transparent modulo bandwidth concerns as it is not added to + // the timeline. + if membership := getMembershipFromEvent(&ev.Event, userID); membership != "" { + if membership == gomatrixserverlib.Join { + // send full room state down instead of a delta + var s []types.StreamEvent + s, err = d.currentStateStreamEventsForRoom(ctx, txn, roomID, stateFilter) + if err != nil { + return nil, nil, err + } + state[roomID] = s + continue // we'll add this room in when we do joined rooms + } + + deltas = append(deltas, stateDelta{ + membership: membership, + membershipPos: ev.StreamPosition, + stateEvents: d.StreamEventsToEvents(device, stateStreamEvents), + roomID: roomID, + }) + break + } + } + } + + // Add in currently joined rooms + joinedRoomIDs, err := d.CurrentRoomState.SelectRoomIDsWithMembership(ctx, txn, userID, gomatrixserverlib.Join) + if err != nil { + return nil, nil, err + } + for _, joinedRoomID := range joinedRoomIDs { + deltas = append(deltas, stateDelta{ + membership: gomatrixserverlib.Join, + stateEvents: d.StreamEventsToEvents(device, state[joinedRoomID]), + roomID: joinedRoomID, + }) + } + + return deltas, joinedRoomIDs, nil +} + +// getStateDeltasForFullStateSync is a variant of getStateDeltas used for /sync +// requests with full_state=true. +// Fetches full state for all joined rooms and uses selectStateInRange to get +// updates for other rooms. +func (d *Database) getStateDeltasForFullStateSync( + ctx context.Context, device *authtypes.Device, txn *sql.Tx, + fromPos, toPos types.StreamPosition, userID string, + stateFilter *gomatrixserverlib.StateFilter, +) ([]stateDelta, []string, error) { + joinedRoomIDs, err := d.CurrentRoomState.SelectRoomIDsWithMembership(ctx, txn, userID, gomatrixserverlib.Join) + if err != nil { + return nil, nil, err + } + + // Use a reasonable initial capacity + deltas := make([]stateDelta, 0, len(joinedRoomIDs)) + + // Add full states for all joined rooms + for _, joinedRoomID := range joinedRoomIDs { + s, stateErr := d.currentStateStreamEventsForRoom(ctx, txn, joinedRoomID, stateFilter) + if stateErr != nil { + return nil, nil, stateErr + } + deltas = append(deltas, stateDelta{ + membership: gomatrixserverlib.Join, + stateEvents: d.StreamEventsToEvents(device, s), + roomID: joinedRoomID, + }) + } + + // Get all the state events ever between these two positions + stateNeeded, eventMap, err := d.OutputEvents.SelectStateInRange(ctx, txn, fromPos, toPos, stateFilter) + if err != nil { + return nil, nil, err + } + state, err := d.fetchStateEvents(ctx, txn, stateNeeded, eventMap) + if err != nil { + return nil, nil, err + } + + for roomID, stateStreamEvents := range state { + for _, ev := range stateStreamEvents { + if membership := getMembershipFromEvent(&ev.Event, userID); membership != "" { + if membership != gomatrixserverlib.Join { // We've already added full state for all joined rooms above. + deltas = append(deltas, stateDelta{ + membership: membership, + membershipPos: ev.StreamPosition, + stateEvents: d.StreamEventsToEvents(device, stateStreamEvents), + roomID: roomID, + }) + } + + break + } + } + } + + return deltas, joinedRoomIDs, nil +} + +func (d *Database) currentStateStreamEventsForRoom( + ctx context.Context, txn *sql.Tx, roomID string, + stateFilter *gomatrixserverlib.StateFilter, +) ([]types.StreamEvent, error) { + allState, err := d.CurrentRoomState.SelectCurrentState(ctx, txn, roomID, stateFilter) + if err != nil { + return nil, err + } + s := make([]types.StreamEvent, len(allState)) + for i := 0; i < len(s); i++ { + s[i] = types.StreamEvent{HeaderedEvent: allState[i], StreamPosition: 0} + } + return s, nil +} + +// There may be some overlap where events in stateEvents are already in recentEvents, so filter +// them out so we don't include them twice in the /sync response. They should be in recentEvents +// only, so clients get to the correct state once they have rolled forward. +func removeDuplicates(stateEvents, recentEvents []gomatrixserverlib.HeaderedEvent) []gomatrixserverlib.HeaderedEvent { + for _, recentEv := range recentEvents { + if recentEv.StateKey() == nil { + continue // not a state event + } + // TODO: This is a linear scan over all the current state events in this room. This will + // be slow for big rooms. We should instead sort the state events by event ID (ORDER BY) + // then do a binary search to find matching events, similar to what roomserver does. + for j := 0; j < len(stateEvents); j++ { + if stateEvents[j].EventID() == recentEv.EventID() { + // overwrite the element to remove with the last element then pop the last element. + // This is orders of magnitude faster than re-slicing, but doesn't preserve ordering + // (we don't care about the order of stateEvents) + stateEvents[j] = stateEvents[len(stateEvents)-1] + stateEvents = stateEvents[:len(stateEvents)-1] + break // there shouldn't be multiple events with the same event ID + } + } + } + return stateEvents +} + +// getMembershipFromEvent returns the value of content.membership iff the event is a state event +// with type 'm.room.member' and state_key of userID. Otherwise, an empty string is returned. +func getMembershipFromEvent(ev *gomatrixserverlib.Event, userID string) string { + if ev.Type() == "m.room.member" && ev.StateKeyEquals(userID) { + membership, err := ev.Membership() + if err != nil { + return "" + } + return membership + } + return "" +} + +type stateDelta struct { + roomID string + stateEvents []gomatrixserverlib.HeaderedEvent + membership string + // The PDU stream position of the latest membership event for this user, if applicable. + // Can be 0 if there is no membership event in this delta. + membershipPos types.StreamPosition +} diff --git a/syncapi/storage/sqlite3/backwards_extremities_table.go b/syncapi/storage/sqlite3/backwards_extremities_table.go new file mode 100644 index 000000000..a172f6bf7 --- /dev/null +++ b/syncapi/storage/sqlite3/backwards_extremities_table.go @@ -0,0 +1,106 @@ +// Copyright 2018 New Vector Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + + "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/syncapi/storage/tables" +) + +const backwardExtremitiesSchema = ` +-- Stores output room events received from the roomserver. +CREATE TABLE IF NOT EXISTS syncapi_backward_extremities ( + -- The 'room_id' key for the event. + room_id TEXT NOT NULL, + -- The event ID for the last known event. This is the backwards extremity. + event_id TEXT NOT NULL, + -- The prev_events for the last known event. This is used to update extremities. + prev_event_id TEXT NOT NULL, + PRIMARY KEY(room_id, event_id, prev_event_id) +); +` + +const insertBackwardExtremitySQL = "" + + "INSERT INTO syncapi_backward_extremities (room_id, event_id, prev_event_id)" + + " VALUES ($1, $2, $3)" + + " ON CONFLICT (room_id, event_id, prev_event_id) DO NOTHING" + +const selectBackwardExtremitiesForRoomSQL = "" + + "SELECT DISTINCT event_id FROM syncapi_backward_extremities WHERE room_id = $1" + +const deleteBackwardExtremitySQL = "" + + "DELETE FROM syncapi_backward_extremities WHERE room_id = $1 AND prev_event_id = $2" + +type backwardExtremitiesStatements struct { + insertBackwardExtremityStmt *sql.Stmt + selectBackwardExtremitiesForRoomStmt *sql.Stmt + deleteBackwardExtremityStmt *sql.Stmt +} + +func NewSqliteBackwardsExtremitiesTable(db *sql.DB) (tables.BackwardsExtremities, error) { + s := &backwardExtremitiesStatements{} + _, err := db.Exec(backwardExtremitiesSchema) + if err != nil { + return nil, err + } + if s.insertBackwardExtremityStmt, err = db.Prepare(insertBackwardExtremitySQL); err != nil { + return nil, err + } + if s.selectBackwardExtremitiesForRoomStmt, err = db.Prepare(selectBackwardExtremitiesForRoomSQL); err != nil { + return nil, err + } + if s.deleteBackwardExtremityStmt, err = db.Prepare(deleteBackwardExtremitySQL); err != nil { + return nil, err + } + return s, nil +} + +func (s *backwardExtremitiesStatements) InsertsBackwardExtremity( + ctx context.Context, txn *sql.Tx, roomID, eventID string, prevEventID string, +) (err error) { + _, err = txn.Stmt(s.insertBackwardExtremityStmt).ExecContext(ctx, roomID, eventID, prevEventID) + return +} + +func (s *backwardExtremitiesStatements) SelectBackwardExtremitiesForRoom( + ctx context.Context, roomID string, +) (eventIDs []string, err error) { + rows, err := s.selectBackwardExtremitiesForRoomStmt.QueryContext(ctx, roomID) + if err != nil { + return + } + defer common.CloseAndLogIfError(ctx, rows, "selectBackwardExtremitiesForRoom: rows.close() failed") + + for rows.Next() { + var eID string + if err = rows.Scan(&eID); err != nil { + return + } + + eventIDs = append(eventIDs, eID) + } + + return eventIDs, rows.Err() +} + +func (s *backwardExtremitiesStatements) DeleteBackwardExtremity( + ctx context.Context, txn *sql.Tx, roomID, knownEventID string, +) (err error) { + _, err = txn.Stmt(s.deleteBackwardExtremityStmt).ExecContext(ctx, roomID, knownEventID) + return +} diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go index 9fafdbede..b540fbc69 100644 --- a/syncapi/storage/sqlite3/current_room_state_table.go +++ b/syncapi/storage/sqlite3/current_room_state_table.go @@ -22,6 +22,7 @@ import ( "strings" "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -91,35 +92,37 @@ type currentRoomStateStatements struct { selectStateEventStmt *sql.Stmt } -func (s *currentRoomStateStatements) prepare(db *sql.DB, streamID *streamIDStatements) (err error) { - s.streamIDStatements = streamID - _, err = db.Exec(currentRoomStateSchema) +func NewSqliteCurrentRoomStateTable(db *sql.DB, streamID *streamIDStatements) (tables.CurrentRoomState, error) { + s := ¤tRoomStateStatements{ + streamIDStatements: streamID, + } + _, err := db.Exec(currentRoomStateSchema) if err != nil { - return + return nil, err } if s.upsertRoomStateStmt, err = db.Prepare(upsertRoomStateSQL); err != nil { - return + return nil, err } if s.deleteRoomStateByEventIDStmt, err = db.Prepare(deleteRoomStateByEventIDSQL); err != nil { - return + return nil, err } if s.selectRoomIDsWithMembershipStmt, err = db.Prepare(selectRoomIDsWithMembershipSQL); err != nil { - return + return nil, err } if s.selectCurrentStateStmt, err = db.Prepare(selectCurrentStateSQL); err != nil { - return + return nil, err } if s.selectJoinedUsersStmt, err = db.Prepare(selectJoinedUsersSQL); err != nil { - return + return nil, err } if s.selectStateEventStmt, err = db.Prepare(selectStateEventSQL); err != nil { - return + return nil, err } - return + return s, nil } // JoinedMemberLists returns a map of room ID to a list of joined user IDs. -func (s *currentRoomStateStatements) selectJoinedUsers( +func (s *currentRoomStateStatements) SelectJoinedUsers( ctx context.Context, ) (map[string][]string, error) { rows, err := s.selectJoinedUsersStmt.QueryContext(ctx) @@ -143,7 +146,7 @@ func (s *currentRoomStateStatements) selectJoinedUsers( } // SelectRoomIDsWithMembership returns the list of room IDs which have the given user in the given membership state. -func (s *currentRoomStateStatements) selectRoomIDsWithMembership( +func (s *currentRoomStateStatements) SelectRoomIDsWithMembership( ctx context.Context, txn *sql.Tx, userID string, @@ -168,7 +171,7 @@ func (s *currentRoomStateStatements) selectRoomIDsWithMembership( } // CurrentState returns all the current state events for the given room. -func (s *currentRoomStateStatements) selectCurrentState( +func (s *currentRoomStateStatements) SelectCurrentState( ctx context.Context, txn *sql.Tx, roomID string, stateFilterPart *gomatrixserverlib.StateFilter, ) ([]gomatrixserverlib.HeaderedEvent, error) { @@ -189,7 +192,7 @@ func (s *currentRoomStateStatements) selectCurrentState( return rowsToEvents(rows) } -func (s *currentRoomStateStatements) deleteRoomStateByEventID( +func (s *currentRoomStateStatements) DeleteRoomStateByEventID( ctx context.Context, txn *sql.Tx, eventID string, ) error { stmt := common.TxStmt(txn, s.deleteRoomStateByEventIDStmt) @@ -197,7 +200,7 @@ func (s *currentRoomStateStatements) deleteRoomStateByEventID( return err } -func (s *currentRoomStateStatements) upsertRoomState( +func (s *currentRoomStateStatements) UpsertRoomState( ctx context.Context, txn *sql.Tx, event gomatrixserverlib.HeaderedEvent, membership *string, addedAt types.StreamPosition, ) error { @@ -231,7 +234,7 @@ func (s *currentRoomStateStatements) upsertRoomState( return err } -func (s *currentRoomStateStatements) selectEventsWithEventIDs( +func (s *currentRoomStateStatements) SelectEventsWithEventIDs( ctx context.Context, txn *sql.Tx, eventIDs []string, ) ([]types.StreamEvent, error) { iEventIDs := make([]interface{}, len(eventIDs)) @@ -264,7 +267,7 @@ func rowsToEvents(rows *sql.Rows) ([]gomatrixserverlib.HeaderedEvent, error) { return result, nil } -func (s *currentRoomStateStatements) selectStateEvent( +func (s *currentRoomStateStatements) SelectStateEvent( ctx context.Context, roomID, evType, stateKey string, ) (*gomatrixserverlib.HeaderedEvent, error) { stmt := s.selectStateEventStmt diff --git a/syncapi/storage/sqlite3/output_room_events_topology_table.go b/syncapi/storage/sqlite3/output_room_events_topology_table.go index 0d313d7c6..4469f5b76 100644 --- a/syncapi/storage/sqlite3/output_room_events_topology_table.go +++ b/syncapi/storage/sqlite3/output_room_events_topology_table.go @@ -19,6 +19,7 @@ import ( "database/sql" "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -77,35 +78,36 @@ type outputRoomEventsTopologyStatements struct { selectEventIDsFromPositionStmt *sql.Stmt } -func (s *outputRoomEventsTopologyStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(outputRoomEventsTopologySchema) +func NewSqliteTopologyTable(db *sql.DB) (tables.Topology, error) { + s := &outputRoomEventsTopologyStatements{} + _, err := db.Exec(outputRoomEventsTopologySchema) if err != nil { - return + return nil, err } if s.insertEventInTopologyStmt, err = db.Prepare(insertEventInTopologySQL); err != nil { - return + return nil, err } if s.selectEventIDsInRangeASCStmt, err = db.Prepare(selectEventIDsInRangeASCSQL); err != nil { - return + return nil, err } if s.selectEventIDsInRangeDESCStmt, err = db.Prepare(selectEventIDsInRangeDESCSQL); err != nil { - return + return nil, err } if s.selectPositionInTopologyStmt, err = db.Prepare(selectPositionInTopologySQL); err != nil { - return + return nil, err } if s.selectMaxPositionInTopologyStmt, err = db.Prepare(selectMaxPositionInTopologySQL); err != nil { - return + return nil, err } if s.selectEventIDsFromPositionStmt, err = db.Prepare(selectEventIDsFromPositionSQL); err != nil { - return + return nil, err } - return + return s, nil } // insertEventInTopology inserts the given event in the room's topology, based // on the event's depth. -func (s *outputRoomEventsTopologyStatements) insertEventInTopology( +func (s *outputRoomEventsTopologyStatements) InsertEventInTopology( ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.HeaderedEvent, pos types.StreamPosition, ) (err error) { stmt := common.TxStmt(txn, s.insertEventInTopologyStmt) @@ -118,7 +120,7 @@ func (s *outputRoomEventsTopologyStatements) insertEventInTopology( // selectEventIDsInRange selects the IDs of events which positions are within a // given range in a given room's topological order. // Returns an empty slice if no events match the given range. -func (s *outputRoomEventsTopologyStatements) selectEventIDsInRange( +func (s *outputRoomEventsTopologyStatements) SelectEventIDsInRange( ctx context.Context, txn *sql.Tx, roomID string, fromPos, toPos, toMicroPos types.StreamPosition, limit int, chronologicalOrder bool, @@ -155,7 +157,7 @@ func (s *outputRoomEventsTopologyStatements) selectEventIDsInRange( // selectPositionInTopology returns the position of a given event in the // topology of the room it belongs to. -func (s *outputRoomEventsTopologyStatements) selectPositionInTopology( +func (s *outputRoomEventsTopologyStatements) SelectPositionInTopology( ctx context.Context, txn *sql.Tx, eventID string, ) (pos types.StreamPosition, spos types.StreamPosition, err error) { stmt := common.TxStmt(txn, s.selectPositionInTopologyStmt) @@ -163,7 +165,7 @@ func (s *outputRoomEventsTopologyStatements) selectPositionInTopology( return } -func (s *outputRoomEventsTopologyStatements) selectMaxPositionInTopology( +func (s *outputRoomEventsTopologyStatements) SelectMaxPositionInTopology( ctx context.Context, txn *sql.Tx, roomID string, ) (pos types.StreamPosition, spos types.StreamPosition, err error) { stmt := common.TxStmt(txn, s.selectMaxPositionInTopologyStmt) @@ -173,7 +175,7 @@ func (s *outputRoomEventsTopologyStatements) selectMaxPositionInTopology( // selectEventIDsFromPosition returns the IDs of all events that have a given // position in the topology of a given room. -func (s *outputRoomEventsTopologyStatements) selectEventIDsFromPosition( +func (s *outputRoomEventsTopologyStatements) SelectEventIDsFromPosition( ctx context.Context, txn *sql.Tx, roomID string, pos types.StreamPosition, ) (eventIDs []string, err error) { // Query the event IDs. diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index 0da05eab3..e23aeca4e 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -16,18 +16,11 @@ package sqlite3 import ( - "context" "database/sql" - "encoding/json" "errors" - "fmt" "net/url" - "github.com/sirupsen/logrus" - - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/roomserver/api" // Import the sqlite3 package _ "github.com/mattn/go-sqlite3" @@ -35,35 +28,20 @@ import ( "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/syncapi/storage/shared" - "github.com/matrix-org/dendrite/syncapi/storage/tables" - "github.com/matrix-org/dendrite/syncapi/types" - "github.com/matrix-org/gomatrixserverlib" ) -type stateDelta struct { - roomID string - stateEvents []gomatrixserverlib.HeaderedEvent - membership string - // The PDU stream position of the latest membership event for this user, if applicable. - // Can be 0 if there is no membership event in this delta. - membershipPos types.StreamPosition -} - // SyncServerDatasource represents a sync server datasource which manages // both the database for PDUs and caches for EDUs. type SyncServerDatasource struct { shared.Database db *sql.DB common.PartitionOffsetStatements - streamID streamIDStatements - roomstate currentRoomStateStatements - topology outputRoomEventsTopologyStatements - backwardExtremities tables.BackwardsExtremities + streamID streamIDStatements } -// NewSyncServerDatasource creates a new sync server database +// NewDatabase creates a new sync server database // nolint: gocyclo -func NewSyncServerDatasource(dataSourceName string) (*SyncServerDatasource, error) { +func NewDatabase(dataSourceName string) (*SyncServerDatasource, error) { var d SyncServerDatasource uri, err := url.Parse(dataSourceName) if err != nil { @@ -101,982 +79,31 @@ func (d *SyncServerDatasource) prepare() (err error) { if err != nil { return err } - if err = d.roomstate.prepare(d.db, &d.streamID); err != nil { + roomState, err := NewSqliteCurrentRoomStateTable(d.db, &d.streamID) + if err != nil { return err } invites, err := NewSqliteInvitesTable(d.db, &d.streamID) if err != nil { return err } - if err = d.topology.prepare(d.db); err != nil { + topology, err := NewSqliteTopologyTable(d.db) + if err != nil { return err } - d.backwardExtremities, err = tables.NewBackwardsExtremities(d.db, &tables.SqliteBackwardsExtremitiesStatements{}) + bwExtrem, err := NewSqliteBackwardsExtremitiesTable(d.db) if err != nil { return err } d.Database = shared.Database{ - DB: d.db, - Invites: invites, - AccountData: accountData, - OutputEvents: events, - EDUCache: cache.New(), + DB: d.db, + Invites: invites, + AccountData: accountData, + OutputEvents: events, + BackwardExtremities: bwExtrem, + CurrentRoomState: roomState, + Topology: topology, + EDUCache: cache.New(), } return nil } - -// AllJoinedUsersInRooms returns a map of room ID to a list of all joined user IDs. -func (d *SyncServerDatasource) AllJoinedUsersInRooms(ctx context.Context) (map[string][]string, error) { - return d.roomstate.selectJoinedUsers(ctx) -} - -// handleBackwardExtremities adds this event as a backwards extremity if and only if we do not have all of -// the events listed in the event's 'prev_events'. This function also updates the backwards extremities table -// to account for the fact that the given event is no longer a backwards extremity, but may be marked as such. -func (d *SyncServerDatasource) handleBackwardExtremities(ctx context.Context, txn *sql.Tx, ev *gomatrixserverlib.HeaderedEvent) error { - if err := d.backwardExtremities.DeleteBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID()); err != nil { - return err - } - - // Check if we have all of the event's previous events. If an event is - // missing, add it to the room's backward extremities. - prevEvents, err := d.Database.OutputEvents.SelectEvents(ctx, txn, ev.PrevEventIDs()) - if err != nil { - return err - } - var found bool - for _, eID := range ev.PrevEventIDs() { - found = false - for _, prevEv := range prevEvents { - if eID == prevEv.EventID() { - found = true - } - } - - // If the event is missing, consider it a backward extremity. - if !found { - if err = d.backwardExtremities.InsertsBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID(), eID); err != nil { - return err - } - } - } - - return nil -} - -// WriteEvent into the database. It is not safe to call this function from multiple goroutines, as it would create races -// when generating the sync stream position for this event. Returns the sync stream position for the inserted event. -// Returns an error if there was a problem inserting this event. -func (d *SyncServerDatasource) WriteEvent( - ctx context.Context, - ev *gomatrixserverlib.HeaderedEvent, - addStateEvents []gomatrixserverlib.HeaderedEvent, - addStateEventIDs, removeStateEventIDs []string, - transactionID *api.TransactionID, excludeFromSync bool, -) (pduPosition types.StreamPosition, returnErr error) { - returnErr = common.WithTransaction(d.db, func(txn *sql.Tx) error { - var err error - pos, err := d.Database.OutputEvents.InsertEvent( - ctx, txn, ev, addStateEventIDs, removeStateEventIDs, transactionID, excludeFromSync, - ) - if err != nil { - return err - } - pduPosition = pos - - if err = d.topology.insertEventInTopology(ctx, txn, ev, pos); err != nil { - return err - } - - if err = d.handleBackwardExtremities(ctx, txn, ev); err != nil { - return err - } - - if len(addStateEvents) == 0 && len(removeStateEventIDs) == 0 { - // Nothing to do, the event may have just been a message event. - return nil - } - - return d.updateRoomState(ctx, txn, removeStateEventIDs, addStateEvents, pduPosition) - }) - - return pduPosition, returnErr -} - -func (d *SyncServerDatasource) updateRoomState( - ctx context.Context, txn *sql.Tx, - removedEventIDs []string, - addedEvents []gomatrixserverlib.HeaderedEvent, - pduPosition types.StreamPosition, -) error { - // remove first, then add, as we do not ever delete state, but do replace state which is a remove followed by an add. - for _, eventID := range removedEventIDs { - if err := d.roomstate.deleteRoomStateByEventID(ctx, txn, eventID); err != nil { - return err - } - } - - for _, event := range addedEvents { - if event.StateKey() == nil { - // ignore non state events - continue - } - var membership *string - if event.Type() == "m.room.member" { - value, err := event.Membership() - if err != nil { - return err - } - membership = &value - } - if err := d.roomstate.upsertRoomState(ctx, txn, event, membership, pduPosition); err != nil { - return err - } - } - - return nil -} - -// SyncPosition returns the latest positions for syncing. -func (d *SyncServerDatasource) SyncPosition(ctx context.Context) (tok types.StreamingToken, err error) { - err = common.WithTransaction(d.db, func(txn *sql.Tx) error { - pos, err := d.syncPositionTx(ctx, txn) - if err != nil { - return err - } - tok = *pos - return nil - }) - return -} - -// GetStateEvent returns the Matrix state event of a given type for a given room with a given state key -// If no event could be found, returns nil -// If there was an issue during the retrieval, returns an error -func (d *SyncServerDatasource) GetStateEvent( - ctx context.Context, roomID, evType, stateKey string, -) (*gomatrixserverlib.HeaderedEvent, error) { - return d.roomstate.selectStateEvent(ctx, roomID, evType, stateKey) -} - -// GetStateEventsForRoom fetches the state events for a given room. -// Returns an empty slice if no state events could be found for this room. -// Returns an error if there was an issue with the retrieval. -func (d *SyncServerDatasource) GetStateEventsForRoom( - ctx context.Context, roomID string, stateFilterPart *gomatrixserverlib.StateFilter, -) (stateEvents []gomatrixserverlib.HeaderedEvent, err error) { - err = common.WithTransaction(d.db, func(txn *sql.Tx) error { - stateEvents, err = d.roomstate.selectCurrentState(ctx, txn, roomID, stateFilterPart) - return err - }) - return -} - -// GetEventsInTopologicalRange retrieves all of the events on a given ordering using the -// given extremities and limit. -func (d *SyncServerDatasource) GetEventsInTopologicalRange( - ctx context.Context, - from, to *types.TopologyToken, - roomID string, limit int, - backwardOrdering bool, -) (events []types.StreamEvent, err error) { - // TODO: ARGH CONFUSING - // Determine the backward and forward limit, i.e. the upper and lower - // limits to the selection in the room's topology, from the direction. - var backwardLimit, forwardLimit, forwardMicroLimit types.StreamPosition - if backwardOrdering { - // Backward ordering is antichronological (latest event to oldest - // one). - backwardLimit = to.Depth() - forwardLimit = from.Depth() - forwardMicroLimit = from.PDUPosition() - } else { - // Forward ordering is chronological (oldest event to latest one). - backwardLimit = from.Depth() - forwardLimit = to.Depth() - } - - // Select the event IDs from the defined range. - var eIDs []string - eIDs, err = d.topology.selectEventIDsInRange( - ctx, nil, roomID, backwardLimit, forwardLimit, forwardMicroLimit, limit, !backwardOrdering, - ) - if err != nil { - return - } - - // Retrieve the events' contents using their IDs. - events, err = d.Database.OutputEvents.SelectEvents(ctx, nil, eIDs) - return -} - -// BackwardExtremitiesForRoom returns the event IDs of all of the backward -// extremities we know of for a given room. -func (d *SyncServerDatasource) BackwardExtremitiesForRoom( - ctx context.Context, roomID string, -) (backwardExtremities []string, err error) { - return d.backwardExtremities.SelectBackwardExtremitiesForRoom(ctx, roomID) -} - -// MaxTopologicalPosition returns the highest topological position for a given -// room. -func (d *SyncServerDatasource) MaxTopologicalPosition( - ctx context.Context, roomID string, -) (types.StreamPosition, types.StreamPosition, error) { - return d.topology.selectMaxPositionInTopology(ctx, nil, roomID) -} - -// EventsAtTopologicalPosition returns all of the events matching a given -// position in the topology of a given room. -func (d *SyncServerDatasource) EventsAtTopologicalPosition( - ctx context.Context, roomID string, pos types.StreamPosition, -) ([]types.StreamEvent, error) { - eIDs, err := d.topology.selectEventIDsFromPosition(ctx, nil, roomID, pos) - if err != nil { - return nil, err - } - - return d.Database.OutputEvents.SelectEvents(ctx, nil, eIDs) -} - -func (d *SyncServerDatasource) EventPositionInTopology( - ctx context.Context, eventID string, -) (depth types.StreamPosition, stream types.StreamPosition, err error) { - return d.topology.selectPositionInTopology(ctx, nil, eventID) -} - -// SyncStreamPosition returns the latest position in the sync stream. Returns 0 if there are no events yet. -func (d *SyncServerDatasource) SyncStreamPosition(ctx context.Context) (pos types.StreamPosition, err error) { - err = common.WithTransaction(d.db, func(txn *sql.Tx) error { - pos, err = d.syncStreamPositionTx(ctx, txn) - return err - }) - return -} - -func (d *SyncServerDatasource) syncStreamPositionTx( - ctx context.Context, txn *sql.Tx, -) (types.StreamPosition, error) { - maxID, err := d.Database.OutputEvents.SelectMaxEventID(ctx, txn) - if err != nil { - return 0, err - } - maxAccountDataID, err := d.Database.AccountData.SelectMaxAccountDataID(ctx, txn) - if err != nil { - return 0, err - } - if maxAccountDataID > maxID { - maxID = maxAccountDataID - } - maxInviteID, err := d.Database.Invites.SelectMaxInviteID(ctx, txn) - if err != nil { - return 0, err - } - if maxInviteID > maxID { - maxID = maxInviteID - } - return types.StreamPosition(maxID), nil -} - -func (d *SyncServerDatasource) syncPositionTx( - ctx context.Context, txn *sql.Tx, -) (*types.StreamingToken, error) { - - maxEventID, err := d.Database.OutputEvents.SelectMaxEventID(ctx, txn) - if err != nil { - return nil, err - } - maxAccountDataID, err := d.Database.AccountData.SelectMaxAccountDataID(ctx, txn) - if err != nil { - return nil, err - } - if maxAccountDataID > maxEventID { - maxEventID = maxAccountDataID - } - maxInviteID, err := d.Database.Invites.SelectMaxInviteID(ctx, txn) - if err != nil { - return nil, err - } - if maxInviteID > maxEventID { - maxEventID = maxInviteID - } - sp := types.NewStreamToken( - types.StreamPosition(maxEventID), - types.StreamPosition(d.Database.EDUCache.GetLatestSyncPosition()), - ) - return &sp, nil -} - -// addPDUDeltaToResponse adds all PDU deltas to a sync response. -// IDs of all rooms the user joined are returned so EDU deltas can be added for them. -func (d *SyncServerDatasource) addPDUDeltaToResponse( - ctx context.Context, - device authtypes.Device, - fromPos, toPos types.StreamPosition, - numRecentEventsPerRoom int, - wantFullState bool, - res *types.Response, -) (joinedRoomIDs []string, err error) { - txn, err := d.db.BeginTx(ctx, &txReadOnlySnapshot) - if err != nil { - return nil, err - } - var succeeded bool - defer func() { - txerr := common.EndTransaction(txn, &succeeded) - if err == nil && txerr != nil { - err = txerr - } - }() - - stateFilterPart := gomatrixserverlib.DefaultStateFilter() // TODO: use filter provided in request - - // Work out which rooms to return in the response. This is done by getting not only the currently - // joined rooms, but also which rooms have membership transitions for this user between the 2 PDU stream positions. - // This works out what the 'state' key should be for each room as well as which membership block - // to put the room into. - var deltas []stateDelta - if !wantFullState { - deltas, joinedRoomIDs, err = d.getStateDeltas( - ctx, &device, txn, fromPos, toPos, device.UserID, &stateFilterPart, - ) - } else { - deltas, joinedRoomIDs, err = d.getStateDeltasForFullStateSync( - ctx, &device, txn, fromPos, toPos, device.UserID, &stateFilterPart, - ) - } - if err != nil { - return nil, err - } - - for _, delta := range deltas { - err = d.addRoomDeltaToResponse(ctx, &device, txn, fromPos, toPos, delta, numRecentEventsPerRoom, res) - if err != nil { - return nil, err - } - } - - // TODO: This should be done in getStateDeltas - if err = d.addInvitesToResponse(ctx, txn, device.UserID, fromPos, toPos, res); err != nil { - return nil, err - } - - succeeded = true - return joinedRoomIDs, nil -} - -// addTypingDeltaToResponse adds all typing notifications to a sync response -// since the specified position. -func (d *SyncServerDatasource) addTypingDeltaToResponse( - since types.StreamingToken, - joinedRoomIDs []string, - res *types.Response, -) error { - var jr types.JoinResponse - var ok bool - var err error - for _, roomID := range joinedRoomIDs { - if typingUsers, updated := d.Database.EDUCache.GetTypingUsersIfUpdatedAfter( - roomID, int64(since.EDUPosition()), - ); updated { - ev := gomatrixserverlib.ClientEvent{ - Type: gomatrixserverlib.MTyping, - } - ev.Content, err = json.Marshal(map[string]interface{}{ - "user_ids": typingUsers, - }) - if err != nil { - return err - } - - if jr, ok = res.Rooms.Join[roomID]; !ok { - jr = *types.NewJoinResponse() - } - jr.Ephemeral.Events = append(jr.Ephemeral.Events, ev) - res.Rooms.Join[roomID] = jr - } - } - return nil -} - -// addEDUDeltaToResponse adds updates for EDUs of each type since fromPos if -// the positions of that type are not equal in fromPos and toPos. -func (d *SyncServerDatasource) addEDUDeltaToResponse( - fromPos, toPos types.StreamingToken, - joinedRoomIDs []string, - res *types.Response, -) (err error) { - - if fromPos.EDUPosition() != toPos.EDUPosition() { - err = d.addTypingDeltaToResponse( - fromPos, joinedRoomIDs, res, - ) - } - - return -} - -// IncrementalSync returns all the data needed in order to create an incremental -// sync response for the given user. Events returned will include any client -// transaction IDs associated with the given device. These transaction IDs come -// from when the device sent the event via an API that included a transaction -// ID. -func (d *SyncServerDatasource) IncrementalSync( - ctx context.Context, - device authtypes.Device, - fromPos, toPos types.StreamingToken, - numRecentEventsPerRoom int, - wantFullState bool, -) (*types.Response, error) { - nextBatchPos := fromPos.WithUpdates(toPos) - res := types.NewResponse(nextBatchPos) - - var joinedRoomIDs []string - var err error - fmt.Println("from", fromPos.PDUPosition(), "to", toPos.PDUPosition()) - if fromPos.PDUPosition() != toPos.PDUPosition() || wantFullState { - joinedRoomIDs, err = d.addPDUDeltaToResponse( - ctx, device, fromPos.PDUPosition(), toPos.PDUPosition(), numRecentEventsPerRoom, wantFullState, res, - ) - } else { - joinedRoomIDs, err = d.roomstate.selectRoomIDsWithMembership( - ctx, nil, device.UserID, gomatrixserverlib.Join, - ) - } - if err != nil { - return nil, err - } - - err = d.addEDUDeltaToResponse( - fromPos, toPos, joinedRoomIDs, res, - ) - if err != nil { - return nil, err - } - - return res, nil -} - -// getResponseWithPDUsForCompleteSync creates a response and adds all PDUs needed -// to it. It returns toPos and joinedRoomIDs for use of adding EDUs. -func (d *SyncServerDatasource) getResponseWithPDUsForCompleteSync( - ctx context.Context, - userID string, - numRecentEventsPerRoom int, -) ( - res *types.Response, - toPos *types.StreamingToken, - joinedRoomIDs []string, - err error, -) { - // This needs to be all done in a transaction as we need to do multiple SELECTs, and we need to have - // a consistent view of the database throughout. This includes extracting the sync position. - // This does have the unfortunate side-effect that all the matrixy logic resides in this function, - // but it's better to not hide the fact that this is being done in a transaction. - txn, err := d.db.BeginTx(ctx, &txReadOnlySnapshot) - if err != nil { - return - } - var succeeded bool - defer func() { - txerr := common.EndTransaction(txn, &succeeded) - if err == nil && txerr != nil { - err = txerr - } - }() - - // Get the current sync position which we will base the sync response on. - toPos, err = d.syncPositionTx(ctx, txn) - if err != nil { - return - } - - res = types.NewResponse(*toPos) - - // Extract room state and recent events for all rooms the user is joined to. - joinedRoomIDs, err = d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, gomatrixserverlib.Join) - if err != nil { - return - } - - stateFilterPart := gomatrixserverlib.DefaultStateFilter() // TODO: use filter provided in request - - // Build up a /sync response. Add joined rooms. - for _, roomID := range joinedRoomIDs { - var stateEvents []gomatrixserverlib.HeaderedEvent - stateEvents, err = d.roomstate.selectCurrentState(ctx, txn, roomID, &stateFilterPart) - if err != nil { - return - } - //fmt.Println("State events:", stateEvents) - // TODO: When filters are added, we may need to call this multiple times to get enough events. - // See: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L316 - var recentStreamEvents []types.StreamEvent - recentStreamEvents, err = d.Database.OutputEvents.SelectRecentEvents( - ctx, txn, roomID, types.StreamPosition(0), toPos.PDUPosition(), - numRecentEventsPerRoom, true, true, - ) - if err != nil { - return - } - //fmt.Println("Recent stream events:", recentStreamEvents) - - // Retrieve the backward topology position, i.e. the position of the - // oldest event in the room's topology. - var prevBatchStr string - if len(recentStreamEvents) > 0 { - var backwardTopologyPos, backwardStreamPos types.StreamPosition - backwardTopologyPos, backwardStreamPos, err = d.topology.selectPositionInTopology(ctx, txn, recentStreamEvents[0].EventID()) - if err != nil { - return - } - prevBatch := types.NewTopologyToken(backwardTopologyPos, backwardStreamPos) - prevBatch.Decrement() - prevBatchStr = prevBatch.String() - } - - // We don't include a device here as we don't need to send down - // transaction IDs for complete syncs - recentEvents := d.StreamEventsToEvents(nil, recentStreamEvents) - stateEvents = removeDuplicates(stateEvents, recentEvents) - jr := types.NewJoinResponse() - jr.Timeline.PrevBatch = prevBatchStr - jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) - jr.Timeline.Limited = true - jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(stateEvents, gomatrixserverlib.FormatSync) - res.Rooms.Join[roomID] = *jr - } - - if err = d.addInvitesToResponse(ctx, txn, userID, 0, toPos.PDUPosition(), res); err != nil { - return - } - - succeeded = true - return res, toPos, joinedRoomIDs, err -} - -// CompleteSync returns a complete /sync API response for the given user. -func (d *SyncServerDatasource) CompleteSync( - ctx context.Context, userID string, numRecentEventsPerRoom int, -) (*types.Response, error) { - res, toPos, joinedRoomIDs, err := d.getResponseWithPDUsForCompleteSync( - ctx, userID, numRecentEventsPerRoom, - ) - if err != nil { - return nil, err - } - - // Use a zero value SyncPosition for fromPos so all EDU states are added. - err = d.addEDUDeltaToResponse( - types.NewStreamToken(0, 0), *toPos, joinedRoomIDs, res, - ) - if err != nil { - return nil, err - } - - return res, nil -} - -var txReadOnlySnapshot = sql.TxOptions{ - // Set the isolation level so that we see a snapshot of the database. - // In PostgreSQL repeatable read transactions will see a snapshot taken - // at the first query, and since the transaction is read-only it can't - // run into any serialisation errors. - // https://www.postgresql.org/docs/9.5/static/transaction-iso.html#XACT-REPEATABLE-READ - Isolation: sql.LevelRepeatableRead, - ReadOnly: true, -} - -func (d *SyncServerDatasource) addInvitesToResponse( - ctx context.Context, txn *sql.Tx, - userID string, - fromPos, toPos types.StreamPosition, - res *types.Response, -) error { - invites, err := d.Database.Invites.SelectInviteEventsInRange( - ctx, txn, userID, fromPos, toPos, - ) - if err != nil { - return err - } - for roomID, inviteEvent := range invites { - ir := types.NewInviteResponse(inviteEvent) - res.Rooms.Invite[roomID] = *ir - } - return nil -} - -// Retrieve the backward topology position, i.e. the position of the -// oldest event in the room's topology. -func (d *SyncServerDatasource) getBackwardTopologyPos( - ctx context.Context, txn *sql.Tx, - events []types.StreamEvent, -) (pos, spos types.StreamPosition) { - if len(events) > 0 { - pos, spos, _ = d.topology.selectPositionInTopology(ctx, txn, events[0].EventID()) - } - // go to the previous position so we don't pull out the same event twice - // FIXME: This could be done more nicely by being explicit with inclusive/exclusive rules - if pos-1 <= 0 { - pos = types.StreamPosition(1) - } else { - pos = pos - 1 - spos += 1000 // this has to be bigger than the number of events we backfill per request - } - return -} - -// addRoomDeltaToResponse adds a room state delta to a sync response -func (d *SyncServerDatasource) addRoomDeltaToResponse( - ctx context.Context, - device *authtypes.Device, - txn *sql.Tx, - fromPos, toPos types.StreamPosition, - delta stateDelta, - numRecentEventsPerRoom int, - res *types.Response, -) error { - endPos := toPos - if delta.membershipPos > 0 && delta.membership == gomatrixserverlib.Leave { - // make sure we don't leak recent events after the leave event. - // TODO: History visibility makes this somewhat complex to handle correctly. For example: - // TODO: This doesn't work for join -> leave in a single /sync request (see events prior to join). - // TODO: This will fail on join -> leave -> sensitive msg -> join -> leave - // in a single /sync request - // This is all "okay" assuming history_visibility == "shared" which it is by default. - endPos = delta.membershipPos - } - recentStreamEvents, err := d.Database.OutputEvents.SelectRecentEvents( - ctx, txn, delta.roomID, types.StreamPosition(fromPos), types.StreamPosition(endPos), - numRecentEventsPerRoom, true, true, - ) - if err != nil { - return err - } - recentEvents := d.StreamEventsToEvents(device, recentStreamEvents) - delta.stateEvents = removeDuplicates(delta.stateEvents, recentEvents) - backwardTopologyPos, backwardStreamPos := d.getBackwardTopologyPos(ctx, txn, recentStreamEvents) - prevBatch := types.NewTopologyToken( - backwardTopologyPos, backwardStreamPos, - ) - - switch delta.membership { - case gomatrixserverlib.Join: - jr := types.NewJoinResponse() - jr.Timeline.PrevBatch = prevBatch.String() - jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) - jr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true - jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) - res.Rooms.Join[delta.roomID] = *jr - case gomatrixserverlib.Leave: - fallthrough // transitions to leave are the same as ban - case gomatrixserverlib.Ban: - // TODO: recentEvents may contain events that this user is not allowed to see because they are - // no longer in the room. - lr := types.NewLeaveResponse() - lr.Timeline.PrevBatch = prevBatch.String() - lr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) - lr.Timeline.Limited = false // TODO: if len(events) >= numRecents + 1 and then set limited:true - lr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.stateEvents, gomatrixserverlib.FormatSync) - res.Rooms.Leave[delta.roomID] = *lr - } - - return nil -} - -// fetchStateEvents converts the set of event IDs into a set of events. It will fetch any which are missing from the database. -// Returns a map of room ID to list of events. -func (d *SyncServerDatasource) fetchStateEvents( - ctx context.Context, txn *sql.Tx, - roomIDToEventIDSet map[string]map[string]bool, - eventIDToEvent map[string]types.StreamEvent, -) (map[string][]types.StreamEvent, error) { - stateBetween := make(map[string][]types.StreamEvent) - missingEvents := make(map[string][]string) - for roomID, ids := range roomIDToEventIDSet { - events := stateBetween[roomID] - for id, need := range ids { - if !need { - continue // deleted state - } - e, ok := eventIDToEvent[id] - if ok { - events = append(events, e) - } else { - m := missingEvents[roomID] - m = append(m, id) - missingEvents[roomID] = m - } - } - stateBetween[roomID] = events - } - - if len(missingEvents) > 0 { - // This happens when add_state_ids has an event ID which is not in the provided range. - // We need to explicitly fetch them. - allMissingEventIDs := []string{} - for _, missingEvIDs := range missingEvents { - allMissingEventIDs = append(allMissingEventIDs, missingEvIDs...) - } - evs, err := d.fetchMissingStateEvents(ctx, txn, allMissingEventIDs) - if err != nil { - return nil, err - } - // we know we got them all otherwise an error would've been returned, so just loop the events - for _, ev := range evs { - roomID := ev.RoomID() - stateBetween[roomID] = append(stateBetween[roomID], ev) - } - } - return stateBetween, nil -} - -func (d *SyncServerDatasource) fetchMissingStateEvents( - ctx context.Context, txn *sql.Tx, eventIDs []string, -) ([]types.StreamEvent, error) { - // Fetch from the events table first so we pick up the stream ID for the - // event. - events, err := d.Database.OutputEvents.SelectEvents(ctx, txn, eventIDs) - if err != nil { - return nil, err - } - - have := map[string]bool{} - for _, event := range events { - have[event.EventID()] = true - } - var missing []string - for _, eventID := range eventIDs { - if !have[eventID] { - missing = append(missing, eventID) - } - } - if len(missing) == 0 { - return events, nil - } - - // If they are missing from the events table then they should be state - // events that we received from outside the main event stream. - // These should be in the room state table. - stateEvents, err := d.roomstate.selectEventsWithEventIDs(ctx, txn, missing) - - if err != nil { - return nil, err - } - if len(stateEvents) != len(missing) { - return nil, fmt.Errorf("failed to map all event IDs to events: (got %d, wanted %d)", len(stateEvents), len(missing)) - } - events = append(events, stateEvents...) - return events, nil -} - -// getStateDeltas returns the state deltas between fromPos and toPos, -// exclusive of oldPos, inclusive of newPos, for the rooms in which -// the user has new membership events. -// A list of joined room IDs is also returned in case the caller needs it. -func (d *SyncServerDatasource) getStateDeltas( - ctx context.Context, device *authtypes.Device, txn *sql.Tx, - fromPos, toPos types.StreamPosition, userID string, - stateFilterPart *gomatrixserverlib.StateFilter, -) ([]stateDelta, []string, error) { - // Implement membership change algorithm: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L821 - // - Get membership list changes for this user in this sync response - // - For each room which has membership list changes: - // * Check if the room is 'newly joined' (insufficient to just check for a join event because we allow dupe joins TODO). - // If it is, then we need to send the full room state down (and 'limited' is always true). - // * Check if user is still CURRENTLY invited to the room. If so, add room to 'invited' block. - // * Check if the user is CURRENTLY (TODO) left/banned. If so, add room to 'archived' block. - // - Get all CURRENTLY joined rooms, and add them to 'joined' block. - var deltas []stateDelta - - // get all the state events ever between these two positions - stateNeeded, eventMap, err := d.Database.OutputEvents.SelectStateInRange(ctx, txn, fromPos, toPos, stateFilterPart) - if err != nil { - return nil, nil, err - } - state, err := d.fetchStateEvents(ctx, txn, stateNeeded, eventMap) - if err != nil { - return nil, nil, err - } - - for roomID, stateStreamEvents := range state { - for _, ev := range stateStreamEvents { - // TODO: Currently this will incorrectly add rooms which were ALREADY joined but they sent another no-op join event. - // We should be checking if the user was already joined at fromPos and not proceed if so. As a result of this, - // dupe join events will result in the entire room state coming down to the client again. This is added in - // the 'state' part of the response though, so is transparent modulo bandwidth concerns as it is not added to - // the timeline. - if membership := getMembershipFromEvent(&ev.HeaderedEvent, userID); membership != "" { - if membership == gomatrixserverlib.Join { - // send full room state down instead of a delta - var s []types.StreamEvent - s, err = d.currentStateStreamEventsForRoom(ctx, txn, roomID, stateFilterPart) - if err != nil { - return nil, nil, err - } - state[roomID] = s - continue // we'll add this room in when we do joined rooms - } - - deltas = append(deltas, stateDelta{ - membership: membership, - membershipPos: ev.StreamPosition, - stateEvents: d.StreamEventsToEvents(device, stateStreamEvents), - roomID: roomID, - }) - break - } - } - } - - // Add in currently joined rooms - joinedRoomIDs, err := d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, gomatrixserverlib.Join) - if err != nil { - return nil, nil, err - } - for _, joinedRoomID := range joinedRoomIDs { - deltas = append(deltas, stateDelta{ - membership: gomatrixserverlib.Join, - stateEvents: d.StreamEventsToEvents(device, state[joinedRoomID]), - roomID: joinedRoomID, - }) - } - - return deltas, joinedRoomIDs, nil -} - -// getStateDeltasForFullStateSync is a variant of getStateDeltas used for /sync -// requests with full_state=true. -// Fetches full state for all joined rooms and uses selectStateInRange to get -// updates for other rooms. -func (d *SyncServerDatasource) getStateDeltasForFullStateSync( - ctx context.Context, device *authtypes.Device, txn *sql.Tx, - fromPos, toPos types.StreamPosition, userID string, - stateFilterPart *gomatrixserverlib.StateFilter, -) ([]stateDelta, []string, error) { - joinedRoomIDs, err := d.roomstate.selectRoomIDsWithMembership(ctx, txn, userID, gomatrixserverlib.Join) - if err != nil { - return nil, nil, err - } - - // Use a reasonable initial capacity - deltas := make([]stateDelta, 0, len(joinedRoomIDs)) - - // Add full states for all joined rooms - for _, joinedRoomID := range joinedRoomIDs { - s, stateErr := d.currentStateStreamEventsForRoom(ctx, txn, joinedRoomID, stateFilterPart) - if stateErr != nil { - return nil, nil, stateErr - } - deltas = append(deltas, stateDelta{ - membership: gomatrixserverlib.Join, - stateEvents: d.StreamEventsToEvents(device, s), - roomID: joinedRoomID, - }) - } - - // Get all the state events ever between these two positions - stateNeeded, eventMap, err := d.Database.OutputEvents.SelectStateInRange(ctx, txn, fromPos, toPos, stateFilterPart) - if err != nil { - return nil, nil, err - } - state, err := d.fetchStateEvents(ctx, txn, stateNeeded, eventMap) - if err != nil { - return nil, nil, err - } - - for roomID, stateStreamEvents := range state { - for _, ev := range stateStreamEvents { - if membership := getMembershipFromEvent(&ev.HeaderedEvent, userID); membership != "" { - if membership != gomatrixserverlib.Join { // We've already added full state for all joined rooms above. - deltas = append(deltas, stateDelta{ - membership: membership, - membershipPos: ev.StreamPosition, - stateEvents: d.StreamEventsToEvents(device, stateStreamEvents), - roomID: roomID, - }) - } - - break - } - } - } - - return deltas, joinedRoomIDs, nil -} - -func (d *SyncServerDatasource) currentStateStreamEventsForRoom( - ctx context.Context, txn *sql.Tx, roomID string, - stateFilterPart *gomatrixserverlib.StateFilter, -) ([]types.StreamEvent, error) { - allState, err := d.roomstate.selectCurrentState(ctx, txn, roomID, stateFilterPart) - if err != nil { - return nil, err - } - s := make([]types.StreamEvent, len(allState)) - for i := 0; i < len(s); i++ { - s[i] = types.StreamEvent{HeaderedEvent: allState[i], StreamPosition: 0} - } - return s, nil -} - -// StreamEventsToEvents converts streamEvent to Event. If device is non-nil and -// matches the streamevent.transactionID device then the transaction ID gets -// added to the unsigned section of the output event. -func (d *SyncServerDatasource) StreamEventsToEvents(device *authtypes.Device, in []types.StreamEvent) []gomatrixserverlib.HeaderedEvent { - out := make([]gomatrixserverlib.HeaderedEvent, len(in)) - for i := 0; i < len(in); i++ { - out[i] = in[i].HeaderedEvent - if device != nil && in[i].TransactionID != nil { - if device.UserID == in[i].Sender() && device.SessionID == in[i].TransactionID.SessionID { - err := out[i].SetUnsignedField( - "transaction_id", in[i].TransactionID.TransactionID, - ) - if err != nil { - logrus.WithFields(logrus.Fields{ - "event_id": out[i].EventID(), - }).WithError(err).Warnf("Failed to add transaction ID to event") - } - } - } - } - return out -} - -// There may be some overlap where events in stateEvents are already in recentEvents, so filter -// them out so we don't include them twice in the /sync response. They should be in recentEvents -// only, so clients get to the correct state once they have rolled forward. -func removeDuplicates(stateEvents, recentEvents []gomatrixserverlib.HeaderedEvent) []gomatrixserverlib.HeaderedEvent { - for _, recentEv := range recentEvents { - if recentEv.StateKey() == nil { - continue // not a state event - } - // TODO: This is a linear scan over all the current state events in this room. This will - // be slow for big rooms. We should instead sort the state events by event ID (ORDER BY) - // then do a binary search to find matching events, similar to what roomserver does. - for j := 0; j < len(stateEvents); j++ { - if stateEvents[j].EventID() == recentEv.EventID() { - // overwrite the element to remove with the last element then pop the last element. - // This is orders of magnitude faster than re-slicing, but doesn't preserve ordering - // (we don't care about the order of stateEvents) - stateEvents[j] = stateEvents[len(stateEvents)-1] - stateEvents = stateEvents[:len(stateEvents)-1] - break // there shouldn't be multiple events with the same event ID - } - } - } - return stateEvents -} - -// getMembershipFromEvent returns the value of content.membership iff the event is a state event -// with type 'm.room.member' and state_key of userID. Otherwise, an empty string is returned. -func getMembershipFromEvent(ev *gomatrixserverlib.HeaderedEvent, userID string) string { - if ev.Type() == "m.room.member" && ev.StateKeyEquals(userID) { - membership, err := ev.Membership() - if err != nil { - return "" - } - return membership - } - return "" -} diff --git a/syncapi/storage/storage.go b/syncapi/storage/storage.go index 9574f37bd..76a4b7a4e 100644 --- a/syncapi/storage/storage.go +++ b/syncapi/storage/storage.go @@ -28,14 +28,14 @@ import ( func NewSyncServerDatasource(dataSourceName string, dbProperties common.DbProperties) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { - return postgres.NewSyncServerDatasource(dataSourceName, dbProperties) + return postgres.NewDatabase(dataSourceName, dbProperties) } switch uri.Scheme { case "postgres": - return postgres.NewSyncServerDatasource(dataSourceName, dbProperties) + return postgres.NewDatabase(dataSourceName, dbProperties) case "file": - return sqlite3.NewSyncServerDatasource(dataSourceName) + return sqlite3.NewDatabase(dataSourceName) default: - return postgres.NewSyncServerDatasource(dataSourceName, dbProperties) + return postgres.NewDatabase(dataSourceName, dbProperties) } } diff --git a/syncapi/storage/storage_test.go b/syncapi/storage/storage_test.go index f7fa1a870..bffcfd053 100644 --- a/syncapi/storage/storage_test.go +++ b/syncapi/storage/storage_test.go @@ -51,7 +51,7 @@ func MustCreateEvent(t *testing.T, roomID string, prevs []gomatrixserverlib.Head } func MustCreateDatabase(t *testing.T) storage.Database { - db, err := sqlite3.NewSyncServerDatasource("file::memory:") + db, err := sqlite3.NewDatabase("file::memory:") if err != nil { t.Fatalf("NewSyncServerDatasource returned %s", err) } diff --git a/syncapi/storage/storage_wasm.go b/syncapi/storage/storage_wasm.go index 84bd9df96..666179afb 100644 --- a/syncapi/storage/storage_wasm.go +++ b/syncapi/storage/storage_wasm.go @@ -35,7 +35,7 @@ func NewSyncServerDatasource( case "postgres": return nil, fmt.Errorf("Cannot use postgres implementation") case "file": - return sqlite3.NewSyncServerDatasource(dataSourceName) + return sqlite3.NewDatabase(dataSourceName) default: return nil, fmt.Errorf("Cannot use postgres implementation") } diff --git a/syncapi/storage/tables/backward_extremities.go b/syncapi/storage/tables/backward_extremities.go deleted file mode 100644 index babd9aaa1..000000000 --- a/syncapi/storage/tables/backward_extremities.go +++ /dev/null @@ -1,175 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tables - -import ( - "context" - "database/sql" - - "github.com/matrix-org/dendrite/common" -) - -// BackwardsExtremitiesStatements contains the SQL statements to implement. -// See BackwardsExtremities to see the parameter and response types. -type BackwardsExtremitiesStatements interface { - Schema() string - InsertBackwardExtremity() string - SelectBackwardExtremitiesForRoom() string - DeleteBackwardExtremity() string -} - -type PostgresBackwardsExtremitiesStatements struct{} - -func (s *PostgresBackwardsExtremitiesStatements) Schema() string { - return `-- Stores output room events received from the roomserver. - CREATE TABLE IF NOT EXISTS syncapi_backward_extremities ( - -- The 'room_id' key for the event. - room_id TEXT NOT NULL, - -- The event ID for the last known event. This is the backwards extremity. - event_id TEXT NOT NULL, - -- The prev_events for the last known event. This is used to update extremities. - prev_event_id TEXT NOT NULL, - - PRIMARY KEY(room_id, event_id, prev_event_id) - ); - ` -} -func (s *PostgresBackwardsExtremitiesStatements) InsertBackwardExtremity() string { - return "" + - "INSERT INTO syncapi_backward_extremities (room_id, event_id, prev_event_id)" + - " VALUES ($1, $2, $3)" + - " ON CONFLICT DO NOTHING" -} -func (s *PostgresBackwardsExtremitiesStatements) SelectBackwardExtremitiesForRoom() string { - return "SELECT DISTINCT event_id FROM syncapi_backward_extremities WHERE room_id = $1" -} -func (s *PostgresBackwardsExtremitiesStatements) DeleteBackwardExtremity() string { - return "DELETE FROM syncapi_backward_extremities WHERE room_id = $1 AND prev_event_id = $2" -} - -type SqliteBackwardsExtremitiesStatements struct{} - -func (s *SqliteBackwardsExtremitiesStatements) Schema() string { - return `-- Stores output room events received from the roomserver. -CREATE TABLE IF NOT EXISTS syncapi_backward_extremities ( - -- The 'room_id' key for the event. - room_id TEXT NOT NULL, - -- The event ID for the last known event. This is the backwards extremity. - event_id TEXT NOT NULL, - -- The prev_events for the last known event. This is used to update extremities. - prev_event_id TEXT NOT NULL, - - PRIMARY KEY(room_id, event_id, prev_event_id) -); -` -} - -func (s *SqliteBackwardsExtremitiesStatements) InsertBackwardExtremity() string { - return "" + - "INSERT INTO syncapi_backward_extremities (room_id, event_id, prev_event_id)" + - " VALUES ($1, $2, $3)" + - " ON CONFLICT (room_id, event_id, prev_event_id) DO NOTHING" -} - -func (s *SqliteBackwardsExtremitiesStatements) SelectBackwardExtremitiesForRoom() string { - return "" + - "SELECT DISTINCT event_id FROM syncapi_backward_extremities WHERE room_id = $1" -} - -func (s *SqliteBackwardsExtremitiesStatements) DeleteBackwardExtremity() string { - return "" + - "DELETE FROM syncapi_backward_extremities WHERE room_id = $1 AND prev_event_id = $2" -} - -// BackwardsExtremities keeps track of backwards extremities for a room. -// Backwards extremities are the earliest (DAG-wise) known events which we have -// the entire event JSON. These event IDs are used in federation requests to fetch -// even earlier events. -// -// We persist the previous event IDs as well, one per row, so when we do fetch even -// earlier events we can simply delete rows which referenced it. Consider the graph: -// A -// | Event C has 1 prev_event ID: A. -// B C -// |___| Event D has 2 prev_event IDs: B and C. -// | -// D -// The earliest known event we have is D, so this table has 2 rows. -// A backfill request gives us C but not B. We delete rows where prev_event=C. This -// still means that D is a backwards extremity as we do not have event B. However, event -// C is *also* a backwards extremity at this point as we do not have event A. Later, -// when we fetch event B, we delete rows where prev_event=B. This then removes D as -// a backwards extremity because there are no more rows with event_id=B. -type BackwardsExtremities struct { - insertBackwardExtremityStmt *sql.Stmt - selectBackwardExtremitiesForRoomStmt *sql.Stmt - deleteBackwardExtremityStmt *sql.Stmt -} - -// NewBackwardsExtremities prepares the table -func NewBackwardsExtremities(db *sql.DB, stmts BackwardsExtremitiesStatements) (table BackwardsExtremities, err error) { - _, err = db.Exec(stmts.Schema()) - if err != nil { - return - } - if table.insertBackwardExtremityStmt, err = db.Prepare(stmts.InsertBackwardExtremity()); err != nil { - return - } - if table.selectBackwardExtremitiesForRoomStmt, err = db.Prepare(stmts.SelectBackwardExtremitiesForRoom()); err != nil { - return - } - if table.deleteBackwardExtremityStmt, err = db.Prepare(stmts.DeleteBackwardExtremity()); err != nil { - return - } - return -} - -// InsertsBackwardExtremity inserts a new backwards extremity. -func (s *BackwardsExtremities) InsertsBackwardExtremity( - ctx context.Context, txn *sql.Tx, roomID, eventID string, prevEventID string, -) (err error) { - _, err = txn.Stmt(s.insertBackwardExtremityStmt).ExecContext(ctx, roomID, eventID, prevEventID) - return -} - -// SelectBackwardExtremitiesForRoom retrieves all backwards extremities for the room. -func (s *BackwardsExtremities) SelectBackwardExtremitiesForRoom( - ctx context.Context, roomID string, -) (eventIDs []string, err error) { - rows, err := s.selectBackwardExtremitiesForRoomStmt.QueryContext(ctx, roomID) - if err != nil { - return - } - defer common.CloseAndLogIfError(ctx, rows, "selectBackwardExtremitiesForRoom: rows.close() failed") - - for rows.Next() { - var eID string - if err = rows.Scan(&eID); err != nil { - return - } - - eventIDs = append(eventIDs, eID) - } - - return eventIDs, rows.Err() -} - -// DeleteBackwardExtremity removes a backwards extremity for a room, if one existed. -func (s *BackwardsExtremities) DeleteBackwardExtremity( - ctx context.Context, txn *sql.Tx, roomID, knownEventID string, -) (err error) { - _, err = txn.Stmt(s.deleteBackwardExtremityStmt).ExecContext(ctx, roomID, knownEventID) - return -} diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go index ffd502165..8f0b8b895 100644 --- a/syncapi/storage/tables/interface.go +++ b/syncapi/storage/tables/interface.go @@ -30,3 +30,61 @@ type Events interface { SelectEarlyEvents(ctx context.Context, txn *sql.Tx, roomID string, fromPos, toPos types.StreamPosition, limit int) ([]types.StreamEvent, error) SelectEvents(ctx context.Context, txn *sql.Tx, eventIDs []string) ([]types.StreamEvent, error) } + +type Topology interface { + // InsertEventInTopology inserts the given event in the room's topology, based + // on the event's depth. + InsertEventInTopology(ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.HeaderedEvent, pos types.StreamPosition) (err error) + // SelectEventIDsInRange selects the IDs of events which positions are within a + // given range in a given room's topological order. + // Returns an empty slice if no events match the given range. + SelectEventIDsInRange(ctx context.Context, txn *sql.Tx, roomID string, fromPos, toPos, toMicroPos types.StreamPosition, limit int, chronologicalOrder bool) (eventIDs []string, err error) + // SelectPositionInTopology returns the position of a given event in the + // topology of the room it belongs to. + SelectPositionInTopology(ctx context.Context, txn *sql.Tx, eventID string) (pos, spos types.StreamPosition, err error) + SelectMaxPositionInTopology(ctx context.Context, txn *sql.Tx, roomID string) (pos types.StreamPosition, spos types.StreamPosition, err error) + // SelectEventIDsFromPosition returns the IDs of all events that have a given + // position in the topology of a given room. + SelectEventIDsFromPosition(ctx context.Context, txn *sql.Tx, roomID string, pos types.StreamPosition) (eventIDs []string, err error) +} + +type CurrentRoomState interface { + SelectStateEvent(ctx context.Context, roomID, evType, stateKey string) (*gomatrixserverlib.HeaderedEvent, error) + SelectEventsWithEventIDs(ctx context.Context, txn *sql.Tx, eventIDs []string) ([]types.StreamEvent, error) + UpsertRoomState(ctx context.Context, txn *sql.Tx, event gomatrixserverlib.HeaderedEvent, membership *string, addedAt types.StreamPosition) error + DeleteRoomStateByEventID(ctx context.Context, txn *sql.Tx, eventID string) error + // SelectCurrentState returns all the current state events for the given room. + SelectCurrentState(ctx context.Context, txn *sql.Tx, roomID string, stateFilter *gomatrixserverlib.StateFilter) ([]gomatrixserverlib.HeaderedEvent, error) + // SelectRoomIDsWithMembership returns the list of room IDs which have the given user in the given membership state. + SelectRoomIDsWithMembership(ctx context.Context, txn *sql.Tx, userID string, membership string) ([]string, error) + // SelectJoinedUsers returns a map of room ID to a list of joined user IDs. + SelectJoinedUsers(ctx context.Context) (map[string][]string, error) +} + +// BackwardsExtremities keeps track of backwards extremities for a room. +// Backwards extremities are the earliest (DAG-wise) known events which we have +// the entire event JSON. These event IDs are used in federation requests to fetch +// even earlier events. +// +// We persist the previous event IDs as well, one per row, so when we do fetch even +// earlier events we can simply delete rows which referenced it. Consider the graph: +// A +// | Event C has 1 prev_event ID: A. +// B C +// |___| Event D has 2 prev_event IDs: B and C. +// | +// D +// The earliest known event we have is D, so this table has 2 rows. +// A backfill request gives us C but not B. We delete rows where prev_event=C. This +// still means that D is a backwards extremity as we do not have event B. However, event +// C is *also* a backwards extremity at this point as we do not have event A. Later, +// when we fetch event B, we delete rows where prev_event=B. This then removes D as +// a backwards extremity because there are no more rows with event_id=B. +type BackwardsExtremities interface { + // InsertsBackwardExtremity inserts a new backwards extremity. + InsertsBackwardExtremity(ctx context.Context, txn *sql.Tx, roomID, eventID string, prevEventID string) (err error) + // SelectBackwardExtremitiesForRoom retrieves all backwards extremities for the room. + SelectBackwardExtremitiesForRoom(ctx context.Context, roomID string) (eventIDs []string, err error) + // DeleteBackwardExtremity removes a backwards extremity for a room, if one existed. + DeleteBackwardExtremity(ctx context.Context, txn *sql.Tx, roomID, knownEventID string) (err error) +} From 3cb04e80042104d14ccfa162b3e40a5b08819eac Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 14 May 2020 16:49:18 +0100 Subject: [PATCH 059/200] Update INSTALL.md, move docs (#1034) * Update INSTALL.md * Move some things * Clean up * Move some more things * Don't build all the things for the monolith * Update INSTALL.md * Nuke hooks --- .editorconfig | 18 -- README.md | 6 +- CODE_STYLE.md => docs/CODE_STYLE.md | 0 CONTRIBUTING.md => docs/CONTRIBUTING.md | 0 DESIGN.md => docs/DESIGN.md | 0 INSTALL.md => docs/INSTALL.md | 236 +++++++++++++++--------- WIRING.md => docs/WIRING.md | 0 p2p.md => docs/p2p.md | 0 hooks/install.sh | 5 - hooks/pre-commit | 22 --- 10 files changed, 153 insertions(+), 134 deletions(-) delete mode 100644 .editorconfig rename CODE_STYLE.md => docs/CODE_STYLE.md (100%) rename CONTRIBUTING.md => docs/CONTRIBUTING.md (100%) rename DESIGN.md => docs/DESIGN.md (100%) rename INSTALL.md => docs/INSTALL.md (64%) rename WIRING.md => docs/WIRING.md (100%) rename p2p.md => docs/p2p.md (100%) delete mode 100755 hooks/install.sh delete mode 100755 hooks/pre-commit diff --git a/.editorconfig b/.editorconfig deleted file mode 100644 index 1fee56179..000000000 --- a/.editorconfig +++ /dev/null @@ -1,18 +0,0 @@ -root = true - -[*] -charset = utf-8 - -end_of_line = lf -insert_final_newline = true -trim_trailing_whitespace = true - -[*.go] -indent_style = tab -indent_size = 4 - -[*.md] -trim_trailing_whitespace = false - -[*.{yml,yaml}] -indent_style = space diff --git a/README.md b/README.md index 49f9ca840..aa9060f1b 100644 --- a/README.md +++ b/README.md @@ -3,15 +3,15 @@ Dendrite will be a second-generation Matrix homeserver written in Go. It's still very much a work in progress, but installation instructions can be -found in [INSTALL.md](INSTALL.md). It is not recommended to use Dendrite as a +found in [INSTALL.md](docs/INSTALL.md). It is not recommended to use Dendrite as a production homeserver at this time. -An overview of the design can be found in [DESIGN.md](DESIGN.md). +An overview of the design can be found in [DESIGN.md](docs/DESIGN.md). # Contributing Everyone is welcome to help out and contribute! See -[CONTRIBUTING.md](CONTRIBUTING.md) to get started! +[CONTRIBUTING.md](docs/CONTRIBUTING.md) to get started! Please note that, as of February 2020, Dendrite now only targets Go 1.13 or later. Please ensure that you are using at least Go 1.13 when developing for diff --git a/CODE_STYLE.md b/docs/CODE_STYLE.md similarity index 100% rename from CODE_STYLE.md rename to docs/CODE_STYLE.md diff --git a/CONTRIBUTING.md b/docs/CONTRIBUTING.md similarity index 100% rename from CONTRIBUTING.md rename to docs/CONTRIBUTING.md diff --git a/DESIGN.md b/docs/DESIGN.md similarity index 100% rename from DESIGN.md rename to docs/DESIGN.md diff --git a/INSTALL.md b/docs/INSTALL.md similarity index 64% rename from INSTALL.md rename to docs/INSTALL.md index 4173e705e..184b777d8 100644 --- a/INSTALL.md +++ b/docs/INSTALL.md @@ -2,38 +2,66 @@ Dendrite can be run in one of two configurations: - * A cluster of individual components, dealing with different aspects of the - Matrix protocol (see [WIRING.md](./WIRING.md)). Components communicate with - one another via [Apache Kafka](https://kafka.apache.org). +* **Polylith mode**: A cluster of individual components, dealing with different + aspects of the Matrix protocol (see [WIRING.md](WIRING.md)). Components communicate with each other using internal HTTP APIs and [Apache Kafka](https://kafka.apache.org). This will almost certainly be the preferred model + for large-scale deployments. - * A monolith server, in which all components run in the same process. In this - configuration, Kafka can be replaced with an in-process implementation - called [naffka](https://github.com/matrix-org/naffka). +* **Monolith mode**: All components run in the same process. In this mode, + Kafka is completely optional and can instead be replaced with an in-process + lightweight implementation called [Naffka](https://github.com/matrix-org/naffka). This will usually be the preferred model for low-volume, low-user + or experimental deployments. + +Regardless of whether you are running in polylith or monolith mode, each Dendrite component that requires storage has its own database. Both Postgres +and SQLite are supported and can be mixed-and-matched across components as +needed in the configuration file. + +Be advised that Dendrite is still developmental and it's not recommended for +use in production environments yet! ## Requirements - - Go 1.13+ - - Postgres 9.5+ - - For Kafka (optional if using the monolith server): - - Unix-based system (https://kafka.apache.org/documentation/#os) - - JDK 1.8+ / OpenJDK 1.8+ - - Apache Kafka 0.10.2+ (see [scripts/install-local-kafka.sh](scripts/install-local-kafka.sh) for up-to-date version numbers) +Dendrite requires: +* Go 1.13 or higher +* Postgres 9.5 or higher (if using Postgres databases, not needed for SQLite) -## Setting up a development environment +If you want to run a polylith deployment, you also need: -Assumes Go 1.13+ and JDK 1.8+ are already installed and are on PATH. +* Apache Kafka 0.10.2+ + +## Building up a monolith deploment + +Start by cloning the code: ```bash -# Get the code git clone https://github.com/matrix-org/dendrite cd dendrite +``` -# Build it +Then build it: + +```bash +go build -o bin/dendrite-monolith-server ./cmd/dendrite-monolith-server +go build -o bin/generate-keys ./cmd/generate-keys +``` + +## Building up a polylith deployment + +Start by cloning the code: + +```bash +git clone https://github.com/matrix-org/dendrite +cd dendrite +``` + +Then build it: + +```bash ./build.sh ``` -If using Kafka, install and start it (c.f. [scripts/install-local-kafka.sh](scripts/install-local-kafka.sh)): +Install and start Kafka (c.f. [scripts/install-local-kafka.sh](scripts/install-local-kafka.sh)): + ```bash KAFKA_URL=http://archive.apache.org/dist/kafka/2.1.0/kafka_2.11-2.1.0.tgz @@ -51,7 +79,7 @@ kafka/bin/zookeeper-server-start.sh -daemon kafka/config/zookeeper.properties kafka/bin/kafka-server-start.sh -daemon kafka/config/server.properties ``` -On MacOS, you can use [homebrew](https://brew.sh/) for easier setup of kafka +On macOS, you can use [Homebrew](https://brew.sh/) for easier setup of Kafka: ```bash brew install kafka @@ -61,15 +89,24 @@ brew services start kafka ## Configuration +### SQLite database setup + +Dendrite can use the built-in SQLite database engine for small setups. +The SQLite databases do not need to be preconfigured - Dendrite will +create them automatically at startup. + ### Postgres database setup -Dendrite requires a postgres database engine, version 9.5 or later. +Assuming that Postgres 9.5 (or later) is installed: + +* Create role, choosing a new password when prompted: -* Create role: ```bash - sudo -u postgres createuser -P dendrite # prompts for password + sudo -u postgres createuser -P dendrite ``` -* Create databases: + +* Create the component databases: + ```bash for i in account device mediaapi syncapi roomserver serverkey federationsender publicroomsapi appservice naffka; do sudo -u postgres createdb -O dendrite dendrite_$i @@ -78,42 +115,56 @@ Dendrite requires a postgres database engine, version 9.5 or later. (On macOS, omit `sudo -u postgres` from the above commands.) -### Crypto key generation +### Server key generation -Generate the keys: +Each Dendrite server requires unique server keys. + +Generate the self-signed SSL certificate for federation: ```bash -# Generate a self-signed SSL cert for federation: test -f server.key || openssl req -x509 -newkey rsa:4096 -keyout server.key -out server.crt -days 3650 -nodes -subj /CN=localhost +``` -# generate ed25519 signing key +Generate the server signing key: + +``` test -f matrix_key.pem || ./bin/generate-keys -private-key matrix_key.pem ``` -### Configuration +### Configuration file Create config file, based on `dendrite-config.yaml`. Call it `dendrite.yaml`. Things that will need editing include *at least*: -* `server_name` -* `database/*` (All lines in the database section must have the username and password of the user created with the `createuser` command above. eg:`dendrite:password@localhost`) +* The `server_name` entry to reflect the hostname of your Dendrite server +* The `database` lines with an updated connection string based on your + desired setup, e.g. replacing `component` with the name of the component: + * For Postgres: `postgres://dendrite:password@localhost/component` + * For SQLite on disk: `file:component.db` or `file:///path/to/component.db` + * Postgres and SQLite can be mixed and matched. +* The `use_naffka` option if using Naffka in a monolith deployment + +There are other options which may be useful so review them all. In particular, +if you are trying to federate from your Dendrite instance into public rooms +then configuring `key_perspectives` (like `matrix.org` in the sample) can +help to improve reliability considerably by allowing your homeserver to fetch +public keys for dead homeservers from somewhere else. ## Starting a monolith server -It is possible to use 'naffka' as an in-process replacement to Kafka when using -the monolith server. To do this, set `use_naffka: true` in `dendrite.yaml` and uncomment -the necessary line related to naffka in the `database` section. Be sure to update the -database username and password if needed. +It is possible to use Naffka as an in-process replacement to Kafka when using +the monolith server. To do this, set `use_naffka: true` in your `dendrite.yaml` configuration and uncomment the relevant Naffka line in the `database` section. +Be sure to update the database username and password if needed. The monolith server can be started as shown below. By default it listens for -HTTP connections on port 8008, so point your client at -`http://localhost:8008`. If you set `--tls-cert` and `--tls-key` as shown -below, it will also listen for HTTPS connections on port 8448. +HTTP connections on port 8008, so you can configure your Matrix client to use +`http://localhost:8008` as the server. If you set `--tls-cert` and `--tls-key` +as shown below, it will also listen for HTTPS connections on port 8448. ```bash ./bin/dendrite-monolith-server --tls-cert=server.crt --tls-key=server.key ``` -## Starting a multiprocess server +## Starting a polylith deployment The following contains scripts which will run all the required processes in order to point a Matrix client at Dendrite. Conceptually, you are wiring together to form the following diagram: @@ -170,9 +221,10 @@ Servers --->| federation-api-proxy |--------->| dendrite-federation-api-server | A ==> B = Kafka (A = producer, B = consumer) ``` -### Run a client api proxy +### Client proxy -This is what Matrix clients will talk to. If you use the script below, point your client at `http://localhost:8008`. +This is what Matrix clients will talk to. If you use the script below, point +your client at `http://localhost:8008`. ```bash ./bin/client-api-proxy \ @@ -183,51 +235,10 @@ This is what Matrix clients will talk to. If you use the script below, point you --public-rooms-api-server-url "http://localhost:7775" \ ``` -### Run a client api +### Federation proxy -This is what implements message sending. Clients talk to this via the proxy in order to send messages. - -```bash -./bin/dendrite-client-api-server --config=dendrite.yaml -``` - -(If this fails with `pq: syntax error at or near "ON"`, check you are using at least postgres 9.5.) - -### Run a room server - -This is what implements the room DAG. Clients do not talk to this. - -```bash -./bin/dendrite-room-server --config=dendrite.yaml -``` - -### Run a sync server - -This is what implements `/sync` requests. Clients talk to this via the proxy in order to receive messages. - -```bash -./bin/dendrite-sync-api-server --config dendrite.yaml -``` - -### Run a media server - -This implements `/media` requests. Clients talk to this via the proxy in order to upload and retrieve media. - -```bash -./bin/dendrite-media-api-server --config dendrite.yaml -``` - -### Run public room server - -This implements `/directory` requests. Clients talk to this via the proxy in order to retrieve room directory listings. - -```bash -./bin/dendrite-public-rooms-api-server --config dendrite.yaml -``` - -### Run a federation api proxy - -This is what Matrix servers will talk to. This is only required if you want to support federation. +This is what Matrix servers will talk to. This is only required if you want +to support federation. ```bash ./bin/federation-api-proxy \ @@ -236,7 +247,51 @@ This is what Matrix servers will talk to. This is only required if you want to s --media-api-server-url "http://localhost:7774" \ ``` -### Run a federation api server +### Client API server + +This is what implements message sending. Clients talk to this via the proxy in +order to send messages. + +```bash +./bin/dendrite-client-api-server --config=dendrite.yaml +``` + +### Room server + +This is what implements the room DAG. Clients do not talk to this. + +```bash +./bin/dendrite-room-server --config=dendrite.yaml +``` + +### Sync server + +This is what implements `/sync` requests. Clients talk to this via the proxy +in order to receive messages. + +```bash +./bin/dendrite-sync-api-server --config dendrite.yaml +``` + +### Media server + +This implements `/media` requests. Clients talk to this via the proxy in +order to upload and retrieve media. + +```bash +./bin/dendrite-media-api-server --config dendrite.yaml +``` + +### Public room server + +This implements `/directory` requests. Clients talk to this via the proxy +in order to retrieve room directory listings. + +```bash +./bin/dendrite-public-rooms-api-server --config dendrite.yaml +``` + +### Federation API server This implements federation requests. Servers talk to this via the proxy in order to send transactions. This is only required if you want to support @@ -246,7 +301,7 @@ federation. ./bin/dendrite-federation-api-server --config dendrite.yaml ``` -### Run a federation sender server +### Federation sender This sends events from our users to other servers. This is only required if you want to support federation. @@ -255,7 +310,7 @@ you want to support federation. ./bin/dendrite-federation-sender-server --config dendrite.yaml ``` -### Run an appservice server +### Appservice server This sends events from the network to [application services](https://matrix.org/docs/spec/application_service/unstable.html) @@ -265,3 +320,12 @@ application services on your homeserver. ```bash ./bin/dendrite-appservice-server --config dendrite.yaml ``` + +### Key server + +This manages end-to-end encryption keys (or rather, it will do when it's +finished). + +```bash +./bin/dendrite-key-server --config dendrite.yaml +``` diff --git a/WIRING.md b/docs/WIRING.md similarity index 100% rename from WIRING.md rename to docs/WIRING.md diff --git a/p2p.md b/docs/p2p.md similarity index 100% rename from p2p.md rename to docs/p2p.md diff --git a/hooks/install.sh b/hooks/install.sh deleted file mode 100755 index f8aa331ff..000000000 --- a/hooks/install.sh +++ /dev/null @@ -1,5 +0,0 @@ -#! /bin/bash - -DOT_GIT="$(dirname $0)/../.git" - -ln -s "../../hooks/pre-commit" "$DOT_GIT/hooks/pre-commit" \ No newline at end of file diff --git a/hooks/pre-commit b/hooks/pre-commit deleted file mode 100755 index 6f98b813c..000000000 --- a/hooks/pre-commit +++ /dev/null @@ -1,22 +0,0 @@ -#! /bin/bash - -set -eu - -# make the GIT_DIR and GIT_INDEX_FILE absolute, before we change dir -export GIT_DIR=$(readlink -f `git rev-parse --git-dir`) -if [ -n "${GIT_INDEX_FILE:+x}" ]; then - export GIT_INDEX_FILE=$(readlink -f "$GIT_INDEX_FILE") -fi - -# create a temp dir. The `trap` incantation will ensure that it is removed -# again when this script completes. -tmpdir=`mktemp -d` -trap 'rm -rf "$tmpdir"' EXIT -cd "$tmpdir" - -# get a clean copy of the index (ie, what has been `git add`ed), so that we can -# run the checks against what we are about to commit, rather than what is in -# the working copy. -git checkout-index -a - -./scripts/find-lint.sh fast From 7ca230e931faceb0b8a9e60314258d6cd59f33d4 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Thu, 14 May 2020 17:30:16 +0100 Subject: [PATCH 060/200] Cleanup syncapi topology logic (#1035) * Cleanup syncapi topology logic * Variable renaming * comments --- syncapi/routing/messages.go | 16 ++--- syncapi/storage/interface.go | 7 +- .../output_room_events_topology_table.go | 37 +--------- syncapi/storage/shared/syncserver.go | 68 +++++++++---------- .../sqlite3/output_room_events_table.go | 5 -- .../output_room_events_topology_table.go | 40 +---------- syncapi/storage/storage_test.go | 13 ++-- syncapi/storage/tables/interface.go | 32 +++++---- syncapi/types/types.go | 3 + 9 files changed, 67 insertions(+), 154 deletions(-) diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index 72c306d4f..67de6df7e 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -245,24 +245,20 @@ func (r *messagesReq) retrieveEvents() ( // change the way topological positions are defined (as depth isn't the most // reliable way to define it), it would be easier and less troublesome to // only have to change it in one place, i.e. the database. - startPos, startStreamPos, err := r.db.EventPositionInTopology( + start, err = r.db.EventPositionInTopology( r.ctx, events[0].EventID(), ) if err != nil { err = fmt.Errorf("EventPositionInTopology: for start event %s: %w", events[0].EventID(), err) return } - endPos, endStreamPos, err := r.db.EventPositionInTopology( + end, err = r.db.EventPositionInTopology( r.ctx, events[len(events)-1].EventID(), ) if err != nil { err = fmt.Errorf("EventPositionInTopology: for end event %s: %w", events[len(events)-1].EventID(), err) return } - // Generate pagination tokens to send to the client using the positions - // retrieved previously. - start = types.NewTopologyToken(startPos, startStreamPos) - end = types.NewTopologyToken(endPos, endStreamPos) if r.backwardOrdering { // A stream/topological position is a cursor located between two events. @@ -431,14 +427,10 @@ func setToDefault( ) (to types.TopologyToken, err error) { if backwardOrdering { // go 1 earlier than the first event so we correctly fetch the earliest event + // this is because Database.GetEventsInTopologicalRange is exclusive of the lower-bound. to = types.NewTopologyToken(0, 0) } else { - var depth, stream types.StreamPosition - depth, stream, err = db.MaxTopologicalPosition(ctx, roomID) - if err != nil { - return - } - to = types.NewTopologyToken(depth, stream) + to, err = db.MaxTopologicalPosition(ctx, roomID) } return diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index 63af11365..227167899 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -93,15 +93,12 @@ type Database interface { // GetEventsInTopologicalRange retrieves all of the events on a given ordering using the given extremities and limit. GetEventsInTopologicalRange(ctx context.Context, from, to *types.TopologyToken, roomID string, limit int, backwardOrdering bool) (events []types.StreamEvent, err error) // EventPositionInTopology returns the depth and stream position of the given event. - EventPositionInTopology(ctx context.Context, eventID string) (depth types.StreamPosition, stream types.StreamPosition, err error) - // EventsAtTopologicalPosition returns all of the events matching a given - // position in the topology of a given room. - EventsAtTopologicalPosition(ctx context.Context, roomID string, pos types.StreamPosition) ([]types.StreamEvent, error) + EventPositionInTopology(ctx context.Context, eventID string) (types.TopologyToken, error) // BackwardExtremitiesForRoom returns the event IDs of all of the backward // extremities we know of for a given room. BackwardExtremitiesForRoom(ctx context.Context, roomID string) (backwardExtremities []string, err error) // MaxTopologicalPosition returns the highest topological position for a given room. - MaxTopologicalPosition(ctx context.Context, roomID string) (depth types.StreamPosition, stream types.StreamPosition, err error) + MaxTopologicalPosition(ctx context.Context, roomID string) (types.TopologyToken, error) // StreamEventsToEvents converts streamEvent to Event. If device is non-nil and // matches the streamevent.transactionID device then the transaction ID gets // added to the unsigned section of the output event. diff --git a/syncapi/storage/postgres/output_room_events_topology_table.go b/syncapi/storage/postgres/output_room_events_topology_table.go index 48ebcf251..8a9cc49c9 100644 --- a/syncapi/storage/postgres/output_room_events_topology_table.go +++ b/syncapi/storage/postgres/output_room_events_topology_table.go @@ -72,17 +72,12 @@ const selectMaxPositionInTopologySQL = "" + "SELECT MAX(topological_position) FROM syncapi_output_room_events_topology WHERE room_id=$1" + ") ORDER BY stream_position DESC LIMIT 1" -const selectEventIDsFromPositionSQL = "" + - "SELECT event_id FROM syncapi_output_room_events_topology" + - " WHERE room_id = $1 AND topological_position = $2" - type outputRoomEventsTopologyStatements struct { insertEventInTopologyStmt *sql.Stmt selectEventIDsInRangeASCStmt *sql.Stmt selectEventIDsInRangeDESCStmt *sql.Stmt selectPositionInTopologyStmt *sql.Stmt selectMaxPositionInTopologyStmt *sql.Stmt - selectEventIDsFromPositionStmt *sql.Stmt } func NewPostgresTopologyTable(db *sql.DB) (tables.Topology, error) { @@ -106,9 +101,6 @@ func NewPostgresTopologyTable(db *sql.DB) (tables.Topology, error) { if s.selectMaxPositionInTopologyStmt, err = db.Prepare(selectMaxPositionInTopologySQL); err != nil { return nil, err } - if s.selectEventIDsFromPositionStmt, err = db.Prepare(selectEventIDsFromPositionSQL); err != nil { - return nil, err - } return s, nil } @@ -127,7 +119,7 @@ func (s *outputRoomEventsTopologyStatements) InsertEventInTopology( // given range in a given room's topological order. // Returns an empty slice if no events match the given range. func (s *outputRoomEventsTopologyStatements) SelectEventIDsInRange( - ctx context.Context, txn *sql.Tx, roomID string, fromPos, toPos, toMicroPos types.StreamPosition, + ctx context.Context, txn *sql.Tx, roomID string, minDepth, maxDepth, maxStreamPos types.StreamPosition, limit int, chronologicalOrder bool, ) (eventIDs []string, err error) { // Decide on the selection's order according to whether chronological order @@ -140,7 +132,7 @@ func (s *outputRoomEventsTopologyStatements) SelectEventIDsInRange( } // Query the event IDs. - rows, err := stmt.QueryContext(ctx, roomID, fromPos, toPos, toPos, toMicroPos, limit) + rows, err := stmt.QueryContext(ctx, roomID, minDepth, maxDepth, maxDepth, maxStreamPos, limit) if err == sql.ErrNoRows { // If no event matched the request, return an empty slice. return []string{}, nil @@ -176,28 +168,3 @@ func (s *outputRoomEventsTopologyStatements) SelectMaxPositionInTopology( err = s.selectMaxPositionInTopologyStmt.QueryRowContext(ctx, roomID).Scan(&pos, &spos) return } - -// SelectEventIDsFromPosition returns the IDs of all events that have a given -// position in the topology of a given room. -func (s *outputRoomEventsTopologyStatements) SelectEventIDsFromPosition( - ctx context.Context, txn *sql.Tx, roomID string, pos types.StreamPosition, -) (eventIDs []string, err error) { - // Query the event IDs. - rows, err := s.selectEventIDsFromPositionStmt.QueryContext(ctx, roomID, pos) - if err == sql.ErrNoRows { - // If no event matched the request, return an empty slice. - return []string{}, nil - } else if err != nil { - return - } - defer common.CloseAndLogIfError(ctx, rows, "selectEventIDsFromPosition: rows.close() failed") - // Return the IDs. - var eventID string - for rows.Next() { - if err = rows.Scan(&eventID); err != nil { - return - } - eventIDs = append(eventIDs, eventID) - } - return eventIDs, rows.Err() -} diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go index 96e9ff619..e03a6b9ff 100644 --- a/syncapi/storage/shared/syncserver.go +++ b/syncapi/storage/shared/syncserver.go @@ -319,25 +319,25 @@ func (d *Database) GetEventsInTopologicalRange( roomID string, limit int, backwardOrdering bool, ) (events []types.StreamEvent, err error) { - // Determine the backward and forward limit, i.e. the upper and lower - // limits to the selection in the room's topology, from the direction. - var backwardLimit, forwardLimit, forwardMicroLimit types.StreamPosition + var minDepth, maxDepth, maxStreamPosForMaxDepth types.StreamPosition if backwardOrdering { - // Backward ordering is antichronological (latest event to oldest - // one). - backwardLimit = to.Depth() - forwardLimit = from.Depth() - forwardMicroLimit = from.PDUPosition() + // Backward ordering means the 'from' token has a higher depth than the 'to' token + minDepth = to.Depth() + maxDepth = from.Depth() + // for cases where we have say 5 events with the same depth, the TopologyToken needs to + // know which of the 5 the client has seen. This is done by using the PDU position. + // Events with the same maxDepth but less than this PDU position will be returned. + maxStreamPosForMaxDepth = from.PDUPosition() } else { - // Forward ordering is chronological (oldest event to latest one). - backwardLimit = from.Depth() - forwardLimit = to.Depth() + // Forward ordering means the 'from' token has a lower depth than the 'to' token. + minDepth = from.Depth() + maxDepth = to.Depth() } // Select the event IDs from the defined range. var eIDs []string eIDs, err = d.Topology.SelectEventIDsInRange( - ctx, nil, roomID, backwardLimit, forwardLimit, forwardMicroLimit, limit, !backwardOrdering, + ctx, nil, roomID, minDepth, maxDepth, maxStreamPosForMaxDepth, limit, !backwardOrdering, ) if err != nil { return @@ -350,7 +350,7 @@ func (d *Database) GetEventsInTopologicalRange( func (d *Database) SyncPosition(ctx context.Context) (tok types.StreamingToken, err error) { err = common.WithTransaction(d.DB, func(txn *sql.Tx) error { - pos, err := d.SyncPositionTx(ctx, txn) + pos, err := d.syncPositionTx(ctx, txn) if err != nil { return err } @@ -368,29 +368,25 @@ func (d *Database) BackwardExtremitiesForRoom( func (d *Database) MaxTopologicalPosition( ctx context.Context, roomID string, -) (depth types.StreamPosition, stream types.StreamPosition, err error) { - return d.Topology.SelectMaxPositionInTopology(ctx, nil, roomID) -} - -func (d *Database) EventsAtTopologicalPosition( - ctx context.Context, roomID string, pos types.StreamPosition, -) ([]types.StreamEvent, error) { - eIDs, err := d.Topology.SelectEventIDsFromPosition(ctx, nil, roomID, pos) +) (types.TopologyToken, error) { + depth, streamPos, err := d.Topology.SelectMaxPositionInTopology(ctx, nil, roomID) if err != nil { - return nil, err + return types.NewTopologyToken(0, 0), err } - - return d.OutputEvents.SelectEvents(ctx, nil, eIDs) + return types.NewTopologyToken(depth, streamPos), nil } func (d *Database) EventPositionInTopology( ctx context.Context, eventID string, -) (depth types.StreamPosition, stream types.StreamPosition, err error) { - return d.Topology.SelectPositionInTopology(ctx, nil, eventID) +) (types.TopologyToken, error) { + depth, stream, err := d.Topology.SelectPositionInTopology(ctx, nil, eventID) + if err != nil { + return types.NewTopologyToken(0, 0), err + } + return types.NewTopologyToken(depth, stream), nil } -// TODO FIXME TEMPORARY PUBLIC -func (d *Database) SyncPositionTx( +func (d *Database) syncPositionTx( ctx context.Context, txn *sql.Tx, ) (sp types.StreamingToken, err error) { @@ -466,7 +462,7 @@ func (d *Database) addPDUDeltaToResponse( } // TODO: This should be done in getStateDeltas - if err = d.AddInvitesToResponse(ctx, txn, device.UserID, fromPos, toPos, res); err != nil { + if err = d.addInvitesToResponse(ctx, txn, device.UserID, fromPos, toPos, res); err != nil { return nil, err } @@ -510,8 +506,7 @@ func (d *Database) addTypingDeltaToResponse( // addEDUDeltaToResponse adds updates for EDUs of each type since fromPos if // the positions of that type are not equal in fromPos and toPos. -// TODO FIXME TEMPORARY PUBLIC -func (d *Database) AddEDUDeltaToResponse( +func (d *Database) addEDUDeltaToResponse( fromPos, toPos types.StreamingToken, joinedRoomIDs []string, res *types.Response, @@ -551,7 +546,7 @@ func (d *Database) IncrementalSync( return nil, err } - err = d.AddEDUDeltaToResponse( + err = d.addEDUDeltaToResponse( fromPos, toPos, joinedRoomIDs, res, ) if err != nil { @@ -590,7 +585,7 @@ func (d *Database) getResponseWithPDUsForCompleteSync( }() // Get the current sync position which we will base the sync response on. - toPos, err = d.SyncPositionTx(ctx, txn) + toPos, err = d.syncPositionTx(ctx, txn) if err != nil { return } @@ -649,7 +644,7 @@ func (d *Database) getResponseWithPDUsForCompleteSync( res.Rooms.Join[roomID] = *jr } - if err = d.AddInvitesToResponse(ctx, txn, userID, 0, toPos.PDUPosition(), res); err != nil { + if err = d.addInvitesToResponse(ctx, txn, userID, 0, toPos.PDUPosition(), res); err != nil { return } @@ -668,7 +663,7 @@ func (d *Database) CompleteSync( } // Use a zero value SyncPosition for fromPos so all EDU states are added. - err = d.AddEDUDeltaToResponse( + err = d.addEDUDeltaToResponse( types.NewStreamToken(0, 0), toPos, joinedRoomIDs, res, ) if err != nil { @@ -688,8 +683,7 @@ var txReadOnlySnapshot = sql.TxOptions{ ReadOnly: true, } -// TODO FIXME temporary public -func (d *Database) AddInvitesToResponse( +func (d *Database) addInvitesToResponse( ctx context.Context, txn *sql.Tx, userID string, fromPos, toPos types.StreamPosition, diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index d3e88a549..c1647a14c 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -306,9 +306,6 @@ func (s *outputRoomEventsStatements) InsertEvent( return } -// selectRecentEvents returns the most recent events in the given room, up to a maximum of 'limit'. -// If onlySyncEvents has a value of true, only returns the events that aren't marked as to exclude -// from sync. func (s *outputRoomEventsStatements) SelectRecentEvents( ctx context.Context, txn *sql.Tx, roomID string, fromPos, toPos types.StreamPosition, limit int, @@ -341,8 +338,6 @@ func (s *outputRoomEventsStatements) SelectRecentEvents( return events, nil } -// selectEarlyEvents returns the earliest events in the given room, starting -// from a given position, up to a maximum of 'limit'. func (s *outputRoomEventsStatements) SelectEarlyEvents( ctx context.Context, txn *sql.Tx, roomID string, fromPos, toPos types.StreamPosition, limit int, diff --git a/syncapi/storage/sqlite3/output_room_events_topology_table.go b/syncapi/storage/sqlite3/output_room_events_topology_table.go index 4469f5b76..f25ac623c 100644 --- a/syncapi/storage/sqlite3/output_room_events_topology_table.go +++ b/syncapi/storage/sqlite3/output_room_events_topology_table.go @@ -65,17 +65,12 @@ const selectMaxPositionInTopologySQL = "" + "SELECT MAX(topological_position), stream_position FROM syncapi_output_room_events_topology" + " WHERE room_id = $1 ORDER BY stream_position DESC" -const selectEventIDsFromPositionSQL = "" + - "SELECT event_id FROM syncapi_output_room_events_topology" + - " WHERE room_id = $1 AND topological_position = $2" - type outputRoomEventsTopologyStatements struct { insertEventInTopologyStmt *sql.Stmt selectEventIDsInRangeASCStmt *sql.Stmt selectEventIDsInRangeDESCStmt *sql.Stmt selectPositionInTopologyStmt *sql.Stmt selectMaxPositionInTopologyStmt *sql.Stmt - selectEventIDsFromPositionStmt *sql.Stmt } func NewSqliteTopologyTable(db *sql.DB) (tables.Topology, error) { @@ -99,9 +94,6 @@ func NewSqliteTopologyTable(db *sql.DB) (tables.Topology, error) { if s.selectMaxPositionInTopologyStmt, err = db.Prepare(selectMaxPositionInTopologySQL); err != nil { return nil, err } - if s.selectEventIDsFromPositionStmt, err = db.Prepare(selectEventIDsFromPositionSQL); err != nil { - return nil, err - } return s, nil } @@ -117,12 +109,9 @@ func (s *outputRoomEventsTopologyStatements) InsertEventInTopology( return } -// selectEventIDsInRange selects the IDs of events which positions are within a -// given range in a given room's topological order. -// Returns an empty slice if no events match the given range. func (s *outputRoomEventsTopologyStatements) SelectEventIDsInRange( ctx context.Context, txn *sql.Tx, roomID string, - fromPos, toPos, toMicroPos types.StreamPosition, + minDepth, maxDepth, maxStreamPos types.StreamPosition, limit int, chronologicalOrder bool, ) (eventIDs []string, err error) { // Decide on the selection's order according to whether chronological order @@ -135,7 +124,7 @@ func (s *outputRoomEventsTopologyStatements) SelectEventIDsInRange( } // Query the event IDs. - rows, err := stmt.QueryContext(ctx, roomID, fromPos, toPos, toPos, toMicroPos, limit) + rows, err := stmt.QueryContext(ctx, roomID, minDepth, maxDepth, maxDepth, maxStreamPos, limit) if err == sql.ErrNoRows { // If no event matched the request, return an empty slice. return []string{}, nil @@ -172,28 +161,3 @@ func (s *outputRoomEventsTopologyStatements) SelectMaxPositionInTopology( err = stmt.QueryRowContext(ctx, roomID).Scan(&pos, &spos) return } - -// selectEventIDsFromPosition returns the IDs of all events that have a given -// position in the topology of a given room. -func (s *outputRoomEventsTopologyStatements) SelectEventIDsFromPosition( - ctx context.Context, txn *sql.Tx, roomID string, pos types.StreamPosition, -) (eventIDs []string, err error) { - // Query the event IDs. - stmt := common.TxStmt(txn, s.selectEventIDsFromPositionStmt) - rows, err := stmt.QueryContext(ctx, roomID, pos) - if err == sql.ErrNoRows { - // If no event matched the request, return an empty slice. - return []string{}, nil - } else if err != nil { - return - } - // Return the IDs. - var eventID string - for rows.Next() { - if err = rows.Scan(&eventID); err != nil { - return - } - eventIDs = append(eventIDs, eventID) - } - return -} diff --git a/syncapi/storage/storage_test.go b/syncapi/storage/storage_test.go index bffcfd053..69ac5e6f5 100644 --- a/syncapi/storage/storage_test.go +++ b/syncapi/storage/storage_test.go @@ -292,11 +292,10 @@ func TestGetEventsInRangeWithTopologyToken(t *testing.T) { db := MustCreateDatabase(t) events, _ := SimpleRoom(t, testRoomID, testUserIDA, testUserIDB) MustWriteEvents(t, db, events) - latest, latestStream, err := db.MaxTopologicalPosition(ctx, testRoomID) + from, err := db.MaxTopologicalPosition(ctx, testRoomID) if err != nil { t.Fatalf("failed to get MaxTopologicalPosition: %s", err) } - from := types.NewTopologyToken(latest, latestStream) // head towards the beginning of time to := types.NewTopologyToken(0, 0) @@ -358,16 +357,14 @@ func TestGetEventsInRangeWithEventsSameDepth(t *testing.T) { Depth: depth + 1, })) MustWriteEvents(t, db, events) - latestPos, latestStreamPos, err := db.EventPositionInTopology(ctx, events[len(events)-1].EventID()) + fromLatest, err := db.EventPositionInTopology(ctx, events[len(events)-1].EventID()) if err != nil { t.Fatalf("failed to get EventPositionInTopology: %s", err) } - topoPos, streamPos, err := db.EventPositionInTopology(ctx, events[len(events)-3].EventID()) // Message 2 + fromFork, err := db.EventPositionInTopology(ctx, events[len(events)-3].EventID()) // Message 2 if err != nil { t.Fatalf("failed to get EventPositionInTopology for event: %s", err) } - fromLatest := types.NewTopologyToken(latestPos, latestStreamPos) - fromFork := types.NewTopologyToken(topoPos, streamPos) // head towards the beginning of time to := types.NewTopologyToken(0, 0) @@ -507,12 +504,10 @@ func assertEventsEqual(t *testing.T, msg string, checkRoomID bool, gots []gomatr } func topologyTokenBefore(t *testing.T, db storage.Database, eventID string) *types.TopologyToken { - pos, spos, err := db.EventPositionInTopology(ctx, eventID) + tok, err := db.EventPositionInTopology(ctx, eventID) if err != nil { t.Fatalf("failed to get EventPositionInTopology: %s", err) } - - tok := types.NewTopologyToken(pos, spos) tok.Decrement() return &tok } diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go index 8f0b8b895..1e5351b5b 100644 --- a/syncapi/storage/tables/interface.go +++ b/syncapi/storage/tables/interface.go @@ -11,13 +11,15 @@ import ( type AccountData interface { InsertAccountData(ctx context.Context, txn *sql.Tx, userID, roomID, dataType string) (pos types.StreamPosition, err error) - SelectAccountDataInRange(ctx context.Context, userID string, oldPos, newPos types.StreamPosition, accountDataEventFilter *gomatrixserverlib.EventFilter) (data map[string][]string, err error) + // SelectAccountDataInRange returns a map of room ID to a list of `dataType`. The range is exclusive of `lowPos` and inclusive of `hiPos`. + SelectAccountDataInRange(ctx context.Context, userID string, lowPos, hiPos types.StreamPosition, accountDataEventFilter *gomatrixserverlib.EventFilter) (data map[string][]string, err error) SelectMaxAccountDataID(ctx context.Context, txn *sql.Tx) (id int64, err error) } type Invites interface { InsertInviteEvent(ctx context.Context, txn *sql.Tx, inviteEvent gomatrixserverlib.HeaderedEvent) (streamPos types.StreamPosition, err error) DeleteInviteEvent(ctx context.Context, inviteEventID string) error + // SelectInviteEventsInRange returns a map of room ID to invite events. The range is exclusive of `startPos` and inclusive of `endPos`. SelectInviteEventsInRange(ctx context.Context, txn *sql.Tx, targetUserID string, startPos, endPos types.StreamPosition) (map[string]gomatrixserverlib.HeaderedEvent, error) SelectMaxInviteID(ctx context.Context, txn *sql.Tx) (id int64, err error) } @@ -26,26 +28,30 @@ type Events interface { SelectStateInRange(ctx context.Context, txn *sql.Tx, oldPos, newPos types.StreamPosition, stateFilter *gomatrixserverlib.StateFilter) (map[string]map[string]bool, map[string]types.StreamEvent, error) SelectMaxEventID(ctx context.Context, txn *sql.Tx) (id int64, err error) InsertEvent(ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.HeaderedEvent, addState, removeState []string, transactionID *api.TransactionID, excludeFromSync bool) (streamPos types.StreamPosition, err error) + // SelectRecentEvents returns events between the two stream positions: exclusive of `fromPos` and inclusive of `toPos`. + // If onlySyncEvents has a value of true, only returns the events that aren't marked as to exclude from sync. + // Returns up to `limit` events. SelectRecentEvents(ctx context.Context, txn *sql.Tx, roomID string, fromPos, toPos types.StreamPosition, limit int, chronologicalOrder bool, onlySyncEvents bool) ([]types.StreamEvent, error) + // SelectEarlyEvents returns the earliest events in the given room, exclusive of `fromPos` and inclusive of `toPos`. SelectEarlyEvents(ctx context.Context, txn *sql.Tx, roomID string, fromPos, toPos types.StreamPosition, limit int) ([]types.StreamEvent, error) SelectEvents(ctx context.Context, txn *sql.Tx, eventIDs []string) ([]types.StreamEvent, error) } +// Topology keeps track of the depths and stream positions for all events. +// These positions are used as types.TopologyToken when backfilling events locally. type Topology interface { - // InsertEventInTopology inserts the given event in the room's topology, based - // on the event's depth. + // InsertEventInTopology inserts the given event in the room's topology, based on the event's depth. + // `pos` is the stream position of this event in the events table, and is used to order events which have the same depth. InsertEventInTopology(ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.HeaderedEvent, pos types.StreamPosition) (err error) - // SelectEventIDsInRange selects the IDs of events which positions are within a - // given range in a given room's topological order. + // SelectEventIDsInRange selects the IDs of events whose depths are within a given range in a given room's topological order. + // Events with `minDepth` are *exclusive*, as is the event which has exactly `minDepth`,`maxStreamPos`. + // `maxStreamPos` is only used when events have the same depth as `maxDepth`, which results in events less than `maxStreamPos` being returned. // Returns an empty slice if no events match the given range. - SelectEventIDsInRange(ctx context.Context, txn *sql.Tx, roomID string, fromPos, toPos, toMicroPos types.StreamPosition, limit int, chronologicalOrder bool) (eventIDs []string, err error) - // SelectPositionInTopology returns the position of a given event in the - // topology of the room it belongs to. - SelectPositionInTopology(ctx context.Context, txn *sql.Tx, eventID string) (pos, spos types.StreamPosition, err error) - SelectMaxPositionInTopology(ctx context.Context, txn *sql.Tx, roomID string) (pos types.StreamPosition, spos types.StreamPosition, err error) - // SelectEventIDsFromPosition returns the IDs of all events that have a given - // position in the topology of a given room. - SelectEventIDsFromPosition(ctx context.Context, txn *sql.Tx, roomID string, pos types.StreamPosition) (eventIDs []string, err error) + SelectEventIDsInRange(ctx context.Context, txn *sql.Tx, roomID string, minDepth, maxDepth, maxStreamPos types.StreamPosition, limit int, chronologicalOrder bool) (eventIDs []string, err error) + // SelectPositionInTopology returns the depth and stream position of a given event in the topology of the room it belongs to. + SelectPositionInTopology(ctx context.Context, txn *sql.Tx, eventID string) (depth, spos types.StreamPosition, err error) + // SelectMaxPositionInTopology returns the event which has the highest depth, and if there are multiple, the event with the highest stream position. + SelectMaxPositionInTopology(ctx context.Context, txn *sql.Tx, roomID string) (depth types.StreamPosition, spos types.StreamPosition, err error) } type CurrentRoomState interface { diff --git a/syncapi/types/types.go b/syncapi/types/types.go index c1b6d7dd5..8a79ccd43 100644 --- a/syncapi/types/types.go +++ b/syncapi/types/types.go @@ -119,8 +119,11 @@ func (t *TopologyToken) Decrement() { depth := t.Positions[0] pduPos := t.Positions[1] if depth-1 <= 0 { + // nothing can be lower than this depth = 1 } else { + // this assumes that we will never have 1000 events all with the same + // depth. TODO: work out what the right PDU position is to use, probably needs a db hit. depth-- pduPos += 1000 } From 419ff150d41a3d0de25f0e8e66baf36948bcfbc1 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 15 May 2020 09:32:40 +0100 Subject: [PATCH 061/200] Implement key caching directly (#1038) * Use gomatrixserverlib key caching * Implement key caching wrapper * Add caching wrapper in BaseComponent * Review comments --- common/basecomponent/base.go | 7 ++- common/caching/immutablecache.go | 3 ++ common/caching/immutableinmemorylru.go | 28 ++++++++++ common/keydb/cache/keydb.go | 69 ++++++++++++++++++++++++ common/keydb/postgres/keydb.go | 2 +- common/keydb/sqlite3/keydb.go | 2 +- common/keydb/sqlite3/server_key_table.go | 24 --------- 7 files changed, 108 insertions(+), 27 deletions(-) create mode 100644 common/keydb/cache/keydb.go diff --git a/common/basecomponent/base.go b/common/basecomponent/base.go index cb04a308e..4342e25a3 100644 --- a/common/basecomponent/base.go +++ b/common/basecomponent/base.go @@ -25,6 +25,7 @@ import ( "github.com/matrix-org/dendrite/common/caching" "github.com/matrix-org/dendrite/common/keydb" + "github.com/matrix-org/dendrite/common/keydb/cache" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/naffka" @@ -186,7 +187,11 @@ func (b *BaseDendrite) CreateKeyDB() keydb.Database { logrus.WithError(err).Panicf("failed to connect to keys db") } - return db + cachedDB, err := cache.NewKeyDatabase(db, b.ImmutableCache) + if err != nil { + logrus.WithError(err).Panicf("failed to create key cache wrapper") + } + return cachedDB } // CreateFederationClient creates a new federation client. Should only be called diff --git a/common/caching/immutablecache.go b/common/caching/immutablecache.go index 9620667a2..362e4349a 100644 --- a/common/caching/immutablecache.go +++ b/common/caching/immutablecache.go @@ -4,9 +4,12 @@ import "github.com/matrix-org/gomatrixserverlib" const ( RoomVersionMaxCacheEntries = 128 + ServerKeysMaxCacheEntries = 128 ) type ImmutableCache interface { GetRoomVersion(roomId string) (gomatrixserverlib.RoomVersion, bool) StoreRoomVersion(roomId string, roomVersion gomatrixserverlib.RoomVersion) + GetServerKey(request gomatrixserverlib.PublicKeyLookupRequest) (gomatrixserverlib.PublicKeyLookupResult, bool) + StoreServerKey(request gomatrixserverlib.PublicKeyLookupRequest, response gomatrixserverlib.PublicKeyLookupResult) } diff --git a/common/caching/immutableinmemorylru.go b/common/caching/immutableinmemorylru.go index 3e8f4aadb..6d2a785f3 100644 --- a/common/caching/immutableinmemorylru.go +++ b/common/caching/immutableinmemorylru.go @@ -9,6 +9,7 @@ import ( type ImmutableInMemoryLRUCache struct { roomVersions *lru.Cache + serverKeys *lru.Cache } func NewImmutableInMemoryLRUCache() (*ImmutableInMemoryLRUCache, error) { @@ -16,8 +17,13 @@ func NewImmutableInMemoryLRUCache() (*ImmutableInMemoryLRUCache, error) { if rvErr != nil { return nil, rvErr } + serverKeysCache, rvErr := lru.New(ServerKeysMaxCacheEntries) + if rvErr != nil { + return nil, rvErr + } return &ImmutableInMemoryLRUCache{ roomVersions: roomVersionCache, + serverKeys: serverKeysCache, }, nil } @@ -41,3 +47,25 @@ func (c *ImmutableInMemoryLRUCache) StoreRoomVersion(roomID string, roomVersion checkForInvalidMutation(c.roomVersions, roomID, roomVersion) c.roomVersions.Add(roomID, roomVersion) } + +func (c *ImmutableInMemoryLRUCache) GetServerKey( + request gomatrixserverlib.PublicKeyLookupRequest, +) (gomatrixserverlib.PublicKeyLookupResult, bool) { + key := fmt.Sprintf("%s/%s", request.ServerName, request.KeyID) + val, found := c.serverKeys.Get(key) + if found && val != nil { + if keyLookupResult, ok := val.(gomatrixserverlib.PublicKeyLookupResult); ok { + return keyLookupResult, true + } + } + return gomatrixserverlib.PublicKeyLookupResult{}, false +} + +func (c *ImmutableInMemoryLRUCache) StoreServerKey( + request gomatrixserverlib.PublicKeyLookupRequest, + response gomatrixserverlib.PublicKeyLookupResult, +) { + key := fmt.Sprintf("%s/%s", request.ServerName, request.KeyID) + checkForInvalidMutation(c.roomVersions, key, response) + c.serverKeys.Add(request, response) +} diff --git a/common/keydb/cache/keydb.go b/common/keydb/cache/keydb.go new file mode 100644 index 000000000..ae929fa4c --- /dev/null +++ b/common/keydb/cache/keydb.go @@ -0,0 +1,69 @@ +package cache + +import ( + "context" + "errors" + + "github.com/matrix-org/dendrite/common/caching" + "github.com/matrix-org/dendrite/common/keydb" + "github.com/matrix-org/gomatrixserverlib" +) + +// A Database implements gomatrixserverlib.KeyDatabase and is used to store +// the public keys for other matrix servers. +type KeyDatabase struct { + inner keydb.Database + cache caching.ImmutableCache +} + +func NewKeyDatabase(inner keydb.Database, cache caching.ImmutableCache) (*KeyDatabase, error) { + if inner == nil { + return nil, errors.New("inner database can't be nil") + } + if cache == nil { + return nil, errors.New("cache can't be nil") + } + return &KeyDatabase{ + inner: inner, + cache: cache, + }, nil +} + +// FetcherName implements KeyFetcher +func (d KeyDatabase) FetcherName() string { + return "InMemoryKeyCache" +} + +// FetchKeys implements gomatrixserverlib.KeyDatabase +func (d *KeyDatabase) FetchKeys( + ctx context.Context, + requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp, +) (map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, error) { + results := make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult) + for req := range requests { + if res, cached := d.cache.GetServerKey(req); cached { + results[req] = res + delete(requests, req) + } + } + fromDB, err := d.inner.FetchKeys(ctx, requests) + if err != nil { + return results, err + } + for req, res := range fromDB { + results[req] = res + d.cache.StoreServerKey(req, res) + } + return results, nil +} + +// StoreKeys implements gomatrixserverlib.KeyDatabase +func (d *KeyDatabase) StoreKeys( + ctx context.Context, + keyMap map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, +) error { + for req, res := range keyMap { + d.cache.StoreServerKey(req, res) + } + return d.inner.StoreKeys(ctx, keyMap) +} diff --git a/common/keydb/postgres/keydb.go b/common/keydb/postgres/keydb.go index 6149d8778..706ca0052 100644 --- a/common/keydb/postgres/keydb.go +++ b/common/keydb/postgres/keydb.go @@ -79,7 +79,7 @@ func NewDatabase( // FetcherName implements KeyFetcher func (d Database) FetcherName() string { - return "KeyDatabase" + return "PostgresKeyDatabase" } // FetchKeys implements gomatrixserverlib.KeyDatabase diff --git a/common/keydb/sqlite3/keydb.go b/common/keydb/sqlite3/keydb.go index 1405836a4..94a32e29f 100644 --- a/common/keydb/sqlite3/keydb.go +++ b/common/keydb/sqlite3/keydb.go @@ -80,7 +80,7 @@ func NewDatabase( // FetcherName implements KeyFetcher func (d Database) FetcherName() string { - return "KeyDatabase" + return "SqliteKeyDatabase" } // FetchKeys implements gomatrixserverlib.KeyDatabase diff --git a/common/keydb/sqlite3/server_key_table.go b/common/keydb/sqlite3/server_key_table.go index ba1cc0606..883d3cd0f 100644 --- a/common/keydb/sqlite3/server_key_table.go +++ b/common/keydb/sqlite3/server_key_table.go @@ -20,10 +20,8 @@ import ( "database/sql" "strings" - lru "github.com/hashicorp/golang-lru" "github.com/matrix-org/dendrite/common" "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" ) const serverKeysSchema = ` @@ -66,16 +64,10 @@ type serverKeyStatements struct { db *sql.DB bulkSelectServerKeysStmt *sql.Stmt upsertServerKeysStmt *sql.Stmt - - cache *lru.Cache // nameAndKeyID => gomatrixserverlib.PublicKeyLookupResult } func (s *serverKeyStatements) prepare(db *sql.DB) (err error) { s.db = db - s.cache, err = lru.New(64) - if err != nil { - return - } _, err = db.Exec(serverKeysSchema) if err != nil { return @@ -98,21 +90,6 @@ func (s *serverKeyStatements) bulkSelectServerKeys( nameAndKeyIDs = append(nameAndKeyIDs, nameAndKeyID(request)) } - // If we can satisfy all of the requests from the cache, do so. TODO: Allow partial matches with merges. - cacheResults := map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult{} - for request := range requests { - r, ok := s.cache.Get(nameAndKeyID(request)) - if !ok { - break - } - cacheResult := r.(gomatrixserverlib.PublicKeyLookupResult) - cacheResults[request] = cacheResult - } - if len(cacheResults) == len(requests) { - util.GetLogger(ctx).Infof("KeyDB cache hit for %d keys", len(cacheResults)) - return cacheResults, nil - } - query := strings.Replace(bulkSelectServerKeysSQL, "($1)", common.QueryVariadic(len(nameAndKeyIDs)), 1) iKeyIDs := make([]interface{}, len(nameAndKeyIDs)) @@ -158,7 +135,6 @@ func (s *serverKeyStatements) upsertServerKeys( request gomatrixserverlib.PublicKeyLookupRequest, key gomatrixserverlib.PublicKeyLookupResult, ) error { - s.cache.Add(nameAndKeyID(request), key) _, err := s.upsertServerKeysStmt.ExecContext( ctx, string(request.ServerName), From 2b5052eccfca49e736b39a9879ebde2ab196f6da Mon Sep 17 00:00:00 2001 From: Kegsay Date: Fri, 15 May 2020 09:41:12 +0100 Subject: [PATCH 062/200] Add Range (#1037) * Add Range * Use Range --- syncapi/storage/interface.go | 2 +- .../storage/postgres/account_data_table.go | 11 +--- syncapi/storage/postgres/invites_table.go | 4 +- .../postgres/output_room_events_table.go | 16 +++--- syncapi/storage/shared/syncserver.go | 57 +++++++++++-------- syncapi/storage/sqlite3/account_data_table.go | 11 +--- syncapi/storage/sqlite3/invites_table.go | 4 +- .../sqlite3/output_room_events_table.go | 16 +++--- syncapi/storage/tables/interface.go | 18 +++--- syncapi/sync/requestpool.go | 15 ++++- syncapi/types/types.go | 28 +++++++++ 11 files changed, 108 insertions(+), 74 deletions(-) diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index 227167899..eba008b34 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -64,7 +64,7 @@ type Database interface { // Returns a map following the format data[roomID] = []dataTypes // If no data is retrieved, returns an empty map // If there was an issue with the retrieval, returns an error - GetAccountDataInRange(ctx context.Context, userID string, oldPos, newPos types.StreamPosition, accountDataFilterPart *gomatrixserverlib.EventFilter) (map[string][]string, error) + GetAccountDataInRange(ctx context.Context, userID string, r types.Range, accountDataFilterPart *gomatrixserverlib.EventFilter) (map[string][]string, error) // UpsertAccountData keeps track of new or updated account data, by saving the type // of the new/updated data, and the user ID and room ID the data is related to (empty) // room ID means the data isn't specific to any room) diff --git a/syncapi/storage/postgres/account_data_table.go b/syncapi/storage/postgres/account_data_table.go index 58fb21983..a5e0c1214 100644 --- a/syncapi/storage/postgres/account_data_table.go +++ b/syncapi/storage/postgres/account_data_table.go @@ -100,19 +100,12 @@ func (s *accountDataStatements) InsertAccountData( func (s *accountDataStatements) SelectAccountDataInRange( ctx context.Context, userID string, - oldPos, newPos types.StreamPosition, + r types.Range, accountDataEventFilter *gomatrixserverlib.EventFilter, ) (data map[string][]string, err error) { data = make(map[string][]string) - // If both positions are the same, it means that the data was saved after the - // latest room event. In that case, we need to decrement the old position as - // it would prevent the SQL request from returning anything. - if oldPos == newPos { - oldPos-- - } - - rows, err := s.selectAccountDataInRangeStmt.QueryContext(ctx, userID, oldPos, newPos, + rows, err := s.selectAccountDataInRangeStmt.QueryContext(ctx, userID, r.Low(), r.High(), pq.StringArray(filterConvertTypeWildcardToSQL(accountDataEventFilter.Types)), pq.StringArray(filterConvertTypeWildcardToSQL(accountDataEventFilter.NotTypes)), accountDataEventFilter.Limit, diff --git a/syncapi/storage/postgres/invites_table.go b/syncapi/storage/postgres/invites_table.go index 78ca4d6d8..01f2e7f44 100644 --- a/syncapi/storage/postgres/invites_table.go +++ b/syncapi/storage/postgres/invites_table.go @@ -117,10 +117,10 @@ func (s *inviteEventsStatements) DeleteInviteEvent( // selectInviteEventsInRange returns a map of room ID to invite event for the // active invites for the target user ID in the supplied range. func (s *inviteEventsStatements) SelectInviteEventsInRange( - ctx context.Context, txn *sql.Tx, targetUserID string, startPos, endPos types.StreamPosition, + ctx context.Context, txn *sql.Tx, targetUserID string, r types.Range, ) (map[string]gomatrixserverlib.HeaderedEvent, error) { stmt := common.TxStmt(txn, s.selectInviteEventsInRangeStmt) - rows, err := stmt.QueryContext(ctx, targetUserID, startPos, endPos) + rows, err := stmt.QueryContext(ctx, targetUserID, r.Low(), r.High()) if err != nil { return nil, err } diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go index 5870bfd52..5020d1e7d 100644 --- a/syncapi/storage/postgres/output_room_events_table.go +++ b/syncapi/storage/postgres/output_room_events_table.go @@ -155,13 +155,13 @@ func NewPostgresEventsTable(db *sql.DB) (tables.Events, error) { // Results are bucketed based on the room ID. If the same state is overwritten multiple times between the // two positions, only the most recent state is returned. func (s *outputRoomEventsStatements) SelectStateInRange( - ctx context.Context, txn *sql.Tx, oldPos, newPos types.StreamPosition, + ctx context.Context, txn *sql.Tx, r types.Range, stateFilter *gomatrixserverlib.StateFilter, ) (map[string]map[string]bool, map[string]types.StreamEvent, error) { stmt := common.TxStmt(txn, s.selectStateInRangeStmt) rows, err := stmt.QueryContext( - ctx, oldPos, newPos, + ctx, r.Low(), r.High(), pq.StringArray(stateFilter.Senders), pq.StringArray(stateFilter.NotSenders), pq.StringArray(filterConvertTypeWildcardToSQL(stateFilter.Types)), @@ -198,8 +198,8 @@ func (s *outputRoomEventsStatements) SelectStateInRange( // since it'll just mark the event as not being needed. if len(addIDs) < len(delIDs) { log.WithFields(log.Fields{ - "since": oldPos, - "current": newPos, + "since": r.From, + "current": r.To, "adds": addIDs, "dels": delIDs, }).Warn("StateBetween: ignoring deleted state") @@ -298,7 +298,7 @@ func (s *outputRoomEventsStatements) InsertEvent( // from sync. func (s *outputRoomEventsStatements) SelectRecentEvents( ctx context.Context, txn *sql.Tx, - roomID string, fromPos, toPos types.StreamPosition, limit int, + roomID string, r types.Range, limit int, chronologicalOrder bool, onlySyncEvents bool, ) ([]types.StreamEvent, error) { var stmt *sql.Stmt @@ -307,7 +307,7 @@ func (s *outputRoomEventsStatements) SelectRecentEvents( } else { stmt = common.TxStmt(txn, s.selectRecentEventsStmt) } - rows, err := stmt.QueryContext(ctx, roomID, fromPos, toPos, limit) + rows, err := stmt.QueryContext(ctx, roomID, r.Low(), r.High(), limit) if err != nil { return nil, err } @@ -331,10 +331,10 @@ func (s *outputRoomEventsStatements) SelectRecentEvents( // from a given position, up to a maximum of 'limit'. func (s *outputRoomEventsStatements) SelectEarlyEvents( ctx context.Context, txn *sql.Tx, - roomID string, fromPos, toPos types.StreamPosition, limit int, + roomID string, r types.Range, limit int, ) ([]types.StreamEvent, error) { stmt := common.TxStmt(txn, s.selectEarlyEventsStmt) - rows, err := stmt.QueryContext(ctx, roomID, fromPos, toPos, limit) + rows, err := stmt.QueryContext(ctx, roomID, r.Low(), r.High(), limit) if err != nil { return nil, err } diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go index e03a6b9ff..be1f9c7aa 100644 --- a/syncapi/storage/shared/syncserver.go +++ b/syncapi/storage/shared/syncserver.go @@ -54,17 +54,22 @@ func (d *Database) GetEventsInStreamingRange( roomID string, limit int, backwardOrdering bool, ) (events []types.StreamEvent, err error) { + r := types.Range{ + From: from.PDUPosition(), + To: to.PDUPosition(), + Backwards: backwardOrdering, + } if backwardOrdering { // When using backward ordering, we want the most recent events first. if events, err = d.OutputEvents.SelectRecentEvents( - ctx, nil, roomID, to.PDUPosition(), from.PDUPosition(), limit, false, false, + ctx, nil, roomID, r, limit, false, false, ); err != nil { return } } else { // When using forward ordering, we want the least recent events first. if events, err = d.OutputEvents.SelectEarlyEvents( - ctx, nil, roomID, from.PDUPosition(), to.PDUPosition(), limit, + ctx, nil, roomID, r, limit, ); err != nil { return } @@ -167,10 +172,10 @@ func (d *Database) RetireInviteEvent( // If no data is retrieved, returns an empty map // If there was an issue with the retrieval, returns an error func (d *Database) GetAccountDataInRange( - ctx context.Context, userID string, oldPos, newPos types.StreamPosition, + ctx context.Context, userID string, r types.Range, accountDataFilterPart *gomatrixserverlib.EventFilter, ) (map[string][]string, error) { - return d.AccountData.SelectAccountDataInRange(ctx, userID, oldPos, newPos, accountDataFilterPart) + return d.AccountData.SelectAccountDataInRange(ctx, userID, r, accountDataFilterPart) } // UpsertAccountData keeps track of new or updated account data, by saving the type @@ -417,7 +422,7 @@ func (d *Database) syncPositionTx( func (d *Database) addPDUDeltaToResponse( ctx context.Context, device authtypes.Device, - fromPos, toPos types.StreamPosition, + r types.Range, numRecentEventsPerRoom int, wantFullState bool, res *types.Response, @@ -443,11 +448,11 @@ func (d *Database) addPDUDeltaToResponse( var deltas []stateDelta if !wantFullState { deltas, joinedRoomIDs, err = d.getStateDeltas( - ctx, &device, txn, fromPos, toPos, device.UserID, &stateFilter, + ctx, &device, txn, r, device.UserID, &stateFilter, ) } else { deltas, joinedRoomIDs, err = d.getStateDeltasForFullStateSync( - ctx, &device, txn, fromPos, toPos, device.UserID, &stateFilter, + ctx, &device, txn, r, device.UserID, &stateFilter, ) } if err != nil { @@ -455,14 +460,14 @@ func (d *Database) addPDUDeltaToResponse( } for _, delta := range deltas { - err = d.addRoomDeltaToResponse(ctx, &device, txn, fromPos, toPos, delta, numRecentEventsPerRoom, res) + err = d.addRoomDeltaToResponse(ctx, &device, txn, r, delta, numRecentEventsPerRoom, res) if err != nil { return nil, err } } // TODO: This should be done in getStateDeltas - if err = d.addInvitesToResponse(ctx, txn, device.UserID, fromPos, toPos, res); err != nil { + if err = d.addInvitesToResponse(ctx, txn, device.UserID, r, res); err != nil { return nil, err } @@ -534,8 +539,12 @@ func (d *Database) IncrementalSync( var joinedRoomIDs []string var err error if fromPos.PDUPosition() != toPos.PDUPosition() || wantFullState { + r := types.Range{ + From: fromPos.PDUPosition(), + To: toPos.PDUPosition(), + } joinedRoomIDs, err = d.addPDUDeltaToResponse( - ctx, device, fromPos.PDUPosition(), toPos.PDUPosition(), numRecentEventsPerRoom, wantFullState, res, + ctx, device, r, numRecentEventsPerRoom, wantFullState, res, ) } else { joinedRoomIDs, err = d.CurrentRoomState.SelectRoomIDsWithMembership( @@ -589,6 +598,10 @@ func (d *Database) getResponseWithPDUsForCompleteSync( if err != nil { return } + r := types.Range{ + From: 0, + To: toPos.PDUPosition(), + } res = types.NewResponse(toPos) @@ -611,8 +624,7 @@ func (d *Database) getResponseWithPDUsForCompleteSync( // See: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L316 var recentStreamEvents []types.StreamEvent recentStreamEvents, err = d.OutputEvents.SelectRecentEvents( - ctx, txn, roomID, types.StreamPosition(0), toPos.PDUPosition(), - numRecentEventsPerRoom, true, true, + ctx, txn, roomID, r, numRecentEventsPerRoom, true, true, ) if err != nil { return @@ -644,7 +656,7 @@ func (d *Database) getResponseWithPDUsForCompleteSync( res.Rooms.Join[roomID] = *jr } - if err = d.addInvitesToResponse(ctx, txn, userID, 0, toPos.PDUPosition(), res); err != nil { + if err = d.addInvitesToResponse(ctx, txn, userID, r, res); err != nil { return } @@ -686,11 +698,11 @@ var txReadOnlySnapshot = sql.TxOptions{ func (d *Database) addInvitesToResponse( ctx context.Context, txn *sql.Tx, userID string, - fromPos, toPos types.StreamPosition, + r types.Range, res *types.Response, ) error { invites, err := d.Invites.SelectInviteEventsInRange( - ctx, txn, userID, fromPos, toPos, + ctx, txn, userID, r, ) if err != nil { return err @@ -726,12 +738,11 @@ func (d *Database) addRoomDeltaToResponse( ctx context.Context, device *authtypes.Device, txn *sql.Tx, - fromPos, toPos types.StreamPosition, + r types.Range, delta stateDelta, numRecentEventsPerRoom int, res *types.Response, ) error { - endPos := toPos if delta.membershipPos > 0 && delta.membership == gomatrixserverlib.Leave { // make sure we don't leak recent events after the leave event. // TODO: History visibility makes this somewhat complex to handle correctly. For example: @@ -739,10 +750,10 @@ func (d *Database) addRoomDeltaToResponse( // TODO: This will fail on join -> leave -> sensitive msg -> join -> leave // in a single /sync request // This is all "okay" assuming history_visibility == "shared" which it is by default. - endPos = delta.membershipPos + r.To = delta.membershipPos } recentStreamEvents, err := d.OutputEvents.SelectRecentEvents( - ctx, txn, delta.roomID, types.StreamPosition(fromPos), types.StreamPosition(endPos), + ctx, txn, delta.roomID, r, numRecentEventsPerRoom, true, true, ) if err != nil { @@ -872,7 +883,7 @@ func (d *Database) fetchMissingStateEvents( // A list of joined room IDs is also returned in case the caller needs it. func (d *Database) getStateDeltas( ctx context.Context, device *authtypes.Device, txn *sql.Tx, - fromPos, toPos types.StreamPosition, userID string, + r types.Range, userID string, stateFilter *gomatrixserverlib.StateFilter, ) ([]stateDelta, []string, error) { // Implement membership change algorithm: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L821 @@ -886,7 +897,7 @@ func (d *Database) getStateDeltas( var deltas []stateDelta // get all the state events ever between these two positions - stateNeeded, eventMap, err := d.OutputEvents.SelectStateInRange(ctx, txn, fromPos, toPos, stateFilter) + stateNeeded, eventMap, err := d.OutputEvents.SelectStateInRange(ctx, txn, r, stateFilter) if err != nil { return nil, nil, err } @@ -947,7 +958,7 @@ func (d *Database) getStateDeltas( // updates for other rooms. func (d *Database) getStateDeltasForFullStateSync( ctx context.Context, device *authtypes.Device, txn *sql.Tx, - fromPos, toPos types.StreamPosition, userID string, + r types.Range, userID string, stateFilter *gomatrixserverlib.StateFilter, ) ([]stateDelta, []string, error) { joinedRoomIDs, err := d.CurrentRoomState.SelectRoomIDsWithMembership(ctx, txn, userID, gomatrixserverlib.Join) @@ -972,7 +983,7 @@ func (d *Database) getStateDeltasForFullStateSync( } // Get all the state events ever between these two positions - stateNeeded, eventMap, err := d.OutputEvents.SelectStateInRange(ctx, txn, fromPos, toPos, stateFilter) + stateNeeded, eventMap, err := d.OutputEvents.SelectStateInRange(ctx, txn, r, stateFilter) if err != nil { return nil, nil, err } diff --git a/syncapi/storage/sqlite3/account_data_table.go b/syncapi/storage/sqlite3/account_data_table.go index e5f2417b8..435a399c3 100644 --- a/syncapi/storage/sqlite3/account_data_table.go +++ b/syncapi/storage/sqlite3/account_data_table.go @@ -91,19 +91,12 @@ func (s *accountDataStatements) InsertAccountData( func (s *accountDataStatements) SelectAccountDataInRange( ctx context.Context, userID string, - oldPos, newPos types.StreamPosition, + r types.Range, accountDataFilterPart *gomatrixserverlib.EventFilter, ) (data map[string][]string, err error) { data = make(map[string][]string) - // If both positions are the same, it means that the data was saved after the - // latest room event. In that case, we need to decrement the old position as - // it would prevent the SQL request from returning anything. - if oldPos == newPos { - oldPos-- - } - - rows, err := s.selectAccountDataInRangeStmt.QueryContext(ctx, userID, oldPos, newPos) + rows, err := s.selectAccountDataInRangeStmt.QueryContext(ctx, userID, r.Low(), r.High()) if err != nil { return } diff --git a/syncapi/storage/sqlite3/invites_table.go b/syncapi/storage/sqlite3/invites_table.go index 26b3a3169..9cb184f20 100644 --- a/syncapi/storage/sqlite3/invites_table.go +++ b/syncapi/storage/sqlite3/invites_table.go @@ -121,10 +121,10 @@ func (s *inviteEventsStatements) DeleteInviteEvent( // selectInviteEventsInRange returns a map of room ID to invite event for the // active invites for the target user ID in the supplied range. func (s *inviteEventsStatements) SelectInviteEventsInRange( - ctx context.Context, txn *sql.Tx, targetUserID string, startPos, endPos types.StreamPosition, + ctx context.Context, txn *sql.Tx, targetUserID string, r types.Range, ) (map[string]gomatrixserverlib.HeaderedEvent, error) { stmt := common.TxStmt(txn, s.selectInviteEventsInRangeStmt) - rows, err := stmt.QueryContext(ctx, targetUserID, startPos, endPos) + rows, err := stmt.QueryContext(ctx, targetUserID, r.Low(), r.High()) if err != nil { return nil, err } diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index c1647a14c..c89e007e6 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -146,13 +146,13 @@ func NewSqliteEventsTable(db *sql.DB, streamID *streamIDStatements) (tables.Even // Results are bucketed based on the room ID. If the same state is overwritten multiple times between the // two positions, only the most recent state is returned. func (s *outputRoomEventsStatements) SelectStateInRange( - ctx context.Context, txn *sql.Tx, oldPos, newPos types.StreamPosition, + ctx context.Context, txn *sql.Tx, r types.Range, stateFilterPart *gomatrixserverlib.StateFilter, ) (map[string]map[string]bool, map[string]types.StreamEvent, error) { stmt := common.TxStmt(txn, s.selectStateInRangeStmt) rows, err := stmt.QueryContext( - ctx, oldPos, newPos, + ctx, r.Low(), r.High(), /*pq.StringArray(stateFilterPart.Senders), pq.StringArray(stateFilterPart.NotSenders), pq.StringArray(filterConvertTypeWildcardToSQL(stateFilterPart.Types)), @@ -195,8 +195,8 @@ func (s *outputRoomEventsStatements) SelectStateInRange( // since it'll just mark the event as not being needed. if len(addIDs) < len(delIDs) { log.WithFields(log.Fields{ - "since": oldPos, - "current": newPos, + "since": r.From, + "current": r.To, "adds": addIDsJSON, "dels": delIDsJSON, }).Warn("StateBetween: ignoring deleted state") @@ -308,7 +308,7 @@ func (s *outputRoomEventsStatements) InsertEvent( func (s *outputRoomEventsStatements) SelectRecentEvents( ctx context.Context, txn *sql.Tx, - roomID string, fromPos, toPos types.StreamPosition, limit int, + roomID string, r types.Range, limit int, chronologicalOrder bool, onlySyncEvents bool, ) ([]types.StreamEvent, error) { var stmt *sql.Stmt @@ -318,7 +318,7 @@ func (s *outputRoomEventsStatements) SelectRecentEvents( stmt = common.TxStmt(txn, s.selectRecentEventsStmt) } - rows, err := stmt.QueryContext(ctx, roomID, fromPos, toPos, limit) + rows, err := stmt.QueryContext(ctx, roomID, r.Low(), r.High(), limit) if err != nil { return nil, err } @@ -340,10 +340,10 @@ func (s *outputRoomEventsStatements) SelectRecentEvents( func (s *outputRoomEventsStatements) SelectEarlyEvents( ctx context.Context, txn *sql.Tx, - roomID string, fromPos, toPos types.StreamPosition, limit int, + roomID string, r types.Range, limit int, ) ([]types.StreamEvent, error) { stmt := common.TxStmt(txn, s.selectEarlyEventsStmt) - rows, err := stmt.QueryContext(ctx, roomID, fromPos, toPos, limit) + rows, err := stmt.QueryContext(ctx, roomID, r.Low(), r.High(), limit) if err != nil { return nil, err } diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go index 1e5351b5b..f31bdc2e9 100644 --- a/syncapi/storage/tables/interface.go +++ b/syncapi/storage/tables/interface.go @@ -11,29 +11,29 @@ import ( type AccountData interface { InsertAccountData(ctx context.Context, txn *sql.Tx, userID, roomID, dataType string) (pos types.StreamPosition, err error) - // SelectAccountDataInRange returns a map of room ID to a list of `dataType`. The range is exclusive of `lowPos` and inclusive of `hiPos`. - SelectAccountDataInRange(ctx context.Context, userID string, lowPos, hiPos types.StreamPosition, accountDataEventFilter *gomatrixserverlib.EventFilter) (data map[string][]string, err error) + // SelectAccountDataInRange returns a map of room ID to a list of `dataType`. + SelectAccountDataInRange(ctx context.Context, userID string, r types.Range, accountDataEventFilter *gomatrixserverlib.EventFilter) (data map[string][]string, err error) SelectMaxAccountDataID(ctx context.Context, txn *sql.Tx) (id int64, err error) } type Invites interface { InsertInviteEvent(ctx context.Context, txn *sql.Tx, inviteEvent gomatrixserverlib.HeaderedEvent) (streamPos types.StreamPosition, err error) DeleteInviteEvent(ctx context.Context, inviteEventID string) error - // SelectInviteEventsInRange returns a map of room ID to invite events. The range is exclusive of `startPos` and inclusive of `endPos`. - SelectInviteEventsInRange(ctx context.Context, txn *sql.Tx, targetUserID string, startPos, endPos types.StreamPosition) (map[string]gomatrixserverlib.HeaderedEvent, error) + // SelectInviteEventsInRange returns a map of room ID to invite events. + SelectInviteEventsInRange(ctx context.Context, txn *sql.Tx, targetUserID string, r types.Range) (map[string]gomatrixserverlib.HeaderedEvent, error) SelectMaxInviteID(ctx context.Context, txn *sql.Tx) (id int64, err error) } type Events interface { - SelectStateInRange(ctx context.Context, txn *sql.Tx, oldPos, newPos types.StreamPosition, stateFilter *gomatrixserverlib.StateFilter) (map[string]map[string]bool, map[string]types.StreamEvent, error) + SelectStateInRange(ctx context.Context, txn *sql.Tx, r types.Range, stateFilter *gomatrixserverlib.StateFilter) (map[string]map[string]bool, map[string]types.StreamEvent, error) SelectMaxEventID(ctx context.Context, txn *sql.Tx) (id int64, err error) InsertEvent(ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.HeaderedEvent, addState, removeState []string, transactionID *api.TransactionID, excludeFromSync bool) (streamPos types.StreamPosition, err error) - // SelectRecentEvents returns events between the two stream positions: exclusive of `fromPos` and inclusive of `toPos`. + // SelectRecentEvents returns events between the two stream positions: exclusive of low and inclusive of high. // If onlySyncEvents has a value of true, only returns the events that aren't marked as to exclude from sync. // Returns up to `limit` events. - SelectRecentEvents(ctx context.Context, txn *sql.Tx, roomID string, fromPos, toPos types.StreamPosition, limit int, chronologicalOrder bool, onlySyncEvents bool) ([]types.StreamEvent, error) - // SelectEarlyEvents returns the earliest events in the given room, exclusive of `fromPos` and inclusive of `toPos`. - SelectEarlyEvents(ctx context.Context, txn *sql.Tx, roomID string, fromPos, toPos types.StreamPosition, limit int) ([]types.StreamEvent, error) + SelectRecentEvents(ctx context.Context, txn *sql.Tx, roomID string, r types.Range, limit int, chronologicalOrder bool, onlySyncEvents bool) ([]types.StreamEvent, error) + // SelectEarlyEvents returns the earliest events in the given room. + SelectEarlyEvents(ctx context.Context, txn *sql.Tx, roomID string, r types.Range, limit int) ([]types.StreamEvent, error) SelectEvents(ctx context.Context, txn *sql.Tx, eventIDs []string) ([]types.StreamEvent, error) } diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index 126e76f5b..6e0f44e9a 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -184,11 +184,20 @@ func (rp *RequestPool) appendAccountData( return data, nil } + r := types.Range{ + From: req.since.PDUPosition(), + To: currentPos, + } + // If both positions are the same, it means that the data was saved after the + // latest room event. In that case, we need to decrement the old position as + // results are exclusive of Low. + if r.Low() == r.High() { + r.From-- + } + // Sync is not initial, get all account data since the latest sync dataTypes, err := rp.db.GetAccountDataInRange( - req.ctx, userID, - types.StreamPosition(req.since.PDUPosition()), types.StreamPosition(currentPos), - accountDataFilter, + req.ctx, userID, r, accountDataFilter, ) if err != nil { return nil, err diff --git a/syncapi/types/types.go b/syncapi/types/types.go index 8a79ccd43..caa1b3ade 100644 --- a/syncapi/types/types.go +++ b/syncapi/types/types.go @@ -47,6 +47,34 @@ type StreamEvent struct { ExcludeFromSync bool } +// Range represents a range between two stream positions. +type Range struct { + // From is the position the client has already received. + From StreamPosition + // To is the position the client is going towards. + To StreamPosition + // True if the client is going backwards + Backwards bool +} + +// Low returns the low number of the range. +// This represents the position the client already has and hence is exclusive. +func (r *Range) Low() StreamPosition { + if !r.Backwards { + return r.From + } + return r.To +} + +// High returns the high number of the range +// This represents the position the client is going towards and hence is inclusive. +func (r *Range) High() StreamPosition { + if !r.Backwards { + return r.To + } + return r.From +} + // SyncTokenType represents the type of a sync token. // It can be either "s" (representing a position in the whole stream of events) // or "t" (representing a position in a room's topology/depth). From f0e0a6668f5c44e384c26d5afb3b70c816e3fbff Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 15 May 2020 11:27:10 +0100 Subject: [PATCH 063/200] Prometheus metrics for LRU cache (#1039) * Add prom metrics for the in-memory LRU cache * Increase cache sizes --- common/caching/immutablecache.go | 8 +++++--- common/caching/immutableinmemorylru.go | 28 ++++++++++++++++++++++++-- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/common/caching/immutablecache.go b/common/caching/immutablecache.go index 362e4349a..fea05dd1b 100644 --- a/common/caching/immutablecache.go +++ b/common/caching/immutablecache.go @@ -1,10 +1,12 @@ package caching -import "github.com/matrix-org/gomatrixserverlib" +import ( + "github.com/matrix-org/gomatrixserverlib" +) const ( - RoomVersionMaxCacheEntries = 128 - ServerKeysMaxCacheEntries = 128 + RoomVersionMaxCacheEntries = 1024 + ServerKeysMaxCacheEntries = 1024 ) type ImmutableCache interface { diff --git a/common/caching/immutableinmemorylru.go b/common/caching/immutableinmemorylru.go index 6d2a785f3..36cd56dc2 100644 --- a/common/caching/immutableinmemorylru.go +++ b/common/caching/immutableinmemorylru.go @@ -5,6 +5,8 @@ import ( lru "github.com/hashicorp/golang-lru" "github.com/matrix-org/gomatrixserverlib" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" ) type ImmutableInMemoryLRUCache struct { @@ -21,10 +23,32 @@ func NewImmutableInMemoryLRUCache() (*ImmutableInMemoryLRUCache, error) { if rvErr != nil { return nil, rvErr } - return &ImmutableInMemoryLRUCache{ + cache := &ImmutableInMemoryLRUCache{ roomVersions: roomVersionCache, serverKeys: serverKeysCache, - }, nil + } + cache.configureMetrics() + return cache, nil +} + +func (c *ImmutableInMemoryLRUCache) configureMetrics() { + promauto.NewGaugeFunc(prometheus.GaugeOpts{ + Namespace: "dendrite", + Subsystem: "caching", + Name: "number_room_version_entries", + Help: "The number of room version entries cached.", + }, func() float64 { + return float64(c.roomVersions.Len()) + }) + + promauto.NewGaugeFunc(prometheus.GaugeOpts{ + Namespace: "dendrite", + Subsystem: "caching", + Name: "number_server_key_entries", + Help: "The number of server key entries cached.", + }, func() float64 { + return float64(c.serverKeys.Len()) + }) } func checkForInvalidMutation(cache *lru.Cache, key string, value interface{}) { From 773d5bb9f99a6cc36d284a2fbea933f8d9e7753b Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 15 May 2020 12:07:51 +0100 Subject: [PATCH 064/200] Return user_id and stream_id in federated devices query (#1040) --- federationapi/routing/devices.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/federationapi/routing/devices.go b/federationapi/routing/devices.go index 01647a61e..49686c761 100644 --- a/federationapi/routing/devices.go +++ b/federationapi/routing/devices.go @@ -23,7 +23,9 @@ import ( ) type userDevicesResponse struct { - Devices []authtypes.Device `json:"devices"` + UserID string `json:"user_id"` + StreamID int `json:"stream_id"` + Devices []authtypes.Device `json:"devices"` } // GetUserDevices for the given user id @@ -48,6 +50,12 @@ func GetUserDevices( return util.JSONResponse{ Code: 200, - JSON: userDevicesResponse{devs}, + // TODO: we should return an incrementing stream ID each time the device + // list changes for delta changes to be recognised + JSON: userDevicesResponse{ + UserID: userID, + StreamID: 0, + Devices: devs, + }, } } From f4f032381b2bb44ae8d99bb06584d226f78cc584 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Fri, 15 May 2020 13:39:01 +0100 Subject: [PATCH 065/200] Set canonical alias on room creation (#1042) --- clientapi/routing/createroom.go | 38 +++++++++++++++++++++++++-------- common/eventcontent.go | 5 +++++ 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index 43a16945d..f789ae100 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -213,6 +213,25 @@ func createRoom( return jsonerror.InternalServerError() } + var roomAlias string + if r.RoomAliasName != "" { + roomAlias = fmt.Sprintf("#%s:%s", r.RoomAliasName, cfg.Matrix.ServerName) + // check it's free TODO: This races but is better than nothing + hasAliasReq := roomserverAPI.GetRoomIDForAliasRequest{ + Alias: roomAlias, + } + + var aliasResp roomserverAPI.GetRoomIDForAliasResponse + err = rsAPI.GetRoomIDForAlias(req.Context(), &hasAliasReq, &aliasResp) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("aliasAPI.GetRoomIDForAlias failed") + return jsonerror.InternalServerError() + } + if aliasResp.RoomID != "" { + return util.MessageResponse(400, "Alias already exists") + } + } + membershipContent := gomatrixserverlib.MemberContent{ Membership: gomatrixserverlib.Join, DisplayName: profile.DisplayName, @@ -244,9 +263,9 @@ func createRoom( // 1- m.room.create // 2- room creator join member // 3- m.room.power_levels - // 4- m.room.canonical_alias (opt) TODO - // 5- m.room.join_rules - // 6- m.room.history_visibility + // 4- m.room.join_rules + // 5- m.room.history_visibility + // 6- m.room.canonical_alias (opt) // 7- m.room.guest_access (opt) // 8- other initial state items // 9- m.room.name (opt) @@ -262,10 +281,15 @@ func createRoom( {"m.room.create", "", r.CreationContent}, {"m.room.member", userID, membershipContent}, {"m.room.power_levels", "", common.InitialPowerLevelsContent(userID)}, - // TODO: m.room.canonical_alias {"m.room.join_rules", "", gomatrixserverlib.JoinRuleContent{JoinRule: joinRules}}, {"m.room.history_visibility", "", common.HistoryVisibilityContent{HistoryVisibility: historyVisibility}}, } + if roomAlias != "" { + // TODO: bit of a chicken and egg problem here as the alias doesn't exist and cannot until we have made the room. + // This means we might fail creating the alias but say the canonical alias is something that doesn't exist. + // m.room.aliases is handled when we call roomserver.SetRoomAlias + eventsToMake = append(eventsToMake, fledglingEvent{"m.room.canonical_alias", "", common.CanonicalAlias{Alias: roomAlias}}) + } if r.GuestCanJoin { eventsToMake = append(eventsToMake, fledglingEvent{"m.room.guest_access", "", common.GuestAccessContent{GuestAccess: "can_join"}}) } @@ -278,7 +302,6 @@ func createRoom( } // TODO: invite events // TODO: 3pid invite events - // TODO: m.room.aliases authEvents := gomatrixserverlib.NewAuthEvents(nil) for i, e := range eventsToMake { @@ -330,10 +353,7 @@ func createRoom( // TODO(#269): Reserve room alias while we create the room. This stops us // from creating the room but still failing due to the alias having already // been taken. - var roomAlias string - if r.RoomAliasName != "" { - roomAlias = fmt.Sprintf("#%s:%s", r.RoomAliasName, cfg.Matrix.ServerName) - + if roomAlias != "" { aliasReq := roomserverAPI.SetRoomAliasRequest{ Alias: roomAlias, RoomID: roomID, diff --git a/common/eventcontent.go b/common/eventcontent.go index f3817ba68..e7e1a2c8b 100644 --- a/common/eventcontent.go +++ b/common/eventcontent.go @@ -36,6 +36,11 @@ type HistoryVisibilityContent struct { HistoryVisibility string `json:"history_visibility"` } +// CanonicalAlias is the event content for https://matrix.org/docs/spec/client_server/r0.6.0#m-room-canonical-alias +type CanonicalAlias struct { + Alias string `json:"alias"` +} + // InitialPowerLevelsContent returns the initial values for m.room.power_levels on room creation // if they have not been specified. // http://matrix.org/docs/spec/client_server/r0.2.0.html#m-room-power-levels From 5f6f8adaa5f44bc8c38d0f55e3f5d2dda58baf2b Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 15 May 2020 13:55:14 +0100 Subject: [PATCH 066/200] Don't prematurely stop trying to join using servers (#1041) * Don't prematurely stop trying to join using servers * Factor out performJoinUsingServer --- federationsender/internal/perform.go | 185 +++++++++++++++------------ 1 file changed, 104 insertions(+), 81 deletions(-) diff --git a/federationsender/internal/perform.go b/federationsender/internal/perform.go index 7c5fa73fc..9791afefa 100644 --- a/federationsender/internal/perform.go +++ b/federationsender/internal/perform.go @@ -52,96 +52,22 @@ func (r *FederationSenderInternalAPI) PerformJoin( // Try each server that we were provided until we land on one that // successfully completes the make-join send-join dance. for _, serverName := range request.ServerNames { - // Try to perform a make_join using the information supplied in the - // request. - respMakeJoin, err := r.federation.MakeJoin( + if err := r.performJoinUsingServer( ctx, - serverName, request.RoomID, request.UserID, - supportedVersions, - ) - if err != nil { - // TODO: Check if the user was not allowed to join the room. - r.statistics.ForServer(serverName).Failure() - return fmt.Errorf("r.federation.MakeJoin: %w", err) - } - - // Set all the fields to be what they should be, this should be a no-op - // but it's possible that the remote server returned us something "odd" - respMakeJoin.JoinEvent.Type = gomatrixserverlib.MRoomMember - respMakeJoin.JoinEvent.Sender = request.UserID - respMakeJoin.JoinEvent.StateKey = &request.UserID - respMakeJoin.JoinEvent.RoomID = request.RoomID - respMakeJoin.JoinEvent.Redacts = "" - if request.Content == nil { - request.Content = map[string]interface{}{} - } - request.Content["membership"] = "join" - if err = respMakeJoin.JoinEvent.SetContent(request.Content); err != nil { - return fmt.Errorf("respMakeJoin.JoinEvent.SetContent: %w", err) - } - if err = respMakeJoin.JoinEvent.SetUnsigned(struct{}{}); err != nil { - return fmt.Errorf("respMakeJoin.JoinEvent.SetUnsigned: %w", err) - } - - // Work out if we support the room version that has been supplied in - // the make_join response. - if respMakeJoin.RoomVersion == "" { - respMakeJoin.RoomVersion = gomatrixserverlib.RoomVersionV1 - } - if _, err = respMakeJoin.RoomVersion.EventFormat(); err != nil { - return fmt.Errorf("respMakeJoin.RoomVersion.EventFormat: %w", err) - } - - // Build the join event. - event, err := respMakeJoin.JoinEvent.Build( - time.Now(), - r.cfg.Matrix.ServerName, - r.cfg.Matrix.KeyID, - r.cfg.Matrix.PrivateKey, - respMakeJoin.RoomVersion, - ) - if err != nil { - return fmt.Errorf("respMakeJoin.JoinEvent.Build: %w", err) - } - - // Try to perform a send_join using the newly built event. - respSendJoin, err := r.federation.SendJoin( - ctx, + request.Content, serverName, - event, - respMakeJoin.RoomVersion, - ) - if err != nil { - logrus.WithError(err).Warnf("r.federation.SendJoin failed") - r.statistics.ForServer(serverName).Failure() - continue - } - - // Check that the send_join response was valid. - joinCtx := perform.JoinContext(r.federation, r.keyRing) - if err = joinCtx.CheckSendJoinResponse( - ctx, event, serverName, respMakeJoin, respSendJoin, + supportedVersions, ); err != nil { - logrus.WithError(err).Warnf("joinCtx.CheckSendJoinResponse failed") - continue - } - - // If we successfully performed a send_join above then the other - // server now thinks we're a part of the room. Send the newly - // returned state to the roomserver to update our local view. - if err = r.producer.SendEventWithState( - ctx, - respSendJoin.ToRespState(), - event.Headered(respMakeJoin.RoomVersion), - ); err != nil { - logrus.WithError(err).Warnf("r.producer.SendEventWithState failed") + logrus.WithError(err).WithFields(logrus.Fields{ + "server_name": serverName, + "room_id": request.RoomID, + }).Warnf("Failed to join room through server") continue } // We're all good. - r.statistics.ForServer(serverName).Success() return nil } @@ -152,6 +78,103 @@ func (r *FederationSenderInternalAPI) PerformJoin( ) } +func (r *FederationSenderInternalAPI) performJoinUsingServer( + ctx context.Context, + roomID, userID string, + content map[string]interface{}, + serverName gomatrixserverlib.ServerName, + supportedVersions []gomatrixserverlib.RoomVersion, +) error { + // Try to perform a make_join using the information supplied in the + // request. + respMakeJoin, err := r.federation.MakeJoin( + ctx, + serverName, + roomID, + userID, + supportedVersions, + ) + if err != nil { + // TODO: Check if the user was not allowed to join the room. + r.statistics.ForServer(serverName).Failure() + return fmt.Errorf("r.federation.MakeJoin: %w", err) + } + r.statistics.ForServer(serverName).Success() + + // Set all the fields to be what they should be, this should be a no-op + // but it's possible that the remote server returned us something "odd" + respMakeJoin.JoinEvent.Type = gomatrixserverlib.MRoomMember + respMakeJoin.JoinEvent.Sender = userID + respMakeJoin.JoinEvent.StateKey = &userID + respMakeJoin.JoinEvent.RoomID = roomID + respMakeJoin.JoinEvent.Redacts = "" + if content == nil { + content = map[string]interface{}{} + } + content["membership"] = "join" + if err = respMakeJoin.JoinEvent.SetContent(content); err != nil { + return fmt.Errorf("respMakeJoin.JoinEvent.SetContent: %w", err) + } + if err = respMakeJoin.JoinEvent.SetUnsigned(struct{}{}); err != nil { + return fmt.Errorf("respMakeJoin.JoinEvent.SetUnsigned: %w", err) + } + + // Work out if we support the room version that has been supplied in + // the make_join response. + if respMakeJoin.RoomVersion == "" { + respMakeJoin.RoomVersion = gomatrixserverlib.RoomVersionV1 + } + if _, err = respMakeJoin.RoomVersion.EventFormat(); err != nil { + return fmt.Errorf("respMakeJoin.RoomVersion.EventFormat: %w", err) + } + + // Build the join event. + event, err := respMakeJoin.JoinEvent.Build( + time.Now(), + r.cfg.Matrix.ServerName, + r.cfg.Matrix.KeyID, + r.cfg.Matrix.PrivateKey, + respMakeJoin.RoomVersion, + ) + if err != nil { + return fmt.Errorf("respMakeJoin.JoinEvent.Build: %w", err) + } + + // Try to perform a send_join using the newly built event. + respSendJoin, err := r.federation.SendJoin( + ctx, + serverName, + event, + respMakeJoin.RoomVersion, + ) + if err != nil { + r.statistics.ForServer(serverName).Failure() + return fmt.Errorf("r.federation.SendJoin: %w", err) + } + r.statistics.ForServer(serverName).Success() + + // Check that the send_join response was valid. + joinCtx := perform.JoinContext(r.federation, r.keyRing) + if err = joinCtx.CheckSendJoinResponse( + ctx, event, serverName, respMakeJoin, respSendJoin, + ); err != nil { + return fmt.Errorf("joinCtx.CheckSendJoinResponse: %w", err) + } + + // If we successfully performed a send_join above then the other + // server now thinks we're a part of the room. Send the newly + // returned state to the roomserver to update our local view. + if err = r.producer.SendEventWithState( + ctx, + respSendJoin.ToRespState(), + event.Headered(respMakeJoin.RoomVersion), + ); err != nil { + return fmt.Errorf("r.producer.SendEventWithState: %w", err) + } + + return nil +} + // PerformLeaveRequest implements api.FederationSenderInternalAPI func (r *FederationSenderInternalAPI) PerformLeave( ctx context.Context, From 8db60c90bbe064ffe3f85b79a433172d30ab3b5f Mon Sep 17 00:00:00 2001 From: Kegsay Date: Fri, 15 May 2020 16:27:34 +0100 Subject: [PATCH 067/200] Fix a bug whereby backfilling could leak events across rooms (#1043) * Fix a bug whereby backfilling could leak events across rooms Caused by a faulty SQL query. With tests now. * comment --- .../output_room_events_topology_table.go | 8 +-- .../output_room_events_topology_table.go | 8 +-- syncapi/storage/storage_test.go | 49 +++++++++++++++++++ 3 files changed, 57 insertions(+), 8 deletions(-) diff --git a/syncapi/storage/postgres/output_room_events_topology_table.go b/syncapi/storage/postgres/output_room_events_topology_table.go index 8a9cc49c9..9fc5f5194 100644 --- a/syncapi/storage/postgres/output_room_events_topology_table.go +++ b/syncapi/storage/postgres/output_room_events_topology_table.go @@ -48,17 +48,17 @@ const insertEventInTopologySQL = "" + const selectEventIDsInRangeASCSQL = "" + "SELECT event_id FROM syncapi_output_room_events_topology" + - " WHERE room_id = $1 AND" + + " WHERE room_id = $1 AND (" + "(topological_position > $2 AND topological_position < $3) OR" + "(topological_position = $4 AND stream_position <= $5)" + - " ORDER BY topological_position ASC, stream_position ASC LIMIT $6" + ") ORDER BY topological_position ASC, stream_position ASC LIMIT $6" const selectEventIDsInRangeDESCSQL = "" + "SELECT event_id FROM syncapi_output_room_events_topology" + - " WHERE room_id = $1 AND" + + " WHERE room_id = $1 AND (" + "(topological_position > $2 AND topological_position < $3) OR" + "(topological_position = $4 AND stream_position <= $5)" + - " ORDER BY topological_position DESC, stream_position DESC LIMIT $6" + ") ORDER BY topological_position DESC, stream_position DESC LIMIT $6" const selectPositionInTopologySQL = "" + "SELECT topological_position, stream_position FROM syncapi_output_room_events_topology" + diff --git a/syncapi/storage/sqlite3/output_room_events_topology_table.go b/syncapi/storage/sqlite3/output_room_events_topology_table.go index f25ac623c..22779238c 100644 --- a/syncapi/storage/sqlite3/output_room_events_topology_table.go +++ b/syncapi/storage/sqlite3/output_room_events_topology_table.go @@ -45,17 +45,17 @@ const insertEventInTopologySQL = "" + const selectEventIDsInRangeASCSQL = "" + "SELECT event_id FROM syncapi_output_room_events_topology" + - " WHERE room_id = $1 AND" + + " WHERE room_id = $1 AND (" + "(topological_position > $2 AND topological_position < $3) OR" + "(topological_position = $4 AND stream_position <= $5)" + - " ORDER BY topological_position ASC, stream_position ASC LIMIT $6" + ") ORDER BY topological_position ASC, stream_position ASC LIMIT $6" const selectEventIDsInRangeDESCSQL = "" + "SELECT event_id FROM syncapi_output_room_events_topology" + - " WHERE room_id = $1 AND" + + " WHERE room_id = $1 AND (" + "(topological_position > $2 AND topological_position < $3) OR" + "(topological_position = $4 AND stream_position <= $5)" + - " ORDER BY topological_position DESC, stream_position DESC LIMIT $6" + ") ORDER BY topological_position DESC, stream_position DESC LIMIT $6" const selectPositionInTopologySQL = "" + "SELECT topological_position, stream_position FROM syncapi_output_room_events_topology" + diff --git a/syncapi/storage/storage_test.go b/syncapi/storage/storage_test.go index 69ac5e6f5..6b0458527 100644 --- a/syncapi/storage/storage_test.go +++ b/syncapi/storage/storage_test.go @@ -405,6 +405,55 @@ func TestGetEventsInRangeWithEventsSameDepth(t *testing.T) { } } +// The purpose of this test is to make sure that the query to pull out events is honouring the room ID correctly. +// It works by creating two rooms with the same events in them, then selecting events by topological range. +// Specifically, we know that events with the same depth but lower stream positions are selected, and it's possible +// that this check isn't using the room ID if the brackets are wrong in the SQL query. +func TestGetEventsInTopologicalRangeMultiRoom(t *testing.T) { + t.Parallel() + db := MustCreateDatabase(t) + + makeEvents := func(roomID string) (events []gomatrixserverlib.HeaderedEvent) { + events = append(events, MustCreateEvent(t, roomID, nil, &gomatrixserverlib.EventBuilder{ + Content: []byte(fmt.Sprintf(`{"room_version":"4","creator":"%s"}`, testUserIDA)), + Type: "m.room.create", + StateKey: &emptyStateKey, + Sender: testUserIDA, + Depth: int64(len(events) + 1), + })) + events = append(events, MustCreateEvent(t, roomID, []gomatrixserverlib.HeaderedEvent{events[len(events)-1]}, &gomatrixserverlib.EventBuilder{ + Content: []byte(fmt.Sprintf(`{"membership":"join"}`)), + Type: "m.room.member", + StateKey: &testUserIDA, + Sender: testUserIDA, + Depth: int64(len(events) + 1), + })) + return + } + + roomA := "!room_a:" + string(testOrigin) + roomB := "!room_b:" + string(testOrigin) + eventsA := makeEvents(roomA) + eventsB := makeEvents(roomB) + MustWriteEvents(t, db, eventsA) + MustWriteEvents(t, db, eventsB) + from, err := db.MaxTopologicalPosition(ctx, roomB) + if err != nil { + t.Fatalf("failed to get MaxTopologicalPosition: %s", err) + } + // head towards the beginning of time + to := types.NewTopologyToken(0, 0) + + // Query using room B as room A was inserted first and hence A will have lower stream positions but identical depths, + // allowing this bug to surface. + paginatedEvents, err := db.GetEventsInTopologicalRange(ctx, &from, &to, roomB, 5, true) + if err != nil { + t.Fatalf("GetEventsInRange returned an error: %s", err) + } + gots := gomatrixserverlib.HeaderedToClientEvents(db.StreamEventsToEvents(&testUserDeviceA, paginatedEvents), gomatrixserverlib.FormatAll) + assertEventsEqual(t, "", true, gots, reversed(eventsB)) +} + // The purpose of this test is to make sure that events are returned in the right *order* when they have been inserted in a manner similar to // how any kind of backfill operation will insert the events. This test inserts the SimpleRoom events in a manner similar to how backfill over // federation would: From bfb954519bdf172451d999ac4c654b3d15eff124 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 15 May 2020 17:39:20 +0100 Subject: [PATCH 068/200] Only keep keys valid for 100 years rather than the heat death of the universe WASM builds really dislike thinking that far into the future, causing WASM to trap with "float unrepresentable in integer range", which corrupts the Go stack in wasm_exec which then leads to a segfault and the program exiting. --- common/keydb/postgres/keydb.go | 4 ++-- common/keydb/sqlite3/keydb.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/common/keydb/postgres/keydb.go b/common/keydb/postgres/keydb.go index 706ca0052..a8a304376 100644 --- a/common/keydb/postgres/keydb.go +++ b/common/keydb/postgres/keydb.go @@ -17,7 +17,7 @@ package postgres import ( "context" - "math" + "time" "golang.org/x/crypto/ed25519" @@ -62,7 +62,7 @@ func NewDatabase( VerifyKey: gomatrixserverlib.VerifyKey{ Key: gomatrixserverlib.Base64String(serverKey), }, - ValidUntilTS: math.MaxUint64 >> 1, + ValidUntilTS: gomatrixserverlib.AsTimestamp(time.Now().Add(100 * 365 * 24 * time.Hour)), ExpiredTS: gomatrixserverlib.PublicKeyNotExpired, } err = d.StoreKeys( diff --git a/common/keydb/sqlite3/keydb.go b/common/keydb/sqlite3/keydb.go index 94a32e29f..1ff315d7c 100644 --- a/common/keydb/sqlite3/keydb.go +++ b/common/keydb/sqlite3/keydb.go @@ -17,7 +17,7 @@ package sqlite3 import ( "context" - "math" + "time" "golang.org/x/crypto/ed25519" @@ -63,7 +63,7 @@ func NewDatabase( VerifyKey: gomatrixserverlib.VerifyKey{ Key: gomatrixserverlib.Base64String(serverKey), }, - ValidUntilTS: math.MaxUint64 >> 1, + ValidUntilTS: gomatrixserverlib.AsTimestamp(time.Now().Add(100 * 365 * 24 * time.Hour)), ExpiredTS: gomatrixserverlib.PublicKeyNotExpired, } err = d.StoreKeys( From dce4f436f77b71155483a47a6ad14da5c0ae5d6a Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 18 May 2020 10:56:43 +0100 Subject: [PATCH 069/200] Add -api flag to monolith (#1044) * Add flag for enabling HTTP APIs in monolith mode * Flag -api * Only start HTTP APIs if needed --- appservice/appservice.go | 4 +++- cmd/dendrite-appservice-server/main.go | 2 +- cmd/dendrite-client-api-server/main.go | 2 +- cmd/dendrite-demo-libp2p/p2pdendrite.go | 2 +- cmd/dendrite-edu-server/main.go | 2 +- cmd/dendrite-federation-api-server/main.go | 2 +- cmd/dendrite-federation-sender-server/main.go | 2 +- cmd/dendrite-key-server/main.go | 2 +- cmd/dendrite-media-api-server/main.go | 2 +- cmd/dendrite-monolith-server/main.go | 11 ++++++----- cmd/dendrite-public-rooms-api-server/main.go | 2 +- cmd/dendrite-room-server/main.go | 2 +- cmd/dendrite-sync-api-server/main.go | 2 +- cmd/dendritejs/main.go | 2 +- common/basecomponent/base.go | 4 +++- eduserver/eduserver.go | 5 ++++- federationsender/federationsender.go | 5 ++++- roomserver/roomserver.go | 4 +++- 18 files changed, 35 insertions(+), 22 deletions(-) diff --git a/appservice/appservice.go b/appservice/appservice.go index e52db2c28..3a1c4677a 100644 --- a/appservice/appservice.go +++ b/appservice/appservice.go @@ -82,7 +82,9 @@ func SetupAppServiceAPIComponent( Cfg: base.Cfg, } - appserviceQueryAPI.SetupHTTP(http.DefaultServeMux) + if base.EnableHTTPAPIs { + appserviceQueryAPI.SetupHTTP(http.DefaultServeMux) + } consumer := consumers.NewOutputRoomEventConsumer( base.Cfg, base.KafkaConsumer, accountsDB, appserviceDB, diff --git a/cmd/dendrite-appservice-server/main.go b/cmd/dendrite-appservice-server/main.go index 91ca707e7..faf9ade31 100644 --- a/cmd/dendrite-appservice-server/main.go +++ b/cmd/dendrite-appservice-server/main.go @@ -22,7 +22,7 @@ import ( func main() { cfg := basecomponent.ParseFlags() - base := basecomponent.NewBaseDendrite(cfg, "AppServiceAPI") + base := basecomponent.NewBaseDendrite(cfg, "AppServiceAPI", true) defer base.Close() // nolint: errcheck accountDB := base.CreateAccountsDB() diff --git a/cmd/dendrite-client-api-server/main.go b/cmd/dendrite-client-api-server/main.go index 1ce66db2c..d4ab9e42f 100644 --- a/cmd/dendrite-client-api-server/main.go +++ b/cmd/dendrite-client-api-server/main.go @@ -26,7 +26,7 @@ import ( func main() { cfg := basecomponent.ParseFlags() - base := basecomponent.NewBaseDendrite(cfg, "ClientAPI") + base := basecomponent.NewBaseDendrite(cfg, "ClientAPI", true) defer base.Close() // nolint: errcheck accountDB := base.CreateAccountsDB() diff --git a/cmd/dendrite-demo-libp2p/p2pdendrite.go b/cmd/dendrite-demo-libp2p/p2pdendrite.go index a9db3b39c..831b5d7f4 100644 --- a/cmd/dendrite-demo-libp2p/p2pdendrite.go +++ b/cmd/dendrite-demo-libp2p/p2pdendrite.go @@ -54,7 +54,7 @@ type P2PDendrite struct { // The componentName is used for logging purposes, and should be a friendly name // of the component running, e.g. SyncAPI. func NewP2PDendrite(cfg *config.Dendrite, componentName string) *P2PDendrite { - baseDendrite := basecomponent.NewBaseDendrite(cfg, componentName) + baseDendrite := basecomponent.NewBaseDendrite(cfg, componentName, false) ctx, cancel := context.WithCancel(context.Background()) diff --git a/cmd/dendrite-edu-server/main.go b/cmd/dendrite-edu-server/main.go index a4511f1ba..98b05e649 100644 --- a/cmd/dendrite-edu-server/main.go +++ b/cmd/dendrite-edu-server/main.go @@ -23,7 +23,7 @@ import ( func main() { cfg := basecomponent.ParseFlags() - base := basecomponent.NewBaseDendrite(cfg, "EDUServerAPI") + base := basecomponent.NewBaseDendrite(cfg, "EDUServerAPI", true) defer func() { if err := base.Close(); err != nil { logrus.WithError(err).Warn("BaseDendrite close failed") diff --git a/cmd/dendrite-federation-api-server/main.go b/cmd/dendrite-federation-api-server/main.go index d829326a7..b37570201 100644 --- a/cmd/dendrite-federation-api-server/main.go +++ b/cmd/dendrite-federation-api-server/main.go @@ -25,7 +25,7 @@ import ( func main() { cfg := basecomponent.ParseFlags() - base := basecomponent.NewBaseDendrite(cfg, "FederationAPI") + base := basecomponent.NewBaseDendrite(cfg, "FederationAPI", true) defer base.Close() // nolint: errcheck accountDB := base.CreateAccountsDB() diff --git a/cmd/dendrite-federation-sender-server/main.go b/cmd/dendrite-federation-sender-server/main.go index 0daac1bcb..c9bf6688f 100644 --- a/cmd/dendrite-federation-sender-server/main.go +++ b/cmd/dendrite-federation-sender-server/main.go @@ -22,7 +22,7 @@ import ( func main() { cfg := basecomponent.ParseFlags() - base := basecomponent.NewBaseDendrite(cfg, "FederationSender") + base := basecomponent.NewBaseDendrite(cfg, "FederationSender", true) defer base.Close() // nolint: errcheck federation := base.CreateFederationClient() diff --git a/cmd/dendrite-key-server/main.go b/cmd/dendrite-key-server/main.go index 5b2166d9a..aca666fad 100644 --- a/cmd/dendrite-key-server/main.go +++ b/cmd/dendrite-key-server/main.go @@ -21,7 +21,7 @@ import ( func main() { cfg := basecomponent.ParseFlags() - base := basecomponent.NewBaseDendrite(cfg, "KeyServer") + base := basecomponent.NewBaseDendrite(cfg, "KeyServer", true) defer base.Close() // nolint: errcheck accountDB := base.CreateAccountsDB() diff --git a/cmd/dendrite-media-api-server/main.go b/cmd/dendrite-media-api-server/main.go index a818db73a..cfb32173d 100644 --- a/cmd/dendrite-media-api-server/main.go +++ b/cmd/dendrite-media-api-server/main.go @@ -21,7 +21,7 @@ import ( func main() { cfg := basecomponent.ParseFlags() - base := basecomponent.NewBaseDendrite(cfg, "MediaAPI") + base := basecomponent.NewBaseDendrite(cfg, "MediaAPI", true) defer base.Close() // nolint: errcheck deviceDB := base.CreateDeviceDB() diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index f22610616..23959152e 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -41,15 +41,16 @@ import ( ) var ( - httpBindAddr = flag.String("http-bind-address", ":8008", "The HTTP listening port for the server") - httpsBindAddr = flag.String("https-bind-address", ":8448", "The HTTPS listening port for the server") - certFile = flag.String("tls-cert", "", "The PEM formatted X509 certificate to use for TLS") - keyFile = flag.String("tls-key", "", "The PEM private key to use for TLS") + httpBindAddr = flag.String("http-bind-address", ":8008", "The HTTP listening port for the server") + httpsBindAddr = flag.String("https-bind-address", ":8448", "The HTTPS listening port for the server") + certFile = flag.String("tls-cert", "", "The PEM formatted X509 certificate to use for TLS") + keyFile = flag.String("tls-key", "", "The PEM private key to use for TLS") + enableHTTPAPIs = flag.Bool("api", false, "Expose internal HTTP APIs in monolith mode") ) func main() { cfg := basecomponent.ParseMonolithFlags() - base := basecomponent.NewBaseDendrite(cfg, "Monolith") + base := basecomponent.NewBaseDendrite(cfg, "Monolith", *enableHTTPAPIs) defer base.Close() // nolint: errcheck accountDB := base.CreateAccountsDB() diff --git a/cmd/dendrite-public-rooms-api-server/main.go b/cmd/dendrite-public-rooms-api-server/main.go index c3b49f4f7..0eb2fcb57 100644 --- a/cmd/dendrite-public-rooms-api-server/main.go +++ b/cmd/dendrite-public-rooms-api-server/main.go @@ -23,7 +23,7 @@ import ( func main() { cfg := basecomponent.ParseFlags() - base := basecomponent.NewBaseDendrite(cfg, "PublicRoomsAPI") + base := basecomponent.NewBaseDendrite(cfg, "PublicRoomsAPI", true) defer base.Close() // nolint: errcheck deviceDB := base.CreateDeviceDB() diff --git a/cmd/dendrite-room-server/main.go b/cmd/dendrite-room-server/main.go index 172468448..4d450bd47 100644 --- a/cmd/dendrite-room-server/main.go +++ b/cmd/dendrite-room-server/main.go @@ -22,7 +22,7 @@ import ( func main() { cfg := basecomponent.ParseFlags() - base := basecomponent.NewBaseDendrite(cfg, "RoomServerAPI") + base := basecomponent.NewBaseDendrite(cfg, "RoomServerAPI", true) defer base.Close() // nolint: errcheck keyDB := base.CreateKeyDB() federation := base.CreateFederationClient() diff --git a/cmd/dendrite-sync-api-server/main.go b/cmd/dendrite-sync-api-server/main.go index 259447af2..7aadd6a7b 100644 --- a/cmd/dendrite-sync-api-server/main.go +++ b/cmd/dendrite-sync-api-server/main.go @@ -21,7 +21,7 @@ import ( func main() { cfg := basecomponent.ParseFlags() - base := basecomponent.NewBaseDendrite(cfg, "SyncAPI") + base := basecomponent.NewBaseDendrite(cfg, "SyncAPI", true) defer base.Close() // nolint: errcheck deviceDB := base.CreateDeviceDB() diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 5b7ed4807..75fe23619 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -108,7 +108,7 @@ func main() { if err := cfg.Derive(); err != nil { logrus.Fatalf("Failed to derive values from config: %s", err) } - base := basecomponent.NewBaseDendrite(cfg, "Monolith") + base := basecomponent.NewBaseDendrite(cfg, "Monolith", false) defer base.Close() // nolint: errcheck accountDB := base.CreateAccountsDB() diff --git a/common/basecomponent/base.go b/common/basecomponent/base.go index 4342e25a3..234de5e72 100644 --- a/common/basecomponent/base.go +++ b/common/basecomponent/base.go @@ -58,6 +58,7 @@ type BaseDendrite struct { // APIMux should be used to register new public matrix api endpoints APIMux *mux.Router + EnableHTTPAPIs bool httpClient *http.Client Cfg *config.Dendrite ImmutableCache caching.ImmutableCache @@ -71,7 +72,7 @@ const HTTPClientTimeout = time.Second * 30 // NewBaseDendrite creates a new instance to be used by a component. // The componentName is used for logging purposes, and should be a friendly name // of the compontent running, e.g. "SyncAPI" -func NewBaseDendrite(cfg *config.Dendrite, componentName string) *BaseDendrite { +func NewBaseDendrite(cfg *config.Dendrite, componentName string, enableHTTPAPIs bool) *BaseDendrite { common.SetupStdLogging() common.SetupHookLogging(cfg.Logging, componentName) common.SetupPprof() @@ -96,6 +97,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string) *BaseDendrite { return &BaseDendrite{ componentName: componentName, + EnableHTTPAPIs: enableHTTPAPIs, tracerCloser: closer, Cfg: cfg, ImmutableCache: cache, diff --git a/eduserver/eduserver.go b/eduserver/eduserver.go index 8ddd2c527..2a209364a 100644 --- a/eduserver/eduserver.go +++ b/eduserver/eduserver.go @@ -35,6 +35,9 @@ func SetupEDUServerComponent( OutputTypingEventTopic: string(base.Cfg.Kafka.Topics.OutputTypingEvent), } - inputAPI.SetupHTTP(http.DefaultServeMux) + if base.EnableHTTPAPIs { + inputAPI.SetupHTTP(http.DefaultServeMux) + } + return inputAPI } diff --git a/federationsender/federationsender.go b/federationsender/federationsender.go index 8e2f256dc..9c151a8d4 100644 --- a/federationsender/federationsender.go +++ b/federationsender/federationsender.go @@ -71,7 +71,10 @@ func SetupFederationSenderComponent( federationSenderDB, base.Cfg, roomserverProducer, federation, keyRing, statistics, ) - queryAPI.SetupHTTP(http.DefaultServeMux) + + if base.EnableHTTPAPIs { + queryAPI.SetupHTTP(http.DefaultServeMux) + } return queryAPI } diff --git a/roomserver/roomserver.go b/roomserver/roomserver.go index 450da5bb6..aa5d223e9 100644 --- a/roomserver/roomserver.go +++ b/roomserver/roomserver.go @@ -51,7 +51,9 @@ func SetupRoomServerComponent( KeyRing: keyRing, } - internalAPI.SetupHTTP(http.DefaultServeMux) + if base.EnableHTTPAPIs { + internalAPI.SetupHTTP(http.DefaultServeMux) + } return &internalAPI } From 5c221f065565a995f1f21be48f96e2ebf69473ec Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 18 May 2020 15:51:49 +0100 Subject: [PATCH 070/200] Latest go-http-js-libp2p --- cmd/dendritejs/main.go | 2 +- cmd/dendritejs/publicrooms.go | 2 +- go.mod | 6 +++--- go.sum | 9 +++++++++ 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 75fe23619..44f3b5c4d 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -37,7 +37,7 @@ import ( "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/syncapi" - "github.com/matrix-org/go-http-js-libp2p/go_http_js_libp2p" + go_http_js_libp2p "github.com/matrix-org/go-http-js-libp2p" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" diff --git a/cmd/dendritejs/publicrooms.go b/cmd/dendritejs/publicrooms.go index 17822e7ad..5246cf535 100644 --- a/cmd/dendritejs/publicrooms.go +++ b/cmd/dendritejs/publicrooms.go @@ -17,7 +17,7 @@ package main import ( - "github.com/matrix-org/go-http-js-libp2p/go_http_js_libp2p" + go_http_js_libp2p "github.com/matrix-org/go-http-js-libp2p" ) type libp2pPublicRoomsProvider struct { diff --git a/go.mod b/go.mod index 8e6d63cd5..5f5d21980 100644 --- a/go.mod +++ b/go.mod @@ -14,8 +14,8 @@ require ( github.com/libp2p/go-libp2p-pubsub v0.2.5 github.com/libp2p/go-libp2p-record v0.1.2 github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 - github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f - github.com/matrix-org/go-sqlite3-js v0.0.0-20200325174927-327088cdef10 + github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518144534-b64c85260c84 + github.com/matrix-org/go-sqlite3-js v0.0.0-20200326102434-98eda28055bd github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 github.com/matrix-org/gomatrixserverlib v0.0.0-20200511154227-5cc71d36632b github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f @@ -26,7 +26,7 @@ require ( github.com/opentracing/opentracing-go v1.1.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.4.1 - github.com/sirupsen/logrus v1.4.2 + github.com/sirupsen/logrus v1.6.0 github.com/tidwall/gjson v1.6.0 github.com/uber/jaeger-client-go v2.15.0+incompatible github.com/uber/jaeger-lib v1.5.0 diff --git a/go.sum b/go.sum index 53aefed37..f0afbf43b 100644 --- a/go.sum +++ b/go.sum @@ -200,6 +200,8 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3 github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d h1:68u9r4wEvL3gYg2jvAOgROwZ3H+Y3hIDk4tbbmIjcYQ= github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -351,16 +353,21 @@ github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czP github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/matrix-org/dendrite v0.0.0-20200220135450-0352f250b857/go.mod h1:DZ35IoR+ViBNVPe9umdlOSnjvKl7wfyRmZg4QfWGvTo= +github.com/matrix-org/dendrite v0.0.0-20200515084112-2b5052eccfca/go.mod h1:Lx68kzmJOOTauSVXvy2D9vYknEEVlSfNMyD7SvqTX/E= github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 h1:nMX2t7hbGF0NYDYySx0pCqEKGKAeZIiSqlWSspetlhY= github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 h1:nMX2t7hbGF0NYDYySx0pCqEKGKAeZIiSqlWSspetlhY= github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg= github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg= github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f h1:5TOte9uk/epk8L+Pbp6qwaV8YsKYXKjyECPHUhJTWQc= github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f/go.mod h1:qK3LUW7RCLhFM7gC3pabj3EXT9A1DsCK33MHstUhhbk= +github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518144534-b64c85260c84 h1:Vqy4mla3pk5es2gQDVgKDWvMpAWMa3V1DQr0iZwSJJk= +github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518144534-b64c85260c84/go.mod h1:mea6AwBSa+ULU24VIZ3AFWgk0kaneepTVmDCQylME1A= github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658 h1:UlhTKClOgWnSB25Rv+BS/Vc1mRinjNUErfyGEVOBP04= github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/go-sqlite3-js v0.0.0-20200325174927-327088cdef10 h1:SnhC7/o87ueVwEWI3mUYtrs+s8VnYq3KZtpWsFQOLFE= github.com/matrix-org/go-sqlite3-js v0.0.0-20200325174927-327088cdef10/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= +github.com/matrix-org/go-sqlite3-js v0.0.0-20200326102434-98eda28055bd h1:C1FV4dRKF1uuGK8UH01+IoW6zZpfsTV1MvQimZvt418= +github.com/matrix-org/go-sqlite3-js v0.0.0-20200326102434-98eda28055bd/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20190130130140-385f072fe9af h1:piaIBNQGIHnni27xRB7VKkEwoWCgAmeuYf8pxAyG0bI= github.com/matrix-org/gomatrix v0.0.0-20190130130140-385f072fe9af/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bhrnp3Ky1qgx/fzCtCALOoGYylh2tpS9K4= @@ -512,6 +519,8 @@ github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY= github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a/go.mod h1:7AyxJNCJ7SBZ1MfVQCWD6Uqo2oubI2Eq2y2eqf+A5r0= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU= From 9ef30bb13b49db7120c153d42cc67859cf5ed7da Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 18 May 2020 17:49:24 +0100 Subject: [PATCH 071/200] Tweaks to latest events updater (#1045) * Comment out updaters a bit, add overwrite flag to latest events * Make sure we don't send fast-forwarded state changes over federation, start with empty set when overwriting * Remove redundant check for overwrite --- roomserver/internal/input_events.go | 40 +++++++++--- roomserver/internal/input_latest_events.go | 71 +++++++++++++++++++--- roomserver/internal/input_membership.go | 9 ++- roomserver/types/types.go | 4 ++ 4 files changed, 101 insertions(+), 23 deletions(-) diff --git a/roomserver/internal/input_events.go b/roomserver/internal/input_events.go index f4d3bb8f5..f5c678ca6 100644 --- a/roomserver/internal/input_events.go +++ b/roomserver/internal/input_events.go @@ -52,13 +52,15 @@ func processRoomEvent( headered := input.Event event := headered.Unwrap() - // Check that the event passes authentication checks and work out the numeric IDs for the auth events. + // Check that the event passes authentication checks and work out + // the numeric IDs for the auth events. authEventNIDs, err := checkAuthEvents(ctx, db, headered, input.AuthEventIDs) if err != nil { logrus.WithError(err).WithField("event_id", event.EventID()).WithField("auth_event_ids", input.AuthEventIDs).Error("processRoomEvent.checkAuthEvents failed for event") return } + // If we don't have a transaction ID then get one. if input.TransactionID != nil { tdID := input.TransactionID eventID, err = db.GetTransactionEventID( @@ -70,17 +72,21 @@ func processRoomEvent( } } - // Store the event + // Store the event. roomNID, stateAtEvent, err := db.StoreEvent(ctx, event, input.TransactionID, authEventNIDs) if err != nil { return } + // For outliers we can stop after we've stored the event itself as it + // doesn't have any associated state to store and we don't need to + // notify anyone about it. if input.Kind == api.KindOutlier { - // For outliers we can stop after we've stored the event itself as it - // doesn't have any associated state to store and we don't need to - // notify anyone about it. - logrus.WithField("event_id", event.EventID()).WithField("type", event.Type()).WithField("room", event.RoomID()).Info("Stored outlier") + logrus.WithFields(logrus.Fields{ + "event_id": event.EventID(), + "type": event.Type(), + "room": event.RoomID(), + }).Info("Stored outlier") return event.EventID(), nil } @@ -93,10 +99,21 @@ func processRoomEvent( } } + if err = updateLatestEvents( + ctx, // context + db, // roomserver database + ow, // output event writer + roomNID, // room NID to update + stateAtEvent, // state at event (below) + event, // event + input.SendAsServer, // send as server + input.TransactionID, // transaction ID + ); err != nil { + return + } + // Update the extremities of the event graph for the room - return event.EventID(), updateLatestEvents( - ctx, db, ow, roomNID, stateAtEvent, event, input.SendAsServer, input.TransactionID, - ) + return event.EventID(), nil } func calculateAndSetState( @@ -111,6 +128,9 @@ func calculateAndSetState( roomState := state.NewStateResolution(db) if input.HasState { + // TODO: Check here if we think we're in the room already. + stateAtEvent.Overwrite = true + // We've been told what the state at the event is so we don't need to calculate it. // Check that those state events are in the database and store the state. var entries []types.StateEntry @@ -122,6 +142,8 @@ func calculateAndSetState( return err } } else { + stateAtEvent.Overwrite = false + // We haven't been told what the state at the event is so we need to calculate it from the prev_events if stateAtEvent.BeforeStateSnapshotNID, err = roomState.CalculateAndStoreStateBeforeEvent(ctx, event, roomNID); err != nil { return err diff --git a/roomserver/internal/input_latest_events.go b/roomserver/internal/input_latest_events.go index 42be0f408..6eeeedab0 100644 --- a/roomserver/internal/input_latest_events.go +++ b/roomserver/internal/input_latest_events.go @@ -69,10 +69,17 @@ func updateLatestEvents( }() u := latestEventsUpdater{ - ctx: ctx, db: db, updater: updater, ow: ow, roomNID: roomNID, - stateAtEvent: stateAtEvent, event: event, sendAsServer: sendAsServer, + ctx: ctx, + db: db, + updater: updater, + ow: ow, + roomNID: roomNID, + stateAtEvent: stateAtEvent, + event: event, + sendAsServer: sendAsServer, transactionID: transactionID, } + if err = u.doUpdateLatestEvents(); err != nil { return err } @@ -115,38 +122,65 @@ type latestEventsUpdater struct { func (u *latestEventsUpdater) doUpdateLatestEvents() error { prevEvents := u.event.PrevEvents() - oldLatest := u.updater.LatestEvents() u.lastEventIDSent = u.updater.LastEventIDSent() u.oldStateNID = u.updater.CurrentStateSnapshotNID() + // If we are doing a regular event update then we will get the + // previous latest events to use as a part of the calculation. If + // we are overwriting the latest events because we have a complete + // state snapshot from somewhere else, e.g. a federated room join, + // then start with an empty set - none of the forward extremities + // that we knew about before matter anymore. + oldLatest := []types.StateAtEventAndReference{} + if !u.stateAtEvent.Overwrite { + oldLatest = u.updater.LatestEvents() + } + + // If the event has already been written to the output log then we + // don't need to do anything, as we've handled it already. hasBeenSent, err := u.updater.HasEventBeenSent(u.stateAtEvent.EventNID) if err != nil { return err } else if hasBeenSent { - // Already sent this event so we can stop processing return nil } + // Update the roomserver_previous_events table with references. This + // is effectively tracking the structure of the DAG. if err = u.updater.StorePreviousEvents(u.stateAtEvent.EventNID, prevEvents); err != nil { return err } + // Get the event reference for our new event. This will be used when + // determining if the event is referenced by an existing event. eventReference := u.event.EventReference() - // Check if this event is already referenced by another event in the room. + + // Check if our new event is already referenced by an existing event + // in the room. If it is then it isn't a latest event. alreadyReferenced, err := u.updater.IsReferenced(eventReference) if err != nil { return err } - u.latest = calculateLatest(oldLatest, alreadyReferenced, prevEvents, types.StateAtEventAndReference{ - EventReference: eventReference, - StateAtEvent: u.stateAtEvent, - }) + // Work out what the latest events are. + u.latest = calculateLatest( + oldLatest, + alreadyReferenced, + prevEvents, + types.StateAtEventAndReference{ + EventReference: eventReference, + StateAtEvent: u.stateAtEvent, + }, + ) + // Now that we know what the latest events are, it's time to get the + // latest state. if err = u.latestState(); err != nil { return err } + // If we need to generate any output events then here's where we do it. + // TODO: Move this! updates, err := updateMemberships(u.ctx, u.db, u.updater, u.removed, u.added) if err != nil { return err @@ -181,10 +215,15 @@ func (u *latestEventsUpdater) latestState() error { var err error roomState := state.NewStateResolution(u.db) + // Get a list of the current latest events. latestStateAtEvents := make([]types.StateAtEvent, len(u.latest)) for i := range u.latest { latestStateAtEvents[i] = u.latest[i].StateAtEvent } + + // Takes the NIDs of the latest events and creates a state snapshot + // of the state after the events. The snapshot state will be resolved + // using the correct state resolution algorithm for the room. u.newStateNID, err = roomState.CalculateAndStoreStateAfterEvents( u.ctx, u.roomNID, latestStateAtEvents, ) @@ -192,6 +231,18 @@ func (u *latestEventsUpdater) latestState() error { return err } + // If we are overwriting the state then we should make sure that we + // don't send anything out over federation again, it will very likely + // be a repeat. + if u.stateAtEvent.Overwrite { + u.sendAsServer = "" + } + + // Now that we have a new state snapshot based on the latest events, + // we can compare that new snapshot to the previous one and see what + // has changed. This gives us one list of removed state events and + // another list of added ones. Replacing a value for a state-key tuple + // will result one removed (the old event) and one added (the new event). u.removed, u.added, err = roomState.DifferenceBetweeenStateSnapshots( u.ctx, u.oldStateNID, u.newStateNID, ) @@ -199,6 +250,8 @@ func (u *latestEventsUpdater) latestState() error { return err } + // Also work out the state before the event removes and the event + // adds. u.stateBeforeEventRemoves, u.stateBeforeEventAdds, err = roomState.DifferenceBetweeenStateSnapshots( u.ctx, u.newStateNID, u.stateAtEvent.BeforeStateSnapshotNID, ) diff --git a/roomserver/internal/input_membership.go b/roomserver/internal/input_membership.go index a0029a289..666e7ebcc 100644 --- a/roomserver/internal/input_membership.go +++ b/roomserver/internal/input_membership.go @@ -16,7 +16,6 @@ package internal import ( "context" - "errors" "fmt" "github.com/matrix-org/dendrite/roomserver/api" @@ -108,10 +107,10 @@ func updateMembership( } if add == nil { - // This shouldn't happen. Returning an error here is better than panicking - // in the membership updater functions later on. - // TODO: Why does this happen to begin with? - return updates, errors.New("add should not be nil") + // This can happen when we have rejoined a room and suddenly we have a + // divergence between the former state and the new one. We don't want to + // act on removals and apparently there are no adds, so stop here. + return updates, nil } mu, err := updater.MembershipUpdater(targetUserNID) diff --git a/roomserver/types/types.go b/roomserver/types/types.go index dfc112cfd..da83f614c 100644 --- a/roomserver/types/types.go +++ b/roomserver/types/types.go @@ -75,6 +75,10 @@ func (a StateEntry) LessThan(b StateEntry) bool { // StateAtEvent is the state before and after a matrix event. type StateAtEvent struct { + // Should this state overwrite the latest events and memberships of the room? + // This might be necessary when rejoining a federated room after a period of + // absence, as our state and latest events will be out of date. + Overwrite bool // The state before the event. BeforeStateSnapshotNID StateSnapshotNID // The state entry for the event itself, allows us to calculate the state after the event. From b730e2189282d122f9a157a08f29d5d672f752cb Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 18 May 2020 18:12:11 +0100 Subject: [PATCH 072/200] bump go-http-js-libp2p --- cmd/dendritejs/main.go | 2 +- go.mod | 4 +++- go.sum | 47 ++---------------------------------------- 3 files changed, 6 insertions(+), 47 deletions(-) diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 44f3b5c4d..6339bfbca 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -72,7 +72,7 @@ func createFederationClient(cfg *config.Dendrite, node *go_http_js_libp2p.P2pLoc func createP2PNode(privKey ed25519.PrivateKey) (serverName string, node *go_http_js_libp2p.P2pLocalNode) { hosted := "/dns4/rendezvous.matrix.org/tcp/8443/wss/p2p-websocket-star/" - node = go_http_js_libp2p.NewP2pLocalNode("org.matrix.p2p.experiment", privKey.Seed(), []string{hosted}) + node = go_http_js_libp2p.NewP2pLocalNode("org.matrix.p2p.experiment", privKey.Seed(), []string{hosted}, "p2p") serverName = node.Id fmt.Println("p2p assigned ServerName: ", serverName) return diff --git a/go.mod b/go.mod index 5f5d21980..bbac53eca 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,7 @@ module github.com/matrix-org/dendrite require ( github.com/Shopify/sarama v1.26.1 + github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect github.com/gorilla/mux v1.7.3 github.com/hashicorp/golang-lru v0.5.4 github.com/lib/pq v1.2.0 @@ -14,7 +15,7 @@ require ( github.com/libp2p/go-libp2p-pubsub v0.2.5 github.com/libp2p/go-libp2p-record v0.1.2 github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 - github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518144534-b64c85260c84 + github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20200326102434-98eda28055bd github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 github.com/matrix-org/gomatrixserverlib v0.0.0-20200511154227-5cc71d36632b @@ -28,6 +29,7 @@ require ( github.com/prometheus/client_golang v1.4.1 github.com/sirupsen/logrus v1.6.0 github.com/tidwall/gjson v1.6.0 + github.com/uber-go/atomic v1.3.0 // indirect github.com/uber/jaeger-client-go v2.15.0+incompatible github.com/uber/jaeger-lib v1.5.0 go.uber.org/atomic v1.4.0 diff --git a/go.sum b/go.sum index f0afbf43b..d487f42a3 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,6 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/DataDog/zstd v1.4.4 h1:+IawcoXhCBylN7ccwdwf8LOH2jKq7NavGpEPanrlTzE= -github.com/DataDog/zstd v1.4.4/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= github.com/Shopify/sarama v1.26.1 h1:3jnfWKD7gVwbB1KSy/lE0szA9duPuSFLViK0o/d3DgA= github.com/Shopify/sarama v1.26.1/go.mod h1:NbSGBSSndYaIhRcBtY9V0U7AyH+x71bG668AuWys/yU= @@ -29,14 +27,11 @@ github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13P github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= -github.com/btcsuite/btcutil v1.0.1 h1:GKOz8BnRjYrb/JTKgaOk+zh26NWNdSNvdvv0xoAZMSA= -github.com/btcsuite/btcutil v1.0.1/go.mod h1:j9HUFwoQRsZL3V4n+qG+CUnEGHOarIxfC3Le2Yhbcts= github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= -github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -184,7 +179,6 @@ github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= @@ -352,36 +346,20 @@ github.com/libp2p/go-yamux v1.3.0/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZ github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/matrix-org/dendrite v0.0.0-20200220135450-0352f250b857/go.mod h1:DZ35IoR+ViBNVPe9umdlOSnjvKl7wfyRmZg4QfWGvTo= -github.com/matrix-org/dendrite v0.0.0-20200515084112-2b5052eccfca/go.mod h1:Lx68kzmJOOTauSVXvy2D9vYknEEVlSfNMyD7SvqTX/E= github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 h1:nMX2t7hbGF0NYDYySx0pCqEKGKAeZIiSqlWSspetlhY= github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 h1:nMX2t7hbGF0NYDYySx0pCqEKGKAeZIiSqlWSspetlhY= github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg= github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg= -github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f h1:5TOte9uk/epk8L+Pbp6qwaV8YsKYXKjyECPHUhJTWQc= -github.com/matrix-org/go-http-js-libp2p v0.0.0-20200318135427-31631a9ef51f/go.mod h1:qK3LUW7RCLhFM7gC3pabj3EXT9A1DsCK33MHstUhhbk= -github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518144534-b64c85260c84 h1:Vqy4mla3pk5es2gQDVgKDWvMpAWMa3V1DQr0iZwSJJk= -github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518144534-b64c85260c84/go.mod h1:mea6AwBSa+ULU24VIZ3AFWgk0kaneepTVmDCQylME1A= -github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658 h1:UlhTKClOgWnSB25Rv+BS/Vc1mRinjNUErfyGEVOBP04= -github.com/matrix-org/go-sqlite3-js v0.0.0-20200304164012-aa524245b658/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= -github.com/matrix-org/go-sqlite3-js v0.0.0-20200325174927-327088cdef10 h1:SnhC7/o87ueVwEWI3mUYtrs+s8VnYq3KZtpWsFQOLFE= -github.com/matrix-org/go-sqlite3-js v0.0.0-20200325174927-327088cdef10/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= +github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 h1:eqE5OnGx9ZMWmrRbD3KF/3KtTunw0iQulI7YxOIdxo4= +github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4/go.mod h1:3WluEZ9QXSwU30tWYqktnpC1x9mwZKx1r8uAv8Iq+a4= github.com/matrix-org/go-sqlite3-js v0.0.0-20200326102434-98eda28055bd h1:C1FV4dRKF1uuGK8UH01+IoW6zZpfsTV1MvQimZvt418= github.com/matrix-org/go-sqlite3-js v0.0.0-20200326102434-98eda28055bd/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= -github.com/matrix-org/gomatrix v0.0.0-20190130130140-385f072fe9af h1:piaIBNQGIHnni27xRB7VKkEwoWCgAmeuYf8pxAyG0bI= -github.com/matrix-org/gomatrix v0.0.0-20190130130140-385f072fe9af/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bhrnp3Ky1qgx/fzCtCALOoGYylh2tpS9K4= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5 h1:kmRjpmFOenVpOaV/DRlo9p6z/IbOKlUC+hhKsAAh8Qg= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200124100636-0c2ec91d1df5/go.mod h1:FsKa2pWE/bpQql9H7U4boOPXFoJX/QcqaZZ6ijLkaZI= github.com/matrix-org/gomatrixserverlib v0.0.0-20200511154227-5cc71d36632b h1:nAmSc1KvQOumoRTz/LD68KyrB6Q5/6q7CmQ5Bswc2nM= github.com/matrix-org/gomatrixserverlib v0.0.0-20200511154227-5cc71d36632b/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= -github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1 h1:osLoFdOy+ChQqVUn2PeTDETFftVkl4w9t/OW18g3lnk= -github.com/matrix-org/naffka v0.0.0-20200127221512-0716baaabaf1/go.mod h1:cXoYQIENbdWIQHt1SyCo6Bl3C3raHwJ0wgVrXHSqf+A= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f/go.mod h1:y0oDTjZDv5SM9a2rp3bl+CU+bvTRINQsdb7YlDql5Go= -github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5 h1:W7l5CP4V7wPyPb4tYE11dbmeAOwtFQBTW0rf4OonOS8= -github.com/matrix-org/util v0.0.0-20171127121716-2e2df66af2f5/go.mod h1:lePuOiXLNDott7NZfnQvJk0lAZ5HgvIuWGhel6J+RLA= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= @@ -395,8 +373,6 @@ github.com/mattn/go-sqlite3 v2.0.2+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= -github.com/miekg/dns v1.1.4 h1:rCMZsU2ScVSYcAsOXgmC6+AKOK+6pmQTOcw03nfwYV0= -github.com/miekg/dns v1.1.4/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.12 h1:WMhc1ik4LNkTg8U9l3hI1LvxKmIL+f1+WV/SZtCbDDA= github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g= @@ -494,29 +470,23 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U= github.com/prometheus/client_golang v1.4.1 h1:FFSuS004yOQEtDdTq+TAOLP5xUq63KqAFYyOi8zA+Y8= github.com/prometheus/client_golang v1.4.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME= -github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= @@ -541,8 +511,6 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= -github.com/tidwall/gjson v1.1.5 h1:QysILxBeUEY3GTLA0fQVgkQG1zme8NxGvhh2SSqWNwI= -github.com/tidwall/gjson v1.1.5/go.mod h1:c/nTNbUr0E0OrXEhq1pwa8iEgc2DOt4ZZqAt1HtCkPA= github.com/tidwall/gjson v1.6.0 h1:9VEQWz6LLMUsUl6PueE49ir4Ka6CzLymOAZDxpFsTDc= github.com/tidwall/gjson v1.6.0/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= @@ -580,8 +548,6 @@ go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.uber.org/atomic v1.3.0 h1:vs7fgriifsPbGdK3bNuMWapNn3qnZhCRXc19NRdq010= -go.uber.org/atomic v1.3.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= @@ -592,8 +558,6 @@ golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613 h1:MQ/ZZiDsUapFFiMS+vzwXkCTeEKaum+Do5rINYJDmxc= -golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= @@ -604,8 +568,6 @@ golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d h1:2+ZP7EfsZV7Vvmx3TIqSlSzATMkTAKqM14YGFPoSKjI= -golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d h1:1ZiEyfaQIg3Qh0EoqpwAakHVhecoE5wlSg5GjnafJGw= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -620,8 +582,6 @@ golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190301231341-16b79f2e4e95 h1:fY7Dsw114eJN4boqzVSbpVHO6rTdhq6/GnXeu+PKnzU= -golang.org/x/net v0.0.0-20190301231341-16b79f2e4e95/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -655,7 +615,6 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 h1:ywK/j/KkyTHcdyYSZNXGjMwgmDSfjglYZ3vStQ/gSCU= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -684,8 +643,6 @@ google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -gopkg.in/Shopify/sarama.v1 v1.20.1 h1:Gi09A3fJXm0Jgt8kuKZ8YK+r60GfYn7MQuEmI3oq6hE= -gopkg.in/Shopify/sarama.v1 v1.20.1/go.mod h1:AxnvoaevB2nBjNK17cG61A3LleFcWFwVBHBt+cot4Oc= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= From 8b3100935ccc2efec67b26251176c2f8b6ad2be2 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 19 May 2020 12:09:52 +0100 Subject: [PATCH 073/200] dendritejs: Persist ed25519 keys in localforage (#1046) --- cmd/dendritejs/main.go | 81 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 80 insertions(+), 1 deletion(-) diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 6339bfbca..618ebbc60 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -20,6 +20,7 @@ import ( "crypto/ed25519" "fmt" "net/http" + "syscall/js" "github.com/matrix-org/dendrite/appservice" "github.com/matrix-org/dendrite/clientapi" @@ -49,11 +50,89 @@ func init() { fmt.Println("dendrite.js starting...") } +const keyNameEd25519 = "_go_ed25519_key" + +func readKeyFromLocalStorage() (key ed25519.PrivateKey, err error) { + localforage := js.Global().Get("localforage") + if !localforage.Truthy() { + err = fmt.Errorf("readKeyFromLocalStorage: no localforage") + return + } + // https://localforage.github.io/localForage/ + item, ok := await(localforage.Call("getItem", keyNameEd25519)) + if !ok || !item.Truthy() { + err = fmt.Errorf("readKeyFromLocalStorage: no key in localforage") + return + } + fmt.Println("Found key in localforage") + // extract []byte and make an ed25519 key + seed := make([]byte, 32, 32) + js.CopyBytesToGo(seed, item) + + return ed25519.NewKeyFromSeed(seed), nil +} + +func writeKeyToLocalStorage(key ed25519.PrivateKey) error { + localforage := js.Global().Get("localforage") + if !localforage.Truthy() { + return fmt.Errorf("writeKeyToLocalStorage: no localforage") + } + + // make a Uint8Array from the key's seed + seed := key.Seed() + jsSeed := js.Global().Get("Uint8Array").New(len(seed)) + js.CopyBytesToJS(jsSeed, seed) + // write it + localforage.Call("setItem", keyNameEd25519, jsSeed) + return nil +} + +// taken from https://go-review.googlesource.com/c/go/+/150917 + +// await waits until the promise v has been resolved or rejected and returns the promise's result value. +// The boolean value ok is true if the promise has been resolved, false if it has been rejected. +// If v is not a promise, v itself is returned as the value and ok is true. +func await(v js.Value) (result js.Value, ok bool) { + if v.Type() != js.TypeObject || v.Get("then").Type() != js.TypeFunction { + return v, true + } + done := make(chan struct{}) + onResolve := js.FuncOf(func(this js.Value, args []js.Value) interface{} { + result = args[0] + ok = true + close(done) + return nil + }) + defer onResolve.Release() + onReject := js.FuncOf(func(this js.Value, args []js.Value) interface{} { + result = args[0] + ok = false + close(done) + return nil + }) + defer onReject.Release() + v.Call("then", onResolve, onReject) + <-done + return +} + func generateKey() ed25519.PrivateKey { - _, priv, err := ed25519.GenerateKey(nil) + // attempt to look for a seed in JS-land and if it exists use it. + priv, err := readKeyFromLocalStorage() + if err == nil { + fmt.Println("Read key from localStorage") + return priv + } + // generate a new key + fmt.Println(err, " : Generating new ed25519 key") + _, priv, err = ed25519.GenerateKey(nil) if err != nil { logrus.Fatalf("Failed to generate ed25519 key: %s", err) } + if err := writeKeyToLocalStorage(priv); err != nil { + fmt.Println("failed to write key to localStorage: ", err) + // non-fatal, we'll just have amnesia for a while + } return priv } From 5faecdac828592b6cf9d2851e13af8fb6d1df30a Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 19 May 2020 16:42:30 +0100 Subject: [PATCH 074/200] Bake in git commit into dendritejs binary (#1048) --- build-dendritejs.sh | 4 ++++ cmd/dendritejs/main.go | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100755 build-dendritejs.sh diff --git a/build-dendritejs.sh b/build-dendritejs.sh new file mode 100755 index 000000000..cd42a6bee --- /dev/null +++ b/build-dendritejs.sh @@ -0,0 +1,4 @@ +#!/bin/bash -eu + +export GIT_COMMIT=$(git rev-list -1 HEAD) && \ +GOOS=js GOARCH=wasm go build -ldflags "-X main.GitCommit=$GIT_COMMIT" -o main.wasm ./cmd/dendritejs \ No newline at end of file diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 618ebbc60..5bdf18efa 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -46,8 +46,10 @@ import ( _ "github.com/matrix-org/go-sqlite3-js" ) +var GitCommit string + func init() { - fmt.Println("dendrite.js starting...") + fmt.Printf("[%s] dendrite.js starting...\n", GitCommit) } const keyNameEd25519 = "_go_ed25519_key" From 260e69d138fb82003b0a7e77aeb5cac6b3281777 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 19 May 2020 18:42:55 +0100 Subject: [PATCH 075/200] Make "Outbound federation can backfill events" pass sytest (#1049) - Use a backfill limit of 100 regardless of what was asked. - Special case the create event for `StateIDsBeforeEvent` - Trim to the limit in `syncapi` --- roomserver/internal/query.go | 7 ++++++- roomserver/internal/query_backfill.go | 6 ++++++ syncapi/routing/messages.go | 9 ++++++++- sytest-whitelist | 1 + 4 files changed, 21 insertions(+), 2 deletions(-) diff --git a/roomserver/internal/query.go b/roomserver/internal/query.go index 8a8c8e7d0..8043746ca 100644 --- a/roomserver/internal/query.go +++ b/roomserver/internal/query.go @@ -535,9 +535,14 @@ func (r *RoomserverInternalAPI) backfillViaFederation(ctx context.Context, req * return fmt.Errorf("backfillViaFederation: unknown room version for room %s : %w", req.RoomID, err) } requester := newBackfillRequester(r.DB, r.FedClient, r.ServerName) + // Request 100 items regardless of what the query asks for. + // We don't want to go much higher than this. + // We can't honour exactly the limit as some sytests rely on requesting more for tests to pass + // (so we don't need to hit /state_ids which the test has no listener for) + // Specifically the test "Outbound federation can backfill events" events, err := gomatrixserverlib.RequestBackfill( ctx, requester, - r.KeyRing, req.RoomID, roomVer, req.EarliestEventsIDs, req.Limit) + r.KeyRing, req.RoomID, roomVer, req.EarliestEventsIDs, 100) if err != nil { return err } diff --git a/roomserver/internal/query_backfill.go b/roomserver/internal/query_backfill.go index d42038e74..82f7238d7 100644 --- a/roomserver/internal/query_backfill.go +++ b/roomserver/internal/query_backfill.go @@ -7,6 +7,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" "github.com/sirupsen/logrus" ) @@ -37,6 +38,11 @@ func (b *backfillRequester) StateIDsBeforeEvent(ctx context.Context, targetEvent if ids, ok := b.eventIDToBeforeStateIDs[targetEvent.EventID()]; ok { return ids, nil } + if len(targetEvent.PrevEventIDs()) == 0 && targetEvent.Type() == "m.room.create" && targetEvent.StateKeyEquals("") { + util.GetLogger(ctx).WithField("room_id", targetEvent.RoomID()).Info("Backfilled to the beginning of the room") + b.eventIDToBeforeStateIDs[targetEvent.EventID()] = []string{} + return nil, nil + } // if we have exactly 1 prev event and we know the state of the room at that prev event, then just roll forward the prev event. // Else, we have to hit /state_ids because either we don't know the state at all at this event (new backwards extremity) or // we don't know the result of state res to merge forks (2 or more prev_events) diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index 67de6df7e..811188cc8 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -412,7 +412,14 @@ func (r *messagesReq) backfill(roomID string, fromEventIDs []string, limit int) } } - return res.Events, nil + // we may have got more than the requested limit so resize now + events := res.Events + if len(events) > limit { + // last `limit` events + events = events[len(events)-limit:] + } + + return events, nil } // setToDefault returns the default value for the "to" query parameter of a diff --git a/sytest-whitelist b/sytest-whitelist index 035b9b36e..4a8af13a6 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -279,3 +279,4 @@ Inbound federation can return missing events for invite visibility Inbound federation can get public room list An event which redacts itself should be ignored A pair of events which redact each other should be ignored +Outbound federation can backfill events From 1414922026636e71ba61acc5c243e9a6cf914981 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Wed, 20 May 2020 16:04:31 +0100 Subject: [PATCH 076/200] sytest: Make 'Inbound federation can backfill events' pass (#1051) * sytest: Make 'Inbound federation can backfill events' pass This breaks 'Outbound federation can backfill events' because now we are returning the right number of events, which the previous test was relying on. Previously, /messages was backfilling the membership event, causing the test to pass. Now we are no longer backfilling the membership event due to the change in this commit, causing the test to fail. The test should instead be returning the membership event locally from synacpis database, but it doesn't do it fast enough, resulting in a no-op /sync response with a next_batch=s0_0 which will never pick up the local membership event when it rolls in. The test does attempt to retry, but doesn't take the new next_batch=s1_0 resulting in it missing from the /messages response. * Linting --- federationapi/routing/backfill.go | 17 +++++++-- roomserver/api/query.go | 15 ++++++-- roomserver/internal/query.go | 14 +++----- roomserver/internal/query_backfill.go | 35 +++++++++++++++---- syncapi/routing/messages.go | 11 +++--- syncapi/storage/interface.go | 5 ++- .../postgres/backwards_extremities_table.go | 13 +++---- syncapi/storage/shared/syncserver.go | 3 +- .../sqlite3/backwards_extremities_table.go | 13 +++---- syncapi/storage/tables/interface.go | 4 +-- sytest-whitelist | 1 + 11 files changed, 86 insertions(+), 45 deletions(-) diff --git a/federationapi/routing/backfill.go b/federationapi/routing/backfill.go index 651a4a2d2..1f46b240f 100644 --- a/federationapi/routing/backfill.go +++ b/federationapi/routing/backfill.go @@ -69,9 +69,14 @@ func Backfill( // Populate the request. req := api.QueryBackfillRequest{ - RoomID: roomID, - EarliestEventsIDs: eIDs, - ServerName: request.Origin(), + RoomID: roomID, + // we don't know who the successors are for these events, which won't + // be a problem because we don't use that information when servicing /backfill requests, + // only when making them. TODO: Think of a better API shape + BackwardsExtremities: map[string][]string{ + "": eIDs, + }, + ServerName: request.Origin(), } if req.Limit, err = strconv.Atoi(limit); err != nil { util.GetLogger(httpReq.Context()).WithError(err).Error("strconv.Atoi failed") @@ -105,6 +110,12 @@ func Backfill( eventJSONs = append(eventJSONs, e.JSON()) } + // sytest wants these in reversed order, similar to /messages, so reverse them now. + for i := len(eventJSONs)/2 - 1; i >= 0; i-- { + opp := len(eventJSONs) - 1 - i + eventJSONs[i], eventJSONs[opp] = eventJSONs[opp], eventJSONs[i] + } + txn := gomatrixserverlib.Transaction{ Origin: cfg.Matrix.ServerName, PDUs: eventJSONs, diff --git a/roomserver/api/query.go b/roomserver/api/query.go index 9afc51f4e..e92d2a992 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -21,6 +21,7 @@ import ( commonHTTP "github.com/matrix-org/dendrite/common/http" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" opentracing "github.com/opentracing/opentracing-go" ) @@ -228,14 +229,24 @@ type QueryStateAndAuthChainResponse struct { type QueryBackfillRequest struct { // The room to backfill RoomID string `json:"room_id"` - // Events to start paginating from. - EarliestEventsIDs []string `json:"earliest_event_ids"` + // A map of backwards extremity event ID to a list of its prev_event IDs. + BackwardsExtremities map[string][]string `json:"backwards_extremities"` // The maximum number of events to retrieve. Limit int `json:"limit"` // The server interested in the events. ServerName gomatrixserverlib.ServerName `json:"server_name"` } +// PrevEventIDs returns the prev_event IDs of all backwards extremities, de-duplicated in a lexicographically sorted order. +func (r *QueryBackfillRequest) PrevEventIDs() []string { + var prevEventIDs []string + for _, pes := range r.BackwardsExtremities { + prevEventIDs = append(prevEventIDs, pes...) + } + prevEventIDs = util.UniqueStrings(prevEventIDs) + return prevEventIDs +} + // QueryBackfillResponse is a response to QueryBackfill. type QueryBackfillResponse struct { // Missing events, arbritrary order. diff --git a/roomserver/internal/query.go b/roomserver/internal/query.go index 8043746ca..2d1c21c57 100644 --- a/roomserver/internal/query.go +++ b/roomserver/internal/query.go @@ -495,14 +495,8 @@ func (r *RoomserverInternalAPI) QueryBackfill( // defines the highest number of elements in the map below. visited := make(map[string]bool, request.Limit) - // The provided event IDs have already been seen by the request's emitter, - // and will be retrieved anyway, so there's no need to care about them if - // they appear in our exploration of the event tree. - for _, id := range request.EarliestEventsIDs { - visited[id] = true - } - - front = request.EarliestEventsIDs + // this will include these events which is what we want + front = request.PrevEventIDs() // Scan the event tree for events to send back. resultNIDs, err := r.scanEventTree(ctx, front, visited, request.Limit, request.ServerName) @@ -534,7 +528,7 @@ func (r *RoomserverInternalAPI) backfillViaFederation(ctx context.Context, req * if err != nil { return fmt.Errorf("backfillViaFederation: unknown room version for room %s : %w", req.RoomID, err) } - requester := newBackfillRequester(r.DB, r.FedClient, r.ServerName) + requester := newBackfillRequester(r.DB, r.FedClient, r.ServerName, req.BackwardsExtremities) // Request 100 items regardless of what the query asks for. // We don't want to go much higher than this. // We can't honour exactly the limit as some sytests rely on requesting more for tests to pass @@ -542,7 +536,7 @@ func (r *RoomserverInternalAPI) backfillViaFederation(ctx context.Context, req * // Specifically the test "Outbound federation can backfill events" events, err := gomatrixserverlib.RequestBackfill( ctx, requester, - r.KeyRing, req.RoomID, roomVer, req.EarliestEventsIDs, 100) + r.KeyRing, req.RoomID, roomVer, req.PrevEventIDs(), 100) if err != nil { return err } diff --git a/roomserver/internal/query_backfill.go b/roomserver/internal/query_backfill.go index 82f7238d7..49e0af34a 100644 --- a/roomserver/internal/query_backfill.go +++ b/roomserver/internal/query_backfill.go @@ -16,6 +16,7 @@ type backfillRequester struct { db storage.Database fedClient *gomatrixserverlib.FederationClient thisServer gomatrixserverlib.ServerName + bwExtrems map[string][]string // per-request state servers []gomatrixserverlib.ServerName @@ -23,13 +24,14 @@ type backfillRequester struct { eventIDMap map[string]gomatrixserverlib.Event } -func newBackfillRequester(db storage.Database, fedClient *gomatrixserverlib.FederationClient, thisServer gomatrixserverlib.ServerName) *backfillRequester { +func newBackfillRequester(db storage.Database, fedClient *gomatrixserverlib.FederationClient, thisServer gomatrixserverlib.ServerName, bwExtrems map[string][]string) *backfillRequester { return &backfillRequester{ db: db, fedClient: fedClient, thisServer: thisServer, eventIDToBeforeStateIDs: make(map[string][]string), eventIDMap: make(map[string]gomatrixserverlib.Event), + bwExtrems: bwExtrems, } } @@ -160,26 +162,44 @@ func (b *backfillRequester) StateBeforeEvent(ctx context.Context, roomVer gomatr // It returns a list of servers which can be queried for backfill requests. These servers // will be servers that are in the room already. The entries at the beginning are preferred servers // and will be tried first. An empty list will fail the request. -func (b *backfillRequester) ServersAtEvent(ctx context.Context, roomID, eventID string) (servers []gomatrixserverlib.ServerName) { +func (b *backfillRequester) ServersAtEvent(ctx context.Context, roomID, eventID string) []gomatrixserverlib.ServerName { + // eventID will be a prev_event ID of a backwards extremity, meaning we will not have a database entry for it. Instead, use + // its successor, so look it up. + successor := "" +FindSuccessor: + for sucID, prevEventIDs := range b.bwExtrems { + for _, pe := range prevEventIDs { + if pe == eventID { + successor = sucID + break FindSuccessor + } + } + } + if successor == "" { + logrus.WithField("event_id", eventID).Error("ServersAtEvent: failed to find successor of this event to determine room state") + return nil + } + eventID = successor + // getMembershipsBeforeEventNID requires a NID, so retrieving the NID for // the event is necessary. NIDs, err := b.db.EventNIDs(ctx, []string{eventID}) if err != nil { logrus.WithField("event_id", eventID).WithError(err).Error("ServersAtEvent: failed to get event NID for event") - return + return nil } stateEntries, err := stateBeforeEvent(ctx, b.db, NIDs[eventID]) if err != nil { logrus.WithField("event_id", eventID).WithError(err).Error("ServersAtEvent: failed to load state before event") - return + return nil } // possibly return all joined servers depending on history visiblity memberEventsFromVis, err := joinEventsFromHistoryVisibility(ctx, b.db, roomID, stateEntries) if err != nil { logrus.WithError(err).Error("ServersAtEvent: failed calculate servers from history visibility rules") - return + return nil } logrus.Infof("ServersAtEvent including %d current events from history visibility", len(memberEventsFromVis)) @@ -189,7 +209,7 @@ func (b *backfillRequester) ServersAtEvent(ctx context.Context, roomID, eventID memberEvents, err := getMembershipsAtState(ctx, b.db, stateEntries, true) if err != nil { logrus.WithField("event_id", eventID).WithError(err).Error("ServersAtEvent: failed to get memberships before event") - return + return nil } memberEvents = append(memberEvents, memberEventsFromVis...) @@ -198,6 +218,7 @@ func (b *backfillRequester) ServersAtEvent(ctx context.Context, roomID, eventID for _, event := range memberEvents { serverSet[event.Origin()] = true } + var servers []gomatrixserverlib.ServerName for server := range serverSet { if server == b.thisServer { continue @@ -205,7 +226,7 @@ func (b *backfillRequester) ServersAtEvent(ctx context.Context, roomID, eventID servers = append(servers, server) } b.servers = servers - return + return servers } // Backfill performs a backfill request to the given server. diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index 811188cc8..88e16fe57 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -205,6 +205,7 @@ func (r *messagesReq) retrieveEvents() ( } var events []gomatrixserverlib.HeaderedEvent + util.GetLogger(r.ctx).WithField("start", start).WithField("end", end).Infof("Fetched %d events locally", len(streamEvents)) // There can be two reasons for streamEvents to be empty: either we've // reached the oldest event in the room (or the most recent one, depending @@ -373,13 +374,13 @@ func (e eventsByDepth) Less(i, j int) bool { // event, or if there is no remote homeserver to contact. // Returns an error if there was an issue with retrieving the list of servers in // the room or sending the request. -func (r *messagesReq) backfill(roomID string, fromEventIDs []string, limit int) ([]gomatrixserverlib.HeaderedEvent, error) { +func (r *messagesReq) backfill(roomID string, backwardsExtremities map[string][]string, limit int) ([]gomatrixserverlib.HeaderedEvent, error) { var res api.QueryBackfillResponse err := r.rsAPI.QueryBackfill(context.Background(), &api.QueryBackfillRequest{ - RoomID: roomID, - EarliestEventsIDs: fromEventIDs, - Limit: limit, - ServerName: r.cfg.Matrix.ServerName, + RoomID: roomID, + BackwardsExtremities: backwardsExtremities, + Limit: limit, + ServerName: r.cfg.Matrix.ServerName, }, &res) if err != nil { return nil, fmt.Errorf("QueryBackfill failed: %w", err) diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index eba008b34..121e94c71 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -94,9 +94,8 @@ type Database interface { GetEventsInTopologicalRange(ctx context.Context, from, to *types.TopologyToken, roomID string, limit int, backwardOrdering bool) (events []types.StreamEvent, err error) // EventPositionInTopology returns the depth and stream position of the given event. EventPositionInTopology(ctx context.Context, eventID string) (types.TopologyToken, error) - // BackwardExtremitiesForRoom returns the event IDs of all of the backward - // extremities we know of for a given room. - BackwardExtremitiesForRoom(ctx context.Context, roomID string) (backwardExtremities []string, err error) + // BackwardExtremitiesForRoom returns a map of backwards extremity event ID to a list of its prev_events. + BackwardExtremitiesForRoom(ctx context.Context, roomID string) (backwardExtremities map[string][]string, err error) // MaxTopologicalPosition returns the highest topological position for a given room. MaxTopologicalPosition(ctx context.Context, roomID string) (types.TopologyToken, error) // StreamEventsToEvents converts streamEvent to Event. If device is non-nil and diff --git a/syncapi/storage/postgres/backwards_extremities_table.go b/syncapi/storage/postgres/backwards_extremities_table.go index 3dd1b2349..fa498a401 100644 --- a/syncapi/storage/postgres/backwards_extremities_table.go +++ b/syncapi/storage/postgres/backwards_extremities_table.go @@ -41,7 +41,7 @@ const insertBackwardExtremitySQL = "" + " ON CONFLICT DO NOTHING" const selectBackwardExtremitiesForRoomSQL = "" + - "SELECT DISTINCT event_id FROM syncapi_backward_extremities WHERE room_id = $1" + "SELECT event_id, prev_event_id FROM syncapi_backward_extremities WHERE room_id = $1" const deleteBackwardExtremitySQL = "" + "DELETE FROM syncapi_backward_extremities WHERE room_id = $1 AND prev_event_id = $2" @@ -79,23 +79,24 @@ func (s *backwardExtremitiesStatements) InsertsBackwardExtremity( func (s *backwardExtremitiesStatements) SelectBackwardExtremitiesForRoom( ctx context.Context, roomID string, -) (eventIDs []string, err error) { +) (bwExtrems map[string][]string, err error) { rows, err := s.selectBackwardExtremitiesForRoomStmt.QueryContext(ctx, roomID) if err != nil { return } defer common.CloseAndLogIfError(ctx, rows, "selectBackwardExtremitiesForRoom: rows.close() failed") + bwExtrems = make(map[string][]string) for rows.Next() { var eID string - if err = rows.Scan(&eID); err != nil { + var prevEventID string + if err = rows.Scan(&eID, &prevEventID); err != nil { return } - - eventIDs = append(eventIDs, eID) + bwExtrems[eID] = append(bwExtrems[eID], prevEventID) } - return eventIDs, rows.Err() + return bwExtrems, rows.Err() } func (s *backwardExtremitiesStatements) DeleteBackwardExtremity( diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go index be1f9c7aa..543e5b4a3 100644 --- a/syncapi/storage/shared/syncserver.go +++ b/syncapi/storage/shared/syncserver.go @@ -310,6 +310,7 @@ func (d *Database) updateRoomState( } membership = &value } + if err := d.CurrentRoomState.UpsertRoomState(ctx, txn, event, membership, pduPosition); err != nil { return err } @@ -367,7 +368,7 @@ func (d *Database) SyncPosition(ctx context.Context) (tok types.StreamingToken, func (d *Database) BackwardExtremitiesForRoom( ctx context.Context, roomID string, -) (backwardExtremities []string, err error) { +) (backwardExtremities map[string][]string, err error) { return d.BackwardExtremities.SelectBackwardExtremitiesForRoom(ctx, roomID) } diff --git a/syncapi/storage/sqlite3/backwards_extremities_table.go b/syncapi/storage/sqlite3/backwards_extremities_table.go index a172f6bf7..c5d9cae59 100644 --- a/syncapi/storage/sqlite3/backwards_extremities_table.go +++ b/syncapi/storage/sqlite3/backwards_extremities_table.go @@ -41,7 +41,7 @@ const insertBackwardExtremitySQL = "" + " ON CONFLICT (room_id, event_id, prev_event_id) DO NOTHING" const selectBackwardExtremitiesForRoomSQL = "" + - "SELECT DISTINCT event_id FROM syncapi_backward_extremities WHERE room_id = $1" + "SELECT event_id, prev_event_id FROM syncapi_backward_extremities WHERE room_id = $1" const deleteBackwardExtremitySQL = "" + "DELETE FROM syncapi_backward_extremities WHERE room_id = $1 AND prev_event_id = $2" @@ -79,23 +79,24 @@ func (s *backwardExtremitiesStatements) InsertsBackwardExtremity( func (s *backwardExtremitiesStatements) SelectBackwardExtremitiesForRoom( ctx context.Context, roomID string, -) (eventIDs []string, err error) { +) (bwExtrems map[string][]string, err error) { rows, err := s.selectBackwardExtremitiesForRoomStmt.QueryContext(ctx, roomID) if err != nil { return } defer common.CloseAndLogIfError(ctx, rows, "selectBackwardExtremitiesForRoom: rows.close() failed") + bwExtrems = make(map[string][]string) for rows.Next() { var eID string - if err = rows.Scan(&eID); err != nil { + var prevEventID string + if err = rows.Scan(&eID, &prevEventID); err != nil { return } - - eventIDs = append(eventIDs, eID) + bwExtrems[eID] = append(bwExtrems[eID], prevEventID) } - return eventIDs, rows.Err() + return bwExtrems, rows.Err() } func (s *backwardExtremitiesStatements) DeleteBackwardExtremity( diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go index f31bdc2e9..bc3b6941a 100644 --- a/syncapi/storage/tables/interface.go +++ b/syncapi/storage/tables/interface.go @@ -89,8 +89,8 @@ type CurrentRoomState interface { type BackwardsExtremities interface { // InsertsBackwardExtremity inserts a new backwards extremity. InsertsBackwardExtremity(ctx context.Context, txn *sql.Tx, roomID, eventID string, prevEventID string) (err error) - // SelectBackwardExtremitiesForRoom retrieves all backwards extremities for the room. - SelectBackwardExtremitiesForRoom(ctx context.Context, roomID string) (eventIDs []string, err error) + // SelectBackwardExtremitiesForRoom retrieves all backwards extremities for the room, as a map of event_id to list of prev_event_ids. + SelectBackwardExtremitiesForRoom(ctx context.Context, roomID string) (bwExtrems map[string][]string, err error) // DeleteBackwardExtremity removes a backwards extremity for a room, if one existed. DeleteBackwardExtremity(ctx context.Context, txn *sql.Tx, roomID, knownEventID string) (err error) } diff --git a/sytest-whitelist b/sytest-whitelist index 4a8af13a6..2e8205dcd 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -280,3 +280,4 @@ Inbound federation can get public room list An event which redacts itself should be ignored A pair of events which redact each other should be ignored Outbound federation can backfill events +Inbound federation can backfill events From 6091bf044ff41e9f248b1077d5b05f1a4c694412 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Wed, 20 May 2020 17:30:03 +0100 Subject: [PATCH 077/200] sytest: Add remaining backfill tests (#1052) One failed because of `null` instead of `[]` in HTTP responses. One failed because we hadn't implemented in-line filter limits! --- federationapi/routing/backfill.go | 2 +- syncapi/sync/request.go | 22 +++++++++++++++++++++- syncapi/sync/requestpool.go | 1 + sytest-whitelist | 2 ++ 4 files changed, 25 insertions(+), 2 deletions(-) diff --git a/federationapi/routing/backfill.go b/federationapi/routing/backfill.go index 1f46b240f..0d3893de3 100644 --- a/federationapi/routing/backfill.go +++ b/federationapi/routing/backfill.go @@ -102,7 +102,7 @@ func Backfill( } } - var eventJSONs []json.RawMessage + eventJSONs := []json.RawMessage{} for _, e := range gomatrixserverlib.ReverseTopologicalOrdering( evs, gomatrixserverlib.TopologicalOrderByPrevEvents, diff --git a/syncapi/sync/request.go b/syncapi/sync/request.go index 66663cf0a..c7796b561 100644 --- a/syncapi/sync/request.go +++ b/syncapi/sync/request.go @@ -16,6 +16,7 @@ package sync import ( "context" + "encoding/json" "net/http" "strconv" "time" @@ -30,6 +31,14 @@ import ( const defaultSyncTimeout = time.Duration(0) const defaultTimelineLimit = 20 +type filter struct { + Room struct { + Timeline struct { + Limit *int `json:"limit"` + } `json:"timeline"` + } `json:"room"` +} + // syncRequest represents a /sync request, with sensible defaults/sanity checks applied. type syncRequest struct { ctx context.Context @@ -54,6 +63,17 @@ func newSyncRequest(req *http.Request, device authtypes.Device) (*syncRequest, e } since = &tok } + timelineLimit := defaultTimelineLimit + // TODO: read from stored filters too + filterQuery := req.URL.Query().Get("filter") + if filterQuery != "" && filterQuery[0] == '{' { + // attempt to parse the timeline limit at least + var f filter + err := json.Unmarshal([]byte(filterQuery), &f) + if err == nil && f.Room.Timeline.Limit != nil { + timelineLimit = *f.Room.Timeline.Limit + } + } // TODO: Additional query params: set_presence, filter return &syncRequest{ ctx: req.Context(), @@ -61,7 +81,7 @@ func newSyncRequest(req *http.Request, device authtypes.Device) (*syncRequest, e timeout: timeout, since: since, wantFullState: wantFullState, - limit: defaultTimelineLimit, // TODO: read from filter + limit: timelineLimit, log: util.GetLogger(req.Context()), }, nil } diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index 6e0f44e9a..82ce283b6 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -59,6 +59,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *authtype "userID": userID, "since": syncReq.since, "timeout": syncReq.timeout, + "limit": syncReq.limit, }) currPos := rp.notifier.CurrentPosition() diff --git a/sytest-whitelist b/sytest-whitelist index 2e8205dcd..d5b4feaf6 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -281,3 +281,5 @@ An event which redacts itself should be ignored A pair of events which redact each other should be ignored Outbound federation can backfill events Inbound federation can backfill events +Backfill checks the events requested belong to the room +Backfilled events whose prev_events are in a different room do not allow cross-room back-pagination From f2c07437fe3d7f54977f7e645cd045a04e832020 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 20 May 2020 18:03:06 +0100 Subject: [PATCH 078/200] Use memberships to determine whether to reset latest events/state on room join (#1047) * Track local/remote memberships, re-scope some input stuff * Check if we're in the room already before resetting latest events/state * Fix postgres, fix lint * Review comments --- roomserver/internal/input.go | 4 +- roomserver/internal/input_events.go | 53 ++++++++--------- roomserver/internal/input_latest_events.go | 23 +++----- roomserver/internal/input_membership.go | 24 +++++--- roomserver/internal/query.go | 4 +- roomserver/internal/query_backfill.go | 2 +- roomserver/storage/interface.go | 4 +- .../storage/postgres/membership_table.go | 59 ++++++++++++++----- roomserver/storage/postgres/storage.go | 17 +++--- .../storage/sqlite3/membership_table.go | 53 +++++++++++++---- roomserver/storage/sqlite3/storage.go | 17 +++--- roomserver/types/types.go | 2 +- 12 files changed, 162 insertions(+), 100 deletions(-) diff --git a/roomserver/internal/input.go b/roomserver/internal/input.go index ab3d7516b..932b4df46 100644 --- a/roomserver/internal/input.go +++ b/roomserver/internal/input.go @@ -60,7 +60,7 @@ func (r *RoomserverInternalAPI) InputRoomEvents( defer r.mutex.Unlock() for i := range request.InputInviteEvents { var loopback *api.InputRoomEvent - if loopback, err = processInviteEvent(ctx, r.DB, r, request.InputInviteEvents[i]); err != nil { + if loopback, err = r.processInviteEvent(ctx, r, request.InputInviteEvents[i]); err != nil { return err } // The processInviteEvent function can optionally return a @@ -71,7 +71,7 @@ func (r *RoomserverInternalAPI) InputRoomEvents( } } for i := range request.InputRoomEvents { - if response.EventID, err = processRoomEvent(ctx, r.DB, r, request.InputRoomEvents[i]); err != nil { + if response.EventID, err = r.processRoomEvent(ctx, request.InputRoomEvents[i]); err != nil { return err } } diff --git a/roomserver/internal/input_events.go b/roomserver/internal/input_events.go index f5c678ca6..a4167714d 100644 --- a/roomserver/internal/input_events.go +++ b/roomserver/internal/input_events.go @@ -31,21 +31,13 @@ import ( log "github.com/sirupsen/logrus" ) -// OutputRoomEventWriter has the APIs needed to write an event to the output logs. -type OutputRoomEventWriter interface { - // Write a list of events for a room - WriteOutputEvents(roomID string, updates []api.OutputEvent) error -} - // processRoomEvent can only be called once at a time // // TODO(#375): This should be rewritten to allow concurrent calls. The // difficulty is in ensuring that we correctly annotate events with the correct // state deltas when sending to kafka streams -func processRoomEvent( +func (r *RoomserverInternalAPI) processRoomEvent( ctx context.Context, - db storage.Database, - ow OutputRoomEventWriter, input api.InputRoomEvent, ) (eventID string, err error) { // Parse and validate the event JSON @@ -54,7 +46,7 @@ func processRoomEvent( // Check that the event passes authentication checks and work out // the numeric IDs for the auth events. - authEventNIDs, err := checkAuthEvents(ctx, db, headered, input.AuthEventIDs) + authEventNIDs, err := checkAuthEvents(ctx, r.DB, headered, input.AuthEventIDs) if err != nil { logrus.WithError(err).WithField("event_id", event.EventID()).WithField("auth_event_ids", input.AuthEventIDs).Error("processRoomEvent.checkAuthEvents failed for event") return @@ -63,7 +55,7 @@ func processRoomEvent( // If we don't have a transaction ID then get one. if input.TransactionID != nil { tdID := input.TransactionID - eventID, err = db.GetTransactionEventID( + eventID, err = r.DB.GetTransactionEventID( ctx, tdID.TransactionID, tdID.SessionID, event.Sender(), ) // On error OR event with the transaction already processed/processesing @@ -73,7 +65,7 @@ func processRoomEvent( } // Store the event. - roomNID, stateAtEvent, err := db.StoreEvent(ctx, event, input.TransactionID, authEventNIDs) + roomNID, stateAtEvent, err := r.DB.StoreEvent(ctx, event, input.TransactionID, authEventNIDs) if err != nil { return } @@ -93,16 +85,14 @@ func processRoomEvent( if stateAtEvent.BeforeStateSnapshotNID == 0 { // We haven't calculated a state for this event yet. // Lets calculate one. - err = calculateAndSetState(ctx, db, input, roomNID, &stateAtEvent, event) + err = r.calculateAndSetState(ctx, input, roomNID, &stateAtEvent, event) if err != nil { return } } - if err = updateLatestEvents( + if err = r.updateLatestEvents( ctx, // context - db, // roomserver database - ow, // output event writer roomNID, // room NID to update stateAtEvent, // state at event (below) event, // event @@ -116,29 +106,36 @@ func processRoomEvent( return event.EventID(), nil } -func calculateAndSetState( +func (r *RoomserverInternalAPI) calculateAndSetState( ctx context.Context, - db storage.Database, input api.InputRoomEvent, roomNID types.RoomNID, stateAtEvent *types.StateAtEvent, event gomatrixserverlib.Event, ) error { var err error - roomState := state.NewStateResolution(db) + roomState := state.NewStateResolution(r.DB) if input.HasState { - // TODO: Check here if we think we're in the room already. + // Check here if we think we're in the room already. stateAtEvent.Overwrite = true + var joinEventNIDs []types.EventNID + // Request join memberships only for local users only. + if joinEventNIDs, err = r.DB.GetMembershipEventNIDsForRoom(ctx, roomNID, true, true); err == nil { + // If we have no local users that are joined to the room then any state about + // the room that we have is quite possibly out of date. Therefore in that case + // we should overwrite it rather than merge it. + stateAtEvent.Overwrite = len(joinEventNIDs) == 0 + } // We've been told what the state at the event is so we don't need to calculate it. // Check that those state events are in the database and store the state. var entries []types.StateEntry - if entries, err = db.StateEntriesForEventIDs(ctx, input.StateEventIDs); err != nil { + if entries, err = r.DB.StateEntriesForEventIDs(ctx, input.StateEventIDs); err != nil { return err } - if stateAtEvent.BeforeStateSnapshotNID, err = db.AddState(ctx, roomNID, nil, entries); err != nil { + if stateAtEvent.BeforeStateSnapshotNID, err = r.DB.AddState(ctx, roomNID, nil, entries); err != nil { return err } } else { @@ -149,12 +146,11 @@ func calculateAndSetState( return err } } - return db.SetState(ctx, stateAtEvent.EventNID, stateAtEvent.BeforeStateSnapshotNID) + return r.DB.SetState(ctx, stateAtEvent.EventNID, stateAtEvent.BeforeStateSnapshotNID) } -func processInviteEvent( +func (r *RoomserverInternalAPI) processInviteEvent( ctx context.Context, - db storage.Database, ow *RoomserverInternalAPI, input api.InputInviteEvent, ) (*api.InputRoomEvent, error) { @@ -172,7 +168,10 @@ func processInviteEvent( "target_user_id": targetUserID, }).Info("processing invite event") - updater, err := db.MembershipUpdater(ctx, roomID, targetUserID, input.RoomVersion) + _, domain, _ := gomatrixserverlib.SplitID('@', targetUserID) + isTargetLocalUser := domain == r.Cfg.Matrix.ServerName + + updater, err := r.DB.MembershipUpdater(ctx, roomID, targetUserID, isTargetLocalUser, input.RoomVersion) if err != nil { return nil, err } @@ -239,7 +238,7 @@ func processInviteEvent( // up from local data (which is most likely to be if the event came // from the CS API). If we know about the room then we can insert // the invite room state, if we don't then we just fail quietly. - if irs, ierr := buildInviteStrippedState(ctx, db, input); ierr == nil { + if irs, ierr := buildInviteStrippedState(ctx, r.DB, input); ierr == nil { if err = event.SetUnsignedField("invite_room_state", irs); err != nil { return nil, err } diff --git a/roomserver/internal/input_latest_events.go b/roomserver/internal/input_latest_events.go index 6eeeedab0..d7c9a5cb6 100644 --- a/roomserver/internal/input_latest_events.go +++ b/roomserver/internal/input_latest_events.go @@ -23,7 +23,6 @@ import ( "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/state" - "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -46,17 +45,15 @@ import ( // 7 <----- latest // // Can only be called once at a time -func updateLatestEvents( +func (r *RoomserverInternalAPI) updateLatestEvents( ctx context.Context, - db storage.Database, - ow OutputRoomEventWriter, roomNID types.RoomNID, stateAtEvent types.StateAtEvent, event gomatrixserverlib.Event, sendAsServer string, transactionID *api.TransactionID, ) (err error) { - updater, err := db.GetLatestEventsForUpdate(ctx, roomNID) + updater, err := r.DB.GetLatestEventsForUpdate(ctx, roomNID) if err != nil { return } @@ -70,9 +67,8 @@ func updateLatestEvents( u := latestEventsUpdater{ ctx: ctx, - db: db, + api: r, updater: updater, - ow: ow, roomNID: roomNID, stateAtEvent: stateAtEvent, event: event, @@ -94,9 +90,8 @@ func updateLatestEvents( // when there are so many variables to pass around. type latestEventsUpdater struct { ctx context.Context - db storage.Database + api *RoomserverInternalAPI updater types.RoomRecentEventsUpdater - ow OutputRoomEventWriter roomNID types.RoomNID stateAtEvent types.StateAtEvent event gomatrixserverlib.Event @@ -181,7 +176,7 @@ func (u *latestEventsUpdater) doUpdateLatestEvents() error { // If we need to generate any output events then here's where we do it. // TODO: Move this! - updates, err := updateMemberships(u.ctx, u.db, u.updater, u.removed, u.added) + updates, err := u.api.updateMemberships(u.ctx, u.updater, u.removed, u.added) if err != nil { return err } @@ -200,7 +195,7 @@ func (u *latestEventsUpdater) doUpdateLatestEvents() error { // send the event asynchronously but we would need to ensure that 1) the events are written to the log in // the correct order, 2) that pending writes are resent across restarts. In order to avoid writing all the // necessary bookkeeping we'll keep the event sending synchronous for now. - if err = u.ow.WriteOutputEvents(u.event.RoomID(), updates); err != nil { + if err = u.api.WriteOutputEvents(u.event.RoomID(), updates); err != nil { return err } @@ -213,7 +208,7 @@ func (u *latestEventsUpdater) doUpdateLatestEvents() error { func (u *latestEventsUpdater) latestState() error { var err error - roomState := state.NewStateResolution(u.db) + roomState := state.NewStateResolution(u.api.DB) // Get a list of the current latest events. latestStateAtEvents := make([]types.StateAtEvent, len(u.latest)) @@ -303,7 +298,7 @@ func (u *latestEventsUpdater) makeOutputNewRoomEvent() (*api.OutputEvent, error) latestEventIDs[i] = u.latest[i].EventID } - roomVersion, err := u.db.GetRoomVersionForRoom(u.ctx, u.event.RoomID()) + roomVersion, err := u.api.DB.GetRoomVersionForRoom(u.ctx, u.event.RoomID()) if err != nil { return nil, err } @@ -329,7 +324,7 @@ func (u *latestEventsUpdater) makeOutputNewRoomEvent() (*api.OutputEvent, error) stateEventNIDs = append(stateEventNIDs, entry.EventNID) } stateEventNIDs = stateEventNIDs[:util.SortAndUnique(eventNIDSorter(stateEventNIDs))] - eventIDMap, err := u.db.EventIDs(u.ctx, stateEventNIDs) + eventIDMap, err := u.api.DB.EventIDs(u.ctx, stateEventNIDs) if err != nil { return nil, err } diff --git a/roomserver/internal/input_membership.go b/roomserver/internal/input_membership.go index 666e7ebcc..af0c7f8b3 100644 --- a/roomserver/internal/input_membership.go +++ b/roomserver/internal/input_membership.go @@ -19,7 +19,6 @@ import ( "fmt" "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -28,9 +27,8 @@ import ( // user affected by a change in the current state of the room. // Returns a list of output events to write to the kafka log to inform the // consumers about the invites added or retired by the change in current state. -func updateMemberships( +func (r *RoomserverInternalAPI) updateMemberships( ctx context.Context, - db storage.Database, updater types.RoomRecentEventsUpdater, removed, added []types.StateEntry, ) ([]api.OutputEvent, error) { @@ -48,7 +46,7 @@ func updateMemberships( // Load the event JSON so we can look up the "membership" key. // TODO: Maybe add a membership key to the events table so we can load that // key without having to load the entire event JSON? - events, err := db.Events(ctx, eventNIDs) + events, err := r.DB.Events(ctx, eventNIDs) if err != nil { return nil, err } @@ -71,15 +69,16 @@ func updateMemberships( ae = &ev.Event } } - if updates, err = updateMembership(updater, targetUserNID, re, ae, updates); err != nil { + if updates, err = r.updateMembership(updater, targetUserNID, re, ae, updates); err != nil { return nil, err } } return updates, nil } -func updateMembership( - updater types.RoomRecentEventsUpdater, targetUserNID types.EventStateKeyNID, +func (r *RoomserverInternalAPI) updateMembership( + updater types.RoomRecentEventsUpdater, + targetUserNID types.EventStateKeyNID, remove, add *gomatrixserverlib.Event, updates []api.OutputEvent, ) ([]api.OutputEvent, error) { @@ -113,7 +112,7 @@ func updateMembership( return updates, nil } - mu, err := updater.MembershipUpdater(targetUserNID) + mu, err := updater.MembershipUpdater(targetUserNID, r.isLocalTarget(add)) if err != nil { return nil, err } @@ -132,6 +131,15 @@ func updateMembership( } } +func (r *RoomserverInternalAPI) isLocalTarget(event *gomatrixserverlib.Event) bool { + isTargetLocalUser := false + if statekey := event.StateKey(); statekey != nil { + _, domain, _ := gomatrixserverlib.SplitID('@', *statekey) + isTargetLocalUser = domain == r.Cfg.Matrix.ServerName + } + return isTargetLocalUser +} + func updateToInviteMembership( mu types.MembershipUpdater, add *gomatrixserverlib.Event, updates []api.OutputEvent, roomVersion gomatrixserverlib.RoomVersion, diff --git a/roomserver/internal/query.go b/roomserver/internal/query.go index 2d1c21c57..fce2ae907 100644 --- a/roomserver/internal/query.go +++ b/roomserver/internal/query.go @@ -267,7 +267,7 @@ func (r *RoomserverInternalAPI) QueryMembershipsForRoom( var stateEntries []types.StateEntry if stillInRoom { var eventNIDs []types.EventNID - eventNIDs, err = r.DB.GetMembershipEventNIDsForRoom(ctx, roomNID, request.JoinedOnly) + eventNIDs, err = r.DB.GetMembershipEventNIDsForRoom(ctx, roomNID, request.JoinedOnly, false) if err != nil { return err } @@ -591,7 +591,7 @@ func (r *RoomserverInternalAPI) isServerCurrentlyInRoom(ctx context.Context, ser return false, err } - eventNIDs, err := r.DB.GetMembershipEventNIDsForRoom(ctx, roomNID, true) + eventNIDs, err := r.DB.GetMembershipEventNIDsForRoom(ctx, roomNID, true, false) if err != nil { return false, err } diff --git a/roomserver/internal/query_backfill.go b/roomserver/internal/query_backfill.go index 49e0af34a..23ae9455a 100644 --- a/roomserver/internal/query_backfill.go +++ b/roomserver/internal/query_backfill.go @@ -297,7 +297,7 @@ func joinEventsFromHistoryVisibility( if err != nil { return nil, err } - joinEventNIDs, err := db.GetMembershipEventNIDsForRoom(ctx, roomNID, true) + joinEventNIDs, err := db.GetMembershipEventNIDsForRoom(ctx, roomNID, true, false) if err != nil { return nil, err } diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go index fb39eca63..1e0232d20 100644 --- a/roomserver/storage/interface.go +++ b/roomserver/storage/interface.go @@ -83,9 +83,9 @@ type Database interface { GetAliasesForRoomID(ctx context.Context, roomID string) ([]string, error) GetCreatorIDForAlias(ctx context.Context, alias string) (string, error) RemoveRoomAlias(ctx context.Context, alias string) error - MembershipUpdater(ctx context.Context, roomID, targetUserID string, roomVersion gomatrixserverlib.RoomVersion) (types.MembershipUpdater, error) + MembershipUpdater(ctx context.Context, roomID, targetUserID string, targetLocal bool, roomVersion gomatrixserverlib.RoomVersion) (types.MembershipUpdater, error) GetMembership(ctx context.Context, roomNID types.RoomNID, requestSenderUserID string) (membershipEventNID types.EventNID, stillInRoom bool, err error) - GetMembershipEventNIDsForRoom(ctx context.Context, roomNID types.RoomNID, joinOnly bool) ([]types.EventNID, error) + GetMembershipEventNIDsForRoom(ctx context.Context, roomNID types.RoomNID, joinOnly bool, localOnly bool) ([]types.EventNID, error) EventsFromIDs(ctx context.Context, eventIDs []string) ([]types.Event, error) GetRoomVersionForRoom(ctx context.Context, roomID string) (gomatrixserverlib.RoomVersion, error) } diff --git a/roomserver/storage/postgres/membership_table.go b/roomserver/storage/postgres/membership_table.go index 9c8a4c259..820ef4e71 100644 --- a/roomserver/storage/postgres/membership_table.go +++ b/roomserver/storage/postgres/membership_table.go @@ -59,6 +59,10 @@ CREATE TABLE IF NOT EXISTS roomserver_membership ( -- This NID is updated if the join event gets updated (e.g. profile update), -- or if the user leaves/joins the room. event_nid BIGINT NOT NULL DEFAULT 0, + -- Local target is true if the target_nid refers to a local user rather than + -- a federated one. This is an optimisation for resetting state on federated + -- room joins. + target_local BOOLEAN NOT NULL DEFAULT false, UNIQUE (room_nid, target_nid) ); ` @@ -66,8 +70,8 @@ CREATE TABLE IF NOT EXISTS roomserver_membership ( // Insert a row in to membership table so that it can be locked by the // SELECT FOR UPDATE const insertMembershipSQL = "" + - "INSERT INTO roomserver_membership (room_nid, target_nid)" + - " VALUES ($1, $2)" + + "INSERT INTO roomserver_membership (room_nid, target_nid, target_local)" + + " VALUES ($1, $2, $3)" + " ON CONFLICT DO NOTHING" const selectMembershipFromRoomAndTargetSQL = "" + @@ -78,10 +82,20 @@ const selectMembershipsFromRoomAndMembershipSQL = "" + "SELECT event_nid FROM roomserver_membership" + " WHERE room_nid = $1 AND membership_nid = $2" +const selectLocalMembershipsFromRoomAndMembershipSQL = "" + + "SELECT event_nid FROM roomserver_membership" + + " WHERE room_nid = $1 AND membership_nid = $2" + + " AND target_local = true" + const selectMembershipsFromRoomSQL = "" + "SELECT event_nid FROM roomserver_membership" + " WHERE room_nid = $1" +const selectLocalMembershipsFromRoomSQL = "" + + "SELECT event_nid FROM roomserver_membership" + + " WHERE room_nid = $1" + + " AND target_local = true" + const selectMembershipForUpdateSQL = "" + "SELECT membership_nid FROM roomserver_membership" + " WHERE room_nid = $1 AND target_nid = $2 FOR UPDATE" @@ -91,12 +105,14 @@ const updateMembershipSQL = "" + " WHERE room_nid = $1 AND target_nid = $2" type membershipStatements struct { - insertMembershipStmt *sql.Stmt - selectMembershipForUpdateStmt *sql.Stmt - selectMembershipFromRoomAndTargetStmt *sql.Stmt - selectMembershipsFromRoomAndMembershipStmt *sql.Stmt - selectMembershipsFromRoomStmt *sql.Stmt - updateMembershipStmt *sql.Stmt + insertMembershipStmt *sql.Stmt + selectMembershipForUpdateStmt *sql.Stmt + selectMembershipFromRoomAndTargetStmt *sql.Stmt + selectMembershipsFromRoomAndMembershipStmt *sql.Stmt + selectLocalMembershipsFromRoomAndMembershipStmt *sql.Stmt + selectMembershipsFromRoomStmt *sql.Stmt + selectLocalMembershipsFromRoomStmt *sql.Stmt + updateMembershipStmt *sql.Stmt } func (s *membershipStatements) prepare(db *sql.DB) (err error) { @@ -110,7 +126,9 @@ func (s *membershipStatements) prepare(db *sql.DB) (err error) { {&s.selectMembershipForUpdateStmt, selectMembershipForUpdateSQL}, {&s.selectMembershipFromRoomAndTargetStmt, selectMembershipFromRoomAndTargetSQL}, {&s.selectMembershipsFromRoomAndMembershipStmt, selectMembershipsFromRoomAndMembershipSQL}, + {&s.selectLocalMembershipsFromRoomAndMembershipStmt, selectLocalMembershipsFromRoomAndMembershipSQL}, {&s.selectMembershipsFromRoomStmt, selectMembershipsFromRoomSQL}, + {&s.selectLocalMembershipsFromRoomStmt, selectLocalMembershipsFromRoomSQL}, {&s.updateMembershipStmt, updateMembershipSQL}, }.prepare(db) } @@ -118,9 +136,10 @@ func (s *membershipStatements) prepare(db *sql.DB) (err error) { func (s *membershipStatements) insertMembership( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, + localTarget bool, ) error { stmt := common.TxStmt(txn, s.insertMembershipStmt) - _, err := stmt.ExecContext(ctx, roomNID, targetUserNID) + _, err := stmt.ExecContext(ctx, roomNID, targetUserNID, localTarget) return err } @@ -145,9 +164,15 @@ func (s *membershipStatements) selectMembershipFromRoomAndTarget( } func (s *membershipStatements) selectMembershipsFromRoom( - ctx context.Context, roomNID types.RoomNID, + ctx context.Context, roomNID types.RoomNID, localOnly bool, ) (eventNIDs []types.EventNID, err error) { - rows, err := s.selectMembershipsFromRoomStmt.QueryContext(ctx, roomNID) + var stmt *sql.Stmt + if localOnly { + stmt = s.selectLocalMembershipsFromRoomStmt + } else { + stmt = s.selectMembershipsFromRoomStmt + } + rows, err := stmt.QueryContext(ctx, roomNID) if err != nil { return } @@ -165,10 +190,16 @@ func (s *membershipStatements) selectMembershipsFromRoom( func (s *membershipStatements) selectMembershipsFromRoomAndMembership( ctx context.Context, - roomNID types.RoomNID, membership membershipState, + roomNID types.RoomNID, membership membershipState, localOnly bool, ) (eventNIDs []types.EventNID, err error) { - stmt := s.selectMembershipsFromRoomAndMembershipStmt - rows, err := stmt.QueryContext(ctx, roomNID, membership) + var rows *sql.Rows + var stmt *sql.Stmt + if localOnly { + stmt = s.selectLocalMembershipsFromRoomAndMembershipStmt + } else { + stmt = s.selectMembershipsFromRoomAndMembershipStmt + } + rows, err = stmt.QueryContext(ctx, roomNID, membership) if err != nil { return } diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go index 1d825ecc2..d451d6650 100644 --- a/roomserver/storage/postgres/storage.go +++ b/roomserver/storage/postgres/storage.go @@ -459,8 +459,8 @@ func (u *roomRecentEventsUpdater) MarkEventAsSent(eventNID types.EventNID) error return u.d.statements.updateEventSentToOutput(u.ctx, u.txn, eventNID) } -func (u *roomRecentEventsUpdater) MembershipUpdater(targetUserNID types.EventStateKeyNID) (types.MembershipUpdater, error) { - return u.d.membershipUpdaterTxn(u.ctx, u.txn, u.roomNID, targetUserNID) +func (u *roomRecentEventsUpdater) MembershipUpdater(targetUserNID types.EventStateKeyNID, targetLocal bool) (types.MembershipUpdater, error) { + return u.d.membershipUpdaterTxn(u.ctx, u.txn, u.roomNID, targetUserNID, targetLocal) } // RoomNID implements query.RoomserverQueryAPIDB @@ -558,7 +558,7 @@ func (d *Database) StateEntriesForTuples( // MembershipUpdater implements input.RoomEventDatabase func (d *Database) MembershipUpdater( ctx context.Context, roomID, targetUserID string, - roomVersion gomatrixserverlib.RoomVersion, + targetLocal bool, roomVersion gomatrixserverlib.RoomVersion, ) (types.MembershipUpdater, error) { txn, err := d.db.Begin() if err != nil { @@ -581,7 +581,7 @@ func (d *Database) MembershipUpdater( return nil, err } - updater, err := d.membershipUpdaterTxn(ctx, txn, roomNID, targetUserNID) + updater, err := d.membershipUpdaterTxn(ctx, txn, roomNID, targetUserNID, targetLocal) if err != nil { return nil, err } @@ -603,9 +603,10 @@ func (d *Database) membershipUpdaterTxn( txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, + targetLocal bool, ) (types.MembershipUpdater, error) { - if err := d.statements.insertMembership(ctx, txn, roomNID, targetUserNID); err != nil { + if err := d.statements.insertMembership(ctx, txn, roomNID, targetUserNID, targetLocal); err != nil { return nil, err } @@ -748,15 +749,15 @@ func (d *Database) GetMembership( // GetMembershipEventNIDsForRoom implements query.RoomserverQueryAPIDB func (d *Database) GetMembershipEventNIDsForRoom( - ctx context.Context, roomNID types.RoomNID, joinOnly bool, + ctx context.Context, roomNID types.RoomNID, joinOnly bool, localOnly bool, ) ([]types.EventNID, error) { if joinOnly { return d.statements.selectMembershipsFromRoomAndMembership( - ctx, roomNID, membershipStateJoin, + ctx, roomNID, membershipStateJoin, localOnly, ) } - return d.statements.selectMembershipsFromRoom(ctx, roomNID) + return d.statements.selectMembershipsFromRoom(ctx, roomNID, localOnly) } // EventsFromIDs implements query.RoomserverQueryAPIEventDB diff --git a/roomserver/storage/sqlite3/membership_table.go b/roomserver/storage/sqlite3/membership_table.go index 7ae28e4b8..ca4d8fbe9 100644 --- a/roomserver/storage/sqlite3/membership_table.go +++ b/roomserver/storage/sqlite3/membership_table.go @@ -38,6 +38,7 @@ const membershipSchema = ` sender_nid INTEGER NOT NULL DEFAULT 0, membership_nid INTEGER NOT NULL DEFAULT 1, event_nid INTEGER NOT NULL DEFAULT 0, + target_local BOOLEAN NOT NULL DEFAULT false, UNIQUE (room_nid, target_nid) ); ` @@ -45,8 +46,8 @@ const membershipSchema = ` // Insert a row in to membership table so that it can be locked by the // SELECT FOR UPDATE const insertMembershipSQL = "" + - "INSERT INTO roomserver_membership (room_nid, target_nid)" + - " VALUES ($1, $2)" + + "INSERT INTO roomserver_membership (room_nid, target_nid, target_local)" + + " VALUES ($1, $2, $3)" + " ON CONFLICT DO NOTHING" const selectMembershipFromRoomAndTargetSQL = "" + @@ -57,10 +58,20 @@ const selectMembershipsFromRoomAndMembershipSQL = "" + "SELECT event_nid FROM roomserver_membership" + " WHERE room_nid = $1 AND membership_nid = $2" +const selectLocalMembershipsFromRoomAndMembershipSQL = "" + + "SELECT event_nid FROM roomserver_membership" + + " WHERE room_nid = $1 AND membership_nid = $2" + + " AND target_local = true" + const selectMembershipsFromRoomSQL = "" + "SELECT event_nid FROM roomserver_membership" + " WHERE room_nid = $1" +const selectLocalMembershipsFromRoomSQL = "" + + "SELECT event_nid FROM roomserver_membership" + + " WHERE room_nid = $1" + + " AND target_local = true" + const selectMembershipForUpdateSQL = "" + "SELECT membership_nid FROM roomserver_membership" + " WHERE room_nid = $1 AND target_nid = $2" @@ -70,12 +81,14 @@ const updateMembershipSQL = "" + " WHERE room_nid = $4 AND target_nid = $5" type membershipStatements struct { - insertMembershipStmt *sql.Stmt - selectMembershipForUpdateStmt *sql.Stmt - selectMembershipFromRoomAndTargetStmt *sql.Stmt - selectMembershipsFromRoomAndMembershipStmt *sql.Stmt - selectMembershipsFromRoomStmt *sql.Stmt - updateMembershipStmt *sql.Stmt + insertMembershipStmt *sql.Stmt + selectMembershipForUpdateStmt *sql.Stmt + selectMembershipFromRoomAndTargetStmt *sql.Stmt + selectMembershipsFromRoomAndMembershipStmt *sql.Stmt + selectLocalMembershipsFromRoomAndMembershipStmt *sql.Stmt + selectMembershipsFromRoomStmt *sql.Stmt + selectLocalMembershipsFromRoomStmt *sql.Stmt + updateMembershipStmt *sql.Stmt } func (s *membershipStatements) prepare(db *sql.DB) (err error) { @@ -89,7 +102,9 @@ func (s *membershipStatements) prepare(db *sql.DB) (err error) { {&s.selectMembershipForUpdateStmt, selectMembershipForUpdateSQL}, {&s.selectMembershipFromRoomAndTargetStmt, selectMembershipFromRoomAndTargetSQL}, {&s.selectMembershipsFromRoomAndMembershipStmt, selectMembershipsFromRoomAndMembershipSQL}, + {&s.selectLocalMembershipsFromRoomAndMembershipStmt, selectLocalMembershipsFromRoomAndMembershipSQL}, {&s.selectMembershipsFromRoomStmt, selectMembershipsFromRoomSQL}, + {&s.selectLocalMembershipsFromRoomStmt, selectLocalMembershipsFromRoomSQL}, {&s.updateMembershipStmt, updateMembershipSQL}, }.prepare(db) } @@ -97,9 +112,10 @@ func (s *membershipStatements) prepare(db *sql.DB) (err error) { func (s *membershipStatements) insertMembership( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, + localTarget bool, ) error { stmt := common.TxStmt(txn, s.insertMembershipStmt) - _, err := stmt.ExecContext(ctx, roomNID, targetUserNID) + _, err := stmt.ExecContext(ctx, roomNID, targetUserNID, localTarget) return err } @@ -127,9 +143,14 @@ func (s *membershipStatements) selectMembershipFromRoomAndTarget( func (s *membershipStatements) selectMembershipsFromRoom( ctx context.Context, txn *sql.Tx, - roomNID types.RoomNID, + roomNID types.RoomNID, localOnly bool, ) (eventNIDs []types.EventNID, err error) { - selectStmt := common.TxStmt(txn, s.selectMembershipsFromRoomStmt) + var selectStmt *sql.Stmt + if localOnly { + selectStmt = common.TxStmt(txn, s.selectLocalMembershipsFromRoomStmt) + } else { + selectStmt = common.TxStmt(txn, s.selectMembershipsFromRoomStmt) + } rows, err := selectStmt.QueryContext(ctx, roomNID) if err != nil { return nil, err @@ -145,11 +166,17 @@ func (s *membershipStatements) selectMembershipsFromRoom( } return } + func (s *membershipStatements) selectMembershipsFromRoomAndMembership( ctx context.Context, txn *sql.Tx, - roomNID types.RoomNID, membership membershipState, + roomNID types.RoomNID, membership membershipState, localOnly bool, ) (eventNIDs []types.EventNID, err error) { - stmt := common.TxStmt(txn, s.selectMembershipsFromRoomAndMembershipStmt) + var stmt *sql.Stmt + if localOnly { + stmt = common.TxStmt(txn, s.selectLocalMembershipsFromRoomAndMembershipStmt) + } else { + stmt = common.TxStmt(txn, s.selectMembershipsFromRoomAndMembershipStmt) + } rows, err := stmt.QueryContext(ctx, roomNID, membership) if err != nil { return diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index e77fea9cf..209922fa2 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -569,9 +569,9 @@ func (u *roomRecentEventsUpdater) MarkEventAsSent(eventNID types.EventNID) error return err } -func (u *roomRecentEventsUpdater) MembershipUpdater(targetUserNID types.EventStateKeyNID) (mu types.MembershipUpdater, err error) { +func (u *roomRecentEventsUpdater) MembershipUpdater(targetUserNID types.EventStateKeyNID, targetLocal bool) (mu types.MembershipUpdater, err error) { err = common.WithTransaction(u.d.db, func(txn *sql.Tx) error { - mu, err = u.d.membershipUpdaterTxn(u.ctx, txn, u.roomNID, targetUserNID) + mu, err = u.d.membershipUpdaterTxn(u.ctx, txn, u.roomNID, targetUserNID, targetLocal) return err }) return @@ -680,7 +680,7 @@ func (d *Database) StateEntriesForTuples( // MembershipUpdater implements input.RoomEventDatabase func (d *Database) MembershipUpdater( ctx context.Context, roomID, targetUserID string, - roomVersion gomatrixserverlib.RoomVersion, + targetLocal bool, roomVersion gomatrixserverlib.RoomVersion, ) (updater types.MembershipUpdater, err error) { var txn *sql.Tx txn, err = d.db.Begin() @@ -716,7 +716,7 @@ func (d *Database) MembershipUpdater( return nil, err } - updater, err = d.membershipUpdaterTxn(ctx, txn, roomNID, targetUserNID) + updater, err = d.membershipUpdaterTxn(ctx, txn, roomNID, targetUserNID, targetLocal) if err != nil { return nil, err } @@ -738,9 +738,10 @@ func (d *Database) membershipUpdaterTxn( txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, + targetLocal bool, ) (types.MembershipUpdater, error) { - if err := d.statements.insertMembership(ctx, txn, roomNID, targetUserNID); err != nil { + if err := d.statements.insertMembership(ctx, txn, roomNID, targetUserNID, targetLocal); err != nil { return nil, err } @@ -896,17 +897,17 @@ func (d *Database) GetMembership( // GetMembershipEventNIDsForRoom implements query.RoomserverQueryAPIDB func (d *Database) GetMembershipEventNIDsForRoom( - ctx context.Context, roomNID types.RoomNID, joinOnly bool, + ctx context.Context, roomNID types.RoomNID, joinOnly bool, localOnly bool, ) (eventNIDs []types.EventNID, err error) { err = common.WithTransaction(d.db, func(txn *sql.Tx) error { if joinOnly { eventNIDs, err = d.statements.selectMembershipsFromRoomAndMembership( - ctx, txn, roomNID, membershipStateJoin, + ctx, txn, roomNID, membershipStateJoin, localOnly, ) return nil } - eventNIDs, err = d.statements.selectMembershipsFromRoom(ctx, txn, roomNID) + eventNIDs, err = d.statements.selectMembershipsFromRoom(ctx, txn, roomNID, localOnly) return nil }) return diff --git a/roomserver/types/types.go b/roomserver/types/types.go index da83f614c..74e6b0784 100644 --- a/roomserver/types/types.go +++ b/roomserver/types/types.go @@ -172,7 +172,7 @@ type RoomRecentEventsUpdater interface { MarkEventAsSent(eventNID EventNID) error // Build a membership updater for the target user in this room. // It will share the same transaction as this updater. - MembershipUpdater(targetUserNID EventStateKeyNID) (MembershipUpdater, error) + MembershipUpdater(targetUserNID EventStateKeyNID, isTargetLocalUser bool) (MembershipUpdater, error) // Implements Transaction so it can be committed or rolledback common.Transaction } From 7d91ef0616c7eccd7d32bd30c1ee2d779d983d12 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 20 May 2020 18:31:02 +0100 Subject: [PATCH 079/200] This now passes on sytest/develop --- sytest-whitelist | 1 + 1 file changed, 1 insertion(+) diff --git a/sytest-whitelist b/sytest-whitelist index d5b4feaf6..c022b16c3 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -283,3 +283,4 @@ Outbound federation can backfill events Inbound federation can backfill events Backfill checks the events requested belong to the room Backfilled events whose prev_events are in a different room do not allow cross-room back-pagination +Outbound federation can request missing events From 91902df95d031861a65ffadc56aed702bb641e5b Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 21 May 2020 11:51:15 +0100 Subject: [PATCH 080/200] Add tests due to https://github.com/matrix-org/sytest/pull/876 --- sytest-whitelist | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sytest-whitelist b/sytest-whitelist index c022b16c3..1b12aba1b 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -284,3 +284,7 @@ Inbound federation can backfill events Backfill checks the events requested belong to the room Backfilled events whose prev_events are in a different room do not allow cross-room back-pagination Outbound federation can request missing events +New room members see their own join event +Existing members see new members' join events +Inbound federation can receive events +Inbound federation can receive redacted events From 3fdb045116c9cd2f2a3badfebec0645d0381bacb Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 21 May 2020 13:02:28 +0100 Subject: [PATCH 081/200] Docker Hub (#1053) * Initial Docker Hub test * Change upstream from for now * Working polylith setup and build tools * Some readme and other bits * Remove the readme for now * Add readme, rename a couple of things * COPY instead of ADD * Simplify component builds * Fix the docker-compose things * Build the base image again * Update readme, add .dockerignore * Indentation * Whitespace * Update images-push.sh * Update images-push.sh --- .dockerignore | 3 + docker/hub/Dockerfile | 10 ++ docker/hub/Dockerfile.component | 13 +++ docker/hub/README.md | 68 ++++++++++++ docker/hub/config/dendrite-config.yaml | 129 ++++++++++++++++++++++ docker/hub/docker-compose.deps.yml | 36 ++++++ docker/hub/docker-compose.monolith.yml | 18 +++ docker/hub/docker-compose.polylith.yml | 146 +++++++++++++++++++++++++ docker/hub/images-build.sh | 17 +++ docker/hub/images-push.sh | 13 +++ 10 files changed, 453 insertions(+) create mode 100644 .dockerignore create mode 100644 docker/hub/Dockerfile create mode 100644 docker/hub/Dockerfile.component create mode 100644 docker/hub/README.md create mode 100644 docker/hub/config/dendrite-config.yaml create mode 100644 docker/hub/docker-compose.deps.yml create mode 100644 docker/hub/docker-compose.monolith.yml create mode 100644 docker/hub/docker-compose.polylith.yml create mode 100755 docker/hub/images-build.sh create mode 100755 docker/hub/images-push.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..76547e9ee --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +bin +*.wasm +.git \ No newline at end of file diff --git a/docker/hub/Dockerfile b/docker/hub/Dockerfile new file mode 100644 index 000000000..d8e07681f --- /dev/null +++ b/docker/hub/Dockerfile @@ -0,0 +1,10 @@ +FROM docker.io/golang:1.13.7-alpine3.11 AS builder + +RUN apk --update --no-cache add bash build-base + +WORKDIR /build + +COPY . /build + +RUN mkdir -p bin +RUN sh ./build.sh \ No newline at end of file diff --git a/docker/hub/Dockerfile.component b/docker/hub/Dockerfile.component new file mode 100644 index 000000000..13634391a --- /dev/null +++ b/docker/hub/Dockerfile.component @@ -0,0 +1,13 @@ +FROM matrixdotorg/dendrite:latest AS base + +FROM alpine:latest + +ARG component=monolith +ENV entrypoint=${component} + +COPY --from=base /build/bin/${component} /usr/bin + +VOLUME /etc/dendrite +WORKDIR /etc/dendrite + +ENTRYPOINT /usr/bin/${entrypoint} $@ \ No newline at end of file diff --git a/docker/hub/README.md b/docker/hub/README.md new file mode 100644 index 000000000..fd6411439 --- /dev/null +++ b/docker/hub/README.md @@ -0,0 +1,68 @@ +# Docker Hub images + +These are Docker Hub-friendly images for Dendrite. + +## Dockerfiles + +The `Dockerfile` builds the base image which contains all of the Dendrite +components. The `Dockerfile.*` files take components from the base image and +produce smaller component-specific images, which are substantially smaller +and do not contain the Go toolchain etc. + +## Compose files + +There are three sample `docker-compose` files: + +- `docker-compose.deps.yml` which runs the Postgres and Kafka prerequisites +- `docker-compose.monolith.yml` which runs a monolith Dendrite deployment +- `docker-compose.polylith.yml` which runs a polylith Dendrite deployment + +## Configuration + +The `docker-compose` files refer to the `/etc/dendrite` volume as where the +runtime config should come from. The mounted folder must contain: + +- `dendrite.yaml` configuration file (based on the sample `dendrite-config.yaml` + in the `docker/hub/config` folder in the [Dendrite repository](https://github.com/matrix-org/dendrite) +- `matrix_key.pem` server key, as generated using `cmd/generate-keys` +- `server.crt` certificate file +- `server.key` private key file for the above certificate + +To generate keys: + +``` +go run github.com/matrix-org/dendrite/cmd/generate-keys \ + --private-key=matrix_key.pem \ + --tls-cert=server.crt \ + --tls-key=server.key +``` + +## Starting Dendrite + +Once in place, start the dependencies: + +``` +docker-compose -f docker-compose.deps.yml up +``` + +Wait a few seconds for Kafka and Postgres to finish starting up, and then start a monolith: + +``` +docker-compose -f docker-compose.monolith.yml up +``` + +... or start the polylith components: + +``` +docker-compose -f docker-compose.polylith.yml up +``` + +## Building the images + +The `docker/hub/images-build.sh` script will build all of the component images. + +The `docker/hub/images-push.sh` script will push them to Docker Hub (subject +to permissions). + +If you wish to build and push your own images, rename `matrixdotorg/dendrite` to +the name of another Docker Hub repository in `images-build.sh` and `images-push.sh`. diff --git a/docker/hub/config/dendrite-config.yaml b/docker/hub/config/dendrite-config.yaml new file mode 100644 index 000000000..23d6479b5 --- /dev/null +++ b/docker/hub/config/dendrite-config.yaml @@ -0,0 +1,129 @@ +# The config file format version +# This is used by dendrite to tell if it understands the config format. +# This will change if the structure of the config file changes or if the meaning +# of an existing config key changes. +version: 0 + +# The matrix specific config +matrix: + # The name of the server. This is usually the domain name, e.g 'matrix.org', 'localhost'. + server_name: "example.com" + # The path to the PEM formatted matrix private key. + private_key: "matrix_key.pem" + # The x509 certificates used by the federation listeners for this server + federation_certificates: ["server.crt"] + # The list of identity servers trusted to verify third party identifiers by this server. + # Defaults to no trusted servers. + trusted_third_party_id_servers: + - vector.im + - matrix.org + +# The media repository config +media: + # The base path to where the media files will be stored. May be relative or absolute. + base_path: /var/dendrite/media + + # The maximum file size in bytes that is allowed to be stored on this server. + # Note: if max_file_size_bytes is set to 0, the size is unlimited. + # Note: if max_file_size_bytes is not set, it will default to 10485760 (10MB) + max_file_size_bytes: 10485760 + + # Whether to dynamically generate thumbnails on-the-fly if the requested resolution is not already generated + # NOTE: This is a possible denial-of-service attack vector - use at your own risk + dynamic_thumbnails: false + + # A list of thumbnail sizes to be pre-generated for downloaded remote / uploaded content + # method is one of crop or scale. If omitted, it will default to scale. + # crop scales to fill the requested dimensions and crops the excess. + # scale scales to fit the requested dimensions and one dimension may be smaller than requested. + thumbnail_sizes: + - width: 32 + height: 32 + method: crop + - width: 96 + height: 96 + method: crop + - width: 320 + height: 240 + method: scale + - width: 640 + height: 480 + method: scale + - width: 800 + height: 600 + method: scale + +# The config for the TURN server +turn: + # Whether or not guests can request TURN credentials + turn_allow_guests: true + # How long the authorization should last + turn_user_lifetime: "1h" + # The list of TURN URIs to pass to clients + turn_uris: [] + + # Authorization via Shared Secret + # The shared secret from coturn + turn_shared_secret: "" + + # Authorization via Static Username & Password + # Hardcoded Username and Password + turn_username: "" + turn_password: "" + +# The config for communicating with kafka +kafka: + # Where the kafka servers are running. + addresses: ["kafka:9092"] + # Whether to use naffka instead of kafka. + # Naffka can only be used when running dendrite as a single monolithic server. + # Kafka can be used both with a monolithic server and when running the + # components as separate servers. + # If enabled database.naffka must also be specified. + use_naffka: false + # The names of the kafka topics to use. + topics: + output_room_event: roomserverOutput + output_client_data: clientapiOutput + output_typing_event: eduServerOutput + user_updates: userUpdates + + +# The postgres connection configs for connecting to the databases e.g a postgres:// URI +database: + account: "postgres://dendrite:itsasecret@postgres/dendrite_account?sslmode=disable" + device: "postgres://dendrite:itsasecret@postgres/dendrite_device?sslmode=disable" + media_api: "postgres://dendrite:itsasecret@postgres/dendrite_mediaapi?sslmode=disable" + sync_api: "postgres://dendrite:itsasecret@postgres/dendrite_syncapi?sslmode=disable" + room_server: "postgres://dendrite:itsasecret@postgres/dendrite_roomserver?sslmode=disable" + server_key: "postgres://dendrite:itsasecret@postgres/dendrite_serverkey?sslmode=disable" + federation_sender: "postgres://dendrite:itsasecret@postgres/dendrite_federationsender?sslmode=disable" + public_rooms_api: "postgres://dendrite:itsasecret@postgres/dendrite_publicroomsapi?sslmode=disable" + appservice: "postgres://dendrite:itsasecret@postgres/dendrite_appservice?sslmode=disable" + # If using naffka you need to specify a naffka database + #naffka: "postgres://dendrite:itsasecret@postgres/dendrite_naffka?sslmode=disable" + +# The TCP host:port pairs to bind the internal HTTP APIs to. +# These shouldn't be exposed to the public internet. +# These aren't needed when running dendrite as a monolithic server. +listen: + room_server: "room_server:7770" + client_api: "client_api:7771" + federation_api: "federation_api:7772" + sync_api: "sync_api:7773" + media_api: "media_api:7774" + public_rooms_api: "public_rooms_api:7775" + federation_sender: "federation_sender:7776" + edu_server: "edu_server:7777" + +# The configuration for tracing the dendrite components. +tracing: + # Config for the jaeger opentracing reporter. + # See https://godoc.org/github.com/uber/jaeger-client-go/config#Configuration + # for documentation. + jaeger: + disabled: true + +# A list of application service config files to use +application_services: + config_files: [] diff --git a/docker/hub/docker-compose.deps.yml b/docker/hub/docker-compose.deps.yml new file mode 100644 index 000000000..ada625aa3 --- /dev/null +++ b/docker/hub/docker-compose.deps.yml @@ -0,0 +1,36 @@ +version: "3.4" +services: + postgres: + hostname: postgres + image: postgres:9.5 + restart: always + volumes: + - ../postgres/create_db.sh:/docker-entrypoint-initdb.d/20-create_db.sh + environment: + POSTGRES_PASSWORD: itsasecret + POSTGRES_USER: dendrite + networks: + - internal + + zookeeper: + hostname: zookeeper + image: zookeeper + networks: + - internal + + kafka: + container_name: dendrite_kafka + hostname: kafka + image: wurstmeister/kafka + environment: + KAFKA_ADVERTISED_HOST_NAME: "kafka" + KAFKA_DELETE_TOPIC_ENABLE: "true" + KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181" + depends_on: + - zookeeper + networks: + - internal + +networks: + internal: + attachable: true diff --git a/docker/hub/docker-compose.monolith.yml b/docker/hub/docker-compose.monolith.yml new file mode 100644 index 000000000..336a43984 --- /dev/null +++ b/docker/hub/docker-compose.monolith.yml @@ -0,0 +1,18 @@ +version: "3.4" +services: + monolith: + hostname: monolith + image: matrixdotorg/dendrite:monolith + command: [ + "--config=dendrite.yaml", + "--tls-cert=server.crt", + "--tls-key=server.key" + ] + volumes: + - ./config:/etc/dendrite + networks: + - internal + +networks: + internal: + attachable: true diff --git a/docker/hub/docker-compose.polylith.yml b/docker/hub/docker-compose.polylith.yml new file mode 100644 index 000000000..f39670788 --- /dev/null +++ b/docker/hub/docker-compose.polylith.yml @@ -0,0 +1,146 @@ +version: "3.4" +services: + client_api_proxy: + hostname: client_api_proxy + image: matrixdotorg/dendrite:clientproxy + command: [ + "--bind-address=:8008", + "--client-api-server-url=http://client_api:7771", + "--sync-api-server-url=http://sync_api:7773", + "--media-api-server-url=http://media_api:7774", + "--public-rooms-api-server-url=http://public_rooms_api:7775" + ] + volumes: + - ./config:/etc/dendrite + networks: + - internal + depends_on: + - sync_api + - client_api + - media_api + - public_rooms_api + ports: + - "8008:8008" + + client_api: + hostname: client_api + image: matrixdotorg/dendrite:clientapi + command: [ + "--config=dendrite.yaml" + ] + volumes: + - ./config:/etc/dendrite + - room_server + networks: + - internal + + media_api: + hostname: media_api + image: matrixdotorg/dendrite:mediaapi + command: [ + "--config=dendrite.yaml" + ] + volumes: + - ./config:/etc/dendrite + networks: + - internal + + public_rooms_api: + hostname: public_rooms_api + image: matrixdotorg/dendrite:publicroomsapi + command: [ + "--config=dendrite.yaml" + ] + volumes: + - ./config:/etc/dendrite + networks: + - internal + + sync_api: + hostname: sync_api + image: matrixdotorg/dendrite:syncapi + command: [ + "--config=dendrite.yaml" + ] + volumes: + - ./config:/etc/dendrite + networks: + - internal + + room_server: + hostname: room_server + image: matrixdotorg/dendrite:roomserver + command: [ + "--config=dendrite.yaml" + ] + volumes: + - ./config:/etc/dendrite + networks: + - internal + + edu_server: + hostname: edu_server + image: matrixdotorg/dendrite:eduserver + command: [ + "--config=dendrite.yaml" + ] + volumes: + - ./config:/etc/dendrite + networks: + - internal + + federation_api_proxy: + hostname: federation_api_proxy + image: matrixdotorg/dendrite:federationproxy + command: [ + "--bind-address=:8448", + "--federation-api-url=http://federation_api_server:7772", + "--media-api-server-url=http://media_api:7774" + ] + volumes: + - ./config:/etc/dendrite + depends_on: + - federation_api + - federation_sender + - media_api + networks: + - internal + ports: + - "8448:8448" + + federation_api: + hostname: federation_api + image: matrixdotorg/dendrite:federationapi + command: [ + "--config=dendrite.yaml" + ] + volumes: + - ./config:/etc/dendrite + networks: + - internal + + federation_sender: + hostname: federation_sender + image: matrixdotorg/dendrite:federationsender + command: [ + "--config=dendrite.yaml" + ] + volumes: + - ./config:/etc/dendrite + networks: + - internal + + key_server: + hostname: key_serverde + image: matrixdotorg/dendrite:keyserver + command: [ + "--config=dendrite.yaml" + ] + volumes: + - ./config:/etc/dendrite + networks: + - internal + +networks: + internal: + attachable: true diff --git a/docker/hub/images-build.sh b/docker/hub/images-build.sh new file mode 100755 index 000000000..0c6a0eb7a --- /dev/null +++ b/docker/hub/images-build.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +cd $(git rev-parse --show-toplevel) + +docker build -f docker/hub/Dockerfile -t matrixdotorg/dendrite:latest . + +docker build -t matrixdotorg/dendrite:clientapi --build-arg component=dendrite-client-api-server -f docker/hub/Dockerfile.component . +docker build -t matrixdotorg/dendrite:clientproxy --build-arg component=client-api-proxy -f docker/hub/Dockerfile.component . +docker build -t matrixdotorg/dendrite:eduserver --build-arg component=dendrite-edu-server -f docker/hub/Dockerfile.component . +docker build -t matrixdotorg/dendrite:federationapi --build-arg component=dendrite-federation-api-server -f docker/hub/Dockerfile.component . +docker build -t matrixdotorg/dendrite:federationsender --build-arg component=dendrite-federation-sender-server -f docker/hub/Dockerfile.component . +docker build -t matrixdotorg/dendrite:federationproxy --build-arg component=federation-api-proxy -f docker/hub/Dockerfile.component . +docker build -t matrixdotorg/dendrite:keyserver --build-arg component=dendrite-key-server -f docker/hub/Dockerfile.component . +docker build -t matrixdotorg/dendrite:mediaapi --build-arg component=dendrite-media-api-server -f docker/hub/Dockerfile.component . +docker build -t matrixdotorg/dendrite:publicroomsapi --build-arg component=dendrite-public-rooms-api-server -f docker/hub/Dockerfile.component . +docker build -t matrixdotorg/dendrite:roomserver --build-arg component=dendrite-room-server -f docker/hub/Dockerfile.component . +docker build -t matrixdotorg/dendrite:syncapi --build-arg component=dendrite-sync-api-server -f docker/hub/Dockerfile.component . diff --git a/docker/hub/images-push.sh b/docker/hub/images-push.sh new file mode 100755 index 000000000..c62072687 --- /dev/null +++ b/docker/hub/images-push.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +docker push matrixdotorg/dendrite:clientapi +docker push matrixdotorg/dendrite:clientproxy +docker push matrixdotorg/dendrite:eduserver +docker push matrixdotorg/dendrite:federationapi +docker push matrixdotorg/dendrite:federationsender +docker push matrixdotorg/dendrite:federationproxy +docker push matrixdotorg/dendrite:keyserver +docker push matrixdotorg/dendrite:mediaapi +docker push matrixdotorg/dendrite:publicroomsapi +docker push matrixdotorg/dendrite:roomserver +docker push matrixdotorg/dendrite:syncapi From 24d8df664c21fa8bd68d80b5585a496e264c410a Mon Sep 17 00:00:00 2001 From: Kegsay Date: Thu, 21 May 2020 14:40:13 +0100 Subject: [PATCH 082/200] Fix #897 and shuffle directory around (#1054) * Fix #897 and shuffle directory around * Update find-lint * goimports Co-authored-by: Neil Alexander --- appservice/api/query.go | 10 +- appservice/appservice.go | 6 +- appservice/consumers/roomserver.go | 8 +- appservice/query/query.go | 8 +- appservice/routing/routing.go | 10 +- appservice/storage/postgres/storage.go | 4 +- appservice/storage/sqlite3/storage.go | 4 +- appservice/storage/storage.go | 4 +- appservice/storage/storage_wasm.go | 4 +- appservice/types/types.go | 2 +- appservice/workers/transaction_scheduler.go | 2 +- {docker => build/docker}/Dockerfile | 0 {docker => build/docker}/README.md | 0 {docker => build/docker}/build.sh | 0 {docker => build/docker}/dendrite-docker.yml | 0 {docker => build/docker}/docker-compose.yml | 0 {docker => build/docker}/hub/Dockerfile | 0 .../docker}/hub/Dockerfile.component | 0 {docker => build/docker}/hub/README.md | 0 .../docker}/hub/config/dendrite-config.yaml | 0 .../docker}/hub/docker-compose.deps.yml | 0 .../docker}/hub/docker-compose.monolith.yml | 0 .../docker}/hub/docker-compose.polylith.yml | 0 {docker => build/docker}/hub/images-build.sh | 0 {docker => build/docker}/hub/images-push.sh | 0 .../docker}/postgres/create_db.sh | 0 .../docker}/services/client-api-proxy.sh | 0 .../docker}/services/client-api.sh | 0 .../docker}/services/edu-server.sh | 0 .../docker}/services/federation-api-proxy.sh | 0 .../docker}/services/federation-api.sh | 0 .../docker}/services/federation-sender.sh | 0 .../docker}/services/key-server.sh | 0 .../docker}/services/media-api.sh | 0 {docker => build/docker}/services/monolith.sh | 0 .../docker}/services/public-rooms-api.sh | 0 .../docker}/services/room-server.sh | 0 {docker => build/docker}/services/sync-api.sh | 0 {scripts => build/scripts}/README.md | 0 {scripts => build/scripts}/build-test-lint.sh | 0 {scripts => build/scripts}/find-lint.sh | 2 +- .../scripts}/install-local-kafka.sh | 0 clientapi/auth/auth.go | 2 +- clientapi/auth/storage/accounts/interface.go | 4 +- .../accounts/postgres/account_data_table.go | 4 +- .../accounts/postgres/membership_table.go | 4 +- .../auth/storage/accounts/postgres/storage.go | 20 +- .../accounts/postgres/threepid_table.go | 6 +- .../accounts/sqlite3/membership_table.go | 6 +- .../auth/storage/accounts/sqlite3/storage.go | 20 +- .../accounts/sqlite3/threepid_table.go | 8 +- clientapi/auth/storage/accounts/storage.go | 4 +- .../auth/storage/accounts/storage_wasm.go | 4 +- .../storage/devices/postgres/devices_table.go | 14 +- .../auth/storage/devices/postgres/storage.go | 16 +- .../storage/devices/sqlite3/devices_table.go | 16 +- .../auth/storage/devices/sqlite3/storage.go | 16 +- clientapi/auth/storage/devices/storage.go | 4 +- .../auth/storage/devices/storage_wasm.go | 4 +- clientapi/clientapi.go | 4 +- clientapi/consumers/roomserver.go | 8 +- clientapi/producers/syncapi.go | 4 +- clientapi/producers/userupdate.go | 2 +- clientapi/routing/auth_fallback.go | 2 +- clientapi/routing/createroom.go | 18 +- clientapi/routing/directory.go | 2 +- clientapi/routing/getevent.go | 2 +- clientapi/routing/joinroom.go | 2 +- clientapi/routing/login.go | 2 +- clientapi/routing/membership.go | 10 +- clientapi/routing/memberships.go | 2 +- clientapi/routing/profile.go | 26 +-- clientapi/routing/register.go | 8 +- clientapi/routing/register_test.go | 2 +- clientapi/routing/routing.go | 194 +++++++++--------- clientapi/routing/sendevent.go | 12 +- clientapi/routing/threepid.go | 2 +- clientapi/routing/voip.go | 2 +- clientapi/threepid/invites.go | 6 +- clientapi/threepid/threepid.go | 2 +- cmd/dendrite-appservice-server/main.go | 4 +- cmd/dendrite-client-api-server/main.go | 6 +- cmd/dendrite-demo-libp2p/main.go | 10 +- cmd/dendrite-demo-libp2p/mdnslistener.go | 2 +- cmd/dendrite-demo-libp2p/p2pdendrite.go | 4 +- cmd/dendrite-edu-server/main.go | 2 +- cmd/dendrite-federation-api-server/main.go | 4 +- cmd/dendrite-federation-sender-server/main.go | 4 +- cmd/dendrite-key-server/main.go | 2 +- cmd/dendrite-media-api-server/main.go | 2 +- cmd/dendrite-monolith-server/main.go | 12 +- cmd/dendrite-public-rooms-api-server/main.go | 2 +- cmd/dendrite-room-server/main.go | 4 +- cmd/dendrite-sync-api-server/main.go | 2 +- cmd/dendritejs/main.go | 10 +- cmd/generate-keys/main.go | 2 +- cmd/mediaapi-integration-tests/main.go | 2 +- cmd/roomserver-integration-tests/main.go | 4 +- cmd/syncserver-integration-tests/main.go | 4 +- eduserver/api/input.go | 4 +- eduserver/cache/cache_test.go | 2 +- eduserver/eduserver.go | 2 +- eduserver/input/input.go | 4 +- federationapi/federationapi.go | 2 +- federationapi/routing/backfill.go | 2 +- federationapi/routing/invite.go | 2 +- federationapi/routing/join.go | 10 +- federationapi/routing/keys.go | 2 +- federationapi/routing/leave.go | 10 +- federationapi/routing/profile.go | 10 +- federationapi/routing/query.go | 2 +- federationapi/routing/routing.go | 72 +++---- federationapi/routing/send.go | 2 +- federationapi/routing/threepid.go | 2 +- federationsender/api/perform.go | 8 +- federationsender/api/query.go | 6 +- federationsender/consumers/eduserver.go | 8 +- federationsender/consumers/roomserver.go | 8 +- federationsender/federationsender.go | 2 +- federationsender/internal/api.go | 12 +- federationsender/storage/interface.go | 4 +- .../storage/postgres/joined_hosts_table.go | 10 +- .../storage/postgres/room_table.go | 8 +- federationsender/storage/postgres/storage.go | 8 +- .../storage/sqlite3/joined_hosts_table.go | 10 +- .../storage/sqlite3/room_table.go | 8 +- federationsender/storage/sqlite3/storage.go | 8 +- federationsender/storage/storage.go | 4 +- federationsender/storage/storage_wasm.go | 4 +- {common => internal}/basecomponent/base.go | 20 +- {common => internal}/basecomponent/flags.go | 2 +- .../caching/immutablecache.go | 0 .../caching/immutableinmemorylru.go | 0 {common => internal}/config/appservice.go | 0 {common => internal}/config/config.go | 0 {common => internal}/config/config_test.go | 0 {common => internal}/consumers.go | 2 +- {common => internal}/eventcontent.go | 2 +- {common => internal}/events.go | 4 +- {common => internal}/http/http.go | 0 {common => internal}/httpapi.go | 4 +- {common => internal}/httpapi_test.go | 2 +- {common => internal}/keydb/cache/keydb.go | 4 +- {common => internal}/keydb/interface.go | 0 {common => internal}/keydb/keydb.go | 8 +- {common => internal}/keydb/keydb_wasm.go | 6 +- {common => internal}/keydb/keyring.go | 2 +- {common => internal}/keydb/postgres/keydb.go | 4 +- .../keydb/postgres/server_key_table.go | 4 +- {common => internal}/keydb/sqlite3/keydb.go | 4 +- .../keydb/sqlite3/server_key_table.go | 6 +- {common => internal}/log.go | 4 +- .../partition_offset_table.go | 2 +- {common => internal}/postgres.go | 2 +- {common => internal}/postgres_wasm.go | 2 +- {common => internal}/routing.go | 2 +- {common => internal}/sql.go | 2 +- internal/sqlutil/trace.go | 6 +- {common => internal}/test/client.go | 0 {common => internal}/test/config.go | 2 +- {common => internal}/test/kafka.go | 0 {common => internal}/test/server.go | 2 +- {common => internal}/test/slice.go | 0 .../transactions/transactions.go | 0 .../transactions/transactions_test.go | 0 {common => internal}/types.go | 2 +- keyserver/keyserver.go | 2 +- keyserver/routing/routing.go | 6 +- mediaapi/fileutils/fileutils.go | 2 +- mediaapi/mediaapi.go | 2 +- mediaapi/routing/download.go | 2 +- mediaapi/routing/routing.go | 10 +- mediaapi/routing/upload.go | 2 +- mediaapi/storage/postgres/prepare.go | 2 +- mediaapi/storage/postgres/storage.go | 4 +- mediaapi/storage/postgres/thumbnail_table.go | 4 +- mediaapi/storage/sqlite3/prepare.go | 2 +- mediaapi/storage/sqlite3/storage.go | 4 +- mediaapi/storage/sqlite3/thumbnail_table.go | 4 +- mediaapi/storage/storage.go | 4 +- mediaapi/storage/storage_wasm.go | 4 +- mediaapi/thumbnailer/thumbnailer.go | 2 +- mediaapi/thumbnailer/thumbnailer_bimg.go | 2 +- mediaapi/thumbnailer/thumbnailer_nfnt.go | 2 +- mediaapi/types/types.go | 2 +- publicroomsapi/consumers/roomserver.go | 8 +- publicroomsapi/publicroomsapi.go | 2 +- publicroomsapi/routing/routing.go | 14 +- publicroomsapi/storage/interface.go | 4 +- .../storage/postgres/public_rooms_table.go | 4 +- publicroomsapi/storage/postgres/storage.go | 20 +- .../storage/sqlite3/public_rooms_table.go | 4 +- publicroomsapi/storage/sqlite3/storage.go | 20 +- publicroomsapi/storage/storage.go | 4 +- roomserver/api/alias.go | 12 +- roomserver/api/http.go | 2 +- roomserver/api/input.go | 4 +- roomserver/api/perform.go | 6 +- roomserver/api/query.go | 26 +-- roomserver/internal/api.go | 46 ++--- roomserver/internal/input_events.go | 4 +- roomserver/internal/input_latest_events.go | 4 +- roomserver/internal/perform_join.go | 6 +- roomserver/internal/perform_leave.go | 6 +- roomserver/internal/query_test.go | 2 +- roomserver/roomserver.go | 2 +- roomserver/state/state.go | 2 +- .../storage/postgres/event_json_table.go | 4 +- .../postgres/event_state_keys_table.go | 10 +- .../storage/postgres/event_types_table.go | 8 +- roomserver/storage/postgres/events_table.go | 26 +-- roomserver/storage/postgres/invite_table.go | 10 +- .../storage/postgres/membership_table.go | 12 +- .../storage/postgres/previous_events_table.go | 6 +- .../storage/postgres/room_aliases_table.go | 4 +- roomserver/storage/postgres/rooms_table.go | 14 +- .../storage/postgres/state_block_table.go | 8 +- roomserver/storage/postgres/storage.go | 4 +- .../storage/sqlite3/event_json_table.go | 8 +- .../storage/sqlite3/event_state_keys_table.go | 10 +- .../storage/sqlite3/event_types_table.go | 14 +- roomserver/storage/sqlite3/events_table.go | 54 ++--- roomserver/storage/sqlite3/invite_table.go | 6 +- .../storage/sqlite3/membership_table.go | 22 +- .../storage/sqlite3/previous_events_table.go | 6 +- .../storage/sqlite3/room_aliases_table.go | 14 +- roomserver/storage/sqlite3/rooms_table.go | 16 +- .../storage/sqlite3/state_block_table.go | 16 +- .../storage/sqlite3/state_snapshot_table.go | 6 +- roomserver/storage/sqlite3/storage.go | 58 +++--- .../storage/sqlite3/transactions_table.go | 6 +- roomserver/storage/storage.go | 4 +- roomserver/storage/storage_wasm.go | 4 +- roomserver/types/types.go | 6 +- syncapi/api/query.go | 10 +- syncapi/consumers/clientapi.go | 10 +- syncapi/consumers/eduserver.go | 8 +- syncapi/consumers/roomserver.go | 8 +- syncapi/routing/messages.go | 2 +- syncapi/routing/routing.go | 10 +- syncapi/storage/interface.go | 4 +- .../storage/postgres/account_data_table.go | 6 +- .../postgres/backwards_extremities_table.go | 4 +- .../postgres/current_room_state_table.go | 20 +- syncapi/storage/postgres/invites_table.go | 8 +- .../postgres/output_room_events_table.go | 24 +-- .../output_room_events_topology_table.go | 4 +- syncapi/storage/postgres/syncserver.go | 6 +- syncapi/storage/shared/syncserver.go | 18 +- syncapi/storage/sqlite3/account_data_table.go | 4 +- .../sqlite3/backwards_extremities_table.go | 4 +- .../sqlite3/current_room_state_table.go | 20 +- syncapi/storage/sqlite3/invites_table.go | 8 +- .../sqlite3/output_room_events_table.go | 22 +- .../output_room_events_topology_table.go | 12 +- syncapi/storage/sqlite3/stream_id_table.go | 6 +- syncapi/storage/sqlite3/syncserver.go | 6 +- syncapi/storage/storage.go | 4 +- syncapi/storage/storage_wasm.go | 4 +- syncapi/syncapi.go | 4 +- 260 files changed, 901 insertions(+), 901 deletions(-) rename {docker => build/docker}/Dockerfile (100%) rename {docker => build/docker}/README.md (100%) rename {docker => build/docker}/build.sh (100%) rename {docker => build/docker}/dendrite-docker.yml (100%) rename {docker => build/docker}/docker-compose.yml (100%) rename {docker => build/docker}/hub/Dockerfile (100%) rename {docker => build/docker}/hub/Dockerfile.component (100%) rename {docker => build/docker}/hub/README.md (100%) rename {docker => build/docker}/hub/config/dendrite-config.yaml (100%) rename {docker => build/docker}/hub/docker-compose.deps.yml (100%) rename {docker => build/docker}/hub/docker-compose.monolith.yml (100%) rename {docker => build/docker}/hub/docker-compose.polylith.yml (100%) rename {docker => build/docker}/hub/images-build.sh (100%) rename {docker => build/docker}/hub/images-push.sh (100%) rename {docker => build/docker}/postgres/create_db.sh (100%) rename {docker => build/docker}/services/client-api-proxy.sh (100%) rename {docker => build/docker}/services/client-api.sh (100%) rename {docker => build/docker}/services/edu-server.sh (100%) rename {docker => build/docker}/services/federation-api-proxy.sh (100%) rename {docker => build/docker}/services/federation-api.sh (100%) rename {docker => build/docker}/services/federation-sender.sh (100%) rename {docker => build/docker}/services/key-server.sh (100%) rename {docker => build/docker}/services/media-api.sh (100%) rename {docker => build/docker}/services/monolith.sh (100%) rename {docker => build/docker}/services/public-rooms-api.sh (100%) rename {docker => build/docker}/services/room-server.sh (100%) rename {docker => build/docker}/services/sync-api.sh (100%) rename {scripts => build/scripts}/README.md (100%) rename {scripts => build/scripts}/build-test-lint.sh (100%) rename {scripts => build/scripts}/find-lint.sh (98%) rename {scripts => build/scripts}/install-local-kafka.sh (100%) rename {common => internal}/basecomponent/base.go (94%) rename {common => internal}/basecomponent/flags.go (97%) rename {common => internal}/caching/immutablecache.go (100%) rename {common => internal}/caching/immutableinmemorylru.go (100%) rename {common => internal}/config/appservice.go (100%) rename {common => internal}/config/config.go (100%) rename {common => internal}/config/config_test.go (100%) rename {common => internal}/consumers.go (99%) rename {common => internal}/eventcontent.go (99%) rename {common => internal}/events.go (98%) rename {common => internal}/http/http.go (100%) rename {common => internal}/httpapi.go (99%) rename {common => internal}/httpapi_test.go (99%) rename {common => internal}/keydb/cache/keydb.go (94%) rename {common => internal}/keydb/interface.go (100%) rename {common => internal}/keydb/keydb.go (88%) rename {common => internal}/keydb/keydb_wasm.go (89%) rename {common => internal}/keydb/keyring.go (97%) rename {common => internal}/keydb/postgres/keydb.go (97%) rename {common => internal}/keydb/postgres/server_key_table.go (97%) rename {common => internal}/keydb/sqlite3/keydb.go (96%) rename {common => internal}/keydb/sqlite3/server_key_table.go (95%) rename {common => internal}/log.go (98%) rename {common => internal}/partition_offset_table.go (99%) rename {common => internal}/postgres.go (98%) rename {common => internal}/postgres_wasm.go (97%) rename {common => internal}/routing.go (98%) rename {common => internal}/sql.go (99%) rename {common => internal}/test/client.go (100%) rename {common => internal}/test/config.go (99%) rename {common => internal}/test/kafka.go (100%) rename {common => internal}/test/server.go (98%) rename {common => internal}/test/slice.go (100%) rename {common => internal}/transactions/transactions.go (100%) rename {common => internal}/transactions/transactions_test.go (100%) rename {common => internal}/types.go (99%) diff --git a/appservice/api/query.go b/appservice/api/query.go index afd5c5d76..4a4e31a99 100644 --- a/appservice/api/query.go +++ b/appservice/api/query.go @@ -27,8 +27,8 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/dendrite/common" - commonHTTP "github.com/matrix-org/dendrite/common/http" + "github.com/matrix-org/dendrite/internal" + internalHTTP "github.com/matrix-org/dendrite/internal/http" opentracing "github.com/opentracing/opentracing-go" ) @@ -119,7 +119,7 @@ func (h *httpAppServiceQueryAPI) RoomAliasExists( defer span.Finish() apiURL := h.appserviceURL + AppServiceRoomAliasExistsPath - return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // UserIDExists implements AppServiceQueryAPI @@ -132,7 +132,7 @@ func (h *httpAppServiceQueryAPI) UserIDExists( defer span.Finish() apiURL := h.appserviceURL + AppServiceUserIDExistsPath - return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // RetrieveUserProfile is a wrapper that queries both the local database and @@ -165,7 +165,7 @@ func RetrieveUserProfile( // If no user exists, return if !userResp.UserIDExists { - return nil, common.ErrProfileNoExists + return nil, internal.ErrProfileNoExists } // Try to query the user from the local database again diff --git a/appservice/appservice.go b/appservice/appservice.go index 3a1c4677a..3c67ce9c1 100644 --- a/appservice/appservice.go +++ b/appservice/appservice.go @@ -29,9 +29,9 @@ import ( "github.com/matrix-org/dendrite/appservice/workers" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" - "github.com/matrix-org/dendrite/common/basecomponent" - "github.com/matrix-org/dendrite/common/config" - "github.com/matrix-org/dendrite/common/transactions" + "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/transactions" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go index b7f689249..bb4df7906 100644 --- a/appservice/consumers/roomserver.go +++ b/appservice/consumers/roomserver.go @@ -21,8 +21,8 @@ import ( "github.com/matrix-org/dendrite/appservice/storage" "github.com/matrix-org/dendrite/appservice/types" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" - "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" @@ -32,7 +32,7 @@ import ( // OutputRoomEventConsumer consumes events that originated in the room server. type OutputRoomEventConsumer struct { - roomServerConsumer *common.ContinualConsumer + roomServerConsumer *internal.ContinualConsumer db accounts.Database asDB storage.Database rsAPI api.RoomserverInternalAPI @@ -50,7 +50,7 @@ func NewOutputRoomEventConsumer( rsAPI api.RoomserverInternalAPI, workerStates []types.ApplicationServiceWorkerState, ) *OutputRoomEventConsumer { - consumer := common.ContinualConsumer{ + consumer := internal.ContinualConsumer{ Topic: string(cfg.Kafka.Topics.OutputRoomEvent), Consumer: kafkaConsumer, PartitionStore: store, diff --git a/appservice/query/query.go b/appservice/query/query.go index fde3ab09c..a61997b42 100644 --- a/appservice/query/query.go +++ b/appservice/query/query.go @@ -24,8 +24,8 @@ import ( "time" "github.com/matrix-org/dendrite/appservice/api" - "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/util" opentracing "github.com/opentracing/opentracing-go" log "github.com/sirupsen/logrus" @@ -185,7 +185,7 @@ func makeHTTPClient() *http.Client { func (a *AppServiceQueryAPI) SetupHTTP(servMux *http.ServeMux) { servMux.Handle( api.AppServiceRoomAliasExistsPath, - common.MakeInternalAPI("appserviceRoomAliasExists", func(req *http.Request) util.JSONResponse { + internal.MakeInternalAPI("appserviceRoomAliasExists", func(req *http.Request) util.JSONResponse { var request api.RoomAliasExistsRequest var response api.RoomAliasExistsResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -199,7 +199,7 @@ func (a *AppServiceQueryAPI) SetupHTTP(servMux *http.ServeMux) { ) servMux.Handle( api.AppServiceUserIDExistsPath, - common.MakeInternalAPI("appserviceUserIDExists", func(req *http.Request) util.JSONResponse { + internal.MakeInternalAPI("appserviceUserIDExists", func(req *http.Request) util.JSONResponse { var request api.UserIDExistsRequest var response api.UserIDExistsResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { diff --git a/appservice/routing/routing.go b/appservice/routing/routing.go index 9f59e05f7..ac1ff32ea 100644 --- a/appservice/routing/routing.go +++ b/appservice/routing/routing.go @@ -19,9 +19,9 @@ import ( "github.com/gorilla/mux" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" - "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/common/config" - "github.com/matrix-org/dendrite/common/transactions" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/transactions" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -45,7 +45,7 @@ func Setup( appMux := apiMux.PathPrefix(pathPrefixApp).Subrouter() appMux.Handle("/alias", - common.MakeExternalAPI("alias", func(req *http.Request) util.JSONResponse { + internal.MakeExternalAPI("alias", func(req *http.Request) util.JSONResponse { // TODO: Implement return util.JSONResponse{ Code: http.StatusOK, @@ -54,7 +54,7 @@ func Setup( }), ).Methods(http.MethodGet, http.MethodOptions) appMux.Handle("/user", - common.MakeExternalAPI("user", func(req *http.Request) util.JSONResponse { + internal.MakeExternalAPI("user", func(req *http.Request) util.JSONResponse { // TODO: Implement return util.JSONResponse{ Code: http.StatusOK, diff --git a/appservice/storage/postgres/storage.go b/appservice/storage/postgres/storage.go index 475db6fc2..aeead6ed3 100644 --- a/appservice/storage/postgres/storage.go +++ b/appservice/storage/postgres/storage.go @@ -21,7 +21,7 @@ import ( // Import postgres database driver _ "github.com/lib/pq" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) @@ -34,7 +34,7 @@ type Database struct { } // NewDatabase opens a new database -func NewDatabase(dataSourceName string, dbProperties common.DbProperties) (*Database, error) { +func NewDatabase(dataSourceName string, dbProperties internal.DbProperties) (*Database, error) { var result Database var err error if result.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { diff --git a/appservice/storage/sqlite3/storage.go b/appservice/storage/sqlite3/storage.go index 0b0590f6d..275c66526 100644 --- a/appservice/storage/sqlite3/storage.go +++ b/appservice/storage/sqlite3/storage.go @@ -20,7 +20,7 @@ import ( "database/sql" // Import SQLite database driver - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" _ "github.com/mattn/go-sqlite3" @@ -37,7 +37,7 @@ type Database struct { func NewDatabase(dataSourceName string) (*Database, error) { var result Database var err error - if result.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil); err != nil { + if result.db, err = sqlutil.Open(internal.SQLiteDriverName(), dataSourceName, nil); err != nil { return nil, err } if err = result.prepare(); err != nil { diff --git a/appservice/storage/storage.go b/appservice/storage/storage.go index bf0a9b0c8..51ba1de53 100644 --- a/appservice/storage/storage.go +++ b/appservice/storage/storage.go @@ -21,12 +21,12 @@ import ( "github.com/matrix-org/dendrite/appservice/storage/postgres" "github.com/matrix-org/dendrite/appservice/storage/sqlite3" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" ) // NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme) // and sets DB connection parameters -func NewDatabase(dataSourceName string, dbProperties common.DbProperties) (Database, error) { +func NewDatabase(dataSourceName string, dbProperties internal.DbProperties) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return postgres.NewDatabase(dataSourceName, dbProperties) diff --git a/appservice/storage/storage_wasm.go b/appservice/storage/storage_wasm.go index ff0330ae0..a6144b435 100644 --- a/appservice/storage/storage_wasm.go +++ b/appservice/storage/storage_wasm.go @@ -19,12 +19,12 @@ import ( "net/url" "github.com/matrix-org/dendrite/appservice/storage/sqlite3" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" ) func NewDatabase( dataSourceName string, - dbProperties common.DbProperties, // nolint:unparam + dbProperties internal.DbProperties, // nolint:unparam ) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { diff --git a/appservice/types/types.go b/appservice/types/types.go index aac731550..b6386df67 100644 --- a/appservice/types/types.go +++ b/appservice/types/types.go @@ -15,7 +15,7 @@ package types import ( "sync" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal/config" ) const ( diff --git a/appservice/workers/transaction_scheduler.go b/appservice/workers/transaction_scheduler.go index 10c7ef911..208056192 100644 --- a/appservice/workers/transaction_scheduler.go +++ b/appservice/workers/transaction_scheduler.go @@ -25,7 +25,7 @@ import ( "github.com/matrix-org/dendrite/appservice/storage" "github.com/matrix-org/dendrite/appservice/types" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/gomatrixserverlib" log "github.com/sirupsen/logrus" ) diff --git a/docker/Dockerfile b/build/docker/Dockerfile similarity index 100% rename from docker/Dockerfile rename to build/docker/Dockerfile diff --git a/docker/README.md b/build/docker/README.md similarity index 100% rename from docker/README.md rename to build/docker/README.md diff --git a/docker/build.sh b/build/docker/build.sh similarity index 100% rename from docker/build.sh rename to build/docker/build.sh diff --git a/docker/dendrite-docker.yml b/build/docker/dendrite-docker.yml similarity index 100% rename from docker/dendrite-docker.yml rename to build/docker/dendrite-docker.yml diff --git a/docker/docker-compose.yml b/build/docker/docker-compose.yml similarity index 100% rename from docker/docker-compose.yml rename to build/docker/docker-compose.yml diff --git a/docker/hub/Dockerfile b/build/docker/hub/Dockerfile similarity index 100% rename from docker/hub/Dockerfile rename to build/docker/hub/Dockerfile diff --git a/docker/hub/Dockerfile.component b/build/docker/hub/Dockerfile.component similarity index 100% rename from docker/hub/Dockerfile.component rename to build/docker/hub/Dockerfile.component diff --git a/docker/hub/README.md b/build/docker/hub/README.md similarity index 100% rename from docker/hub/README.md rename to build/docker/hub/README.md diff --git a/docker/hub/config/dendrite-config.yaml b/build/docker/hub/config/dendrite-config.yaml similarity index 100% rename from docker/hub/config/dendrite-config.yaml rename to build/docker/hub/config/dendrite-config.yaml diff --git a/docker/hub/docker-compose.deps.yml b/build/docker/hub/docker-compose.deps.yml similarity index 100% rename from docker/hub/docker-compose.deps.yml rename to build/docker/hub/docker-compose.deps.yml diff --git a/docker/hub/docker-compose.monolith.yml b/build/docker/hub/docker-compose.monolith.yml similarity index 100% rename from docker/hub/docker-compose.monolith.yml rename to build/docker/hub/docker-compose.monolith.yml diff --git a/docker/hub/docker-compose.polylith.yml b/build/docker/hub/docker-compose.polylith.yml similarity index 100% rename from docker/hub/docker-compose.polylith.yml rename to build/docker/hub/docker-compose.polylith.yml diff --git a/docker/hub/images-build.sh b/build/docker/hub/images-build.sh similarity index 100% rename from docker/hub/images-build.sh rename to build/docker/hub/images-build.sh diff --git a/docker/hub/images-push.sh b/build/docker/hub/images-push.sh similarity index 100% rename from docker/hub/images-push.sh rename to build/docker/hub/images-push.sh diff --git a/docker/postgres/create_db.sh b/build/docker/postgres/create_db.sh similarity index 100% rename from docker/postgres/create_db.sh rename to build/docker/postgres/create_db.sh diff --git a/docker/services/client-api-proxy.sh b/build/docker/services/client-api-proxy.sh similarity index 100% rename from docker/services/client-api-proxy.sh rename to build/docker/services/client-api-proxy.sh diff --git a/docker/services/client-api.sh b/build/docker/services/client-api.sh similarity index 100% rename from docker/services/client-api.sh rename to build/docker/services/client-api.sh diff --git a/docker/services/edu-server.sh b/build/docker/services/edu-server.sh similarity index 100% rename from docker/services/edu-server.sh rename to build/docker/services/edu-server.sh diff --git a/docker/services/federation-api-proxy.sh b/build/docker/services/federation-api-proxy.sh similarity index 100% rename from docker/services/federation-api-proxy.sh rename to build/docker/services/federation-api-proxy.sh diff --git a/docker/services/federation-api.sh b/build/docker/services/federation-api.sh similarity index 100% rename from docker/services/federation-api.sh rename to build/docker/services/federation-api.sh diff --git a/docker/services/federation-sender.sh b/build/docker/services/federation-sender.sh similarity index 100% rename from docker/services/federation-sender.sh rename to build/docker/services/federation-sender.sh diff --git a/docker/services/key-server.sh b/build/docker/services/key-server.sh similarity index 100% rename from docker/services/key-server.sh rename to build/docker/services/key-server.sh diff --git a/docker/services/media-api.sh b/build/docker/services/media-api.sh similarity index 100% rename from docker/services/media-api.sh rename to build/docker/services/media-api.sh diff --git a/docker/services/monolith.sh b/build/docker/services/monolith.sh similarity index 100% rename from docker/services/monolith.sh rename to build/docker/services/monolith.sh diff --git a/docker/services/public-rooms-api.sh b/build/docker/services/public-rooms-api.sh similarity index 100% rename from docker/services/public-rooms-api.sh rename to build/docker/services/public-rooms-api.sh diff --git a/docker/services/room-server.sh b/build/docker/services/room-server.sh similarity index 100% rename from docker/services/room-server.sh rename to build/docker/services/room-server.sh diff --git a/docker/services/sync-api.sh b/build/docker/services/sync-api.sh similarity index 100% rename from docker/services/sync-api.sh rename to build/docker/services/sync-api.sh diff --git a/scripts/README.md b/build/scripts/README.md similarity index 100% rename from scripts/README.md rename to build/scripts/README.md diff --git a/scripts/build-test-lint.sh b/build/scripts/build-test-lint.sh similarity index 100% rename from scripts/build-test-lint.sh rename to build/scripts/build-test-lint.sh diff --git a/scripts/find-lint.sh b/build/scripts/find-lint.sh similarity index 98% rename from scripts/find-lint.sh rename to build/scripts/find-lint.sh index c9663e4e8..7e37e1548 100755 --- a/scripts/find-lint.sh +++ b/build/scripts/find-lint.sh @@ -14,7 +14,7 @@ set -eux -cd `dirname $0`/.. +cd `dirname $0`/../.. args="" if [ ${1:-""} = "fast" ] diff --git a/scripts/install-local-kafka.sh b/build/scripts/install-local-kafka.sh similarity index 100% rename from scripts/install-local-kafka.sh rename to build/scripts/install-local-kafka.sh diff --git a/clientapi/auth/auth.go b/clientapi/auth/auth.go index 87a2f6677..3482e5018 100644 --- a/clientapi/auth/auth.go +++ b/clientapi/auth/auth.go @@ -28,7 +28,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/userutil" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/util" ) diff --git a/clientapi/auth/storage/accounts/interface.go b/clientapi/auth/storage/accounts/interface.go index a5052b047..a860f809d 100644 --- a/clientapi/auth/storage/accounts/interface.go +++ b/clientapi/auth/storage/accounts/interface.go @@ -19,12 +19,12 @@ import ( "errors" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/gomatrixserverlib" ) type Database interface { - common.PartitionStorer + internal.PartitionStorer GetAccountByPassword(ctx context.Context, localpart, plaintextPassword string) (*authtypes.Account, error) GetProfileByLocalpart(ctx context.Context, localpart string) (*authtypes.Profile, error) SetAvatarURL(ctx context.Context, localpart string, avatarURL string) error diff --git a/clientapi/auth/storage/accounts/postgres/account_data_table.go b/clientapi/auth/storage/accounts/postgres/account_data_table.go index 9198a7440..2e044b367 100644 --- a/clientapi/auth/storage/accounts/postgres/account_data_table.go +++ b/clientapi/auth/storage/accounts/postgres/account_data_table.go @@ -18,7 +18,7 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/gomatrixserverlib" ) @@ -92,7 +92,7 @@ func (s *accountDataStatements) selectAccountData( if err != nil { return } - defer common.CloseAndLogIfError(ctx, rows, "selectAccountData: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "selectAccountData: rows.close() failed") global = []gomatrixserverlib.ClientEvent{} rooms = make(map[string][]gomatrixserverlib.ClientEvent) diff --git a/clientapi/auth/storage/accounts/postgres/membership_table.go b/clientapi/auth/storage/accounts/postgres/membership_table.go index 04e9095e9..d006e916a 100644 --- a/clientapi/auth/storage/accounts/postgres/membership_table.go +++ b/clientapi/auth/storage/accounts/postgres/membership_table.go @@ -18,7 +18,7 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/lib/pq" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" @@ -127,7 +127,7 @@ func (s *membershipStatements) selectMembershipsByLocalpart( memberships = []authtypes.Membership{} - defer common.CloseAndLogIfError(ctx, rows, "selectMembershipsByLocalpart: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "selectMembershipsByLocalpart: rows.close() failed") for rows.Next() { var m authtypes.Membership m.Localpart = localpart diff --git a/clientapi/auth/storage/accounts/postgres/storage.go b/clientapi/auth/storage/accounts/postgres/storage.go index 7e2073ec0..4a1832674 100644 --- a/clientapi/auth/storage/accounts/postgres/storage.go +++ b/clientapi/auth/storage/accounts/postgres/storage.go @@ -21,7 +21,7 @@ import ( "strconv" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" "golang.org/x/crypto/bcrypt" @@ -33,7 +33,7 @@ import ( // Database represents an account database type Database struct { db *sql.DB - common.PartitionOffsetStatements + internal.PartitionOffsetStatements accounts accountsStatements profiles profilesStatements memberships membershipStatements @@ -44,13 +44,13 @@ type Database struct { } // NewDatabase creates a new accounts and profiles database -func NewDatabase(dataSourceName string, dbProperties common.DbProperties, serverName gomatrixserverlib.ServerName) (*Database, error) { +func NewDatabase(dataSourceName string, dbProperties internal.DbProperties, serverName gomatrixserverlib.ServerName) (*Database, error) { var db *sql.DB var err error if db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { return nil, err } - partitions := common.PartitionOffsetStatements{} + partitions := internal.PartitionOffsetStatements{} if err = partitions.Prepare(db, "account"); err != nil { return nil, err } @@ -123,7 +123,7 @@ func (d *Database) SetDisplayName( // CreateGuestAccount makes a new guest account and creates an empty profile // for this account. func (d *Database) CreateGuestAccount(ctx context.Context) (acc *authtypes.Account, err error) { - err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { var numLocalpart int64 numLocalpart, err = d.accounts.selectNewNumericLocalpart(ctx, txn) if err != nil { @@ -142,7 +142,7 @@ func (d *Database) CreateGuestAccount(ctx context.Context) (acc *authtypes.Accou func (d *Database) CreateAccount( ctx context.Context, localpart, plaintextPassword, appserviceID string, ) (acc *authtypes.Account, err error) { - err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { acc, err = d.createAccount(ctx, txn, localpart, plaintextPassword, appserviceID) return err }) @@ -163,7 +163,7 @@ func (d *Database) createAccount( } } if err := d.profiles.insertProfile(ctx, txn, localpart); err != nil { - if common.IsUniqueConstraintViolationErr(err) { + if internal.IsUniqueConstraintViolationErr(err) { return nil, nil } return nil, err @@ -210,7 +210,7 @@ func (d *Database) removeMembershipsByEventIDs( func (d *Database) UpdateMemberships( ctx context.Context, eventsToAdd []gomatrixserverlib.Event, idsToRemove []string, ) error { - return common.WithTransaction(d.db, func(txn *sql.Tx) error { + return internal.WithTransaction(d.db, func(txn *sql.Tx) error { if err := d.removeMembershipsByEventIDs(ctx, txn, idsToRemove); err != nil { return err } @@ -297,7 +297,7 @@ func (d *Database) newMembership( func (d *Database) SaveAccountData( ctx context.Context, localpart, roomID, dataType, content string, ) error { - return common.WithTransaction(d.db, func(txn *sql.Tx) error { + return internal.WithTransaction(d.db, func(txn *sql.Tx) error { return d.accountDatas.insertAccountData(ctx, txn, localpart, roomID, dataType, content) }) } @@ -348,7 +348,7 @@ var Err3PIDInUse = errors.New("This third-party identifier is already in use") func (d *Database) SaveThreePIDAssociation( ctx context.Context, threepid, localpart, medium string, ) (err error) { - return common.WithTransaction(d.db, func(txn *sql.Tx) error { + return internal.WithTransaction(d.db, func(txn *sql.Tx) error { user, err := d.threepids.selectLocalpartForThreePID( ctx, txn, threepid, medium, ) diff --git a/clientapi/auth/storage/accounts/postgres/threepid_table.go b/clientapi/auth/storage/accounts/postgres/threepid_table.go index 851b4a90b..0b12b5c5b 100644 --- a/clientapi/auth/storage/accounts/postgres/threepid_table.go +++ b/clientapi/auth/storage/accounts/postgres/threepid_table.go @@ -18,7 +18,7 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" ) @@ -82,7 +82,7 @@ func (s *threepidStatements) prepare(db *sql.DB) (err error) { func (s *threepidStatements) selectLocalpartForThreePID( ctx context.Context, txn *sql.Tx, threepid string, medium string, ) (localpart string, err error) { - stmt := common.TxStmt(txn, s.selectLocalpartForThreePIDStmt) + stmt := internal.TxStmt(txn, s.selectLocalpartForThreePIDStmt) err = stmt.QueryRowContext(ctx, threepid, medium).Scan(&localpart) if err == sql.ErrNoRows { return "", nil @@ -117,7 +117,7 @@ func (s *threepidStatements) selectThreePIDsForLocalpart( func (s *threepidStatements) insertThreePID( ctx context.Context, txn *sql.Tx, threepid, medium, localpart string, ) (err error) { - stmt := common.TxStmt(txn, s.insertThreePIDStmt) + stmt := internal.TxStmt(txn, s.insertThreePIDStmt) _, err = stmt.ExecContext(ctx, threepid, medium, localpart) return } diff --git a/clientapi/auth/storage/accounts/sqlite3/membership_table.go b/clientapi/auth/storage/accounts/sqlite3/membership_table.go index bd9838b6b..90e16fb01 100644 --- a/clientapi/auth/storage/accounts/sqlite3/membership_table.go +++ b/clientapi/auth/storage/accounts/sqlite3/membership_table.go @@ -20,7 +20,7 @@ import ( "strings" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" ) const membershipSchema = ` @@ -95,7 +95,7 @@ func (s *membershipStatements) insertMembership( func (s *membershipStatements) deleteMembershipsByEventIDs( ctx context.Context, txn *sql.Tx, eventIDs []string, ) (err error) { - sqlStr := strings.Replace(deleteMembershipsByEventIDsSQL, "($1)", common.QueryVariadic(len(eventIDs)), 1) + sqlStr := strings.Replace(deleteMembershipsByEventIDsSQL, "($1)", internal.QueryVariadic(len(eventIDs)), 1) iEventIDs := make([]interface{}, len(eventIDs)) for i, e := range eventIDs { iEventIDs[i] = e @@ -125,7 +125,7 @@ func (s *membershipStatements) selectMembershipsByLocalpart( memberships = []authtypes.Membership{} - defer common.CloseAndLogIfError(ctx, rows, "selectMembershipsByLocalpart: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "selectMembershipsByLocalpart: rows.close() failed") for rows.Next() { var m authtypes.Membership m.Localpart = localpart diff --git a/clientapi/auth/storage/accounts/sqlite3/storage.go b/clientapi/auth/storage/accounts/sqlite3/storage.go index 30a93e7ed..7dec87294 100644 --- a/clientapi/auth/storage/accounts/sqlite3/storage.go +++ b/clientapi/auth/storage/accounts/sqlite3/storage.go @@ -22,7 +22,7 @@ import ( "sync" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" "golang.org/x/crypto/bcrypt" @@ -34,7 +34,7 @@ import ( // Database represents an account database type Database struct { db *sql.DB - common.PartitionOffsetStatements + internal.PartitionOffsetStatements accounts accountsStatements profiles profilesStatements memberships membershipStatements @@ -50,10 +50,10 @@ type Database struct { func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) { var db *sql.DB var err error - if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil); err != nil { + if db, err = sqlutil.Open(internal.SQLiteDriverName(), dataSourceName, nil); err != nil { return nil, err } - partitions := common.PartitionOffsetStatements{} + partitions := internal.PartitionOffsetStatements{} if err = partitions.Prepare(db, "account"); err != nil { return nil, err } @@ -126,7 +126,7 @@ func (d *Database) SetDisplayName( // CreateGuestAccount makes a new guest account and creates an empty profile // for this account. func (d *Database) CreateGuestAccount(ctx context.Context) (acc *authtypes.Account, err error) { - err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { // We need to lock so we sequentially create numeric localparts. If we don't, two calls to // this function will cause the same number to be selected and one will fail with 'database is locked' // when the first txn upgrades to a write txn. @@ -152,7 +152,7 @@ func (d *Database) CreateGuestAccount(ctx context.Context) (acc *authtypes.Accou func (d *Database) CreateAccount( ctx context.Context, localpart, plaintextPassword, appserviceID string, ) (acc *authtypes.Account, err error) { - err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { acc, err = d.createAccount(ctx, txn, localpart, plaintextPassword, appserviceID) return err }) @@ -172,7 +172,7 @@ func (d *Database) createAccount( } } if err := d.profiles.insertProfile(ctx, txn, localpart); err != nil { - if common.IsUniqueConstraintViolationErr(err) { + if internal.IsUniqueConstraintViolationErr(err) { return nil, nil } return nil, err @@ -219,7 +219,7 @@ func (d *Database) removeMembershipsByEventIDs( func (d *Database) UpdateMemberships( ctx context.Context, eventsToAdd []gomatrixserverlib.Event, idsToRemove []string, ) error { - return common.WithTransaction(d.db, func(txn *sql.Tx) error { + return internal.WithTransaction(d.db, func(txn *sql.Tx) error { if err := d.removeMembershipsByEventIDs(ctx, txn, idsToRemove); err != nil { return err } @@ -306,7 +306,7 @@ func (d *Database) newMembership( func (d *Database) SaveAccountData( ctx context.Context, localpart, roomID, dataType, content string, ) error { - return common.WithTransaction(d.db, func(txn *sql.Tx) error { + return internal.WithTransaction(d.db, func(txn *sql.Tx) error { return d.accountDatas.insertAccountData(ctx, txn, localpart, roomID, dataType, content) }) } @@ -357,7 +357,7 @@ var Err3PIDInUse = errors.New("This third-party identifier is already in use") func (d *Database) SaveThreePIDAssociation( ctx context.Context, threepid, localpart, medium string, ) (err error) { - return common.WithTransaction(d.db, func(txn *sql.Tx) error { + return internal.WithTransaction(d.db, func(txn *sql.Tx) error { user, err := d.threepids.selectLocalpartForThreePID( ctx, txn, threepid, medium, ) diff --git a/clientapi/auth/storage/accounts/sqlite3/threepid_table.go b/clientapi/auth/storage/accounts/sqlite3/threepid_table.go index 29ee4c3d0..f78a5ffcd 100644 --- a/clientapi/auth/storage/accounts/sqlite3/threepid_table.go +++ b/clientapi/auth/storage/accounts/sqlite3/threepid_table.go @@ -18,7 +18,7 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" ) @@ -82,7 +82,7 @@ func (s *threepidStatements) prepare(db *sql.DB) (err error) { func (s *threepidStatements) selectLocalpartForThreePID( ctx context.Context, txn *sql.Tx, threepid string, medium string, ) (localpart string, err error) { - stmt := common.TxStmt(txn, s.selectLocalpartForThreePIDStmt) + stmt := internal.TxStmt(txn, s.selectLocalpartForThreePIDStmt) err = stmt.QueryRowContext(ctx, threepid, medium).Scan(&localpart) if err == sql.ErrNoRows { return "", nil @@ -97,7 +97,7 @@ func (s *threepidStatements) selectThreePIDsForLocalpart( if err != nil { return } - defer common.CloseAndLogIfError(ctx, rows, "selectThreePIDsForLocalpart: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "selectThreePIDsForLocalpart: rows.close() failed") threepids = []authtypes.ThreePID{} for rows.Next() { @@ -117,7 +117,7 @@ func (s *threepidStatements) selectThreePIDsForLocalpart( func (s *threepidStatements) insertThreePID( ctx context.Context, txn *sql.Tx, threepid, medium, localpart string, ) (err error) { - stmt := common.TxStmt(txn, s.insertThreePIDStmt) + stmt := internal.TxStmt(txn, s.insertThreePIDStmt) _, err = stmt.ExecContext(ctx, threepid, medium, localpart) return } diff --git a/clientapi/auth/storage/accounts/storage.go b/clientapi/auth/storage/accounts/storage.go index 394cc5e16..37126b30b 100644 --- a/clientapi/auth/storage/accounts/storage.go +++ b/clientapi/auth/storage/accounts/storage.go @@ -21,13 +21,13 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/postgres" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/sqlite3" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/gomatrixserverlib" ) // NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme) // and sets postgres connection parameters -func NewDatabase(dataSourceName string, dbProperties common.DbProperties, serverName gomatrixserverlib.ServerName) (Database, error) { +func NewDatabase(dataSourceName string, dbProperties internal.DbProperties, serverName gomatrixserverlib.ServerName) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return postgres.NewDatabase(dataSourceName, dbProperties, serverName) diff --git a/clientapi/auth/storage/accounts/storage_wasm.go b/clientapi/auth/storage/accounts/storage_wasm.go index 61f9d6997..81e77cf79 100644 --- a/clientapi/auth/storage/accounts/storage_wasm.go +++ b/clientapi/auth/storage/accounts/storage_wasm.go @@ -19,13 +19,13 @@ import ( "net/url" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/sqlite3" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/gomatrixserverlib" ) func NewDatabase( dataSourceName string, - dbProperties common.DbProperties, // nolint:unparam + dbProperties internal.DbProperties, // nolint:unparam serverName gomatrixserverlib.ServerName, ) (Database, error) { uri, err := url.Parse(dataSourceName) diff --git a/clientapi/auth/storage/devices/postgres/devices_table.go b/clientapi/auth/storage/devices/postgres/devices_table.go index ee5591706..67d573f95 100644 --- a/clientapi/auth/storage/devices/postgres/devices_table.go +++ b/clientapi/auth/storage/devices/postgres/devices_table.go @@ -22,7 +22,7 @@ import ( "github.com/lib/pq" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/userutil" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/gomatrixserverlib" ) @@ -137,7 +137,7 @@ func (s *devicesStatements) insertDevice( ) (*authtypes.Device, error) { createdTimeMS := time.Now().UnixNano() / 1000000 var sessionID int64 - stmt := common.TxStmt(txn, s.insertDeviceStmt) + stmt := internal.TxStmt(txn, s.insertDeviceStmt) if err := stmt.QueryRowContext(ctx, id, localpart, accessToken, createdTimeMS, displayName).Scan(&sessionID); err != nil { return nil, err } @@ -153,7 +153,7 @@ func (s *devicesStatements) insertDevice( func (s *devicesStatements) deleteDevice( ctx context.Context, txn *sql.Tx, id, localpart string, ) error { - stmt := common.TxStmt(txn, s.deleteDeviceStmt) + stmt := internal.TxStmt(txn, s.deleteDeviceStmt) _, err := stmt.ExecContext(ctx, id, localpart) return err } @@ -163,7 +163,7 @@ func (s *devicesStatements) deleteDevice( func (s *devicesStatements) deleteDevices( ctx context.Context, txn *sql.Tx, localpart string, devices []string, ) error { - stmt := common.TxStmt(txn, s.deleteDevicesStmt) + stmt := internal.TxStmt(txn, s.deleteDevicesStmt) _, err := stmt.ExecContext(ctx, localpart, pq.Array(devices)) return err } @@ -173,7 +173,7 @@ func (s *devicesStatements) deleteDevices( func (s *devicesStatements) deleteDevicesByLocalpart( ctx context.Context, txn *sql.Tx, localpart string, ) error { - stmt := common.TxStmt(txn, s.deleteDevicesByLocalpartStmt) + stmt := internal.TxStmt(txn, s.deleteDevicesByLocalpartStmt) _, err := stmt.ExecContext(ctx, localpart) return err } @@ -181,7 +181,7 @@ func (s *devicesStatements) deleteDevicesByLocalpart( func (s *devicesStatements) updateDeviceName( ctx context.Context, txn *sql.Tx, localpart, deviceID string, displayName *string, ) error { - stmt := common.TxStmt(txn, s.updateDeviceNameStmt) + stmt := internal.TxStmt(txn, s.updateDeviceNameStmt) _, err := stmt.ExecContext(ctx, displayName, localpart, deviceID) return err } @@ -226,7 +226,7 @@ func (s *devicesStatements) selectDevicesByLocalpart( if err != nil { return devices, err } - defer common.CloseAndLogIfError(ctx, rows, "selectDevicesByLocalpart: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "selectDevicesByLocalpart: rows.close() failed") for rows.Next() { var dev authtypes.Device diff --git a/clientapi/auth/storage/devices/postgres/storage.go b/clientapi/auth/storage/devices/postgres/storage.go index 57c268df8..e54e7c5d7 100644 --- a/clientapi/auth/storage/devices/postgres/storage.go +++ b/clientapi/auth/storage/devices/postgres/storage.go @@ -21,7 +21,7 @@ import ( "encoding/base64" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) @@ -36,7 +36,7 @@ type Database struct { } // NewDatabase creates a new device database -func NewDatabase(dataSourceName string, dbProperties common.DbProperties, serverName gomatrixserverlib.ServerName) (*Database, error) { +func NewDatabase(dataSourceName string, dbProperties internal.DbProperties, serverName gomatrixserverlib.ServerName) (*Database, error) { var db *sql.DB var err error if db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { @@ -83,7 +83,7 @@ func (d *Database) CreateDevice( displayName *string, ) (dev *authtypes.Device, returnErr error) { if deviceID != nil { - returnErr = common.WithTransaction(d.db, func(txn *sql.Tx) error { + returnErr = internal.WithTransaction(d.db, func(txn *sql.Tx) error { var err error // Revoke existing tokens for this device if err = d.devices.deleteDevice(ctx, txn, *deviceID, localpart); err != nil { @@ -103,7 +103,7 @@ func (d *Database) CreateDevice( return } - returnErr = common.WithTransaction(d.db, func(txn *sql.Tx) error { + returnErr = internal.WithTransaction(d.db, func(txn *sql.Tx) error { var err error dev, err = d.devices.insertDevice(ctx, txn, newDeviceID, localpart, accessToken, displayName) return err @@ -133,7 +133,7 @@ func generateDeviceID() (string, error) { func (d *Database) UpdateDevice( ctx context.Context, localpart, deviceID string, displayName *string, ) error { - return common.WithTransaction(d.db, func(txn *sql.Tx) error { + return internal.WithTransaction(d.db, func(txn *sql.Tx) error { return d.devices.updateDeviceName(ctx, txn, localpart, deviceID, displayName) }) } @@ -145,7 +145,7 @@ func (d *Database) UpdateDevice( func (d *Database) RemoveDevice( ctx context.Context, deviceID, localpart string, ) error { - return common.WithTransaction(d.db, func(txn *sql.Tx) error { + return internal.WithTransaction(d.db, func(txn *sql.Tx) error { if err := d.devices.deleteDevice(ctx, txn, deviceID, localpart); err != sql.ErrNoRows { return err } @@ -160,7 +160,7 @@ func (d *Database) RemoveDevice( func (d *Database) RemoveDevices( ctx context.Context, localpart string, devices []string, ) error { - return common.WithTransaction(d.db, func(txn *sql.Tx) error { + return internal.WithTransaction(d.db, func(txn *sql.Tx) error { if err := d.devices.deleteDevices(ctx, txn, localpart, devices); err != sql.ErrNoRows { return err } @@ -174,7 +174,7 @@ func (d *Database) RemoveDevices( func (d *Database) RemoveAllDevices( ctx context.Context, localpart string, ) error { - return common.WithTransaction(d.db, func(txn *sql.Tx) error { + return internal.WithTransaction(d.db, func(txn *sql.Tx) error { if err := d.devices.deleteDevicesByLocalpart(ctx, txn, localpart); err != sql.ErrNoRows { return err } diff --git a/clientapi/auth/storage/devices/sqlite3/devices_table.go b/clientapi/auth/storage/devices/sqlite3/devices_table.go index f69810b7d..6ffc1646f 100644 --- a/clientapi/auth/storage/devices/sqlite3/devices_table.go +++ b/clientapi/auth/storage/devices/sqlite3/devices_table.go @@ -20,7 +20,7 @@ import ( "strings" "time" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/userutil" @@ -128,8 +128,8 @@ func (s *devicesStatements) insertDevice( ) (*authtypes.Device, error) { createdTimeMS := time.Now().UnixNano() / 1000000 var sessionID int64 - countStmt := common.TxStmt(txn, s.selectDevicesCountStmt) - insertStmt := common.TxStmt(txn, s.insertDeviceStmt) + countStmt := internal.TxStmt(txn, s.selectDevicesCountStmt) + insertStmt := internal.TxStmt(txn, s.insertDeviceStmt) if err := countStmt.QueryRowContext(ctx).Scan(&sessionID); err != nil { return nil, err } @@ -148,7 +148,7 @@ func (s *devicesStatements) insertDevice( func (s *devicesStatements) deleteDevice( ctx context.Context, txn *sql.Tx, id, localpart string, ) error { - stmt := common.TxStmt(txn, s.deleteDeviceStmt) + stmt := internal.TxStmt(txn, s.deleteDeviceStmt) _, err := stmt.ExecContext(ctx, id, localpart) return err } @@ -156,12 +156,12 @@ func (s *devicesStatements) deleteDevice( func (s *devicesStatements) deleteDevices( ctx context.Context, txn *sql.Tx, localpart string, devices []string, ) error { - orig := strings.Replace(deleteDevicesSQL, "($1)", common.QueryVariadic(len(devices)), 1) + orig := strings.Replace(deleteDevicesSQL, "($1)", internal.QueryVariadic(len(devices)), 1) prep, err := s.db.Prepare(orig) if err != nil { return err } - stmt := common.TxStmt(txn, prep) + stmt := internal.TxStmt(txn, prep) params := make([]interface{}, len(devices)+1) params[0] = localpart for i, v := range devices { @@ -175,7 +175,7 @@ func (s *devicesStatements) deleteDevices( func (s *devicesStatements) deleteDevicesByLocalpart( ctx context.Context, txn *sql.Tx, localpart string, ) error { - stmt := common.TxStmt(txn, s.deleteDevicesByLocalpartStmt) + stmt := internal.TxStmt(txn, s.deleteDevicesByLocalpartStmt) _, err := stmt.ExecContext(ctx, localpart) return err } @@ -183,7 +183,7 @@ func (s *devicesStatements) deleteDevicesByLocalpart( func (s *devicesStatements) updateDeviceName( ctx context.Context, txn *sql.Tx, localpart, deviceID string, displayName *string, ) error { - stmt := common.TxStmt(txn, s.updateDeviceNameStmt) + stmt := internal.TxStmt(txn, s.updateDeviceNameStmt) _, err := stmt.ExecContext(ctx, displayName, localpart, deviceID) return err } diff --git a/clientapi/auth/storage/devices/sqlite3/storage.go b/clientapi/auth/storage/devices/sqlite3/storage.go index b69d6278c..5cd894517 100644 --- a/clientapi/auth/storage/devices/sqlite3/storage.go +++ b/clientapi/auth/storage/devices/sqlite3/storage.go @@ -21,7 +21,7 @@ import ( "encoding/base64" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" @@ -41,7 +41,7 @@ type Database struct { func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) { var db *sql.DB var err error - if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil); err != nil { + if db, err = sqlutil.Open(internal.SQLiteDriverName(), dataSourceName, nil); err != nil { return nil, err } d := devicesStatements{} @@ -85,7 +85,7 @@ func (d *Database) CreateDevice( displayName *string, ) (dev *authtypes.Device, returnErr error) { if deviceID != nil { - returnErr = common.WithTransaction(d.db, func(txn *sql.Tx) error { + returnErr = internal.WithTransaction(d.db, func(txn *sql.Tx) error { var err error // Revoke existing tokens for this device if err = d.devices.deleteDevice(ctx, txn, *deviceID, localpart); err != nil { @@ -105,7 +105,7 @@ func (d *Database) CreateDevice( return } - returnErr = common.WithTransaction(d.db, func(txn *sql.Tx) error { + returnErr = internal.WithTransaction(d.db, func(txn *sql.Tx) error { var err error dev, err = d.devices.insertDevice(ctx, txn, newDeviceID, localpart, accessToken, displayName) return err @@ -135,7 +135,7 @@ func generateDeviceID() (string, error) { func (d *Database) UpdateDevice( ctx context.Context, localpart, deviceID string, displayName *string, ) error { - return common.WithTransaction(d.db, func(txn *sql.Tx) error { + return internal.WithTransaction(d.db, func(txn *sql.Tx) error { return d.devices.updateDeviceName(ctx, txn, localpart, deviceID, displayName) }) } @@ -147,7 +147,7 @@ func (d *Database) UpdateDevice( func (d *Database) RemoveDevice( ctx context.Context, deviceID, localpart string, ) error { - return common.WithTransaction(d.db, func(txn *sql.Tx) error { + return internal.WithTransaction(d.db, func(txn *sql.Tx) error { if err := d.devices.deleteDevice(ctx, txn, deviceID, localpart); err != sql.ErrNoRows { return err } @@ -162,7 +162,7 @@ func (d *Database) RemoveDevice( func (d *Database) RemoveDevices( ctx context.Context, localpart string, devices []string, ) error { - return common.WithTransaction(d.db, func(txn *sql.Tx) error { + return internal.WithTransaction(d.db, func(txn *sql.Tx) error { if err := d.devices.deleteDevices(ctx, txn, localpart, devices); err != sql.ErrNoRows { return err } @@ -176,7 +176,7 @@ func (d *Database) RemoveDevices( func (d *Database) RemoveAllDevices( ctx context.Context, localpart string, ) error { - return common.WithTransaction(d.db, func(txn *sql.Tx) error { + return internal.WithTransaction(d.db, func(txn *sql.Tx) error { if err := d.devices.deleteDevicesByLocalpart(ctx, txn, localpart); err != sql.ErrNoRows { return err } diff --git a/clientapi/auth/storage/devices/storage.go b/clientapi/auth/storage/devices/storage.go index ec47a3272..c132598bd 100644 --- a/clientapi/auth/storage/devices/storage.go +++ b/clientapi/auth/storage/devices/storage.go @@ -21,13 +21,13 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/storage/devices/postgres" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices/sqlite3" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/gomatrixserverlib" ) // NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme) // and sets postgres connection parameters -func NewDatabase(dataSourceName string, dbProperties common.DbProperties, serverName gomatrixserverlib.ServerName) (Database, error) { +func NewDatabase(dataSourceName string, dbProperties internal.DbProperties, serverName gomatrixserverlib.ServerName) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return postgres.NewDatabase(dataSourceName, dbProperties, serverName) diff --git a/clientapi/auth/storage/devices/storage_wasm.go b/clientapi/auth/storage/devices/storage_wasm.go index e25b7c640..14c19e74b 100644 --- a/clientapi/auth/storage/devices/storage_wasm.go +++ b/clientapi/auth/storage/devices/storage_wasm.go @@ -19,13 +19,13 @@ import ( "net/url" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices/sqlite3" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/gomatrixserverlib" ) func NewDatabase( dataSourceName string, - dbProperties common.DbProperties, // nolint:unparam + dbProperties internal.DbProperties, // nolint:unparam serverName gomatrixserverlib.ServerName, ) (Database, error) { uri, err := url.Parse(dataSourceName) diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go index f81e0242c..66317e92e 100644 --- a/clientapi/clientapi.go +++ b/clientapi/clientapi.go @@ -21,10 +21,10 @@ import ( "github.com/matrix-org/dendrite/clientapi/consumers" "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/clientapi/routing" - "github.com/matrix-org/dendrite/common/basecomponent" - "github.com/matrix-org/dendrite/common/transactions" eduServerAPI "github.com/matrix-org/dendrite/eduserver/api" federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/transactions" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" diff --git a/clientapi/consumers/roomserver.go b/clientapi/consumers/roomserver.go index d0c91e88f..bd8ac1dcd 100644 --- a/clientapi/consumers/roomserver.go +++ b/clientapi/consumers/roomserver.go @@ -19,8 +19,8 @@ import ( "encoding/json" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" - "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" @@ -31,7 +31,7 @@ import ( // OutputRoomEventConsumer consumes events that originated in the room server. type OutputRoomEventConsumer struct { rsAPI api.RoomserverInternalAPI - rsConsumer *common.ContinualConsumer + rsConsumer *internal.ContinualConsumer db accounts.Database serverName string } @@ -44,7 +44,7 @@ func NewOutputRoomEventConsumer( rsAPI api.RoomserverInternalAPI, ) *OutputRoomEventConsumer { - consumer := common.ContinualConsumer{ + consumer := internal.ContinualConsumer{ Topic: string(cfg.Kafka.Topics.OutputRoomEvent), Consumer: kafkaConsumer, PartitionStore: store, diff --git a/clientapi/producers/syncapi.go b/clientapi/producers/syncapi.go index 0a446e2f5..244a61dc2 100644 --- a/clientapi/producers/syncapi.go +++ b/clientapi/producers/syncapi.go @@ -17,7 +17,7 @@ package producers import ( "encoding/json" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/Shopify/sarama" ) @@ -32,7 +32,7 @@ type SyncAPIProducer struct { func (p *SyncAPIProducer) SendData(userID string, roomID string, dataType string) error { var m sarama.ProducerMessage - data := common.AccountData{ + data := internal.AccountData{ RoomID: roomID, Type: dataType, } diff --git a/clientapi/producers/userupdate.go b/clientapi/producers/userupdate.go index 426b6d509..02e1700fd 100644 --- a/clientapi/producers/userupdate.go +++ b/clientapi/producers/userupdate.go @@ -26,7 +26,7 @@ type UserUpdateProducer struct { Producer sarama.SyncProducer } -// TODO: Move this struct to `common` so the components that consume the topic +// TODO: Move this struct to `internal` so the components that consume the topic // can use it when parsing incoming messages type profileUpdate struct { Updated string `json:"updated"` // Which attribute is updated (can be either `avatar_url` or `displayname`) diff --git a/clientapi/routing/auth_fallback.go b/clientapi/routing/auth_fallback.go index 8cb6b3d9b..b7f2cd6d3 100644 --- a/clientapi/routing/auth_fallback.go +++ b/clientapi/routing/auth_fallback.go @@ -20,7 +20,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/util" ) diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index f789ae100..1b4ff184f 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -31,8 +31,8 @@ import ( "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/clientapi/threepid" - "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" log "github.com/sirupsen/logrus" @@ -99,7 +99,7 @@ func (r createRoomRequest) Validate() *util.JSONResponse { // Validate creation_content fields defined in the spec by marshalling the // creation_content map into bytes and then unmarshalling the bytes into - // common.CreateContent. + // internal.CreateContent. creationContentBytes, err := json.Marshal(r.CreationContent) if err != nil { @@ -280,25 +280,25 @@ func createRoom( eventsToMake := []fledglingEvent{ {"m.room.create", "", r.CreationContent}, {"m.room.member", userID, membershipContent}, - {"m.room.power_levels", "", common.InitialPowerLevelsContent(userID)}, + {"m.room.power_levels", "", internal.InitialPowerLevelsContent(userID)}, {"m.room.join_rules", "", gomatrixserverlib.JoinRuleContent{JoinRule: joinRules}}, - {"m.room.history_visibility", "", common.HistoryVisibilityContent{HistoryVisibility: historyVisibility}}, + {"m.room.history_visibility", "", internal.HistoryVisibilityContent{HistoryVisibility: historyVisibility}}, } if roomAlias != "" { // TODO: bit of a chicken and egg problem here as the alias doesn't exist and cannot until we have made the room. // This means we might fail creating the alias but say the canonical alias is something that doesn't exist. // m.room.aliases is handled when we call roomserver.SetRoomAlias - eventsToMake = append(eventsToMake, fledglingEvent{"m.room.canonical_alias", "", common.CanonicalAlias{Alias: roomAlias}}) + eventsToMake = append(eventsToMake, fledglingEvent{"m.room.canonical_alias", "", internal.CanonicalAlias{Alias: roomAlias}}) } if r.GuestCanJoin { - eventsToMake = append(eventsToMake, fledglingEvent{"m.room.guest_access", "", common.GuestAccessContent{GuestAccess: "can_join"}}) + eventsToMake = append(eventsToMake, fledglingEvent{"m.room.guest_access", "", internal.GuestAccessContent{GuestAccess: "can_join"}}) } eventsToMake = append(eventsToMake, r.InitialState...) if r.Name != "" { - eventsToMake = append(eventsToMake, fledglingEvent{"m.room.name", "", common.NameContent{Name: r.Name}}) + eventsToMake = append(eventsToMake, fledglingEvent{"m.room.name", "", internal.NameContent{Name: r.Name}}) } if r.Topic != "" { - eventsToMake = append(eventsToMake, fledglingEvent{"m.room.topic", "", common.TopicContent{Topic: r.Topic}}) + eventsToMake = append(eventsToMake, fledglingEvent{"m.room.topic", "", internal.TopicContent{Topic: r.Topic}}) } // TODO: invite events // TODO: 3pid invite events diff --git a/clientapi/routing/directory.go b/clientapi/routing/directory.go index a0a60a47a..b1dfb56fb 100644 --- a/clientapi/routing/directory.go +++ b/clientapi/routing/directory.go @@ -21,8 +21,8 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/common/config" federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/internal/config" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" diff --git a/clientapi/routing/getevent.go b/clientapi/routing/getevent.go index bf49968de..3ca9d1b62 100644 --- a/clientapi/routing/getevent.go +++ b/clientapi/routing/getevent.go @@ -19,7 +19,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" diff --git a/clientapi/routing/joinroom.go b/clientapi/routing/joinroom.go index 2a2fce185..500df337a 100644 --- a/clientapi/routing/joinroom.go +++ b/clientapi/routing/joinroom.go @@ -72,7 +72,7 @@ func JoinRoomByIDOrAlias( return util.JSONResponse{ Code: http.StatusOK, - // TODO: Put the response struct somewhere common. + // TODO: Put the response struct somewhere internal. JSON: struct { RoomID string `json:"room_id"` }{joinReq.RoomIDOrAlias}, diff --git a/clientapi/routing/login.go b/clientapi/routing/login.go index 21b947200..c0042fa95 100644 --- a/clientapi/routing/login.go +++ b/clientapi/routing/login.go @@ -26,7 +26,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/userutil" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index 0a56eec57..cd0ce7328 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -27,8 +27,8 @@ import ( "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/clientapi/threepid" - "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/roomserver/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" @@ -97,7 +97,7 @@ func SendMembership( Code: http.StatusBadRequest, JSON: jsonerror.BadJSON(err.Error()), } - } else if err == common.ErrRoomNoExists { + } else if err == internal.ErrRoomNoExists { return util.JSONResponse{ Code: http.StatusNotFound, JSON: jsonerror.NotFound(err.Error()), @@ -190,7 +190,7 @@ func buildMembershipEvent( return nil, err } - return common.BuildEvent(ctx, &builder, cfg, evTime, rsAPI, nil) + return internal.BuildEvent(ctx, &builder, cfg, evTime, rsAPI, nil) } // loadProfile lookups the profile of a given user from the database and returns @@ -271,7 +271,7 @@ func checkAndProcessThreepid( Code: http.StatusBadRequest, JSON: jsonerror.NotTrusted(body.IDServer), } - } else if err == common.ErrRoomNoExists { + } else if err == internal.ErrRoomNoExists { return inviteStored, &util.JSONResponse{ Code: http.StatusNotFound, JSON: jsonerror.NotFound(err.Error()), diff --git a/clientapi/routing/memberships.go b/clientapi/routing/memberships.go index f5d9bc4c5..095a85c08 100644 --- a/clientapi/routing/memberships.go +++ b/clientapi/routing/memberships.go @@ -21,7 +21,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index b51533e45..1c1ee8036 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -25,8 +25,8 @@ import ( "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" - "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" @@ -43,7 +43,7 @@ func GetProfile( ) util.JSONResponse { profile, err := getProfile(req.Context(), accountDB, cfg, userID, asAPI, federation) if err != nil { - if err == common.ErrProfileNoExists { + if err == internal.ErrProfileNoExists { return util.JSONResponse{ Code: http.StatusNotFound, JSON: jsonerror.NotFound("The user does not exist or does not have a profile"), @@ -56,7 +56,7 @@ func GetProfile( return util.JSONResponse{ Code: http.StatusOK, - JSON: common.ProfileResponse{ + JSON: internal.ProfileResponse{ AvatarURL: profile.AvatarURL, DisplayName: profile.DisplayName, }, @@ -71,7 +71,7 @@ func GetAvatarURL( ) util.JSONResponse { profile, err := getProfile(req.Context(), accountDB, cfg, userID, asAPI, federation) if err != nil { - if err == common.ErrProfileNoExists { + if err == internal.ErrProfileNoExists { return util.JSONResponse{ Code: http.StatusNotFound, JSON: jsonerror.NotFound("The user does not exist or does not have a profile"), @@ -84,7 +84,7 @@ func GetAvatarURL( return util.JSONResponse{ Code: http.StatusOK, - JSON: common.AvatarURL{ + JSON: internal.AvatarURL{ AvatarURL: profile.AvatarURL, }, } @@ -105,7 +105,7 @@ func SetAvatarURL( changedKey := "avatar_url" - var r common.AvatarURL + var r internal.AvatarURL if resErr := httputil.UnmarshalJSONRequest(req, &r); resErr != nil { return *resErr } @@ -185,7 +185,7 @@ func GetDisplayName( ) util.JSONResponse { profile, err := getProfile(req.Context(), accountDB, cfg, userID, asAPI, federation) if err != nil { - if err == common.ErrProfileNoExists { + if err == internal.ErrProfileNoExists { return util.JSONResponse{ Code: http.StatusNotFound, JSON: jsonerror.NotFound("The user does not exist or does not have a profile"), @@ -198,7 +198,7 @@ func GetDisplayName( return util.JSONResponse{ Code: http.StatusOK, - JSON: common.DisplayName{ + JSON: internal.DisplayName{ DisplayName: profile.DisplayName, }, } @@ -219,7 +219,7 @@ func SetDisplayName( changedKey := "displayname" - var r common.DisplayName + var r internal.DisplayName if resErr := httputil.UnmarshalJSONRequest(req, &r); resErr != nil { return *resErr } @@ -294,7 +294,7 @@ func SetDisplayName( // getProfile gets the full profile of a user by querying the database or a // remote homeserver. // Returns an error when something goes wrong or specifically -// common.ErrProfileNoExists when the profile doesn't exist. +// internal.ErrProfileNoExists when the profile doesn't exist. func getProfile( ctx context.Context, accountDB accounts.Database, cfg *config.Dendrite, userID string, @@ -311,7 +311,7 @@ func getProfile( if fedErr != nil { if x, ok := fedErr.(gomatrix.HTTPError); ok { if x.Code == http.StatusNotFound { - return nil, common.ErrProfileNoExists + return nil, internal.ErrProfileNoExists } } @@ -366,7 +366,7 @@ func buildMembershipEvents( return nil, err } - event, err := common.BuildEvent(ctx, &builder, cfg, evTime, rsAPI, nil) + event, err := internal.BuildEvent(ctx, &builder, cfg, evTime, rsAPI, nil) if err != nil { return nil, err } diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index b67e68e19..40a2862f8 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -32,7 +32,7 @@ import ( "sync" "time" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/clientapi/auth" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" @@ -41,7 +41,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/userutil" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/tokens" "github.com/matrix-org/util" @@ -136,7 +136,7 @@ type registerRequest struct { DeviceID *string `json:"device_id"` // Prevent this user from logging in - InhibitLogin common.WeakBoolean `json:"inhibit_login"` + InhibitLogin internal.WeakBoolean `json:"inhibit_login"` // Application Services place Type in the root of their registration // request, whereas clients place it in the authDict struct. @@ -811,7 +811,7 @@ func completeRegistration( accountDB accounts.Database, deviceDB devices.Database, username, password, appserviceID string, - inhibitLogin common.WeakBoolean, + inhibitLogin internal.WeakBoolean, displayName, deviceID *string, ) util.JSONResponse { if username == "" { diff --git a/clientapi/routing/register_test.go b/clientapi/routing/register_test.go index 6fcf0bc39..a44389f94 100644 --- a/clientapi/routing/register_test.go +++ b/clientapi/routing/register_test.go @@ -19,7 +19,7 @@ import ( "testing" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal/config" ) var ( diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index f8cdcf5b8..377881cb6 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -27,10 +27,10 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" - "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/common/config" - "github.com/matrix-org/dendrite/common/transactions" federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/transactions" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -63,7 +63,7 @@ func Setup( ) { apiMux.Handle("/_matrix/client/versions", - common.MakeExternalAPI("versions", func(req *http.Request) util.JSONResponse { + internal.MakeExternalAPI("versions", func(req *http.Request) util.JSONResponse { return util.JSONResponse{ Code: http.StatusOK, JSON: struct { @@ -89,13 +89,13 @@ func Setup( } r0mux.Handle("/createRoom", - common.MakeAuthAPI("createRoom", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + internal.MakeAuthAPI("createRoom", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return CreateRoom(req, device, cfg, producer, accountDB, rsAPI, asAPI) }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/join/{roomIDOrAlias}", - common.MakeAuthAPI(gomatrixserverlib.Join, authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(req)) + internal.MakeAuthAPI(gomatrixserverlib.Join, authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := internal.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -105,13 +105,13 @@ func Setup( }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/joined_rooms", - common.MakeAuthAPI("joined_rooms", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + internal.MakeAuthAPI("joined_rooms", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return GetJoinedRooms(req, device, accountDB) }), ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/leave", - common.MakeAuthAPI("membership", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(req)) + internal.MakeAuthAPI("membership", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := internal.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -121,8 +121,8 @@ func Setup( }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/{membership:(?:join|kick|ban|unban|invite)}", - common.MakeAuthAPI("membership", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(req)) + internal.MakeAuthAPI("membership", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := internal.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -130,8 +130,8 @@ func Setup( }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/send/{eventType}", - common.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(req)) + internal.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := internal.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -139,8 +139,8 @@ func Setup( }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/send/{eventType}/{txnID}", - common.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(req)) + internal.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := internal.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -150,8 +150,8 @@ func Setup( }), ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/event/{eventID}", - common.MakeAuthAPI("rooms_get_event", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(req)) + internal.MakeAuthAPI("rooms_get_event", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := internal.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -159,24 +159,24 @@ func Setup( }), ).Methods(http.MethodGet, http.MethodOptions) - r0mux.Handle("/rooms/{roomID}/state", common.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(req)) + r0mux.Handle("/rooms/{roomID}/state", internal.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := internal.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } return OnIncomingStateRequest(req.Context(), rsAPI, vars["roomID"]) })).Methods(http.MethodGet, http.MethodOptions) - r0mux.Handle("/rooms/{roomID}/state/{type}", common.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(req)) + r0mux.Handle("/rooms/{roomID}/state/{type}", internal.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := internal.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } return OnIncomingStateTypeRequest(req.Context(), rsAPI, vars["roomID"], vars["type"], "") })).Methods(http.MethodGet, http.MethodOptions) - r0mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", common.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(req)) + r0mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", internal.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := internal.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -184,8 +184,8 @@ func Setup( })).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/state/{eventType:[^/]+/?}", - common.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(req)) + internal.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := internal.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -200,8 +200,8 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/state/{eventType}/{stateKey}", - common.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(req)) + internal.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := internal.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -210,21 +210,21 @@ func Setup( }), ).Methods(http.MethodPut, http.MethodOptions) - r0mux.Handle("/register", common.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse { + r0mux.Handle("/register", internal.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse { return Register(req, accountDB, deviceDB, cfg) })).Methods(http.MethodPost, http.MethodOptions) - v1mux.Handle("/register", common.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse { + v1mux.Handle("/register", internal.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse { return LegacyRegister(req, accountDB, deviceDB, cfg) })).Methods(http.MethodPost, http.MethodOptions) - r0mux.Handle("/register/available", common.MakeExternalAPI("registerAvailable", func(req *http.Request) util.JSONResponse { + r0mux.Handle("/register/available", internal.MakeExternalAPI("registerAvailable", func(req *http.Request) util.JSONResponse { return RegisterAvailable(req, cfg, accountDB) })).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/directory/room/{roomAlias}", - common.MakeExternalAPI("directory_room", func(req *http.Request) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(req)) + internal.MakeExternalAPI("directory_room", func(req *http.Request) util.JSONResponse { + vars, err := internal.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -233,8 +233,8 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/directory/room/{roomAlias}", - common.MakeAuthAPI("directory_room", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(req)) + internal.MakeAuthAPI("directory_room", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := internal.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -243,8 +243,8 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/directory/room/{roomAlias}", - common.MakeAuthAPI("directory_room", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(req)) + internal.MakeAuthAPI("directory_room", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := internal.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -253,20 +253,20 @@ func Setup( ).Methods(http.MethodDelete, http.MethodOptions) r0mux.Handle("/logout", - common.MakeAuthAPI("logout", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + internal.MakeAuthAPI("logout", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return Logout(req, deviceDB, device) }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/logout/all", - common.MakeAuthAPI("logout", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + internal.MakeAuthAPI("logout", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return LogoutAll(req, deviceDB, device) }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/typing/{userID}", - common.MakeAuthAPI("rooms_typing", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(req)) + internal.MakeAuthAPI("rooms_typing", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := internal.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -275,7 +275,7 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/account/whoami", - common.MakeAuthAPI("whoami", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + internal.MakeAuthAPI("whoami", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return Whoami(req, device) }), ).Methods(http.MethodGet, http.MethodOptions) @@ -283,20 +283,20 @@ func Setup( // Stub endpoints required by Riot r0mux.Handle("/login", - common.MakeExternalAPI("login", func(req *http.Request) util.JSONResponse { + internal.MakeExternalAPI("login", func(req *http.Request) util.JSONResponse { return Login(req, accountDB, deviceDB, cfg) }), ).Methods(http.MethodGet, http.MethodPost, http.MethodOptions) r0mux.Handle("/auth/{authType}/fallback/web", - common.MakeHTMLAPI("auth_fallback", func(w http.ResponseWriter, req *http.Request) *util.JSONResponse { + internal.MakeHTMLAPI("auth_fallback", func(w http.ResponseWriter, req *http.Request) *util.JSONResponse { vars := mux.Vars(req) return AuthFallback(w, req, vars["authType"], cfg) }), ).Methods(http.MethodGet, http.MethodPost, http.MethodOptions) r0mux.Handle("/pushrules/", - common.MakeExternalAPI("push_rules", func(req *http.Request) util.JSONResponse { + internal.MakeExternalAPI("push_rules", func(req *http.Request) util.JSONResponse { // TODO: Implement push rules API res := json.RawMessage(`{ "global": { @@ -315,8 +315,8 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/user/{userId}/filter", - common.MakeAuthAPI("put_filter", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(req)) + internal.MakeAuthAPI("put_filter", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := internal.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -325,8 +325,8 @@ func Setup( ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/user/{userId}/filter/{filterId}", - common.MakeAuthAPI("get_filter", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(req)) + internal.MakeAuthAPI("get_filter", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := internal.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -337,8 +337,8 @@ func Setup( // Riot user settings r0mux.Handle("/profile/{userID}", - common.MakeExternalAPI("profile", func(req *http.Request) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(req)) + internal.MakeExternalAPI("profile", func(req *http.Request) util.JSONResponse { + vars, err := internal.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -347,8 +347,8 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/profile/{userID}/avatar_url", - common.MakeExternalAPI("profile_avatar_url", func(req *http.Request) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(req)) + internal.MakeExternalAPI("profile_avatar_url", func(req *http.Request) util.JSONResponse { + vars, err := internal.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -357,8 +357,8 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/profile/{userID}/avatar_url", - common.MakeAuthAPI("profile_avatar_url", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(req)) + internal.MakeAuthAPI("profile_avatar_url", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := internal.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -369,8 +369,8 @@ func Setup( // PUT requests, so we need to allow this method r0mux.Handle("/profile/{userID}/displayname", - common.MakeExternalAPI("profile_displayname", func(req *http.Request) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(req)) + internal.MakeExternalAPI("profile_displayname", func(req *http.Request) util.JSONResponse { + vars, err := internal.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -379,8 +379,8 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/profile/{userID}/displayname", - common.MakeAuthAPI("profile_displayname", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(req)) + internal.MakeAuthAPI("profile_displayname", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := internal.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -391,32 +391,32 @@ func Setup( // PUT requests, so we need to allow this method r0mux.Handle("/account/3pid", - common.MakeAuthAPI("account_3pid", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + internal.MakeAuthAPI("account_3pid", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return GetAssociated3PIDs(req, accountDB, device) }), ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/account/3pid", - common.MakeAuthAPI("account_3pid", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + internal.MakeAuthAPI("account_3pid", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return CheckAndSave3PIDAssociation(req, accountDB, device, cfg) }), ).Methods(http.MethodPost, http.MethodOptions) unstableMux.Handle("/account/3pid/delete", - common.MakeAuthAPI("account_3pid", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + internal.MakeAuthAPI("account_3pid", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return Forget3PID(req, accountDB) }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/{path:(?:account/3pid|register)}/email/requestToken", - common.MakeExternalAPI("account_3pid_request_token", func(req *http.Request) util.JSONResponse { + internal.MakeExternalAPI("account_3pid_request_token", func(req *http.Request) util.JSONResponse { return RequestEmailToken(req, accountDB, cfg) }), ).Methods(http.MethodPost, http.MethodOptions) // Riot logs get flooded unless this is handled r0mux.Handle("/presence/{userID}/status", - common.MakeExternalAPI("presence", func(req *http.Request) util.JSONResponse { + internal.MakeExternalAPI("presence", func(req *http.Request) util.JSONResponse { // TODO: Set presence (probably the responsibility of a presence server not clientapi) return util.JSONResponse{ Code: http.StatusOK, @@ -426,13 +426,13 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/voip/turnServer", - common.MakeAuthAPI("turn_server", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + internal.MakeAuthAPI("turn_server", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return RequestTurnServer(req, device, cfg) }), ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/thirdparty/protocols", - common.MakeExternalAPI("thirdparty_protocols", func(req *http.Request) util.JSONResponse { + internal.MakeExternalAPI("thirdparty_protocols", func(req *http.Request) util.JSONResponse { // TODO: Return the third party protcols return util.JSONResponse{ Code: http.StatusOK, @@ -442,7 +442,7 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/initialSync", - common.MakeExternalAPI("rooms_initial_sync", func(req *http.Request) util.JSONResponse { + internal.MakeExternalAPI("rooms_initial_sync", func(req *http.Request) util.JSONResponse { // TODO: Allow people to peek into rooms. return util.JSONResponse{ Code: http.StatusForbidden, @@ -452,8 +452,8 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/user/{userID}/account_data/{type}", - common.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(req)) + internal.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := internal.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -462,8 +462,8 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/user/{userID}/rooms/{roomID}/account_data/{type}", - common.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(req)) + internal.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := internal.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -472,8 +472,8 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/user/{userID}/account_data/{type}", - common.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(req)) + internal.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := internal.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -482,8 +482,8 @@ func Setup( ).Methods(http.MethodGet) r0mux.Handle("/user/{userID}/rooms/{roomID}/account_data/{type}", - common.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(req)) + internal.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := internal.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -492,8 +492,8 @@ func Setup( ).Methods(http.MethodGet) r0mux.Handle("/rooms/{roomID}/members", - common.MakeAuthAPI("rooms_members", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(req)) + internal.MakeAuthAPI("rooms_members", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := internal.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -502,8 +502,8 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/joined_members", - common.MakeAuthAPI("rooms_members", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(req)) + internal.MakeAuthAPI("rooms_members", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := internal.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -512,21 +512,21 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/read_markers", - common.MakeExternalAPI("rooms_read_markers", func(req *http.Request) util.JSONResponse { + internal.MakeExternalAPI("rooms_read_markers", func(req *http.Request) util.JSONResponse { // TODO: return the read_markers. return util.JSONResponse{Code: http.StatusOK, JSON: struct{}{}} }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/devices", - common.MakeAuthAPI("get_devices", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + internal.MakeAuthAPI("get_devices", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return GetDevicesByLocalpart(req, deviceDB, device) }), ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/devices/{deviceID}", - common.MakeAuthAPI("get_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(req)) + internal.MakeAuthAPI("get_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := internal.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -535,8 +535,8 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/devices/{deviceID}", - common.MakeAuthAPI("device_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(req)) + internal.MakeAuthAPI("device_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := internal.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -545,8 +545,8 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/devices/{deviceID}", - common.MakeAuthAPI("delete_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(req)) + internal.MakeAuthAPI("delete_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := internal.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -555,14 +555,14 @@ func Setup( ).Methods(http.MethodDelete, http.MethodOptions) r0mux.Handle("/delete_devices", - common.MakeAuthAPI("delete_devices", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + internal.MakeAuthAPI("delete_devices", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return DeleteDevices(req, deviceDB, device) }), ).Methods(http.MethodPost, http.MethodOptions) // Stub implementations for sytest r0mux.Handle("/events", - common.MakeExternalAPI("events", func(req *http.Request) util.JSONResponse { + internal.MakeExternalAPI("events", func(req *http.Request) util.JSONResponse { return util.JSONResponse{Code: http.StatusOK, JSON: map[string]interface{}{ "chunk": []interface{}{}, "start": "", @@ -572,7 +572,7 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/initialSync", - common.MakeExternalAPI("initial_sync", func(req *http.Request) util.JSONResponse { + internal.MakeExternalAPI("initial_sync", func(req *http.Request) util.JSONResponse { return util.JSONResponse{Code: http.StatusOK, JSON: map[string]interface{}{ "end": "", }} @@ -580,8 +580,8 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/user/{userId}/rooms/{roomId}/tags", - common.MakeAuthAPI("get_tags", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(req)) + internal.MakeAuthAPI("get_tags", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := internal.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -590,8 +590,8 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/user/{userId}/rooms/{roomId}/tags/{tag}", - common.MakeAuthAPI("put_tag", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(req)) + internal.MakeAuthAPI("put_tag", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := internal.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -600,8 +600,8 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/user/{userId}/rooms/{roomId}/tags/{tag}", - common.MakeAuthAPI("delete_tag", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(req)) + internal.MakeAuthAPI("delete_tag", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := internal.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -610,7 +610,7 @@ func Setup( ).Methods(http.MethodDelete, http.MethodOptions) r0mux.Handle("/capabilities", - common.MakeAuthAPI("capabilities", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + internal.MakeAuthAPI("capabilities", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return GetCapabilities(req, rsAPI) }), ).Methods(http.MethodGet) diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go index 7280dcd94..aa7317094 100644 --- a/clientapi/routing/sendevent.go +++ b/clientapi/routing/sendevent.go @@ -21,9 +21,9 @@ import ( "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" - "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/common/config" - "github.com/matrix-org/dendrite/common/transactions" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/transactions" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -148,14 +148,14 @@ func generateSendEvent( } var queryRes api.QueryLatestEventsAndStateResponse - e, err := common.BuildEvent(req.Context(), &builder, cfg, evTime, rsAPI, &queryRes) - if err == common.ErrRoomNoExists { + e, err := internal.BuildEvent(req.Context(), &builder, cfg, evTime, rsAPI, &queryRes) + if err == internal.ErrRoomNoExists { return nil, &util.JSONResponse{ Code: http.StatusNotFound, JSON: jsonerror.NotFound("Room does not exist"), } } else if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("common.BuildEvent failed") + util.GetLogger(req.Context()).WithError(err).Error("internal.BuildEvent failed") resErr := jsonerror.InternalServerError() return nil, &resErr } diff --git a/clientapi/routing/threepid.go b/clientapi/routing/threepid.go index fed9ae32e..49a9f3047 100644 --- a/clientapi/routing/threepid.go +++ b/clientapi/routing/threepid.go @@ -22,7 +22,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/threepid" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" diff --git a/clientapi/routing/voip.go b/clientapi/routing/voip.go index c1fd741c9..212d9b0a4 100644 --- a/clientapi/routing/voip.go +++ b/clientapi/routing/voip.go @@ -24,7 +24,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/gomatrix" "github.com/matrix-org/util" ) diff --git a/clientapi/threepid/invites.go b/clientapi/threepid/invites.go index 5e7e4f2b4..8fed99565 100644 --- a/clientapi/threepid/invites.go +++ b/clientapi/threepid/invites.go @@ -27,8 +27,8 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/producers" - "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" ) @@ -354,7 +354,7 @@ func emit3PIDInviteEvent( } queryRes := api.QueryLatestEventsAndStateResponse{} - event, err := common.BuildEvent(ctx, builder, cfg, evTime, rsAPI, &queryRes) + event, err := internal.BuildEvent(ctx, builder, cfg, evTime, rsAPI, &queryRes) if err != nil { return err } diff --git a/clientapi/threepid/threepid.go b/clientapi/threepid/threepid.go index a7f26c295..bffe31adc 100644 --- a/clientapi/threepid/threepid.go +++ b/clientapi/threepid/threepid.go @@ -24,7 +24,7 @@ import ( "strconv" "strings" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal/config" ) // EmailAssociationRequest represents the request defined at https://matrix.org/docs/spec/client_server/r0.2.0.html#post-matrix-client-r0-register-email-requesttoken diff --git a/cmd/dendrite-appservice-server/main.go b/cmd/dendrite-appservice-server/main.go index faf9ade31..a508be3b4 100644 --- a/cmd/dendrite-appservice-server/main.go +++ b/cmd/dendrite-appservice-server/main.go @@ -16,8 +16,8 @@ package main import ( "github.com/matrix-org/dendrite/appservice" - "github.com/matrix-org/dendrite/common/basecomponent" - "github.com/matrix-org/dendrite/common/transactions" + "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/transactions" ) func main() { diff --git a/cmd/dendrite-client-api-server/main.go b/cmd/dendrite-client-api-server/main.go index d4ab9e42f..0815be799 100644 --- a/cmd/dendrite-client-api-server/main.go +++ b/cmd/dendrite-client-api-server/main.go @@ -16,11 +16,11 @@ package main import ( "github.com/matrix-org/dendrite/clientapi" - "github.com/matrix-org/dendrite/common/basecomponent" - "github.com/matrix-org/dendrite/common/keydb" - "github.com/matrix-org/dendrite/common/transactions" "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/eduserver/cache" + "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/keydb" + "github.com/matrix-org/dendrite/internal/transactions" ) func main() { diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index 0ff610a5b..a04a28e06 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -32,13 +32,13 @@ import ( "github.com/matrix-org/dendrite/clientapi" "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/cmd/dendrite-demo-libp2p/storage" - "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/common/config" - "github.com/matrix-org/dendrite/common/keydb" - "github.com/matrix-org/dendrite/common/transactions" "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/federationsender" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/keydb" + "github.com/matrix-org/dendrite/internal/transactions" "github.com/matrix-org/dendrite/mediaapi" "github.com/matrix-org/dendrite/publicroomsapi" "github.com/matrix-org/dendrite/roomserver" @@ -178,7 +178,7 @@ func main() { publicroomsapi.SetupPublicRoomsAPIComponent(&base.Base, deviceDB, publicRoomsDB, rsAPI, federation, nil) // Check this later syncapi.SetupSyncAPIComponent(&base.Base, deviceDB, accountDB, rsAPI, federation, &cfg) - httpHandler := common.WrapHandlerInCORS(base.Base.APIMux) + httpHandler := internal.WrapHandlerInCORS(base.Base.APIMux) // Set up the API endpoints we handle. /metrics is for prometheus, and is // not wrapped by CORS, while everything else is diff --git a/cmd/dendrite-demo-libp2p/mdnslistener.go b/cmd/dendrite-demo-libp2p/mdnslistener.go index 3fefbec2c..c05c94d59 100644 --- a/cmd/dendrite-demo-libp2p/mdnslistener.go +++ b/cmd/dendrite-demo-libp2p/mdnslistener.go @@ -21,7 +21,7 @@ import ( "github.com/libp2p/go-libp2p-core/host" "github.com/libp2p/go-libp2p-core/peer" - "github.com/matrix-org/dendrite/common/keydb" + "github.com/matrix-org/dendrite/internal/keydb" "github.com/matrix-org/gomatrixserverlib" ) diff --git a/cmd/dendrite-demo-libp2p/p2pdendrite.go b/cmd/dendrite-demo-libp2p/p2pdendrite.go index 831b5d7f4..b7c5c66b0 100644 --- a/cmd/dendrite-demo-libp2p/p2pdendrite.go +++ b/cmd/dendrite-demo-libp2p/p2pdendrite.go @@ -22,7 +22,7 @@ import ( pstore "github.com/libp2p/go-libp2p-core/peerstore" record "github.com/libp2p/go-libp2p-record" - "github.com/matrix-org/dendrite/common/basecomponent" + "github.com/matrix-org/dendrite/internal/basecomponent" "github.com/libp2p/go-libp2p" circuit "github.com/libp2p/go-libp2p-circuit" @@ -34,7 +34,7 @@ import ( pubsub "github.com/libp2p/go-libp2p-pubsub" "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal/config" ) // P2PDendrite is a Peer-to-Peer variant of BaseDendrite. diff --git a/cmd/dendrite-edu-server/main.go b/cmd/dendrite-edu-server/main.go index 98b05e649..66e17e577 100644 --- a/cmd/dendrite-edu-server/main.go +++ b/cmd/dendrite-edu-server/main.go @@ -15,9 +15,9 @@ package main import ( _ "net/http/pprof" - "github.com/matrix-org/dendrite/common/basecomponent" "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/eduserver/cache" + "github.com/matrix-org/dendrite/internal/basecomponent" "github.com/sirupsen/logrus" ) diff --git a/cmd/dendrite-federation-api-server/main.go b/cmd/dendrite-federation-api-server/main.go index b37570201..be97bc9a0 100644 --- a/cmd/dendrite-federation-api-server/main.go +++ b/cmd/dendrite-federation-api-server/main.go @@ -16,11 +16,11 @@ package main import ( "github.com/matrix-org/dendrite/clientapi/producers" - "github.com/matrix-org/dendrite/common/basecomponent" - "github.com/matrix-org/dendrite/common/keydb" "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/federationapi" + "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/keydb" ) func main() { diff --git a/cmd/dendrite-federation-sender-server/main.go b/cmd/dendrite-federation-sender-server/main.go index c9bf6688f..b6af8deb7 100644 --- a/cmd/dendrite-federation-sender-server/main.go +++ b/cmd/dendrite-federation-sender-server/main.go @@ -15,9 +15,9 @@ package main import ( - "github.com/matrix-org/dendrite/common/basecomponent" - "github.com/matrix-org/dendrite/common/keydb" "github.com/matrix-org/dendrite/federationsender" + "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/keydb" ) func main() { diff --git a/cmd/dendrite-key-server/main.go b/cmd/dendrite-key-server/main.go index aca666fad..17fee2c64 100644 --- a/cmd/dendrite-key-server/main.go +++ b/cmd/dendrite-key-server/main.go @@ -15,7 +15,7 @@ package main import ( - "github.com/matrix-org/dendrite/common/basecomponent" + "github.com/matrix-org/dendrite/internal/basecomponent" "github.com/matrix-org/dendrite/keyserver" ) diff --git a/cmd/dendrite-media-api-server/main.go b/cmd/dendrite-media-api-server/main.go index cfb32173d..1a51717a7 100644 --- a/cmd/dendrite-media-api-server/main.go +++ b/cmd/dendrite-media-api-server/main.go @@ -15,7 +15,7 @@ package main import ( - "github.com/matrix-org/dendrite/common/basecomponent" + "github.com/matrix-org/dendrite/internal/basecomponent" "github.com/matrix-org/dendrite/mediaapi" ) diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 23959152e..6d3a479ac 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -21,14 +21,14 @@ import ( "github.com/matrix-org/dendrite/appservice" "github.com/matrix-org/dendrite/clientapi" "github.com/matrix-org/dendrite/clientapi/producers" - "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/common/basecomponent" - "github.com/matrix-org/dendrite/common/keydb" - "github.com/matrix-org/dendrite/common/transactions" "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/federationsender" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/keydb" + "github.com/matrix-org/dendrite/internal/transactions" "github.com/matrix-org/dendrite/keyserver" "github.com/matrix-org/dendrite/mediaapi" "github.com/matrix-org/dendrite/publicroomsapi" @@ -91,12 +91,12 @@ func main() { publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, rsAPI, federation, nil) syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, rsAPI, federation, cfg) - httpHandler := common.WrapHandlerInCORS(base.APIMux) + httpHandler := internal.WrapHandlerInCORS(base.APIMux) // Set up the API endpoints we handle. /metrics is for prometheus, and is // not wrapped by CORS, while everything else is if cfg.Metrics.Enabled { - http.Handle("/metrics", common.WrapHandlerInBasicAuth(promhttp.Handler(), cfg.Metrics.BasicAuth)) + http.Handle("/metrics", internal.WrapHandlerInBasicAuth(promhttp.Handler(), cfg.Metrics.BasicAuth)) } http.Handle("/", httpHandler) diff --git a/cmd/dendrite-public-rooms-api-server/main.go b/cmd/dendrite-public-rooms-api-server/main.go index 0eb2fcb57..cf9d09c11 100644 --- a/cmd/dendrite-public-rooms-api-server/main.go +++ b/cmd/dendrite-public-rooms-api-server/main.go @@ -15,7 +15,7 @@ package main import ( - "github.com/matrix-org/dendrite/common/basecomponent" + "github.com/matrix-org/dendrite/internal/basecomponent" "github.com/matrix-org/dendrite/publicroomsapi" "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/sirupsen/logrus" diff --git a/cmd/dendrite-room-server/main.go b/cmd/dendrite-room-server/main.go index 4d450bd47..31cf4a042 100644 --- a/cmd/dendrite-room-server/main.go +++ b/cmd/dendrite-room-server/main.go @@ -15,8 +15,8 @@ package main import ( - "github.com/matrix-org/dendrite/common/basecomponent" - "github.com/matrix-org/dendrite/common/keydb" + "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/keydb" "github.com/matrix-org/dendrite/roomserver" ) diff --git a/cmd/dendrite-sync-api-server/main.go b/cmd/dendrite-sync-api-server/main.go index 7aadd6a7b..5a1a80dfd 100644 --- a/cmd/dendrite-sync-api-server/main.go +++ b/cmd/dendrite-sync-api-server/main.go @@ -15,7 +15,7 @@ package main import ( - "github.com/matrix-org/dendrite/common/basecomponent" + "github.com/matrix-org/dendrite/internal/basecomponent" "github.com/matrix-org/dendrite/syncapi" ) diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 5bdf18efa..2609c74cd 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -25,14 +25,14 @@ import ( "github.com/matrix-org/dendrite/appservice" "github.com/matrix-org/dendrite/clientapi" "github.com/matrix-org/dendrite/clientapi/producers" - "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/common/basecomponent" - "github.com/matrix-org/dendrite/common/config" - "github.com/matrix-org/dendrite/common/transactions" "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/federationsender" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/transactions" "github.com/matrix-org/dendrite/mediaapi" "github.com/matrix-org/dendrite/publicroomsapi" "github.com/matrix-org/dendrite/publicroomsapi/storage" @@ -227,7 +227,7 @@ func main() { publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, rsAPI, federation, p2pPublicRoomProvider) syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, rsAPI, federation, cfg) - httpHandler := common.WrapHandlerInCORS(base.APIMux) + httpHandler := internal.WrapHandlerInCORS(base.APIMux) http.Handle("/", httpHandler) diff --git a/cmd/generate-keys/main.go b/cmd/generate-keys/main.go index b807c2673..ceeca5672 100644 --- a/cmd/generate-keys/main.go +++ b/cmd/generate-keys/main.go @@ -20,7 +20,7 @@ import ( "log" "os" - "github.com/matrix-org/dendrite/common/test" + "github.com/matrix-org/dendrite/internal/test" ) const usage = `Usage: %s diff --git a/cmd/mediaapi-integration-tests/main.go b/cmd/mediaapi-integration-tests/main.go index fd6982fd3..e6ce14d28 100644 --- a/cmd/mediaapi-integration-tests/main.go +++ b/cmd/mediaapi-integration-tests/main.go @@ -24,7 +24,7 @@ import ( "path/filepath" "time" - "github.com/matrix-org/dendrite/common/test" + "github.com/matrix-org/dendrite/internal/test" "github.com/matrix-org/gomatrixserverlib" "gopkg.in/yaml.v2" ) diff --git a/cmd/roomserver-integration-tests/main.go b/cmd/roomserver-integration-tests/main.go index 7126405e5..126053bac 100644 --- a/cmd/roomserver-integration-tests/main.go +++ b/cmd/roomserver-integration-tests/main.go @@ -28,8 +28,8 @@ import ( "net/http" - "github.com/matrix-org/dendrite/common/caching" - "github.com/matrix-org/dendrite/common/test" + "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/test" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" ) diff --git a/cmd/syncserver-integration-tests/main.go b/cmd/syncserver-integration-tests/main.go index d14e854c3..cfe8cc165 100644 --- a/cmd/syncserver-integration-tests/main.go +++ b/cmd/syncserver-integration-tests/main.go @@ -24,8 +24,8 @@ import ( "path/filepath" "time" - "github.com/matrix-org/dendrite/common/config" - "github.com/matrix-org/dendrite/common/test" + "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/test" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" ) diff --git a/eduserver/api/input.go b/eduserver/api/input.go index be2d4c56a..999a5b41a 100644 --- a/eduserver/api/input.go +++ b/eduserver/api/input.go @@ -18,7 +18,7 @@ import ( "errors" "net/http" - commonHTTP "github.com/matrix-org/dendrite/common/http" + internalHTTP "github.com/matrix-org/dendrite/internal/http" "github.com/matrix-org/gomatrixserverlib" opentracing "github.com/opentracing/opentracing-go" ) @@ -80,5 +80,5 @@ func (h *httpEDUServerInputAPI) InputTypingEvent( defer span.Finish() apiURL := h.eduServerURL + EDUServerInputTypingEventPath - return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } diff --git a/eduserver/cache/cache_test.go b/eduserver/cache/cache_test.go index 8a1b6f797..d1b2f8bd5 100644 --- a/eduserver/cache/cache_test.go +++ b/eduserver/cache/cache_test.go @@ -16,7 +16,7 @@ import ( "testing" "time" - "github.com/matrix-org/dendrite/common/test" + "github.com/matrix-org/dendrite/internal/test" ) func TestEDUCache(t *testing.T) { diff --git a/eduserver/eduserver.go b/eduserver/eduserver.go index 2a209364a..cba60b8ea 100644 --- a/eduserver/eduserver.go +++ b/eduserver/eduserver.go @@ -15,10 +15,10 @@ package eduserver import ( "net/http" - "github.com/matrix-org/dendrite/common/basecomponent" "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/eduserver/input" + "github.com/matrix-org/dendrite/internal/basecomponent" ) // SetupEDUServerComponent sets up and registers HTTP handlers for the diff --git a/eduserver/input/input.go b/eduserver/input/input.go index 6a4a4bb4e..50837154a 100644 --- a/eduserver/input/input.go +++ b/eduserver/input/input.go @@ -19,9 +19,9 @@ import ( "time" "github.com/Shopify/sarama" - "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/eduserver/cache" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -92,7 +92,7 @@ func (t *EDUServerInputAPI) sendEvent(ite *api.InputTypingEvent) error { // SetupHTTP adds the EDUServerInputAPI handlers to the http.ServeMux. func (t *EDUServerInputAPI) SetupHTTP(servMux *http.ServeMux) { servMux.Handle(api.EDUServerInputTypingEventPath, - common.MakeInternalAPI("inputTypingEvents", func(req *http.Request) util.JSONResponse { + internal.MakeInternalAPI("inputTypingEvents", func(req *http.Request) util.JSONResponse { var request api.InputTypingEventRequest var response api.InputTypingEventResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go index d458b8537..9e3411403 100644 --- a/federationapi/federationapi.go +++ b/federationapi/federationapi.go @@ -18,8 +18,8 @@ import ( appserviceAPI "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" - "github.com/matrix-org/dendrite/common/basecomponent" federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/internal/basecomponent" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" // TODO: Are we really wanting to pull in the producer from clientapi diff --git a/federationapi/routing/backfill.go b/federationapi/routing/backfill.go index 0d3893de3..10bc62630 100644 --- a/federationapi/routing/backfill.go +++ b/federationapi/routing/backfill.go @@ -22,7 +22,7 @@ import ( "time" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" diff --git a/federationapi/routing/invite.go b/federationapi/routing/invite.go index 064abe7e9..05efe5872 100644 --- a/federationapi/routing/invite.go +++ b/federationapi/routing/invite.go @@ -21,7 +21,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal/config" roomserverVersion "github.com/matrix-org/dendrite/roomserver/version" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go index 3533bbba8..617a6cab0 100644 --- a/federationapi/routing/join.go +++ b/federationapi/routing/join.go @@ -22,8 +22,8 @@ import ( "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" - "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -96,14 +96,14 @@ func MakeJoin( queryRes := api.QueryLatestEventsAndStateResponse{ RoomVersion: verRes.RoomVersion, } - event, err := common.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), rsAPI, &queryRes) - if err == common.ErrRoomNoExists { + event, err := internal.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), rsAPI, &queryRes) + if err == internal.ErrRoomNoExists { return util.JSONResponse{ Code: http.StatusNotFound, JSON: jsonerror.NotFound("Room does not exist"), } } else if err != nil { - util.GetLogger(httpReq.Context()).WithError(err).Error("common.BuildEvent failed") + util.GetLogger(httpReq.Context()).WithError(err).Error("internal.BuildEvent failed") return jsonerror.InternalServerError() } diff --git a/federationapi/routing/keys.go b/federationapi/routing/keys.go index 3eb88567d..79cc1f82c 100644 --- a/federationapi/routing/keys.go +++ b/federationapi/routing/keys.go @@ -19,7 +19,7 @@ import ( "net/http" "time" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" "golang.org/x/crypto/ed25519" diff --git a/federationapi/routing/leave.go b/federationapi/routing/leave.go index bab3fd0b5..bd226d7ee 100644 --- a/federationapi/routing/leave.go +++ b/federationapi/routing/leave.go @@ -18,8 +18,8 @@ import ( "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" - "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -70,14 +70,14 @@ func MakeLeave( } var queryRes api.QueryLatestEventsAndStateResponse - event, err := common.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), rsAPI, &queryRes) - if err == common.ErrRoomNoExists { + event, err := internal.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), rsAPI, &queryRes) + if err == internal.ErrRoomNoExists { return util.JSONResponse{ Code: http.StatusNotFound, JSON: jsonerror.NotFound("Room does not exist"), } } else if err != nil { - util.GetLogger(httpReq.Context()).WithError(err).Error("common.BuildEvent failed") + util.GetLogger(httpReq.Context()).WithError(err).Error("internal.BuildEvent failed") return jsonerror.InternalServerError() } diff --git a/federationapi/routing/profile.go b/federationapi/routing/profile.go index 9a81c1b33..832849845 100644 --- a/federationapi/routing/profile.go +++ b/federationapi/routing/profile.go @@ -21,8 +21,8 @@ import ( appserviceAPI "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -72,11 +72,11 @@ func GetProfile( if field != "" { switch field { case "displayname": - res = common.DisplayName{ + res = internal.DisplayName{ DisplayName: profile.DisplayName, } case "avatar_url": - res = common.AvatarURL{ + res = internal.AvatarURL{ AvatarURL: profile.AvatarURL, } default: @@ -84,7 +84,7 @@ func GetProfile( res = jsonerror.InvalidArgumentValue("The request body did not contain an allowed value of argument 'field'. Allowed values are either: 'avatar_url', 'displayname'.") } } else { - res = common.ProfileResponse{ + res = internal.ProfileResponse{ AvatarURL: profile.AvatarURL, DisplayName: profile.DisplayName, } diff --git a/federationapi/routing/query.go b/federationapi/routing/query.go index c58690c63..39fd6d2ee 100644 --- a/federationapi/routing/query.go +++ b/federationapi/routing/query.go @@ -19,8 +19,8 @@ import ( "net/http" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/common/config" federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/internal/config" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index 67f3a957c..c368c16c4 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -22,9 +22,9 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/clientapi/producers" - "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/common/config" federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/config" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -58,7 +58,7 @@ func Setup( v1fedmux := apiMux.PathPrefix(pathPrefixV1Federation).Subrouter() v2fedmux := apiMux.PathPrefix(pathPrefixV2Federation).Subrouter() - localKeys := common.MakeExternalAPI("localkeys", func(req *http.Request) util.JSONResponse { + localKeys := internal.MakeExternalAPI("localkeys", func(req *http.Request) util.JSONResponse { return LocalKeys(cfg) }) @@ -70,10 +70,10 @@ func Setup( v2keysmux.Handle("/server/", localKeys).Methods(http.MethodGet) v2keysmux.Handle("/server", localKeys).Methods(http.MethodGet) - v1fedmux.Handle("/send/{txnID}", common.MakeFedAPI( + v1fedmux.Handle("/send/{txnID}", internal.MakeFedAPI( "federation_send", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) + vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq)) if err != nil { return util.ErrorResponse(err) } @@ -84,10 +84,10 @@ func Setup( }, )).Methods(http.MethodPut, http.MethodOptions) - v2fedmux.Handle("/invite/{roomID}/{eventID}", common.MakeFedAPI( + v2fedmux.Handle("/invite/{roomID}/{eventID}", internal.MakeFedAPI( "federation_invite", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) + vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq)) if err != nil { return util.ErrorResponse(err) } @@ -98,16 +98,16 @@ func Setup( }, )).Methods(http.MethodPut, http.MethodOptions) - v1fedmux.Handle("/3pid/onbind", common.MakeExternalAPI("3pid_onbind", + v1fedmux.Handle("/3pid/onbind", internal.MakeExternalAPI("3pid_onbind", func(req *http.Request) util.JSONResponse { return CreateInvitesFrom3PIDInvites(req, rsAPI, asAPI, cfg, producer, federation, accountDB) }, )).Methods(http.MethodPost, http.MethodOptions) - v1fedmux.Handle("/exchange_third_party_invite/{roomID}", common.MakeFedAPI( + v1fedmux.Handle("/exchange_third_party_invite/{roomID}", internal.MakeFedAPI( "exchange_third_party_invite", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) + vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq)) if err != nil { return util.ErrorResponse(err) } @@ -117,10 +117,10 @@ func Setup( }, )).Methods(http.MethodPut, http.MethodOptions) - v1fedmux.Handle("/event/{eventID}", common.MakeFedAPI( + v1fedmux.Handle("/event/{eventID}", internal.MakeFedAPI( "federation_get_event", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) + vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq)) if err != nil { return util.ErrorResponse(err) } @@ -130,10 +130,10 @@ func Setup( }, )).Methods(http.MethodGet) - v1fedmux.Handle("/state/{roomID}", common.MakeFedAPI( + v1fedmux.Handle("/state/{roomID}", internal.MakeFedAPI( "federation_get_state", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) + vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq)) if err != nil { return util.ErrorResponse(err) } @@ -143,10 +143,10 @@ func Setup( }, )).Methods(http.MethodGet) - v1fedmux.Handle("/state_ids/{roomID}", common.MakeFedAPI( + v1fedmux.Handle("/state_ids/{roomID}", internal.MakeFedAPI( "federation_get_state_ids", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) + vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq)) if err != nil { return util.ErrorResponse(err) } @@ -156,7 +156,7 @@ func Setup( }, )).Methods(http.MethodGet) - v1fedmux.Handle("/event_auth/{roomID}/{eventID}", common.MakeFedAPI( + v1fedmux.Handle("/event_auth/{roomID}/{eventID}", internal.MakeFedAPI( "federation_get_event_auth", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { vars := mux.Vars(httpReq) @@ -166,7 +166,7 @@ func Setup( }, )).Methods(http.MethodGet) - v1fedmux.Handle("/query/directory", common.MakeFedAPI( + v1fedmux.Handle("/query/directory", internal.MakeFedAPI( "federation_query_room_alias", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { return RoomAliasToID( @@ -175,7 +175,7 @@ func Setup( }, )).Methods(http.MethodGet) - v1fedmux.Handle("/query/profile", common.MakeFedAPI( + v1fedmux.Handle("/query/profile", internal.MakeFedAPI( "federation_query_profile", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { return GetProfile( @@ -184,10 +184,10 @@ func Setup( }, )).Methods(http.MethodGet) - v1fedmux.Handle("/user/devices/{userID}", common.MakeFedAPI( + v1fedmux.Handle("/user/devices/{userID}", internal.MakeFedAPI( "federation_user_devices", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) + vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq)) if err != nil { return util.ErrorResponse(err) } @@ -197,10 +197,10 @@ func Setup( }, )).Methods(http.MethodGet) - v1fedmux.Handle("/make_join/{roomID}/{eventID}", common.MakeFedAPI( + v1fedmux.Handle("/make_join/{roomID}/{eventID}", internal.MakeFedAPI( "federation_make_join", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) + vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq)) if err != nil { return util.ErrorResponse(err) } @@ -226,10 +226,10 @@ func Setup( }, )).Methods(http.MethodGet) - v1fedmux.Handle("/send_join/{roomID}/{eventID}", common.MakeFedAPI( + v1fedmux.Handle("/send_join/{roomID}/{eventID}", internal.MakeFedAPI( "federation_send_join", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) + vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq)) if err != nil { return util.ErrorResponse(err) } @@ -248,10 +248,10 @@ func Setup( }, )).Methods(http.MethodPut) - v2fedmux.Handle("/send_join/{roomID}/{eventID}", common.MakeFedAPI( + v2fedmux.Handle("/send_join/{roomID}/{eventID}", internal.MakeFedAPI( "federation_send_join", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) + vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq)) if err != nil { return util.ErrorResponse(err) } @@ -263,10 +263,10 @@ func Setup( }, )).Methods(http.MethodPut) - v1fedmux.Handle("/make_leave/{roomID}/{eventID}", common.MakeFedAPI( + v1fedmux.Handle("/make_leave/{roomID}/{eventID}", internal.MakeFedAPI( "federation_make_leave", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) + vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq)) if err != nil { return util.ErrorResponse(err) } @@ -278,10 +278,10 @@ func Setup( }, )).Methods(http.MethodGet) - v2fedmux.Handle("/send_leave/{roomID}/{eventID}", common.MakeFedAPI( + v2fedmux.Handle("/send_leave/{roomID}/{eventID}", internal.MakeFedAPI( "federation_send_leave", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) + vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq)) if err != nil { return util.ErrorResponse(err) } @@ -293,17 +293,17 @@ func Setup( }, )).Methods(http.MethodPut) - v1fedmux.Handle("/version", common.MakeExternalAPI( + v1fedmux.Handle("/version", internal.MakeExternalAPI( "federation_version", func(httpReq *http.Request) util.JSONResponse { return Version() }, )).Methods(http.MethodGet) - v1fedmux.Handle("/get_missing_events/{roomID}", common.MakeFedAPI( + v1fedmux.Handle("/get_missing_events/{roomID}", internal.MakeFedAPI( "federation_get_missing_events", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) + vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq)) if err != nil { return util.ErrorResponse(err) } @@ -311,10 +311,10 @@ func Setup( }, )).Methods(http.MethodPost) - v1fedmux.Handle("/backfill/{roomID}", common.MakeFedAPI( + v1fedmux.Handle("/backfill/{roomID}", internal.MakeFedAPI( "federation_backfill", cfg.Matrix.ServerName, keys, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(httpReq)) + vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq)) if err != nil { return util.ErrorResponse(err) } diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index e62855abb..6cf44018e 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -22,7 +22,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" diff --git a/federationapi/routing/threepid.go b/federationapi/routing/threepid.go index 3c1d09e11..8053cedde 100644 --- a/federationapi/routing/threepid.go +++ b/federationapi/routing/threepid.go @@ -26,7 +26,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/roomserver/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" diff --git a/federationsender/api/perform.go b/federationsender/api/perform.go index 2f643e5cf..427e70683 100644 --- a/federationsender/api/perform.go +++ b/federationsender/api/perform.go @@ -3,8 +3,8 @@ package api import ( "context" - commonHTTP "github.com/matrix-org/dendrite/common/http" "github.com/matrix-org/dendrite/federationsender/types" + internalHTTP "github.com/matrix-org/dendrite/internal/http" "github.com/matrix-org/gomatrixserverlib" "github.com/opentracing/opentracing-go" ) @@ -40,7 +40,7 @@ func (h *httpFederationSenderInternalAPI) PerformDirectoryLookup( defer span.Finish() apiURL := h.federationSenderURL + FederationSenderPerformDirectoryLookupRequestPath - return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } type PerformJoinRequest struct { @@ -63,7 +63,7 @@ func (h *httpFederationSenderInternalAPI) PerformJoin( defer span.Finish() apiURL := h.federationSenderURL + FederationSenderPerformJoinRequestPath - return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } type PerformLeaveRequest struct { @@ -85,5 +85,5 @@ func (h *httpFederationSenderInternalAPI) PerformLeave( defer span.Finish() apiURL := h.federationSenderURL + FederationSenderPerformLeaveRequestPath - return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } diff --git a/federationsender/api/query.go b/federationsender/api/query.go index 7a58cc863..bf58d5cc9 100644 --- a/federationsender/api/query.go +++ b/federationsender/api/query.go @@ -3,8 +3,8 @@ package api import ( "context" - commonHTTP "github.com/matrix-org/dendrite/common/http" "github.com/matrix-org/dendrite/federationsender/types" + internalHTTP "github.com/matrix-org/dendrite/internal/http" "github.com/matrix-org/gomatrixserverlib" "github.com/opentracing/opentracing-go" @@ -36,7 +36,7 @@ func (h *httpFederationSenderInternalAPI) QueryJoinedHostsInRoom( defer span.Finish() apiURL := h.federationSenderURL + FederationSenderQueryJoinedHostsInRoomPath - return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // QueryJoinedHostServerNamesRequest is a request to QueryJoinedHostServerNames @@ -59,5 +59,5 @@ func (h *httpFederationSenderInternalAPI) QueryJoinedHostServerNamesInRoom( defer span.Finish() apiURL := h.federationSenderURL + FederationSenderQueryJoinedHostServerNamesInRoomPath - return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } diff --git a/federationsender/consumers/eduserver.go b/federationsender/consumers/eduserver.go index 269701d77..bcebb3ce1 100644 --- a/federationsender/consumers/eduserver.go +++ b/federationsender/consumers/eduserver.go @@ -17,18 +17,18 @@ import ( "encoding/json" "github.com/Shopify/sarama" - "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/federationsender/queue" "github.com/matrix-org/dendrite/federationsender/storage" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/gomatrixserverlib" log "github.com/sirupsen/logrus" ) // OutputTypingEventConsumer consumes events that originate in EDU server. type OutputTypingEventConsumer struct { - consumer *common.ContinualConsumer + consumer *internal.ContinualConsumer db storage.Database queues *queue.OutgoingQueues ServerName gomatrixserverlib.ServerName @@ -41,7 +41,7 @@ func NewOutputTypingEventConsumer( queues *queue.OutgoingQueues, store storage.Database, ) *OutputTypingEventConsumer { - consumer := common.ContinualConsumer{ + consumer := internal.ContinualConsumer{ Topic: string(cfg.Kafka.Topics.OutputTypingEvent), Consumer: kafkaConsumer, PartitionStore: store, diff --git a/federationsender/consumers/roomserver.go b/federationsender/consumers/roomserver.go index 901239472..5f8a555bd 100644 --- a/federationsender/consumers/roomserver.go +++ b/federationsender/consumers/roomserver.go @@ -20,11 +20,11 @@ import ( "fmt" "github.com/Shopify/sarama" - "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/federationsender/queue" "github.com/matrix-org/dendrite/federationsender/storage" "github.com/matrix-org/dendrite/federationsender/types" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" log "github.com/sirupsen/logrus" @@ -35,7 +35,7 @@ import ( type OutputRoomEventConsumer struct { cfg *config.Dendrite rsAPI api.RoomserverInternalAPI - rsConsumer *common.ContinualConsumer + rsConsumer *internal.ContinualConsumer db storage.Database queues *queue.OutgoingQueues } @@ -48,7 +48,7 @@ func NewOutputRoomEventConsumer( store storage.Database, rsAPI api.RoomserverInternalAPI, ) *OutputRoomEventConsumer { - consumer := common.ContinualConsumer{ + consumer := internal.ContinualConsumer{ Topic: string(cfg.Kafka.Topics.OutputRoomEvent), Consumer: kafkaConsumer, PartitionStore: store, diff --git a/federationsender/federationsender.go b/federationsender/federationsender.go index 9c151a8d4..446f920c2 100644 --- a/federationsender/federationsender.go +++ b/federationsender/federationsender.go @@ -17,7 +17,6 @@ package federationsender import ( "net/http" - "github.com/matrix-org/dendrite/common/basecomponent" "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/federationsender/consumers" "github.com/matrix-org/dendrite/federationsender/internal" @@ -25,6 +24,7 @@ import ( "github.com/matrix-org/dendrite/federationsender/queue" "github.com/matrix-org/dendrite/federationsender/storage" "github.com/matrix-org/dendrite/federationsender/types" + "github.com/matrix-org/dendrite/internal/basecomponent" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" diff --git a/federationsender/internal/api.go b/federationsender/internal/api.go index 481795220..1f6d857f4 100644 --- a/federationsender/internal/api.go +++ b/federationsender/internal/api.go @@ -4,12 +4,12 @@ import ( "encoding/json" "net/http" - "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/federationsender/producers" "github.com/matrix-org/dendrite/federationsender/storage" "github.com/matrix-org/dendrite/federationsender/types" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -46,7 +46,7 @@ func NewFederationSenderInternalAPI( func (f *FederationSenderInternalAPI) SetupHTTP(servMux *http.ServeMux) { servMux.Handle( api.FederationSenderQueryJoinedHostsInRoomPath, - common.MakeInternalAPI("QueryJoinedHostsInRoom", func(req *http.Request) util.JSONResponse { + internal.MakeInternalAPI("QueryJoinedHostsInRoom", func(req *http.Request) util.JSONResponse { var request api.QueryJoinedHostsInRoomRequest var response api.QueryJoinedHostsInRoomResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -60,7 +60,7 @@ func (f *FederationSenderInternalAPI) SetupHTTP(servMux *http.ServeMux) { ) servMux.Handle( api.FederationSenderQueryJoinedHostServerNamesInRoomPath, - common.MakeInternalAPI("QueryJoinedHostServerNamesInRoom", func(req *http.Request) util.JSONResponse { + internal.MakeInternalAPI("QueryJoinedHostServerNamesInRoom", func(req *http.Request) util.JSONResponse { var request api.QueryJoinedHostServerNamesInRoomRequest var response api.QueryJoinedHostServerNamesInRoomResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -73,7 +73,7 @@ func (f *FederationSenderInternalAPI) SetupHTTP(servMux *http.ServeMux) { }), ) servMux.Handle(api.FederationSenderPerformJoinRequestPath, - common.MakeInternalAPI("PerformJoinRequest", func(req *http.Request) util.JSONResponse { + internal.MakeInternalAPI("PerformJoinRequest", func(req *http.Request) util.JSONResponse { var request api.PerformJoinRequest var response api.PerformJoinResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -86,7 +86,7 @@ func (f *FederationSenderInternalAPI) SetupHTTP(servMux *http.ServeMux) { }), ) servMux.Handle(api.FederationSenderPerformLeaveRequestPath, - common.MakeInternalAPI("PerformLeaveRequest", func(req *http.Request) util.JSONResponse { + internal.MakeInternalAPI("PerformLeaveRequest", func(req *http.Request) util.JSONResponse { var request api.PerformLeaveRequest var response api.PerformLeaveResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { diff --git a/federationsender/storage/interface.go b/federationsender/storage/interface.go index ae2956475..be195382b 100644 --- a/federationsender/storage/interface.go +++ b/federationsender/storage/interface.go @@ -17,12 +17,12 @@ package storage import ( "context" - "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/federationsender/types" + "github.com/matrix-org/dendrite/internal" ) type Database interface { - common.PartitionStorer + internal.PartitionStorer UpdateRoom(ctx context.Context, roomID, oldEventID, newEventID string, addHosts []types.JoinedHost, removeHosts []string) (joinedHosts []types.JoinedHost, err error) GetJoinedHosts(ctx context.Context, roomID string) ([]types.JoinedHost, error) } diff --git a/federationsender/storage/postgres/joined_hosts_table.go b/federationsender/storage/postgres/joined_hosts_table.go index b3c45abda..1b898f485 100644 --- a/federationsender/storage/postgres/joined_hosts_table.go +++ b/federationsender/storage/postgres/joined_hosts_table.go @@ -20,8 +20,8 @@ import ( "database/sql" "github.com/lib/pq" - "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/federationsender/types" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/gomatrixserverlib" ) @@ -85,7 +85,7 @@ func (s *joinedHostsStatements) insertJoinedHosts( roomID, eventID string, serverName gomatrixserverlib.ServerName, ) error { - stmt := common.TxStmt(txn, s.insertJoinedHostsStmt) + stmt := internal.TxStmt(txn, s.insertJoinedHostsStmt) _, err := stmt.ExecContext(ctx, roomID, eventID, serverName) return err } @@ -93,7 +93,7 @@ func (s *joinedHostsStatements) insertJoinedHosts( func (s *joinedHostsStatements) deleteJoinedHosts( ctx context.Context, txn *sql.Tx, eventIDs []string, ) error { - stmt := common.TxStmt(txn, s.deleteJoinedHostsStmt) + stmt := internal.TxStmt(txn, s.deleteJoinedHostsStmt) _, err := stmt.ExecContext(ctx, pq.StringArray(eventIDs)) return err } @@ -101,7 +101,7 @@ func (s *joinedHostsStatements) deleteJoinedHosts( func (s *joinedHostsStatements) selectJoinedHostsWithTx( ctx context.Context, txn *sql.Tx, roomID string, ) ([]types.JoinedHost, error) { - stmt := common.TxStmt(txn, s.selectJoinedHostsStmt) + stmt := internal.TxStmt(txn, s.selectJoinedHostsStmt) return joinedHostsFromStmt(ctx, stmt, roomID) } @@ -118,7 +118,7 @@ func joinedHostsFromStmt( if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "joinedHostsFromStmt: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "joinedHostsFromStmt: rows.close() failed") var result []types.JoinedHost for rows.Next() { diff --git a/federationsender/storage/postgres/room_table.go b/federationsender/storage/postgres/room_table.go index a64424b44..c945475ba 100644 --- a/federationsender/storage/postgres/room_table.go +++ b/federationsender/storage/postgres/room_table.go @@ -19,7 +19,7 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" ) const roomSchema = ` @@ -71,7 +71,7 @@ func (s *roomStatements) prepare(db *sql.DB) (err error) { func (s *roomStatements) insertRoom( ctx context.Context, txn *sql.Tx, roomID string, ) error { - _, err := common.TxStmt(txn, s.insertRoomStmt).ExecContext(ctx, roomID) + _, err := internal.TxStmt(txn, s.insertRoomStmt).ExecContext(ctx, roomID) return err } @@ -82,7 +82,7 @@ func (s *roomStatements) selectRoomForUpdate( ctx context.Context, txn *sql.Tx, roomID string, ) (string, error) { var lastEventID string - stmt := common.TxStmt(txn, s.selectRoomForUpdateStmt) + stmt := internal.TxStmt(txn, s.selectRoomForUpdateStmt) err := stmt.QueryRowContext(ctx, roomID).Scan(&lastEventID) if err != nil { return "", err @@ -95,7 +95,7 @@ func (s *roomStatements) selectRoomForUpdate( func (s *roomStatements) updateRoom( ctx context.Context, txn *sql.Tx, roomID, lastEventID string, ) error { - stmt := common.TxStmt(txn, s.updateRoomStmt) + stmt := internal.TxStmt(txn, s.updateRoomStmt) _, err := stmt.ExecContext(ctx, roomID, lastEventID) return err } diff --git a/federationsender/storage/postgres/storage.go b/federationsender/storage/postgres/storage.go index c3892ac10..a4733a20c 100644 --- a/federationsender/storage/postgres/storage.go +++ b/federationsender/storage/postgres/storage.go @@ -19,8 +19,8 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/federationsender/types" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" ) @@ -28,12 +28,12 @@ import ( type Database struct { joinedHostsStatements roomStatements - common.PartitionOffsetStatements + internal.PartitionOffsetStatements db *sql.DB } // NewDatabase opens a new database -func NewDatabase(dataSourceName string, dbProperties common.DbProperties) (*Database, error) { +func NewDatabase(dataSourceName string, dbProperties internal.DbProperties) (*Database, error) { var result Database var err error if result.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { @@ -70,7 +70,7 @@ func (d *Database) UpdateRoom( addHosts []types.JoinedHost, removeHosts []string, ) (joinedHosts []types.JoinedHost, err error) { - err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { err = d.insertRoom(ctx, txn, roomID) if err != nil { return err diff --git a/federationsender/storage/sqlite3/joined_hosts_table.go b/federationsender/storage/sqlite3/joined_hosts_table.go index 466ae4991..795d33208 100644 --- a/federationsender/storage/sqlite3/joined_hosts_table.go +++ b/federationsender/storage/sqlite3/joined_hosts_table.go @@ -19,8 +19,8 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/federationsender/types" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/gomatrixserverlib" ) @@ -84,7 +84,7 @@ func (s *joinedHostsStatements) insertJoinedHosts( roomID, eventID string, serverName gomatrixserverlib.ServerName, ) error { - stmt := common.TxStmt(txn, s.insertJoinedHostsStmt) + stmt := internal.TxStmt(txn, s.insertJoinedHostsStmt) _, err := stmt.ExecContext(ctx, roomID, eventID, serverName) return err } @@ -93,7 +93,7 @@ func (s *joinedHostsStatements) deleteJoinedHosts( ctx context.Context, txn *sql.Tx, eventIDs []string, ) error { for _, eventID := range eventIDs { - stmt := common.TxStmt(txn, s.deleteJoinedHostsStmt) + stmt := internal.TxStmt(txn, s.deleteJoinedHostsStmt) if _, err := stmt.ExecContext(ctx, eventID); err != nil { return err } @@ -104,7 +104,7 @@ func (s *joinedHostsStatements) deleteJoinedHosts( func (s *joinedHostsStatements) selectJoinedHostsWithTx( ctx context.Context, txn *sql.Tx, roomID string, ) ([]types.JoinedHost, error) { - stmt := common.TxStmt(txn, s.selectJoinedHostsStmt) + stmt := internal.TxStmt(txn, s.selectJoinedHostsStmt) return joinedHostsFromStmt(ctx, stmt, roomID) } @@ -121,7 +121,7 @@ func joinedHostsFromStmt( if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "joinedHostsFromStmt: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "joinedHostsFromStmt: rows.close() failed") var result []types.JoinedHost for rows.Next() { diff --git a/federationsender/storage/sqlite3/room_table.go b/federationsender/storage/sqlite3/room_table.go index 6361400d3..4bf9ab81d 100644 --- a/federationsender/storage/sqlite3/room_table.go +++ b/federationsender/storage/sqlite3/room_table.go @@ -19,7 +19,7 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" ) const roomSchema = ` @@ -71,7 +71,7 @@ func (s *roomStatements) prepare(db *sql.DB) (err error) { func (s *roomStatements) insertRoom( ctx context.Context, txn *sql.Tx, roomID string, ) error { - _, err := common.TxStmt(txn, s.insertRoomStmt).ExecContext(ctx, roomID) + _, err := internal.TxStmt(txn, s.insertRoomStmt).ExecContext(ctx, roomID) return err } @@ -82,7 +82,7 @@ func (s *roomStatements) selectRoomForUpdate( ctx context.Context, txn *sql.Tx, roomID string, ) (string, error) { var lastEventID string - stmt := common.TxStmt(txn, s.selectRoomForUpdateStmt) + stmt := internal.TxStmt(txn, s.selectRoomForUpdateStmt) err := stmt.QueryRowContext(ctx, roomID).Scan(&lastEventID) if err != nil { return "", err @@ -95,7 +95,7 @@ func (s *roomStatements) selectRoomForUpdate( func (s *roomStatements) updateRoom( ctx context.Context, txn *sql.Tx, roomID, lastEventID string, ) error { - stmt := common.TxStmt(txn, s.updateRoomStmt) + stmt := internal.TxStmt(txn, s.updateRoomStmt) _, err := stmt.ExecContext(ctx, roomID, lastEventID) return err } diff --git a/federationsender/storage/sqlite3/storage.go b/federationsender/storage/sqlite3/storage.go index 772744474..dc60ebb01 100644 --- a/federationsender/storage/sqlite3/storage.go +++ b/federationsender/storage/sqlite3/storage.go @@ -21,8 +21,8 @@ import ( _ "github.com/mattn/go-sqlite3" - "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/federationsender/types" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" ) @@ -30,7 +30,7 @@ import ( type Database struct { joinedHostsStatements roomStatements - common.PartitionOffsetStatements + internal.PartitionOffsetStatements db *sql.DB } @@ -38,7 +38,7 @@ type Database struct { func NewDatabase(dataSourceName string) (*Database, error) { var result Database var err error - if result.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil); err != nil { + if result.db, err = sqlutil.Open(internal.SQLiteDriverName(), dataSourceName, nil); err != nil { return nil, err } if err = result.prepare(); err != nil { @@ -72,7 +72,7 @@ func (d *Database) UpdateRoom( addHosts []types.JoinedHost, removeHosts []string, ) (joinedHosts []types.JoinedHost, err error) { - err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { err = d.insertRoom(ctx, txn, roomID) if err != nil { return err diff --git a/federationsender/storage/storage.go b/federationsender/storage/storage.go index d481e58a2..df8433eba 100644 --- a/federationsender/storage/storage.go +++ b/federationsender/storage/storage.go @@ -19,13 +19,13 @@ package storage import ( "net/url" - "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/federationsender/storage/postgres" "github.com/matrix-org/dendrite/federationsender/storage/sqlite3" + "github.com/matrix-org/dendrite/internal" ) // NewDatabase opens a new database -func NewDatabase(dataSourceName string, dbProperties common.DbProperties) (Database, error) { +func NewDatabase(dataSourceName string, dbProperties internal.DbProperties) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return postgres.NewDatabase(dataSourceName, dbProperties) diff --git a/federationsender/storage/storage_wasm.go b/federationsender/storage/storage_wasm.go index 44d4c8060..f593fd448 100644 --- a/federationsender/storage/storage_wasm.go +++ b/federationsender/storage/storage_wasm.go @@ -18,14 +18,14 @@ import ( "fmt" "net/url" - "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/federationsender/storage/sqlite3" + "github.com/matrix-org/dendrite/internal" ) // NewDatabase opens a new database func NewDatabase( dataSourceName string, - dbProperties common.DbProperties, // nolint:unparam + dbProperties internal.DbProperties, // nolint:unparam ) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { diff --git a/common/basecomponent/base.go b/internal/basecomponent/base.go similarity index 94% rename from common/basecomponent/base.go rename to internal/basecomponent/base.go index 234de5e72..e9a375a70 100644 --- a/common/basecomponent/base.go +++ b/internal/basecomponent/base.go @@ -23,24 +23,24 @@ import ( "golang.org/x/crypto/ed25519" - "github.com/matrix-org/dendrite/common/caching" - "github.com/matrix-org/dendrite/common/keydb" - "github.com/matrix-org/dendrite/common/keydb/cache" + "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/keydb" + "github.com/matrix-org/dendrite/internal/keydb/cache" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/naffka" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/Shopify/sarama" "github.com/gorilla/mux" appserviceAPI "github.com/matrix-org/dendrite/appservice/api" - "github.com/matrix-org/dendrite/common/config" eduServerAPI "github.com/matrix-org/dendrite/eduserver/api" federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/internal/config" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/sirupsen/logrus" @@ -73,9 +73,9 @@ const HTTPClientTimeout = time.Second * 30 // The componentName is used for logging purposes, and should be a friendly name // of the compontent running, e.g. "SyncAPI" func NewBaseDendrite(cfg *config.Dendrite, componentName string, enableHTTPAPIs bool) *BaseDendrite { - common.SetupStdLogging() - common.SetupHookLogging(cfg.Logging, componentName) - common.SetupPprof() + internal.SetupStdLogging() + internal.SetupHookLogging(cfg.Logging, componentName) + internal.SetupPprof() closer, err := cfg.SetupTracing("Dendrite" + componentName) if err != nil { @@ -221,7 +221,7 @@ func (b *BaseDendrite) SetupAndServeHTTP(bindaddr string, listenaddr string) { WriteTimeout: HTTPServerTimeout, } - common.SetupHTTPAPI(http.DefaultServeMux, common.WrapHandlerInCORS(b.APIMux), b.Cfg) + internal.SetupHTTPAPI(http.DefaultServeMux, internal.WrapHandlerInCORS(b.APIMux), b.Cfg) logrus.Infof("Starting %s server on %s", b.componentName, serv.Addr) err := serv.ListenAndServe() @@ -255,7 +255,7 @@ func setupNaffka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) { uri, err := url.Parse(string(cfg.Database.Naffka)) if err != nil || uri.Scheme == "file" { - db, err = sqlutil.Open(common.SQLiteDriverName(), string(cfg.Database.Naffka), nil) + db, err = sqlutil.Open(internal.SQLiteDriverName(), string(cfg.Database.Naffka), nil) if err != nil { logrus.WithError(err).Panic("Failed to open naffka database") } diff --git a/common/basecomponent/flags.go b/internal/basecomponent/flags.go similarity index 97% rename from common/basecomponent/flags.go rename to internal/basecomponent/flags.go index 6dcb5601a..ecef81f6b 100644 --- a/common/basecomponent/flags.go +++ b/internal/basecomponent/flags.go @@ -17,7 +17,7 @@ package basecomponent import ( "flag" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal/config" "github.com/sirupsen/logrus" ) diff --git a/common/caching/immutablecache.go b/internal/caching/immutablecache.go similarity index 100% rename from common/caching/immutablecache.go rename to internal/caching/immutablecache.go diff --git a/common/caching/immutableinmemorylru.go b/internal/caching/immutableinmemorylru.go similarity index 100% rename from common/caching/immutableinmemorylru.go rename to internal/caching/immutableinmemorylru.go diff --git a/common/config/appservice.go b/internal/config/appservice.go similarity index 100% rename from common/config/appservice.go rename to internal/config/appservice.go diff --git a/common/config/config.go b/internal/config/config.go similarity index 100% rename from common/config/config.go rename to internal/config/config.go diff --git a/common/config/config_test.go b/internal/config/config_test.go similarity index 100% rename from common/config/config_test.go rename to internal/config/config_test.go diff --git a/common/consumers.go b/internal/consumers.go similarity index 99% rename from common/consumers.go rename to internal/consumers.go index bc759d994..df68cbfaa 100644 --- a/common/consumers.go +++ b/internal/consumers.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package common +package internal import ( "context" diff --git a/common/eventcontent.go b/internal/eventcontent.go similarity index 99% rename from common/eventcontent.go rename to internal/eventcontent.go index e7e1a2c8b..645128369 100644 --- a/common/eventcontent.go +++ b/internal/eventcontent.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package common +package internal import "github.com/matrix-org/gomatrixserverlib" diff --git a/common/events.go b/internal/events.go similarity index 98% rename from common/events.go rename to internal/events.go index 0c8ead3a7..89c82e03b 100644 --- a/common/events.go +++ b/internal/events.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package common +package internal import ( "context" @@ -20,7 +20,7 @@ import ( "fmt" "time" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" diff --git a/common/http/http.go b/internal/http/http.go similarity index 100% rename from common/http/http.go rename to internal/http/http.go diff --git a/common/httpapi.go b/internal/httpapi.go similarity index 99% rename from common/httpapi.go rename to internal/httpapi.go index e5324bd17..b4c53f58d 100644 --- a/common/httpapi.go +++ b/internal/httpapi.go @@ -1,4 +1,4 @@ -package common +package internal import ( "io" @@ -11,7 +11,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" opentracing "github.com/opentracing/opentracing-go" diff --git a/common/httpapi_test.go b/internal/httpapi_test.go similarity index 99% rename from common/httpapi_test.go rename to internal/httpapi_test.go index 7de7ce33c..6f159c8d6 100644 --- a/common/httpapi_test.go +++ b/internal/httpapi_test.go @@ -1,4 +1,4 @@ -package common +package internal import ( "net/http" diff --git a/common/keydb/cache/keydb.go b/internal/keydb/cache/keydb.go similarity index 94% rename from common/keydb/cache/keydb.go rename to internal/keydb/cache/keydb.go index ae929fa4c..87573ed2b 100644 --- a/common/keydb/cache/keydb.go +++ b/internal/keydb/cache/keydb.go @@ -4,8 +4,8 @@ import ( "context" "errors" - "github.com/matrix-org/dendrite/common/caching" - "github.com/matrix-org/dendrite/common/keydb" + "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/keydb" "github.com/matrix-org/gomatrixserverlib" ) diff --git a/common/keydb/interface.go b/internal/keydb/interface.go similarity index 100% rename from common/keydb/interface.go rename to internal/keydb/interface.go diff --git a/common/keydb/keydb.go b/internal/keydb/keydb.go similarity index 88% rename from common/keydb/keydb.go rename to internal/keydb/keydb.go index 397d7849c..ad6d56b86 100644 --- a/common/keydb/keydb.go +++ b/internal/keydb/keydb.go @@ -21,16 +21,16 @@ import ( "golang.org/x/crypto/ed25519" - "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/common/keydb/postgres" - "github.com/matrix-org/dendrite/common/keydb/sqlite3" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/keydb/postgres" + "github.com/matrix-org/dendrite/internal/keydb/sqlite3" "github.com/matrix-org/gomatrixserverlib" ) // NewDatabase opens a database connection. func NewDatabase( dataSourceName string, - dbProperties common.DbProperties, + dbProperties internal.DbProperties, serverName gomatrixserverlib.ServerName, serverKey ed25519.PublicKey, serverKeyID gomatrixserverlib.KeyID, diff --git a/common/keydb/keydb_wasm.go b/internal/keydb/keydb_wasm.go similarity index 89% rename from common/keydb/keydb_wasm.go rename to internal/keydb/keydb_wasm.go index 38e595820..349381ee7 100644 --- a/common/keydb/keydb_wasm.go +++ b/internal/keydb/keydb_wasm.go @@ -20,15 +20,15 @@ import ( "golang.org/x/crypto/ed25519" - "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/common/keydb/sqlite3" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/keydb/sqlite3" "github.com/matrix-org/gomatrixserverlib" ) // NewDatabase opens a database connection. func NewDatabase( dataSourceName string, - dbProperties common.DbProperties, // nolint:unparam + dbProperties internal.DbProperties, // nolint:unparam serverName gomatrixserverlib.ServerName, serverKey ed25519.PublicKey, serverKeyID gomatrixserverlib.KeyID, diff --git a/common/keydb/keyring.go b/internal/keydb/keyring.go similarity index 97% rename from common/keydb/keyring.go rename to internal/keydb/keyring.go index e9cc7903e..d0b1904ed 100644 --- a/common/keydb/keyring.go +++ b/internal/keydb/keyring.go @@ -17,7 +17,7 @@ package keydb import ( "encoding/base64" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" "golang.org/x/crypto/ed25519" diff --git a/common/keydb/postgres/keydb.go b/internal/keydb/postgres/keydb.go similarity index 97% rename from common/keydb/postgres/keydb.go rename to internal/keydb/postgres/keydb.go index a8a304376..da3a4d370 100644 --- a/common/keydb/postgres/keydb.go +++ b/internal/keydb/postgres/keydb.go @@ -21,7 +21,7 @@ import ( "golang.org/x/crypto/ed25519" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) @@ -38,7 +38,7 @@ type Database struct { // Returns an error if there was a problem talking to the database. func NewDatabase( dataSourceName string, - dbProperties common.DbProperties, + dbProperties internal.DbProperties, serverName gomatrixserverlib.ServerName, serverKey ed25519.PublicKey, serverKeyID gomatrixserverlib.KeyID, diff --git a/common/keydb/postgres/server_key_table.go b/internal/keydb/postgres/server_key_table.go similarity index 97% rename from common/keydb/postgres/server_key_table.go rename to internal/keydb/postgres/server_key_table.go index 0434eb8b1..b3c26a480 100644 --- a/common/keydb/postgres/server_key_table.go +++ b/internal/keydb/postgres/server_key_table.go @@ -19,7 +19,7 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/lib/pq" "github.com/matrix-org/gomatrixserverlib" @@ -93,7 +93,7 @@ func (s *serverKeyStatements) bulkSelectServerKeys( if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "bulkSelectServerKeys: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectServerKeys: rows.close() failed") results := map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult{} for rows.Next() { var serverName string diff --git a/common/keydb/sqlite3/keydb.go b/internal/keydb/sqlite3/keydb.go similarity index 96% rename from common/keydb/sqlite3/keydb.go rename to internal/keydb/sqlite3/keydb.go index 1ff315d7c..d1dc61c98 100644 --- a/common/keydb/sqlite3/keydb.go +++ b/internal/keydb/sqlite3/keydb.go @@ -21,7 +21,7 @@ import ( "golang.org/x/crypto/ed25519" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" @@ -44,7 +44,7 @@ func NewDatabase( serverKey ed25519.PublicKey, serverKeyID gomatrixserverlib.KeyID, ) (*Database, error) { - db, err := sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil) + db, err := sqlutil.Open(internal.SQLiteDriverName(), dataSourceName, nil) if err != nil { return nil, err } diff --git a/common/keydb/sqlite3/server_key_table.go b/internal/keydb/sqlite3/server_key_table.go similarity index 95% rename from common/keydb/sqlite3/server_key_table.go rename to internal/keydb/sqlite3/server_key_table.go index 883d3cd0f..ae24a14d4 100644 --- a/common/keydb/sqlite3/server_key_table.go +++ b/internal/keydb/sqlite3/server_key_table.go @@ -20,7 +20,7 @@ import ( "database/sql" "strings" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/gomatrixserverlib" ) @@ -90,7 +90,7 @@ func (s *serverKeyStatements) bulkSelectServerKeys( nameAndKeyIDs = append(nameAndKeyIDs, nameAndKeyID(request)) } - query := strings.Replace(bulkSelectServerKeysSQL, "($1)", common.QueryVariadic(len(nameAndKeyIDs)), 1) + query := strings.Replace(bulkSelectServerKeysSQL, "($1)", internal.QueryVariadic(len(nameAndKeyIDs)), 1) iKeyIDs := make([]interface{}, len(nameAndKeyIDs)) for i, v := range nameAndKeyIDs { @@ -101,7 +101,7 @@ func (s *serverKeyStatements) bulkSelectServerKeys( if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "bulkSelectServerKeys: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectServerKeys: rows.close() failed") results := map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult{} for rows.Next() { var serverName string diff --git a/common/log.go b/internal/log.go similarity index 98% rename from common/log.go rename to internal/log.go index 60e969650..fd2b84ab9 100644 --- a/common/log.go +++ b/internal/log.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package common +package internal import ( "context" @@ -27,7 +27,7 @@ import ( "github.com/matrix-org/util" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dugong" "github.com/sirupsen/logrus" ) diff --git a/common/partition_offset_table.go b/internal/partition_offset_table.go similarity index 99% rename from common/partition_offset_table.go rename to internal/partition_offset_table.go index aa799f8a0..8b72819b7 100644 --- a/common/partition_offset_table.go +++ b/internal/partition_offset_table.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package common +package internal import ( "context" diff --git a/common/postgres.go b/internal/postgres.go similarity index 98% rename from common/postgres.go rename to internal/postgres.go index f8daf5783..7ae40d8fd 100644 --- a/common/postgres.go +++ b/internal/postgres.go @@ -14,7 +14,7 @@ // +build !wasm -package common +package internal import "github.com/lib/pq" diff --git a/common/postgres_wasm.go b/internal/postgres_wasm.go similarity index 97% rename from common/postgres_wasm.go rename to internal/postgres_wasm.go index dcc07b31d..64d328291 100644 --- a/common/postgres_wasm.go +++ b/internal/postgres_wasm.go @@ -14,7 +14,7 @@ // +build wasm -package common +package internal // IsUniqueConstraintViolationErr no-ops for this architecture func IsUniqueConstraintViolationErr(err error) bool { diff --git a/common/routing.go b/internal/routing.go similarity index 98% rename from common/routing.go rename to internal/routing.go index cee0c162a..4462c70ca 100644 --- a/common/routing.go +++ b/internal/routing.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package common +package internal import ( "net/url" diff --git a/common/sql.go b/internal/sql.go similarity index 99% rename from common/sql.go rename to internal/sql.go index e93ff1c2f..d6a5a3086 100644 --- a/common/sql.go +++ b/internal/sql.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package common +package internal import ( "database/sql" diff --git a/internal/sqlutil/trace.go b/internal/sqlutil/trace.go index 42ac4e582..1b008e1b7 100644 --- a/internal/sqlutil/trace.go +++ b/internal/sqlutil/trace.go @@ -25,7 +25,7 @@ import ( "strings" "time" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/ngrok/sqlmw" "github.com/sirupsen/logrus" ) @@ -78,7 +78,7 @@ func (in *traceInterceptor) RowsNext(c context.Context, rows driver.Rows, dest [ // Open opens a database specified by its database driver name and a driver-specific data source name, // usually consisting of at least a database name and connection information. Includes tracing driver // if DENDRITE_TRACE_SQL=1 -func Open(driverName, dsn string, dbProperties common.DbProperties) (*sql.DB, error) { +func Open(driverName, dsn string, dbProperties internal.DbProperties) (*sql.DB, error) { if tracingEnabled { // install the wrapped driver driverName += "-trace" @@ -87,7 +87,7 @@ func Open(driverName, dsn string, dbProperties common.DbProperties) (*sql.DB, er if err != nil { return nil, err } - if driverName != common.SQLiteDriverName() && dbProperties != nil { + if driverName != internal.SQLiteDriverName() && dbProperties != nil { logrus.WithFields(logrus.Fields{ "MaxOpenConns": dbProperties.MaxOpenConns(), "MaxIdleConns": dbProperties.MaxIdleConns(), diff --git a/common/test/client.go b/internal/test/client.go similarity index 100% rename from common/test/client.go rename to internal/test/client.go diff --git a/common/test/config.go b/internal/test/config.go similarity index 99% rename from common/test/config.go rename to internal/test/config.go index f88e45125..06510c8b8 100644 --- a/common/test/config.go +++ b/internal/test/config.go @@ -27,7 +27,7 @@ import ( "path/filepath" "time" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/gomatrixserverlib" "gopkg.in/yaml.v2" ) diff --git a/common/test/kafka.go b/internal/test/kafka.go similarity index 100% rename from common/test/kafka.go rename to internal/test/kafka.go diff --git a/common/test/server.go b/internal/test/server.go similarity index 98% rename from common/test/server.go rename to internal/test/server.go index 4fdd5e638..1493dac6f 100644 --- a/common/test/server.go +++ b/internal/test/server.go @@ -21,7 +21,7 @@ import ( "path/filepath" "strings" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal/config" ) // Defaulting allows assignment of string variables with a fallback default value diff --git a/common/test/slice.go b/internal/test/slice.go similarity index 100% rename from common/test/slice.go rename to internal/test/slice.go diff --git a/common/transactions/transactions.go b/internal/transactions/transactions.go similarity index 100% rename from common/transactions/transactions.go rename to internal/transactions/transactions.go diff --git a/common/transactions/transactions_test.go b/internal/transactions/transactions_test.go similarity index 100% rename from common/transactions/transactions_test.go rename to internal/transactions/transactions_test.go diff --git a/common/types.go b/internal/types.go similarity index 99% rename from common/types.go rename to internal/types.go index 91765be00..be2717f39 100644 --- a/common/types.go +++ b/internal/types.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package common +package internal import ( "errors" diff --git a/keyserver/keyserver.go b/keyserver/keyserver.go index 1e0d5cb42..0f450165f 100644 --- a/keyserver/keyserver.go +++ b/keyserver/keyserver.go @@ -17,7 +17,7 @@ package keyserver import ( "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" - "github.com/matrix-org/dendrite/common/basecomponent" + "github.com/matrix-org/dendrite/internal/basecomponent" "github.com/matrix-org/dendrite/keyserver/routing" ) diff --git a/keyserver/routing/routing.go b/keyserver/routing/routing.go index d79ce6d40..3ca68ade4 100644 --- a/keyserver/routing/routing.go +++ b/keyserver/routing/routing.go @@ -22,8 +22,8 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" - "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/util" ) @@ -49,7 +49,7 @@ func Setup( } r0mux.Handle("/keys/query", - common.MakeAuthAPI("queryKeys", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + internal.MakeAuthAPI("queryKeys", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return QueryKeys(req) }), ).Methods(http.MethodPost, http.MethodOptions) diff --git a/mediaapi/fileutils/fileutils.go b/mediaapi/fileutils/fileutils.go index 1de242a00..39687b9d4 100644 --- a/mediaapi/fileutils/fileutils.go +++ b/mediaapi/fileutils/fileutils.go @@ -25,7 +25,7 @@ import ( "path/filepath" "strings" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/mediaapi/types" log "github.com/sirupsen/logrus" ) diff --git a/mediaapi/mediaapi.go b/mediaapi/mediaapi.go index 4a0f5d188..38572daf8 100644 --- a/mediaapi/mediaapi.go +++ b/mediaapi/mediaapi.go @@ -16,7 +16,7 @@ package mediaapi import ( "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" - "github.com/matrix-org/dendrite/common/basecomponent" + "github.com/matrix-org/dendrite/internal/basecomponent" "github.com/matrix-org/dendrite/mediaapi/routing" "github.com/matrix-org/dendrite/mediaapi/storage" "github.com/matrix-org/gomatrixserverlib" diff --git a/mediaapi/routing/download.go b/mediaapi/routing/download.go index 75df313f6..1a025f6f0 100644 --- a/mediaapi/routing/download.go +++ b/mediaapi/routing/download.go @@ -29,7 +29,7 @@ import ( "sync" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/mediaapi/fileutils" "github.com/matrix-org/dendrite/mediaapi/storage" "github.com/matrix-org/dendrite/mediaapi/thumbnailer" diff --git a/mediaapi/routing/routing.go b/mediaapi/routing/routing.go index 71dad19b6..4f765d173 100644 --- a/mediaapi/routing/routing.go +++ b/mediaapi/routing/routing.go @@ -22,8 +22,8 @@ import ( "github.com/gorilla/mux" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" - "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/mediaapi/storage" "github.com/matrix-org/dendrite/mediaapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -59,7 +59,7 @@ func Setup( } // TODO: Add AS support - r0mux.Handle("/upload", common.MakeAuthAPI( + r0mux.Handle("/upload", internal.MakeAuthAPI( "upload", authData, func(req *http.Request, _ *authtypes.Device) util.JSONResponse { return Upload(req, cfg, db, activeThumbnailGeneration) @@ -95,11 +95,11 @@ func makeDownloadAPI( httpHandler := func(w http.ResponseWriter, req *http.Request) { req = util.RequestWithLogging(req) - // Set common headers returned regardless of the outcome of the request + // Set internal headers returned regardless of the outcome of the request util.SetCORSHeaders(w) // Content-Type will be overridden in case of returning file data, else we respond with JSON-formatted errors w.Header().Set("Content-Type", "application/json") - vars, _ := common.URLDecodeMapValues(mux.Vars(req)) + vars, _ := internal.URLDecodeMapValues(mux.Vars(req)) Download( w, req, diff --git a/mediaapi/routing/upload.go b/mediaapi/routing/upload.go index 91a453190..022f978d6 100644 --- a/mediaapi/routing/upload.go +++ b/mediaapi/routing/upload.go @@ -24,7 +24,7 @@ import ( "strings" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/mediaapi/fileutils" "github.com/matrix-org/dendrite/mediaapi/storage" "github.com/matrix-org/dendrite/mediaapi/thumbnailer" diff --git a/mediaapi/storage/postgres/prepare.go b/mediaapi/storage/postgres/prepare.go index 090c3d17d..a2e01884e 100644 --- a/mediaapi/storage/postgres/prepare.go +++ b/mediaapi/storage/postgres/prepare.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -// FIXME: This should be made common! +// FIXME: This should be made internal! package postgres diff --git a/mediaapi/storage/postgres/storage.go b/mediaapi/storage/postgres/storage.go index 4ddfc8fdf..72ee05cd8 100644 --- a/mediaapi/storage/postgres/storage.go +++ b/mediaapi/storage/postgres/storage.go @@ -21,7 +21,7 @@ import ( // Import the postgres database driver. _ "github.com/lib/pq" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/mediaapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -34,7 +34,7 @@ type Database struct { } // Open opens a postgres database. -func Open(dataSourceName string, dbProperties common.DbProperties) (*Database, error) { +func Open(dataSourceName string, dbProperties internal.DbProperties) (*Database, error) { var d Database var err error if d.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { diff --git a/mediaapi/storage/postgres/thumbnail_table.go b/mediaapi/storage/postgres/thumbnail_table.go index 08bddc36f..dbc97cf22 100644 --- a/mediaapi/storage/postgres/thumbnail_table.go +++ b/mediaapi/storage/postgres/thumbnail_table.go @@ -20,7 +20,7 @@ import ( "database/sql" "time" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/mediaapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -146,7 +146,7 @@ func (s *thumbnailStatements) selectThumbnails( if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "selectThumbnails: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "selectThumbnails: rows.close() failed") var thumbnails []*types.ThumbnailMetadata for rows.Next() { diff --git a/mediaapi/storage/sqlite3/prepare.go b/mediaapi/storage/sqlite3/prepare.go index a6bc24c98..8fb3b56f3 100644 --- a/mediaapi/storage/sqlite3/prepare.go +++ b/mediaapi/storage/sqlite3/prepare.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -// FIXME: This should be made common! +// FIXME: This should be made internal! package sqlite3 diff --git a/mediaapi/storage/sqlite3/storage.go b/mediaapi/storage/sqlite3/storage.go index 8fa2e5375..6da7132d8 100644 --- a/mediaapi/storage/sqlite3/storage.go +++ b/mediaapi/storage/sqlite3/storage.go @@ -20,7 +20,7 @@ import ( "database/sql" // Import the postgres database driver. - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/mediaapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -37,7 +37,7 @@ type Database struct { func Open(dataSourceName string) (*Database, error) { var d Database var err error - if d.db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil); err != nil { + if d.db, err = sqlutil.Open(internal.SQLiteDriverName(), dataSourceName, nil); err != nil { return nil, err } if err = d.statements.prepare(d.db); err != nil { diff --git a/mediaapi/storage/sqlite3/thumbnail_table.go b/mediaapi/storage/sqlite3/thumbnail_table.go index 280fafe8d..a0956f68c 100644 --- a/mediaapi/storage/sqlite3/thumbnail_table.go +++ b/mediaapi/storage/sqlite3/thumbnail_table.go @@ -20,7 +20,7 @@ import ( "database/sql" "time" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/mediaapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -136,7 +136,7 @@ func (s *thumbnailStatements) selectThumbnails( if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "selectThumbnails: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "selectThumbnails: rows.close() failed") var thumbnails []*types.ThumbnailMetadata for rows.Next() { diff --git a/mediaapi/storage/storage.go b/mediaapi/storage/storage.go index 6589c8304..2694e197d 100644 --- a/mediaapi/storage/storage.go +++ b/mediaapi/storage/storage.go @@ -19,13 +19,13 @@ package storage import ( "net/url" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/mediaapi/storage/postgres" "github.com/matrix-org/dendrite/mediaapi/storage/sqlite3" ) // Open opens a postgres database. -func Open(dataSourceName string, dbProperties common.DbProperties) (Database, error) { +func Open(dataSourceName string, dbProperties internal.DbProperties) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return postgres.Open(dataSourceName, dbProperties) diff --git a/mediaapi/storage/storage_wasm.go b/mediaapi/storage/storage_wasm.go index 3c39e5d31..78de2cb82 100644 --- a/mediaapi/storage/storage_wasm.go +++ b/mediaapi/storage/storage_wasm.go @@ -18,14 +18,14 @@ import ( "fmt" "net/url" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/mediaapi/storage/sqlite3" ) // Open opens a postgres database. func Open( dataSourceName string, - dbProperties common.DbProperties, // nolint:unparam + dbProperties internal.DbProperties, // nolint:unparam ) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { diff --git a/mediaapi/thumbnailer/thumbnailer.go b/mediaapi/thumbnailer/thumbnailer.go index ebf5138c5..9a58b5bc1 100644 --- a/mediaapi/thumbnailer/thumbnailer.go +++ b/mediaapi/thumbnailer/thumbnailer.go @@ -22,7 +22,7 @@ import ( "path/filepath" "sync" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/mediaapi/storage" "github.com/matrix-org/dendrite/mediaapi/types" log "github.com/sirupsen/logrus" diff --git a/mediaapi/thumbnailer/thumbnailer_bimg.go b/mediaapi/thumbnailer/thumbnailer_bimg.go index db6f23ace..915d576e3 100644 --- a/mediaapi/thumbnailer/thumbnailer_bimg.go +++ b/mediaapi/thumbnailer/thumbnailer_bimg.go @@ -21,7 +21,7 @@ import ( "os" "time" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/mediaapi/storage" "github.com/matrix-org/dendrite/mediaapi/types" log "github.com/sirupsen/logrus" diff --git a/mediaapi/thumbnailer/thumbnailer_nfnt.go b/mediaapi/thumbnailer/thumbnailer_nfnt.go index 4f1e98aa0..b48551e4e 100644 --- a/mediaapi/thumbnailer/thumbnailer_nfnt.go +++ b/mediaapi/thumbnailer/thumbnailer_nfnt.go @@ -30,7 +30,7 @@ import ( "os" "time" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/mediaapi/storage" "github.com/matrix-org/dendrite/mediaapi/types" "github.com/nfnt/resize" diff --git a/mediaapi/types/types.go b/mediaapi/types/types.go index 855e8fe27..9fa549509 100644 --- a/mediaapi/types/types.go +++ b/mediaapi/types/types.go @@ -17,7 +17,7 @@ package types import ( "sync" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/gomatrixserverlib" ) diff --git a/publicroomsapi/consumers/roomserver.go b/publicroomsapi/consumers/roomserver.go index efd093b7e..c513d3b24 100644 --- a/publicroomsapi/consumers/roomserver.go +++ b/publicroomsapi/consumers/roomserver.go @@ -19,8 +19,8 @@ import ( "encoding/json" "github.com/Shopify/sarama" - "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" @@ -30,7 +30,7 @@ import ( // OutputRoomEventConsumer consumes events that originated in the room server. type OutputRoomEventConsumer struct { rsAPI api.RoomserverInternalAPI - rsConsumer *common.ContinualConsumer + rsConsumer *internal.ContinualConsumer db storage.Database } @@ -41,7 +41,7 @@ func NewOutputRoomEventConsumer( store storage.Database, rsAPI api.RoomserverInternalAPI, ) *OutputRoomEventConsumer { - consumer := common.ContinualConsumer{ + consumer := internal.ContinualConsumer{ Topic: string(cfg.Kafka.Topics.OutputRoomEvent), Consumer: kafkaConsumer, PartitionStore: store, diff --git a/publicroomsapi/publicroomsapi.go b/publicroomsapi/publicroomsapi.go index 6a4e65674..f767a5303 100644 --- a/publicroomsapi/publicroomsapi.go +++ b/publicroomsapi/publicroomsapi.go @@ -16,7 +16,7 @@ package publicroomsapi import ( "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" - "github.com/matrix-org/dendrite/common/basecomponent" + "github.com/matrix-org/dendrite/internal/basecomponent" "github.com/matrix-org/dendrite/publicroomsapi/consumers" "github.com/matrix-org/dendrite/publicroomsapi/routing" "github.com/matrix-org/dendrite/publicroomsapi/storage" diff --git a/publicroomsapi/routing/routing.go b/publicroomsapi/routing/routing.go index 09a8eff76..39a9dcca1 100644 --- a/publicroomsapi/routing/routing.go +++ b/publicroomsapi/routing/routing.go @@ -23,7 +23,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/publicroomsapi/directory" "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/publicroomsapi/types" @@ -51,8 +51,8 @@ func Setup( } r0mux.Handle("/directory/list/room/{roomID}", - common.MakeExternalAPI("directory_list", func(req *http.Request) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(req)) + internal.MakeExternalAPI("directory_list", func(req *http.Request) util.JSONResponse { + vars, err := internal.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -61,8 +61,8 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) // TODO: Add AS support r0mux.Handle("/directory/list/room/{roomID}", - common.MakeAuthAPI("directory_list", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(req)) + internal.MakeAuthAPI("directory_list", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := internal.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -70,7 +70,7 @@ func Setup( }), ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/publicRooms", - common.MakeExternalAPI("public_rooms", func(req *http.Request) util.JSONResponse { + internal.MakeExternalAPI("public_rooms", func(req *http.Request) util.JSONResponse { if extRoomsProvider != nil { return directory.GetPostPublicRoomsWithExternal(req, publicRoomsDB, fedClient, extRoomsProvider) } @@ -80,7 +80,7 @@ func Setup( // Federation - TODO: should this live here or in federation API? It's sure easier if it's here so here it is. apiMux.Handle("/_matrix/federation/v1/publicRooms", - common.MakeExternalAPI("federation_public_rooms", func(req *http.Request) util.JSONResponse { + internal.MakeExternalAPI("federation_public_rooms", func(req *http.Request) util.JSONResponse { return directory.GetPostPublicRooms(req, publicRoomsDB) }), ).Methods(http.MethodGet) diff --git a/publicroomsapi/storage/interface.go b/publicroomsapi/storage/interface.go index 0feca0e20..0ca6f455c 100644 --- a/publicroomsapi/storage/interface.go +++ b/publicroomsapi/storage/interface.go @@ -17,12 +17,12 @@ package storage import ( "context" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/gomatrixserverlib" ) type Database interface { - common.PartitionStorer + internal.PartitionStorer GetRoomVisibility(ctx context.Context, roomID string) (bool, error) SetRoomVisibility(ctx context.Context, visible bool, roomID string) error CountPublicRooms(ctx context.Context) (int64, error) diff --git a/publicroomsapi/storage/postgres/public_rooms_table.go b/publicroomsapi/storage/postgres/public_rooms_table.go index 7e31afd2a..39e355368 100644 --- a/publicroomsapi/storage/postgres/public_rooms_table.go +++ b/publicroomsapi/storage/postgres/public_rooms_table.go @@ -21,7 +21,7 @@ import ( "errors" "fmt" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/gomatrixserverlib" "github.com/lib/pq" @@ -205,7 +205,7 @@ func (s *publicRoomsStatements) selectPublicRooms( if err != nil { return []gomatrixserverlib.PublicRoom{}, nil } - defer common.CloseAndLogIfError(ctx, rows, "selectPublicRooms: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "selectPublicRooms: rows.close() failed") rooms := []gomatrixserverlib.PublicRoom{} for rows.Next() { diff --git a/publicroomsapi/storage/postgres/storage.go b/publicroomsapi/storage/postgres/storage.go index 6242c9d54..691199a0b 100644 --- a/publicroomsapi/storage/postgres/storage.go +++ b/publicroomsapi/storage/postgres/storage.go @@ -20,7 +20,7 @@ import ( "database/sql" "encoding/json" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" @@ -29,14 +29,14 @@ import ( // PublicRoomsServerDatabase represents a public rooms server database. type PublicRoomsServerDatabase struct { db *sql.DB - common.PartitionOffsetStatements + internal.PartitionOffsetStatements statements publicRoomsStatements } type attributeValue interface{} // NewPublicRoomsServerDatabase creates a new public rooms server database. -func NewPublicRoomsServerDatabase(dataSourceName string, dbProperties common.DbProperties) (*PublicRoomsServerDatabase, error) { +func NewPublicRoomsServerDatabase(dataSourceName string, dbProperties internal.DbProperties) (*PublicRoomsServerDatabase, error) { var db *sql.DB var err error if db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { @@ -136,33 +136,33 @@ func (d *PublicRoomsServerDatabase) UpdateRoomFromEvent( case "m.room.aliases": return d.updateRoomAliases(ctx, event) case "m.room.canonical_alias": - var content common.CanonicalAliasContent + var content internal.CanonicalAliasContent field := &(content.Alias) attrName := "canonical_alias" return d.updateStringAttribute(ctx, attrName, event, &content, field) case "m.room.name": - var content common.NameContent + var content internal.NameContent field := &(content.Name) attrName := "name" return d.updateStringAttribute(ctx, attrName, event, &content, field) case "m.room.topic": - var content common.TopicContent + var content internal.TopicContent field := &(content.Topic) attrName := "topic" return d.updateStringAttribute(ctx, attrName, event, &content, field) case "m.room.avatar": - var content common.AvatarContent + var content internal.AvatarContent field := &(content.URL) attrName := "avatar_url" return d.updateStringAttribute(ctx, attrName, event, &content, field) case "m.room.history_visibility": - var content common.HistoryVisibilityContent + var content internal.HistoryVisibilityContent field := &(content.HistoryVisibility) attrName := "world_readable" strForTrue := "world_readable" return d.updateBooleanAttribute(ctx, attrName, event, &content, field, strForTrue) case "m.room.guest_access": - var content common.GuestAccessContent + var content internal.GuestAccessContent field := &(content.GuestAccess) attrName := "guest_can_join" strForTrue := "can_join" @@ -243,7 +243,7 @@ func (d *PublicRoomsServerDatabase) updateBooleanAttribute( func (d *PublicRoomsServerDatabase) updateRoomAliases( ctx context.Context, aliasesEvent gomatrixserverlib.Event, ) error { - var content common.AliasesContent + var content internal.AliasesContent if err := json.Unmarshal(aliasesEvent.Content(), &content); err != nil { return err } diff --git a/publicroomsapi/storage/sqlite3/public_rooms_table.go b/publicroomsapi/storage/sqlite3/public_rooms_table.go index 44679837f..7b332e175 100644 --- a/publicroomsapi/storage/sqlite3/public_rooms_table.go +++ b/publicroomsapi/storage/sqlite3/public_rooms_table.go @@ -22,7 +22,7 @@ import ( "errors" "fmt" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/gomatrixserverlib" ) @@ -193,7 +193,7 @@ func (s *publicRoomsStatements) selectPublicRooms( if err != nil { return []gomatrixserverlib.PublicRoom{}, nil } - defer common.CloseAndLogIfError(ctx, rows, "selectPublicRooms failed to close rows") + defer internal.CloseAndLogIfError(ctx, rows, "selectPublicRooms failed to close rows") rooms := []gomatrixserverlib.PublicRoom{} for rows.Next() { diff --git a/publicroomsapi/storage/sqlite3/storage.go b/publicroomsapi/storage/sqlite3/storage.go index efe35bdde..b81222dd3 100644 --- a/publicroomsapi/storage/sqlite3/storage.go +++ b/publicroomsapi/storage/sqlite3/storage.go @@ -22,7 +22,7 @@ import ( _ "github.com/mattn/go-sqlite3" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" @@ -31,7 +31,7 @@ import ( // PublicRoomsServerDatabase represents a public rooms server database. type PublicRoomsServerDatabase struct { db *sql.DB - common.PartitionOffsetStatements + internal.PartitionOffsetStatements statements publicRoomsStatements } @@ -41,7 +41,7 @@ type attributeValue interface{} func NewPublicRoomsServerDatabase(dataSourceName string) (*PublicRoomsServerDatabase, error) { var db *sql.DB var err error - if db, err = sqlutil.Open(common.SQLiteDriverName(), dataSourceName, nil); err != nil { + if db, err = sqlutil.Open(internal.SQLiteDriverName(), dataSourceName, nil); err != nil { return nil, err } storage := PublicRoomsServerDatabase{ @@ -138,33 +138,33 @@ func (d *PublicRoomsServerDatabase) UpdateRoomFromEvent( case "m.room.aliases": return d.updateRoomAliases(ctx, event) case "m.room.canonical_alias": - var content common.CanonicalAliasContent + var content internal.CanonicalAliasContent field := &(content.Alias) attrName := "canonical_alias" return d.updateStringAttribute(ctx, attrName, event, &content, field) case "m.room.name": - var content common.NameContent + var content internal.NameContent field := &(content.Name) attrName := "name" return d.updateStringAttribute(ctx, attrName, event, &content, field) case "m.room.topic": - var content common.TopicContent + var content internal.TopicContent field := &(content.Topic) attrName := "topic" return d.updateStringAttribute(ctx, attrName, event, &content, field) case "m.room.avatar": - var content common.AvatarContent + var content internal.AvatarContent field := &(content.URL) attrName := "avatar_url" return d.updateStringAttribute(ctx, attrName, event, &content, field) case "m.room.history_visibility": - var content common.HistoryVisibilityContent + var content internal.HistoryVisibilityContent field := &(content.HistoryVisibility) attrName := "world_readable" strForTrue := "world_readable" return d.updateBooleanAttribute(ctx, attrName, event, &content, field, strForTrue) case "m.room.guest_access": - var content common.GuestAccessContent + var content internal.GuestAccessContent field := &(content.GuestAccess) attrName := "guest_can_join" strForTrue := "can_join" @@ -245,7 +245,7 @@ func (d *PublicRoomsServerDatabase) updateBooleanAttribute( func (d *PublicRoomsServerDatabase) updateRoomAliases( ctx context.Context, aliasesEvent gomatrixserverlib.Event, ) error { - var content common.AliasesContent + var content internal.AliasesContent if err := json.Unmarshal(aliasesEvent.Content(), &content); err != nil { return err } diff --git a/publicroomsapi/storage/storage.go b/publicroomsapi/storage/storage.go index 7dcfe5633..978d9a3c9 100644 --- a/publicroomsapi/storage/storage.go +++ b/publicroomsapi/storage/storage.go @@ -19,7 +19,7 @@ package storage import ( "net/url" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/publicroomsapi/storage/postgres" "github.com/matrix-org/dendrite/publicroomsapi/storage/sqlite3" ) @@ -28,7 +28,7 @@ const schemePostgres = "postgres" const schemeFile = "file" // NewPublicRoomsServerDatabase opens a database connection. -func NewPublicRoomsServerDatabase(dataSourceName string, dbProperties common.DbProperties) (Database, error) { +func NewPublicRoomsServerDatabase(dataSourceName string, dbProperties internal.DbProperties) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return postgres.NewPublicRoomsServerDatabase(dataSourceName, dbProperties) diff --git a/roomserver/api/alias.go b/roomserver/api/alias.go index 488e99ab8..93975d36c 100644 --- a/roomserver/api/alias.go +++ b/roomserver/api/alias.go @@ -17,7 +17,7 @@ package api import ( "context" - commonHTTP "github.com/matrix-org/dendrite/common/http" + internalHTTP "github.com/matrix-org/dendrite/internal/http" opentracing "github.com/opentracing/opentracing-go" ) @@ -109,7 +109,7 @@ func (h *httpRoomserverInternalAPI) SetRoomAlias( defer span.Finish() apiURL := h.roomserverURL + RoomserverSetRoomAliasPath - return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // GetRoomIDForAlias implements RoomserverAliasAPI @@ -122,7 +122,7 @@ func (h *httpRoomserverInternalAPI) GetRoomIDForAlias( defer span.Finish() apiURL := h.roomserverURL + RoomserverGetRoomIDForAliasPath - return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // GetAliasesForRoomID implements RoomserverAliasAPI @@ -135,7 +135,7 @@ func (h *httpRoomserverInternalAPI) GetAliasesForRoomID( defer span.Finish() apiURL := h.roomserverURL + RoomserverGetAliasesForRoomIDPath - return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // GetCreatorIDForAlias implements RoomserverAliasAPI @@ -148,7 +148,7 @@ func (h *httpRoomserverInternalAPI) GetCreatorIDForAlias( defer span.Finish() apiURL := h.roomserverURL + RoomserverGetCreatorIDForAliasPath - return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // RemoveRoomAlias implements RoomserverAliasAPI @@ -161,5 +161,5 @@ func (h *httpRoomserverInternalAPI) RemoveRoomAlias( defer span.Finish() apiURL := h.roomserverURL + RoomserverRemoveRoomAliasPath - return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } diff --git a/roomserver/api/http.go b/roomserver/api/http.go index d643526bd..8a4d72e4e 100644 --- a/roomserver/api/http.go +++ b/roomserver/api/http.go @@ -4,8 +4,8 @@ import ( "errors" "net/http" - "github.com/matrix-org/dendrite/common/caching" fsInputAPI "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/internal/caching" ) type httpRoomserverInternalAPI struct { diff --git a/roomserver/api/input.go b/roomserver/api/input.go index 8e8fdae4e..f1eec75a8 100644 --- a/roomserver/api/input.go +++ b/roomserver/api/input.go @@ -18,7 +18,7 @@ package api import ( "context" - commonHTTP "github.com/matrix-org/dendrite/common/http" + internalHTTP "github.com/matrix-org/dendrite/internal/http" "github.com/matrix-org/gomatrixserverlib" opentracing "github.com/opentracing/opentracing-go" ) @@ -115,5 +115,5 @@ func (h *httpRoomserverInternalAPI) InputRoomEvents( defer span.Finish() apiURL := h.roomserverURL + RoomserverInputRoomEventsPath - return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } diff --git a/roomserver/api/perform.go b/roomserver/api/perform.go index e60c078bc..93f36f451 100644 --- a/roomserver/api/perform.go +++ b/roomserver/api/perform.go @@ -3,7 +3,7 @@ package api import ( "context" - commonHTTP "github.com/matrix-org/dendrite/common/http" + internalHTTP "github.com/matrix-org/dendrite/internal/http" "github.com/matrix-org/gomatrixserverlib" "github.com/opentracing/opentracing-go" ) @@ -35,7 +35,7 @@ func (h *httpRoomserverInternalAPI) PerformJoin( defer span.Finish() apiURL := h.roomserverURL + RoomserverPerformJoinPath - return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } type PerformLeaveRequest struct { @@ -55,5 +55,5 @@ func (h *httpRoomserverInternalAPI) PerformLeave( defer span.Finish() apiURL := h.roomserverURL + RoomserverPerformLeavePath - return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } diff --git a/roomserver/api/query.go b/roomserver/api/query.go index e92d2a992..98f46c771 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -19,7 +19,7 @@ package api import ( "context" - commonHTTP "github.com/matrix-org/dendrite/common/http" + internalHTTP "github.com/matrix-org/dendrite/internal/http" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" opentracing "github.com/opentracing/opentracing-go" @@ -318,7 +318,7 @@ func (h *httpRoomserverInternalAPI) QueryLatestEventsAndState( defer span.Finish() apiURL := h.roomserverURL + RoomserverQueryLatestEventsAndStatePath - return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // QueryStateAfterEvents implements RoomserverQueryAPI @@ -331,7 +331,7 @@ func (h *httpRoomserverInternalAPI) QueryStateAfterEvents( defer span.Finish() apiURL := h.roomserverURL + RoomserverQueryStateAfterEventsPath - return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // QueryEventsByID implements RoomserverQueryAPI @@ -344,7 +344,7 @@ func (h *httpRoomserverInternalAPI) QueryEventsByID( defer span.Finish() apiURL := h.roomserverURL + RoomserverQueryEventsByIDPath - return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // QueryMembershipForUser implements RoomserverQueryAPI @@ -357,7 +357,7 @@ func (h *httpRoomserverInternalAPI) QueryMembershipForUser( defer span.Finish() apiURL := h.roomserverURL + RoomserverQueryMembershipForUserPath - return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // QueryMembershipsForRoom implements RoomserverQueryAPI @@ -370,7 +370,7 @@ func (h *httpRoomserverInternalAPI) QueryMembershipsForRoom( defer span.Finish() apiURL := h.roomserverURL + RoomserverQueryMembershipsForRoomPath - return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // QueryInvitesForUser implements RoomserverQueryAPI @@ -383,7 +383,7 @@ func (h *httpRoomserverInternalAPI) QueryInvitesForUser( defer span.Finish() apiURL := h.roomserverURL + RoomserverQueryInvitesForUserPath - return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // QueryServerAllowedToSeeEvent implements RoomserverQueryAPI @@ -396,7 +396,7 @@ func (h *httpRoomserverInternalAPI) QueryServerAllowedToSeeEvent( defer span.Finish() apiURL := h.roomserverURL + RoomserverQueryServerAllowedToSeeEventPath - return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // QueryMissingEvents implements RoomServerQueryAPI @@ -409,7 +409,7 @@ func (h *httpRoomserverInternalAPI) QueryMissingEvents( defer span.Finish() apiURL := h.roomserverURL + RoomserverQueryMissingEventsPath - return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // QueryStateAndAuthChain implements RoomserverQueryAPI @@ -422,7 +422,7 @@ func (h *httpRoomserverInternalAPI) QueryStateAndAuthChain( defer span.Finish() apiURL := h.roomserverURL + RoomserverQueryStateAndAuthChainPath - return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // QueryBackfill implements RoomServerQueryAPI @@ -435,7 +435,7 @@ func (h *httpRoomserverInternalAPI) QueryBackfill( defer span.Finish() apiURL := h.roomserverURL + RoomserverQueryBackfillPath - return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // QueryRoomVersionCapabilities implements RoomServerQueryAPI @@ -448,7 +448,7 @@ func (h *httpRoomserverInternalAPI) QueryRoomVersionCapabilities( defer span.Finish() apiURL := h.roomserverURL + RoomserverQueryRoomVersionCapabilitiesPath - return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // QueryRoomVersionForRoom implements RoomServerQueryAPI @@ -466,7 +466,7 @@ func (h *httpRoomserverInternalAPI) QueryRoomVersionForRoom( defer span.Finish() apiURL := h.roomserverURL + RoomserverQueryRoomVersionForRoomPath - err := commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + err := internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) if err == nil { h.immutableCache.StoreRoomVersion(request.RoomID, response.RoomVersion) } diff --git a/roomserver/internal/api.go b/roomserver/internal/api.go index 62389c237..4d223cf36 100644 --- a/roomserver/internal/api.go +++ b/roomserver/internal/api.go @@ -6,10 +6,10 @@ import ( "sync" "github.com/Shopify/sarama" - "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/common/caching" - "github.com/matrix-org/dendrite/common/config" fsAPI "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/gomatrixserverlib" @@ -34,7 +34,7 @@ type RoomserverInternalAPI struct { // nolint: gocyclo func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { servMux.Handle(api.RoomserverInputRoomEventsPath, - common.MakeInternalAPI("inputRoomEvents", func(req *http.Request) util.JSONResponse { + internal.MakeInternalAPI("inputRoomEvents", func(req *http.Request) util.JSONResponse { var request api.InputRoomEventsRequest var response api.InputRoomEventsResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -47,7 +47,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { }), ) servMux.Handle(api.RoomserverPerformJoinPath, - common.MakeInternalAPI("performJoin", func(req *http.Request) util.JSONResponse { + internal.MakeInternalAPI("performJoin", func(req *http.Request) util.JSONResponse { var request api.PerformJoinRequest var response api.PerformJoinResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -60,7 +60,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { }), ) servMux.Handle(api.RoomserverPerformLeavePath, - common.MakeInternalAPI("performLeave", func(req *http.Request) util.JSONResponse { + internal.MakeInternalAPI("performLeave", func(req *http.Request) util.JSONResponse { var request api.PerformLeaveRequest var response api.PerformLeaveResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -74,7 +74,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { ) servMux.Handle( api.RoomserverQueryLatestEventsAndStatePath, - common.MakeInternalAPI("queryLatestEventsAndState", func(req *http.Request) util.JSONResponse { + internal.MakeInternalAPI("queryLatestEventsAndState", func(req *http.Request) util.JSONResponse { var request api.QueryLatestEventsAndStateRequest var response api.QueryLatestEventsAndStateResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -88,7 +88,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { ) servMux.Handle( api.RoomserverQueryStateAfterEventsPath, - common.MakeInternalAPI("queryStateAfterEvents", func(req *http.Request) util.JSONResponse { + internal.MakeInternalAPI("queryStateAfterEvents", func(req *http.Request) util.JSONResponse { var request api.QueryStateAfterEventsRequest var response api.QueryStateAfterEventsResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -102,7 +102,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { ) servMux.Handle( api.RoomserverQueryEventsByIDPath, - common.MakeInternalAPI("queryEventsByID", func(req *http.Request) util.JSONResponse { + internal.MakeInternalAPI("queryEventsByID", func(req *http.Request) util.JSONResponse { var request api.QueryEventsByIDRequest var response api.QueryEventsByIDResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -116,7 +116,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { ) servMux.Handle( api.RoomserverQueryMembershipForUserPath, - common.MakeInternalAPI("QueryMembershipForUser", func(req *http.Request) util.JSONResponse { + internal.MakeInternalAPI("QueryMembershipForUser", func(req *http.Request) util.JSONResponse { var request api.QueryMembershipForUserRequest var response api.QueryMembershipForUserResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -130,7 +130,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { ) servMux.Handle( api.RoomserverQueryMembershipsForRoomPath, - common.MakeInternalAPI("queryMembershipsForRoom", func(req *http.Request) util.JSONResponse { + internal.MakeInternalAPI("queryMembershipsForRoom", func(req *http.Request) util.JSONResponse { var request api.QueryMembershipsForRoomRequest var response api.QueryMembershipsForRoomResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -144,7 +144,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { ) servMux.Handle( api.RoomserverQueryInvitesForUserPath, - common.MakeInternalAPI("queryInvitesForUser", func(req *http.Request) util.JSONResponse { + internal.MakeInternalAPI("queryInvitesForUser", func(req *http.Request) util.JSONResponse { var request api.QueryInvitesForUserRequest var response api.QueryInvitesForUserResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -158,7 +158,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { ) servMux.Handle( api.RoomserverQueryServerAllowedToSeeEventPath, - common.MakeInternalAPI("queryServerAllowedToSeeEvent", func(req *http.Request) util.JSONResponse { + internal.MakeInternalAPI("queryServerAllowedToSeeEvent", func(req *http.Request) util.JSONResponse { var request api.QueryServerAllowedToSeeEventRequest var response api.QueryServerAllowedToSeeEventResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -172,7 +172,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { ) servMux.Handle( api.RoomserverQueryMissingEventsPath, - common.MakeInternalAPI("queryMissingEvents", func(req *http.Request) util.JSONResponse { + internal.MakeInternalAPI("queryMissingEvents", func(req *http.Request) util.JSONResponse { var request api.QueryMissingEventsRequest var response api.QueryMissingEventsResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -186,7 +186,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { ) servMux.Handle( api.RoomserverQueryStateAndAuthChainPath, - common.MakeInternalAPI("queryStateAndAuthChain", func(req *http.Request) util.JSONResponse { + internal.MakeInternalAPI("queryStateAndAuthChain", func(req *http.Request) util.JSONResponse { var request api.QueryStateAndAuthChainRequest var response api.QueryStateAndAuthChainResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -200,7 +200,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { ) servMux.Handle( api.RoomserverQueryBackfillPath, - common.MakeInternalAPI("QueryBackfill", func(req *http.Request) util.JSONResponse { + internal.MakeInternalAPI("QueryBackfill", func(req *http.Request) util.JSONResponse { var request api.QueryBackfillRequest var response api.QueryBackfillResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -214,7 +214,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { ) servMux.Handle( api.RoomserverQueryRoomVersionCapabilitiesPath, - common.MakeInternalAPI("QueryRoomVersionCapabilities", func(req *http.Request) util.JSONResponse { + internal.MakeInternalAPI("QueryRoomVersionCapabilities", func(req *http.Request) util.JSONResponse { var request api.QueryRoomVersionCapabilitiesRequest var response api.QueryRoomVersionCapabilitiesResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -228,7 +228,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { ) servMux.Handle( api.RoomserverQueryRoomVersionForRoomPath, - common.MakeInternalAPI("QueryRoomVersionForRoom", func(req *http.Request) util.JSONResponse { + internal.MakeInternalAPI("QueryRoomVersionForRoom", func(req *http.Request) util.JSONResponse { var request api.QueryRoomVersionForRoomRequest var response api.QueryRoomVersionForRoomResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -242,7 +242,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { ) servMux.Handle( api.RoomserverSetRoomAliasPath, - common.MakeInternalAPI("setRoomAlias", func(req *http.Request) util.JSONResponse { + internal.MakeInternalAPI("setRoomAlias", func(req *http.Request) util.JSONResponse { var request api.SetRoomAliasRequest var response api.SetRoomAliasResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -256,7 +256,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { ) servMux.Handle( api.RoomserverGetRoomIDForAliasPath, - common.MakeInternalAPI("GetRoomIDForAlias", func(req *http.Request) util.JSONResponse { + internal.MakeInternalAPI("GetRoomIDForAlias", func(req *http.Request) util.JSONResponse { var request api.GetRoomIDForAliasRequest var response api.GetRoomIDForAliasResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -270,7 +270,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { ) servMux.Handle( api.RoomserverGetCreatorIDForAliasPath, - common.MakeInternalAPI("GetCreatorIDForAlias", func(req *http.Request) util.JSONResponse { + internal.MakeInternalAPI("GetCreatorIDForAlias", func(req *http.Request) util.JSONResponse { var request api.GetCreatorIDForAliasRequest var response api.GetCreatorIDForAliasResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -284,7 +284,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { ) servMux.Handle( api.RoomserverGetAliasesForRoomIDPath, - common.MakeInternalAPI("getAliasesForRoomID", func(req *http.Request) util.JSONResponse { + internal.MakeInternalAPI("getAliasesForRoomID", func(req *http.Request) util.JSONResponse { var request api.GetAliasesForRoomIDRequest var response api.GetAliasesForRoomIDResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -298,7 +298,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { ) servMux.Handle( api.RoomserverRemoveRoomAliasPath, - common.MakeInternalAPI("removeRoomAlias", func(req *http.Request) util.JSONResponse { + internal.MakeInternalAPI("removeRoomAlias", func(req *http.Request) util.JSONResponse { var request api.RemoveRoomAliasRequest var response api.RemoveRoomAliasResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { diff --git a/roomserver/internal/input_events.go b/roomserver/internal/input_events.go index a4167714d..3a38636f8 100644 --- a/roomserver/internal/input_events.go +++ b/roomserver/internal/input_events.go @@ -21,7 +21,7 @@ import ( "errors" "fmt" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/state" "github.com/matrix-org/dendrite/roomserver/storage" @@ -177,7 +177,7 @@ func (r *RoomserverInternalAPI) processInviteEvent( } succeeded := false defer func() { - txerr := common.EndTransaction(updater, &succeeded) + txerr := internal.EndTransaction(updater, &succeeded) if err == nil && txerr != nil { err = txerr } diff --git a/roomserver/internal/input_latest_events.go b/roomserver/internal/input_latest_events.go index d7c9a5cb6..aea85ca95 100644 --- a/roomserver/internal/input_latest_events.go +++ b/roomserver/internal/input_latest_events.go @@ -20,7 +20,7 @@ import ( "bytes" "context" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/state" "github.com/matrix-org/dendrite/roomserver/types" @@ -59,7 +59,7 @@ func (r *RoomserverInternalAPI) updateLatestEvents( } succeeded := false defer func() { - txerr := common.EndTransaction(updater, &succeeded) + txerr := internal.EndTransaction(updater, &succeeded) if err == nil && txerr != nil { err = txerr } diff --git a/roomserver/internal/perform_join.go b/roomserver/internal/perform_join.go index 8f2f84e0f..dc786b296 100644 --- a/roomserver/internal/perform_join.go +++ b/roomserver/internal/perform_join.go @@ -6,8 +6,8 @@ import ( "strings" "time" - "github.com/matrix-org/dendrite/common" fsAPI "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" @@ -143,7 +143,7 @@ func (r *RoomserverInternalAPI) performJoinRoomByID( // TODO: Check what happens if the room exists on the server // but everyone has since left. I suspect it does the wrong thing. buildRes := api.QueryLatestEventsAndStateResponse{} - event, err := common.BuildEvent( + event, err := internal.BuildEvent( ctx, // the request context &eb, // the template join event r.Cfg, // the server configuration @@ -186,7 +186,7 @@ func (r *RoomserverInternalAPI) performJoinRoomByID( } } - case common.ErrRoomNoExists: + case internal.ErrRoomNoExists: // The room doesn't exist. First of all check if the room is a local // room. If it is then there's nothing more to do - the room just // hasn't been created yet. diff --git a/roomserver/internal/perform_leave.go b/roomserver/internal/perform_leave.go index 5d9b251c9..b60c2e99f 100644 --- a/roomserver/internal/perform_leave.go +++ b/roomserver/internal/perform_leave.go @@ -6,8 +6,8 @@ import ( "strings" "time" - "github.com/matrix-org/dendrite/common" fsAPI "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" ) @@ -97,7 +97,7 @@ func (r *RoomserverInternalAPI) performLeaveRoomByID( // TODO: Check what happens if the room exists on the server // but everyone has since left. I suspect it does the wrong thing. buildRes := api.QueryLatestEventsAndStateResponse{} - event, err := common.BuildEvent( + event, err := internal.BuildEvent( ctx, // the request context &eb, // the template leave event r.Cfg, // the server configuration @@ -106,7 +106,7 @@ func (r *RoomserverInternalAPI) performLeaveRoomByID( &buildRes, // the query response ) if err != nil { - return fmt.Errorf("common.BuildEvent: %w", err) + return fmt.Errorf("internal.BuildEvent: %w", err) } // Give our leave event to the roomserver input stream. The diff --git a/roomserver/internal/query_test.go b/roomserver/internal/query_test.go index 211ab5083..92e008324 100644 --- a/roomserver/internal/query_test.go +++ b/roomserver/internal/query_test.go @@ -19,7 +19,7 @@ import ( "encoding/json" "testing" - "github.com/matrix-org/dendrite/common/test" + "github.com/matrix-org/dendrite/internal/test" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" ) diff --git a/roomserver/roomserver.go b/roomserver/roomserver.go index aa5d223e9..07250e7ce 100644 --- a/roomserver/roomserver.go +++ b/roomserver/roomserver.go @@ -20,7 +20,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/dendrite/common/basecomponent" + "github.com/matrix-org/dendrite/internal/basecomponent" "github.com/matrix-org/dendrite/roomserver/internal" "github.com/matrix-org/dendrite/roomserver/storage" "github.com/sirupsen/logrus" diff --git a/roomserver/state/state.go b/roomserver/state/state.go index 9b005ee6a..d5be4a901 100644 --- a/roomserver/state/state.go +++ b/roomserver/state/state.go @@ -567,7 +567,7 @@ func (v StateResolution) CalculateAndStoreStateAfterEvents( // 3) None of the previous events were state events and they all // have the same state, so this event has exactly the same state // as the previous events. - // This should be the common case. + // This should be the internal case. metrics.algorithm = "no_change" return metrics.stop(prevState.BeforeStateSnapshotNID, nil) } diff --git a/roomserver/storage/postgres/event_json_table.go b/roomserver/storage/postgres/event_json_table.go index 616eaf318..661c44721 100644 --- a/roomserver/storage/postgres/event_json_table.go +++ b/roomserver/storage/postgres/event_json_table.go @@ -19,7 +19,7 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -88,7 +88,7 @@ func (s *eventJSONStatements) bulkSelectEventJSON( if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventJSON: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectEventJSON: rows.close() failed") // We know that we will only get as many results as event NIDs // because of the unique constraint on event NIDs. diff --git a/roomserver/storage/postgres/event_state_keys_table.go b/roomserver/storage/postgres/event_state_keys_table.go index 4c3496d91..b213e057b 100644 --- a/roomserver/storage/postgres/event_state_keys_table.go +++ b/roomserver/storage/postgres/event_state_keys_table.go @@ -20,7 +20,7 @@ import ( "database/sql" "github.com/lib/pq" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -91,7 +91,7 @@ func (s *eventStateKeyStatements) insertEventStateKeyNID( ctx context.Context, txn *sql.Tx, eventStateKey string, ) (types.EventStateKeyNID, error) { var eventStateKeyNID int64 - stmt := common.TxStmt(txn, s.insertEventStateKeyNIDStmt) + stmt := internal.TxStmt(txn, s.insertEventStateKeyNIDStmt) err := stmt.QueryRowContext(ctx, eventStateKey).Scan(&eventStateKeyNID) return types.EventStateKeyNID(eventStateKeyNID), err } @@ -100,7 +100,7 @@ func (s *eventStateKeyStatements) selectEventStateKeyNID( ctx context.Context, txn *sql.Tx, eventStateKey string, ) (types.EventStateKeyNID, error) { var eventStateKeyNID int64 - stmt := common.TxStmt(txn, s.selectEventStateKeyNIDStmt) + stmt := internal.TxStmt(txn, s.selectEventStateKeyNIDStmt) err := stmt.QueryRowContext(ctx, eventStateKey).Scan(&eventStateKeyNID) return types.EventStateKeyNID(eventStateKeyNID), err } @@ -114,7 +114,7 @@ func (s *eventStateKeyStatements) bulkSelectEventStateKeyNID( if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventStateKeyNID: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectEventStateKeyNID: rows.close() failed") result := make(map[string]types.EventStateKeyNID, len(eventStateKeys)) for rows.Next() { @@ -139,7 +139,7 @@ func (s *eventStateKeyStatements) bulkSelectEventStateKey( if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventStateKey: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectEventStateKey: rows.close() failed") result := make(map[types.EventStateKeyNID]string, len(eventStateKeyNIDs)) for rows.Next() { diff --git a/roomserver/storage/postgres/event_types_table.go b/roomserver/storage/postgres/event_types_table.go index 6537a5457..2b0910e71 100644 --- a/roomserver/storage/postgres/event_types_table.go +++ b/roomserver/storage/postgres/event_types_table.go @@ -19,7 +19,7 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/lib/pq" "github.com/matrix-org/dendrite/roomserver/types" @@ -27,7 +27,7 @@ import ( const eventTypesSchema = ` -- Numeric versions of the event "type"s. Event types tend to be taken from a --- small common pool. Assigning each a numeric ID should reduce the amount of +-- small internal pool. Assigning each a numeric ID should reduce the amount of -- data that needs to be stored and fetched from the database. -- It also means that many operations can work with int64 arrays rather than -- string arrays which may help reduce GC pressure. @@ -42,7 +42,7 @@ const eventTypesSchema = ` -- Picking well-known numeric IDs for the events types that require special -- attention during state conflict resolution means that we write that code -- using numeric constants. --- It also means that the numeric IDs for common event types should be +-- It also means that the numeric IDs for internal event types should be -- consistent between different instances which might make ad-hoc debugging -- easier. -- Other event types are automatically assigned numeric IDs starting from 2**16. @@ -134,7 +134,7 @@ func (s *eventTypeStatements) bulkSelectEventTypeNID( if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventTypeNID: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectEventTypeNID: rows.close() failed") result := make(map[string]types.EventTypeNID, len(eventTypes)) for rows.Next() { diff --git a/roomserver/storage/postgres/events_table.go b/roomserver/storage/postgres/events_table.go index ecc35f37a..c28fa8e6b 100644 --- a/roomserver/storage/postgres/events_table.go +++ b/roomserver/storage/postgres/events_table.go @@ -21,7 +21,7 @@ import ( "fmt" "github.com/lib/pq" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -197,7 +197,7 @@ func (s *eventStatements) bulkSelectStateEventByID( if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "bulkSelectStateEventByID: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectStateEventByID: rows.close() failed") // We know that we will only get as many results as event IDs // because of the unique constraint on event IDs. // So we can allocate an array of the correct size now. @@ -222,7 +222,7 @@ func (s *eventStatements) bulkSelectStateEventByID( // We don't know which ones were missing because we don't return the string IDs in the query. // However it should be possible debug this by replaying queries or entries from the input kafka logs. // If this turns out to be impossible and we do need the debug information here, it would be better - // to do it as a separate query rather than slowing down/complicating the common case. + // to do it as a separate query rather than slowing down/complicating the internal case. return nil, types.MissingEventError( fmt.Sprintf("storage: state event IDs missing from the database (%d != %d)", i, len(eventIDs)), ) @@ -240,7 +240,7 @@ func (s *eventStatements) bulkSelectStateAtEventByID( if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "bulkSelectStateAtEventByID: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectStateAtEventByID: rows.close() failed") results := make([]types.StateAtEvent, len(eventIDs)) i := 0 for ; rows.Next(); i++ { @@ -280,13 +280,13 @@ func (s *eventStatements) updateEventState( func (s *eventStatements) selectEventSentToOutput( ctx context.Context, txn *sql.Tx, eventNID types.EventNID, ) (sentToOutput bool, err error) { - stmt := common.TxStmt(txn, s.selectEventSentToOutputStmt) + stmt := internal.TxStmt(txn, s.selectEventSentToOutputStmt) err = stmt.QueryRowContext(ctx, int64(eventNID)).Scan(&sentToOutput) return } func (s *eventStatements) updateEventSentToOutput(ctx context.Context, txn *sql.Tx, eventNID types.EventNID) error { - stmt := common.TxStmt(txn, s.updateEventSentToOutputStmt) + stmt := internal.TxStmt(txn, s.updateEventSentToOutputStmt) _, err := stmt.ExecContext(ctx, int64(eventNID)) return err } @@ -294,7 +294,7 @@ func (s *eventStatements) updateEventSentToOutput(ctx context.Context, txn *sql. func (s *eventStatements) selectEventID( ctx context.Context, txn *sql.Tx, eventNID types.EventNID, ) (eventID string, err error) { - stmt := common.TxStmt(txn, s.selectEventIDStmt) + stmt := internal.TxStmt(txn, s.selectEventIDStmt) err = stmt.QueryRowContext(ctx, int64(eventNID)).Scan(&eventID) return } @@ -302,12 +302,12 @@ func (s *eventStatements) selectEventID( func (s *eventStatements) bulkSelectStateAtEventAndReference( ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID, ) ([]types.StateAtEventAndReference, error) { - stmt := common.TxStmt(txn, s.bulkSelectStateAtEventAndReferenceStmt) + stmt := internal.TxStmt(txn, s.bulkSelectStateAtEventAndReferenceStmt) rows, err := stmt.QueryContext(ctx, eventNIDsAsArray(eventNIDs)) if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "bulkSelectStateAtEventAndReference: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectStateAtEventAndReference: rows.close() failed") results := make([]types.StateAtEventAndReference, len(eventNIDs)) i := 0 for ; rows.Next(); i++ { @@ -348,7 +348,7 @@ func (s *eventStatements) bulkSelectEventReference( if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventReference: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectEventReference: rows.close() failed") results := make([]gomatrixserverlib.EventReference, len(eventNIDs)) i := 0 for ; rows.Next(); i++ { @@ -372,7 +372,7 @@ func (s *eventStatements) bulkSelectEventID(ctx context.Context, eventNIDs []typ if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventID: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectEventID: rows.close() failed") results := make(map[types.EventNID]string, len(eventNIDs)) i := 0 for ; rows.Next(); i++ { @@ -399,7 +399,7 @@ func (s *eventStatements) bulkSelectEventNID(ctx context.Context, eventIDs []str if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventNID: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectEventNID: rows.close() failed") results := make(map[string]types.EventNID, len(eventIDs)) for rows.Next() { var eventID string @@ -425,7 +425,7 @@ func (s *eventStatements) selectMaxEventDepth(ctx context.Context, eventNIDs []t func (s *eventStatements) selectRoomNIDForEventNID( ctx context.Context, txn *sql.Tx, eventNID types.EventNID, ) (roomNID types.RoomNID, err error) { - selectStmt := common.TxStmt(txn, s.selectRoomNIDForEventNIDStmt) + selectStmt := internal.TxStmt(txn, s.selectRoomNIDForEventNIDStmt) err = selectStmt.QueryRowContext(ctx, int64(eventNID)).Scan(&roomNID) return } diff --git a/roomserver/storage/postgres/invite_table.go b/roomserver/storage/postgres/invite_table.go index f764b1561..f0fb919e6 100644 --- a/roomserver/storage/postgres/invite_table.go +++ b/roomserver/storage/postgres/invite_table.go @@ -19,7 +19,7 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -98,7 +98,7 @@ func (s *inviteStatements) insertInviteEvent( targetUserNID, senderUserNID types.EventStateKeyNID, inviteEventJSON []byte, ) (bool, error) { - result, err := common.TxStmt(txn, s.insertInviteEventStmt).ExecContext( + result, err := internal.TxStmt(txn, s.insertInviteEventStmt).ExecContext( ctx, inviteEventID, roomNID, targetUserNID, senderUserNID, inviteEventJSON, ) if err != nil { @@ -115,12 +115,12 @@ func (s *inviteStatements) updateInviteRetired( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, ) ([]string, error) { - stmt := common.TxStmt(txn, s.updateInviteRetiredStmt) + stmt := internal.TxStmt(txn, s.updateInviteRetiredStmt) rows, err := stmt.QueryContext(ctx, roomNID, targetUserNID) if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "updateInviteRetired: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "updateInviteRetired: rows.close() failed") var eventIDs []string for rows.Next() { @@ -144,7 +144,7 @@ func (s *inviteStatements) selectInviteActiveForUserInRoom( if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "selectInviteActiveForUserInRoom: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "selectInviteActiveForUserInRoom: rows.close() failed") var result []types.EventStateKeyNID for rows.Next() { var senderUserNID int64 diff --git a/roomserver/storage/postgres/membership_table.go b/roomserver/storage/postgres/membership_table.go index 820ef4e71..f290a05f9 100644 --- a/roomserver/storage/postgres/membership_table.go +++ b/roomserver/storage/postgres/membership_table.go @@ -19,7 +19,7 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -138,7 +138,7 @@ func (s *membershipStatements) insertMembership( txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, localTarget bool, ) error { - stmt := common.TxStmt(txn, s.insertMembershipStmt) + stmt := internal.TxStmt(txn, s.insertMembershipStmt) _, err := stmt.ExecContext(ctx, roomNID, targetUserNID, localTarget) return err } @@ -147,7 +147,7 @@ func (s *membershipStatements) selectMembershipForUpdate( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, ) (membership membershipState, err error) { - err = common.TxStmt(txn, s.selectMembershipForUpdateStmt).QueryRowContext( + err = internal.TxStmt(txn, s.selectMembershipForUpdateStmt).QueryRowContext( ctx, roomNID, targetUserNID, ).Scan(&membership) return @@ -176,7 +176,7 @@ func (s *membershipStatements) selectMembershipsFromRoom( if err != nil { return } - defer common.CloseAndLogIfError(ctx, rows, "selectMembershipsFromRoom: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "selectMembershipsFromRoom: rows.close() failed") for rows.Next() { var eNID types.EventNID @@ -203,7 +203,7 @@ func (s *membershipStatements) selectMembershipsFromRoomAndMembership( if err != nil { return } - defer common.CloseAndLogIfError(ctx, rows, "selectMembershipsFromRoomAndMembership: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "selectMembershipsFromRoomAndMembership: rows.close() failed") for rows.Next() { var eNID types.EventNID @@ -221,7 +221,7 @@ func (s *membershipStatements) updateMembership( senderUserNID types.EventStateKeyNID, membership membershipState, eventNID types.EventNID, ) error { - _, err := common.TxStmt(txn, s.updateMembershipStmt).ExecContext( + _, err := internal.TxStmt(txn, s.updateMembershipStmt).ExecContext( ctx, roomNID, targetUserNID, senderUserNID, membership, eventNID, ) return err diff --git a/roomserver/storage/postgres/previous_events_table.go b/roomserver/storage/postgres/previous_events_table.go index 4c21b3081..e3ad5dc80 100644 --- a/roomserver/storage/postgres/previous_events_table.go +++ b/roomserver/storage/postgres/previous_events_table.go @@ -19,7 +19,7 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -82,7 +82,7 @@ func (s *previousEventStatements) insertPreviousEvent( previousEventReferenceSHA256 []byte, eventNID types.EventNID, ) error { - stmt := common.TxStmt(txn, s.insertPreviousEventStmt) + stmt := internal.TxStmt(txn, s.insertPreviousEventStmt) _, err := stmt.ExecContext( ctx, previousEventID, previousEventReferenceSHA256, int64(eventNID), ) @@ -95,6 +95,6 @@ func (s *previousEventStatements) selectPreviousEventExists( ctx context.Context, txn *sql.Tx, eventID string, eventReferenceSHA256 []byte, ) error { var ok int64 - stmt := common.TxStmt(txn, s.selectPreviousEventExistsStmt) + stmt := internal.TxStmt(txn, s.selectPreviousEventExistsStmt) return stmt.QueryRowContext(ctx, eventID, eventReferenceSHA256).Scan(&ok) } diff --git a/roomserver/storage/postgres/room_aliases_table.go b/roomserver/storage/postgres/room_aliases_table.go index c37f383c9..c77edd0e0 100644 --- a/roomserver/storage/postgres/room_aliases_table.go +++ b/roomserver/storage/postgres/room_aliases_table.go @@ -19,7 +19,7 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" ) const roomAliasesSchema = ` @@ -97,7 +97,7 @@ func (s *roomAliasesStatements) selectAliasesFromRoomID( if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "selectAliasesFromRoomID: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "selectAliasesFromRoomID: rows.close() failed") var aliases []string for rows.Next() { diff --git a/roomserver/storage/postgres/rooms_table.go b/roomserver/storage/postgres/rooms_table.go index ef5b510c9..fc64489de 100644 --- a/roomserver/storage/postgres/rooms_table.go +++ b/roomserver/storage/postgres/rooms_table.go @@ -21,7 +21,7 @@ import ( "errors" "github.com/lib/pq" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -103,7 +103,7 @@ func (s *roomStatements) insertRoomNID( roomID string, roomVersion gomatrixserverlib.RoomVersion, ) (types.RoomNID, error) { var roomNID int64 - stmt := common.TxStmt(txn, s.insertRoomNIDStmt) + stmt := internal.TxStmt(txn, s.insertRoomNIDStmt) err := stmt.QueryRowContext(ctx, roomID, roomVersion).Scan(&roomNID) return types.RoomNID(roomNID), err } @@ -112,7 +112,7 @@ func (s *roomStatements) selectRoomNID( ctx context.Context, txn *sql.Tx, roomID string, ) (types.RoomNID, error) { var roomNID int64 - stmt := common.TxStmt(txn, s.selectRoomNIDStmt) + stmt := internal.TxStmt(txn, s.selectRoomNIDStmt) err := stmt.QueryRowContext(ctx, roomID).Scan(&roomNID) return types.RoomNID(roomNID), err } @@ -140,7 +140,7 @@ func (s *roomStatements) selectLatestEventsNIDsForUpdate( var nids pq.Int64Array var lastEventSentNID int64 var stateSnapshotNID int64 - stmt := common.TxStmt(txn, s.selectLatestEventNIDsForUpdateStmt) + stmt := internal.TxStmt(txn, s.selectLatestEventNIDsForUpdateStmt) err := stmt.QueryRowContext(ctx, int64(roomNID)).Scan(&nids, &lastEventSentNID, &stateSnapshotNID) if err != nil { return nil, 0, 0, err @@ -160,7 +160,7 @@ func (s *roomStatements) updateLatestEventNIDs( lastEventSentNID types.EventNID, stateSnapshotNID types.StateSnapshotNID, ) error { - stmt := common.TxStmt(txn, s.updateLatestEventNIDsStmt) + stmt := internal.TxStmt(txn, s.updateLatestEventNIDsStmt) _, err := stmt.ExecContext( ctx, roomNID, @@ -175,7 +175,7 @@ func (s *roomStatements) selectRoomVersionForRoomID( ctx context.Context, txn *sql.Tx, roomID string, ) (gomatrixserverlib.RoomVersion, error) { var roomVersion gomatrixserverlib.RoomVersion - stmt := common.TxStmt(txn, s.selectRoomVersionForRoomIDStmt) + stmt := internal.TxStmt(txn, s.selectRoomVersionForRoomIDStmt) err := stmt.QueryRowContext(ctx, roomID).Scan(&roomVersion) if err == sql.ErrNoRows { return roomVersion, errors.New("room not found") @@ -187,7 +187,7 @@ func (s *roomStatements) selectRoomVersionForRoomNID( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, ) (gomatrixserverlib.RoomVersion, error) { var roomVersion gomatrixserverlib.RoomVersion - stmt := common.TxStmt(txn, s.selectRoomVersionForRoomNIDStmt) + stmt := internal.TxStmt(txn, s.selectRoomVersionForRoomNIDStmt) err := stmt.QueryRowContext(ctx, roomNID).Scan(&roomVersion) if err == sql.ErrNoRows { return roomVersion, errors.New("room not found") diff --git a/roomserver/storage/postgres/state_block_table.go b/roomserver/storage/postgres/state_block_table.go index b9246b763..38334fa9e 100644 --- a/roomserver/storage/postgres/state_block_table.go +++ b/roomserver/storage/postgres/state_block_table.go @@ -21,7 +21,7 @@ import ( "fmt" "sort" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/lib/pq" "github.com/matrix-org/dendrite/roomserver/types" @@ -31,7 +31,7 @@ import ( const stateDataSchema = ` -- The state data map. -- Designed to give enough information to run the state resolution algorithm --- without hitting the database in the common case. +-- without hitting the database in the internal case. -- TODO: Is it worth replacing the unique btree index with a covering index so -- that postgres could lookup the state using an index-only scan? -- The type and state_key are included in the index to make it easier to @@ -140,7 +140,7 @@ func (s *stateBlockStatements) bulkSelectStateBlockEntries( if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "bulkSelectStateBlockEntries: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectStateBlockEntries: rows.close() failed") results := make([]types.StateEntryList, len(stateBlockNIDs)) // current is a pointer to the StateEntryList to append the state entries to. @@ -199,7 +199,7 @@ func (s *stateBlockStatements) bulkSelectFilteredStateBlockEntries( if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "bulkSelectFilteredStateBlockEntries: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectFilteredStateBlockEntries: rows.close() failed") var results []types.StateEntryList var current types.StateEntryList diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go index d451d6650..4d1d603e3 100644 --- a/roomserver/storage/postgres/storage.go +++ b/roomserver/storage/postgres/storage.go @@ -20,7 +20,7 @@ import ( "database/sql" "encoding/json" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" // Import the postgres database driver. @@ -37,7 +37,7 @@ type Database struct { } // Open a postgres database. -func Open(dataSourceName string, dbProperties common.DbProperties) (*Database, error) { +func Open(dataSourceName string, dbProperties internal.DbProperties) (*Database, error) { var d Database var err error if d.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { diff --git a/roomserver/storage/sqlite3/event_json_table.go b/roomserver/storage/sqlite3/event_json_table.go index fc661c1da..fbf35e711 100644 --- a/roomserver/storage/sqlite3/event_json_table.go +++ b/roomserver/storage/sqlite3/event_json_table.go @@ -20,7 +20,7 @@ import ( "database/sql" "strings" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -66,7 +66,7 @@ func (s *eventJSONStatements) prepare(db *sql.DB) (err error) { func (s *eventJSONStatements) insertEventJSON( ctx context.Context, txn *sql.Tx, eventNID types.EventNID, eventJSON []byte, ) error { - _, err := common.TxStmt(txn, s.insertEventJSONStmt).ExecContext(ctx, int64(eventNID), eventJSON) + _, err := internal.TxStmt(txn, s.insertEventJSONStmt).ExecContext(ctx, int64(eventNID), eventJSON) return err } @@ -82,13 +82,13 @@ func (s *eventJSONStatements) bulkSelectEventJSON( for k, v := range eventNIDs { iEventNIDs[k] = v } - selectOrig := strings.Replace(bulkSelectEventJSONSQL, "($1)", common.QueryVariadic(len(iEventNIDs)), 1) + selectOrig := strings.Replace(bulkSelectEventJSONSQL, "($1)", internal.QueryVariadic(len(iEventNIDs)), 1) rows, err := txn.QueryContext(ctx, selectOrig, iEventNIDs...) if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventJSON: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectEventJSON: rows.close() failed") // We know that we will only get as many results as event NIDs // because of the unique constraint on event NIDs. diff --git a/roomserver/storage/sqlite3/event_state_keys_table.go b/roomserver/storage/sqlite3/event_state_keys_table.go index 204b4eb62..f49ebf554 100644 --- a/roomserver/storage/sqlite3/event_state_keys_table.go +++ b/roomserver/storage/sqlite3/event_state_keys_table.go @@ -20,7 +20,7 @@ import ( "database/sql" "strings" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -110,13 +110,13 @@ func (s *eventStateKeyStatements) bulkSelectEventStateKeyNID( for k, v := range eventStateKeys { iEventStateKeys[k] = v } - selectOrig := strings.Replace(bulkSelectEventStateKeySQL, "($1)", common.QueryVariadic(len(eventStateKeys)), 1) + selectOrig := strings.Replace(bulkSelectEventStateKeySQL, "($1)", internal.QueryVariadic(len(eventStateKeys)), 1) rows, err := txn.QueryContext(ctx, selectOrig, iEventStateKeys...) if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventStateKeyNID: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectEventStateKeyNID: rows.close() failed") result := make(map[string]types.EventStateKeyNID, len(eventStateKeys)) for rows.Next() { var stateKey string @@ -136,13 +136,13 @@ func (s *eventStateKeyStatements) bulkSelectEventStateKey( for k, v := range eventStateKeyNIDs { iEventStateKeyNIDs[k] = v } - selectOrig := strings.Replace(bulkSelectEventStateKeyNIDSQL, "($1)", common.QueryVariadic(len(eventStateKeyNIDs)), 1) + selectOrig := strings.Replace(bulkSelectEventStateKeyNIDSQL, "($1)", internal.QueryVariadic(len(eventStateKeyNIDs)), 1) rows, err := txn.QueryContext(ctx, selectOrig, iEventStateKeyNIDs...) if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventStateKey: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectEventStateKey: rows.close() failed") result := make(map[types.EventStateKeyNID]string, len(eventStateKeyNIDs)) for rows.Next() { var stateKey string diff --git a/roomserver/storage/sqlite3/event_types_table.go b/roomserver/storage/sqlite3/event_types_table.go index 777f8be79..13abcd4df 100644 --- a/roomserver/storage/sqlite3/event_types_table.go +++ b/roomserver/storage/sqlite3/event_types_table.go @@ -20,7 +20,7 @@ import ( "database/sql" "strings" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -101,8 +101,8 @@ func (s *eventTypeStatements) insertEventTypeNID( ) (types.EventTypeNID, error) { var eventTypeNID int64 var err error - insertStmt := common.TxStmt(tx, s.insertEventTypeNIDStmt) - resultStmt := common.TxStmt(tx, s.insertEventTypeNIDResultStmt) + insertStmt := internal.TxStmt(tx, s.insertEventTypeNIDStmt) + resultStmt := internal.TxStmt(tx, s.insertEventTypeNIDResultStmt) if _, err = insertStmt.ExecContext(ctx, eventType); err == nil { err = resultStmt.QueryRowContext(ctx).Scan(&eventTypeNID) } @@ -113,7 +113,7 @@ func (s *eventTypeStatements) selectEventTypeNID( ctx context.Context, tx *sql.Tx, eventType string, ) (types.EventTypeNID, error) { var eventTypeNID int64 - selectStmt := common.TxStmt(tx, s.selectEventTypeNIDStmt) + selectStmt := internal.TxStmt(tx, s.selectEventTypeNIDStmt) err := selectStmt.QueryRowContext(ctx, eventType).Scan(&eventTypeNID) return types.EventTypeNID(eventTypeNID), err } @@ -126,19 +126,19 @@ func (s *eventTypeStatements) bulkSelectEventTypeNID( for k, v := range eventTypes { iEventTypes[k] = v } - selectOrig := strings.Replace(bulkSelectEventTypeNIDSQL, "($1)", common.QueryVariadic(len(iEventTypes)), 1) + selectOrig := strings.Replace(bulkSelectEventTypeNIDSQL, "($1)", internal.QueryVariadic(len(iEventTypes)), 1) selectPrep, err := s.db.Prepare(selectOrig) if err != nil { return nil, err } /////////////// - selectStmt := common.TxStmt(tx, selectPrep) + selectStmt := internal.TxStmt(tx, selectPrep) rows, err := selectStmt.QueryContext(ctx, iEventTypes...) if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventTypeNID: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectEventTypeNID: rows.close() failed") result := make(map[string]types.EventTypeNID, len(eventTypes)) for rows.Next() { diff --git a/roomserver/storage/sqlite3/events_table.go b/roomserver/storage/sqlite3/events_table.go index a63596aeb..551134950 100644 --- a/roomserver/storage/sqlite3/events_table.go +++ b/roomserver/storage/sqlite3/events_table.go @@ -22,7 +22,7 @@ import ( "fmt" "strings" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -147,7 +147,7 @@ func (s *eventStatements) insertEvent( depth int64, ) (types.EventNID, error) { // attempt to insert: the last_row_id is the event NID - insertStmt := common.TxStmt(txn, s.insertEventStmt) + insertStmt := internal.TxStmt(txn, s.insertEventStmt) result, err := insertStmt.ExecContext( ctx, int64(roomNID), int64(eventTypeNID), int64(eventStateKeyNID), eventID, referenceSHA256, eventNIDsAsArray(authEventNIDs), depth, @@ -168,7 +168,7 @@ func (s *eventStatements) selectEvent( ) (types.EventNID, types.StateSnapshotNID, error) { var eventNID int64 var stateNID int64 - selectStmt := common.TxStmt(txn, s.selectEventStmt) + selectStmt := internal.TxStmt(txn, s.selectEventStmt) err := selectStmt.QueryRowContext(ctx, eventID).Scan(&eventNID, &stateNID) return types.EventNID(eventNID), types.StateSnapshotNID(stateNID), err } @@ -183,19 +183,19 @@ func (s *eventStatements) bulkSelectStateEventByID( for k, v := range eventIDs { iEventIDs[k] = v } - selectOrig := strings.Replace(bulkSelectStateEventByIDSQL, "($1)", common.QueryVariadic(len(iEventIDs)), 1) + selectOrig := strings.Replace(bulkSelectStateEventByIDSQL, "($1)", internal.QueryVariadic(len(iEventIDs)), 1) selectPrep, err := txn.Prepare(selectOrig) if err != nil { return nil, err } /////////////// - selectStmt := common.TxStmt(txn, selectPrep) + selectStmt := internal.TxStmt(txn, selectPrep) rows, err := selectStmt.QueryContext(ctx, iEventIDs...) if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "bulkSelectStateEventByID: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectStateEventByID: rows.close() failed") // We know that we will only get as many results as event IDs // because of the unique constraint on event IDs. // So we can allocate an array of the correct size now. @@ -217,7 +217,7 @@ func (s *eventStatements) bulkSelectStateEventByID( // We don't know which ones were missing because we don't return the string IDs in the query. // However it should be possible debug this by replaying queries or entries from the input kafka logs. // If this turns out to be impossible and we do need the debug information here, it would be better - // to do it as a separate query rather than slowing down/complicating the common case. + // to do it as a separate query rather than slowing down/complicating the internal case. return nil, types.MissingEventError( fmt.Sprintf("storage: state event IDs missing from the database (%d != %d)", i, len(eventIDs)), ) @@ -236,19 +236,19 @@ func (s *eventStatements) bulkSelectStateAtEventByID( for k, v := range eventIDs { iEventIDs[k] = v } - selectOrig := strings.Replace(bulkSelectStateAtEventByIDSQL, "($1)", common.QueryVariadic(len(iEventIDs)), 1) + selectOrig := strings.Replace(bulkSelectStateAtEventByIDSQL, "($1)", internal.QueryVariadic(len(iEventIDs)), 1) selectPrep, err := txn.Prepare(selectOrig) if err != nil { return nil, err } /////////////// - selectStmt := common.TxStmt(txn, selectPrep) + selectStmt := internal.TxStmt(txn, selectPrep) rows, err := selectStmt.QueryContext(ctx, iEventIDs...) if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "bulkSelectStateAtEventByID: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectStateAtEventByID: rows.close() failed") results := make([]types.StateAtEvent, len(eventIDs)) i := 0 for ; rows.Next(); i++ { @@ -278,7 +278,7 @@ func (s *eventStatements) bulkSelectStateAtEventByID( func (s *eventStatements) updateEventState( ctx context.Context, txn *sql.Tx, eventNID types.EventNID, stateNID types.StateSnapshotNID, ) error { - updateStmt := common.TxStmt(txn, s.updateEventStateStmt) + updateStmt := internal.TxStmt(txn, s.updateEventStateStmt) _, err := updateStmt.ExecContext(ctx, int64(stateNID), int64(eventNID)) return err } @@ -286,7 +286,7 @@ func (s *eventStatements) updateEventState( func (s *eventStatements) selectEventSentToOutput( ctx context.Context, txn *sql.Tx, eventNID types.EventNID, ) (sentToOutput bool, err error) { - selectStmt := common.TxStmt(txn, s.selectEventSentToOutputStmt) + selectStmt := internal.TxStmt(txn, s.selectEventSentToOutputStmt) err = selectStmt.QueryRowContext(ctx, int64(eventNID)).Scan(&sentToOutput) //err = s.selectEventSentToOutputStmt.QueryRowContext(ctx, int64(eventNID)).Scan(&sentToOutput) if err != nil { @@ -295,7 +295,7 @@ func (s *eventStatements) selectEventSentToOutput( } func (s *eventStatements) updateEventSentToOutput(ctx context.Context, txn *sql.Tx, eventNID types.EventNID) error { - updateStmt := common.TxStmt(txn, s.updateEventSentToOutputStmt) + updateStmt := internal.TxStmt(txn, s.updateEventSentToOutputStmt) _, err := updateStmt.ExecContext(ctx, int64(eventNID)) //_, err := s.updateEventSentToOutputStmt.ExecContext(ctx, int64(eventNID)) return err @@ -304,7 +304,7 @@ func (s *eventStatements) updateEventSentToOutput(ctx context.Context, txn *sql. func (s *eventStatements) selectEventID( ctx context.Context, txn *sql.Tx, eventNID types.EventNID, ) (eventID string, err error) { - selectStmt := common.TxStmt(txn, s.selectEventIDStmt) + selectStmt := internal.TxStmt(txn, s.selectEventIDStmt) err = selectStmt.QueryRowContext(ctx, int64(eventNID)).Scan(&eventID) return } @@ -317,14 +317,14 @@ func (s *eventStatements) bulkSelectStateAtEventAndReference( for k, v := range eventNIDs { iEventNIDs[k] = v } - selectOrig := strings.Replace(bulkSelectStateAtEventAndReferenceSQL, "($1)", common.QueryVariadic(len(iEventNIDs)), 1) + selectOrig := strings.Replace(bulkSelectStateAtEventAndReferenceSQL, "($1)", internal.QueryVariadic(len(iEventNIDs)), 1) ////////////// rows, err := txn.QueryContext(ctx, selectOrig, iEventNIDs...) if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "bulkSelectStateAtEventAndReference: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectStateAtEventAndReference: rows.close() failed") results := make([]types.StateAtEventAndReference, len(eventNIDs)) i := 0 for ; rows.Next(); i++ { @@ -363,19 +363,19 @@ func (s *eventStatements) bulkSelectEventReference( for k, v := range eventNIDs { iEventNIDs[k] = v } - selectOrig := strings.Replace(bulkSelectEventReferenceSQL, "($1)", common.QueryVariadic(len(iEventNIDs)), 1) + selectOrig := strings.Replace(bulkSelectEventReferenceSQL, "($1)", internal.QueryVariadic(len(iEventNIDs)), 1) selectPrep, err := txn.Prepare(selectOrig) if err != nil { return nil, err } /////////////// - selectStmt := common.TxStmt(txn, selectPrep) + selectStmt := internal.TxStmt(txn, selectPrep) rows, err := selectStmt.QueryContext(ctx, iEventNIDs...) if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventReference: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectEventReference: rows.close() failed") results := make([]gomatrixserverlib.EventReference, len(eventNIDs)) i := 0 for ; rows.Next(); i++ { @@ -397,19 +397,19 @@ func (s *eventStatements) bulkSelectEventID(ctx context.Context, txn *sql.Tx, ev for k, v := range eventNIDs { iEventNIDs[k] = v } - selectOrig := strings.Replace(bulkSelectEventIDSQL, "($1)", common.QueryVariadic(len(iEventNIDs)), 1) + selectOrig := strings.Replace(bulkSelectEventIDSQL, "($1)", internal.QueryVariadic(len(iEventNIDs)), 1) selectPrep, err := txn.Prepare(selectOrig) if err != nil { return nil, err } /////////////// - selectStmt := common.TxStmt(txn, selectPrep) + selectStmt := internal.TxStmt(txn, selectPrep) rows, err := selectStmt.QueryContext(ctx, iEventNIDs...) if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventID: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectEventID: rows.close() failed") results := make(map[types.EventNID]string, len(eventNIDs)) i := 0 for ; rows.Next(); i++ { @@ -434,19 +434,19 @@ func (s *eventStatements) bulkSelectEventNID(ctx context.Context, txn *sql.Tx, e for k, v := range eventIDs { iEventIDs[k] = v } - selectOrig := strings.Replace(bulkSelectEventNIDSQL, "($1)", common.QueryVariadic(len(iEventIDs)), 1) + selectOrig := strings.Replace(bulkSelectEventNIDSQL, "($1)", internal.QueryVariadic(len(iEventIDs)), 1) selectPrep, err := txn.Prepare(selectOrig) if err != nil { return nil, err } /////////////// - selectStmt := common.TxStmt(txn, selectPrep) + selectStmt := internal.TxStmt(txn, selectPrep) rows, err := selectStmt.QueryContext(ctx, iEventIDs...) if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "bulkSelectEventNID: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectEventNID: rows.close() failed") results := make(map[string]types.EventNID, len(eventIDs)) for rows.Next() { var eventID string @@ -465,7 +465,7 @@ func (s *eventStatements) selectMaxEventDepth(ctx context.Context, txn *sql.Tx, for i, v := range eventNIDs { iEventIDs[i] = v } - sqlStr := strings.Replace(selectMaxEventDepthSQL, "($1)", common.QueryVariadic(len(iEventIDs)), 1) + sqlStr := strings.Replace(selectMaxEventDepthSQL, "($1)", internal.QueryVariadic(len(iEventIDs)), 1) err := txn.QueryRowContext(ctx, sqlStr, iEventIDs...).Scan(&result) if err != nil { return 0, err @@ -476,7 +476,7 @@ func (s *eventStatements) selectMaxEventDepth(ctx context.Context, txn *sql.Tx, func (s *eventStatements) selectRoomNIDForEventNID( ctx context.Context, txn *sql.Tx, eventNID types.EventNID, ) (roomNID types.RoomNID, err error) { - selectStmt := common.TxStmt(txn, s.selectRoomNIDForEventNIDStmt) + selectStmt := internal.TxStmt(txn, s.selectRoomNIDForEventNIDStmt) err = selectStmt.QueryRowContext(ctx, int64(eventNID)).Scan(&roomNID) return } diff --git a/roomserver/storage/sqlite3/invite_table.go b/roomserver/storage/sqlite3/invite_table.go index 0ab3e6f36..a42d18a73 100644 --- a/roomserver/storage/sqlite3/invite_table.go +++ b/roomserver/storage/sqlite3/invite_table.go @@ -19,7 +19,7 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -86,7 +86,7 @@ func (s *inviteStatements) insertInviteEvent( targetUserNID, senderUserNID types.EventStateKeyNID, inviteEventJSON []byte, ) (bool, error) { - stmt := common.TxStmt(txn, s.insertInviteEventStmt) + stmt := internal.TxStmt(txn, s.insertInviteEventStmt) defer stmt.Close() // nolint: errcheck result, err := stmt.ExecContext( ctx, inviteEventID, roomNID, targetUserNID, senderUserNID, inviteEventJSON, @@ -137,7 +137,7 @@ func (s *inviteStatements) selectInviteActiveForUserInRoom( if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "selectInviteActiveForUserInRoom: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "selectInviteActiveForUserInRoom: rows.close() failed") var result []types.EventStateKeyNID for rows.Next() { var senderUserNID int64 diff --git a/roomserver/storage/sqlite3/membership_table.go b/roomserver/storage/sqlite3/membership_table.go index ca4d8fbe9..34108af43 100644 --- a/roomserver/storage/sqlite3/membership_table.go +++ b/roomserver/storage/sqlite3/membership_table.go @@ -19,7 +19,7 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -114,7 +114,7 @@ func (s *membershipStatements) insertMembership( roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, localTarget bool, ) error { - stmt := common.TxStmt(txn, s.insertMembershipStmt) + stmt := internal.TxStmt(txn, s.insertMembershipStmt) _, err := stmt.ExecContext(ctx, roomNID, targetUserNID, localTarget) return err } @@ -123,7 +123,7 @@ func (s *membershipStatements) selectMembershipForUpdate( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, ) (membership membershipState, err error) { - stmt := common.TxStmt(txn, s.selectMembershipForUpdateStmt) + stmt := internal.TxStmt(txn, s.selectMembershipForUpdateStmt) err = stmt.QueryRowContext( ctx, roomNID, targetUserNID, ).Scan(&membership) @@ -134,7 +134,7 @@ func (s *membershipStatements) selectMembershipFromRoomAndTarget( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, ) (eventNID types.EventNID, membership membershipState, err error) { - selectStmt := common.TxStmt(txn, s.selectMembershipFromRoomAndTargetStmt) + selectStmt := internal.TxStmt(txn, s.selectMembershipFromRoomAndTargetStmt) err = selectStmt.QueryRowContext( ctx, roomNID, targetUserNID, ).Scan(&membership, &eventNID) @@ -147,15 +147,15 @@ func (s *membershipStatements) selectMembershipsFromRoom( ) (eventNIDs []types.EventNID, err error) { var selectStmt *sql.Stmt if localOnly { - selectStmt = common.TxStmt(txn, s.selectLocalMembershipsFromRoomStmt) + selectStmt = internal.TxStmt(txn, s.selectLocalMembershipsFromRoomStmt) } else { - selectStmt = common.TxStmt(txn, s.selectMembershipsFromRoomStmt) + selectStmt = internal.TxStmt(txn, s.selectMembershipsFromRoomStmt) } rows, err := selectStmt.QueryContext(ctx, roomNID) if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "selectMembershipsFromRoom: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "selectMembershipsFromRoom: rows.close() failed") for rows.Next() { var eNID types.EventNID @@ -173,15 +173,15 @@ func (s *membershipStatements) selectMembershipsFromRoomAndMembership( ) (eventNIDs []types.EventNID, err error) { var stmt *sql.Stmt if localOnly { - stmt = common.TxStmt(txn, s.selectLocalMembershipsFromRoomAndMembershipStmt) + stmt = internal.TxStmt(txn, s.selectLocalMembershipsFromRoomAndMembershipStmt) } else { - stmt = common.TxStmt(txn, s.selectMembershipsFromRoomAndMembershipStmt) + stmt = internal.TxStmt(txn, s.selectMembershipsFromRoomAndMembershipStmt) } rows, err := stmt.QueryContext(ctx, roomNID, membership) if err != nil { return } - defer common.CloseAndLogIfError(ctx, rows, "selectMembershipsFromRoomAndMembership: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "selectMembershipsFromRoomAndMembership: rows.close() failed") for rows.Next() { var eNID types.EventNID @@ -199,7 +199,7 @@ func (s *membershipStatements) updateMembership( senderUserNID types.EventStateKeyNID, membership membershipState, eventNID types.EventNID, ) error { - stmt := common.TxStmt(txn, s.updateMembershipStmt) + stmt := internal.TxStmt(txn, s.updateMembershipStmt) _, err := stmt.ExecContext( ctx, senderUserNID, membership, eventNID, roomNID, targetUserNID, ) diff --git a/roomserver/storage/sqlite3/previous_events_table.go b/roomserver/storage/sqlite3/previous_events_table.go index 9ed64a38e..f344bda95 100644 --- a/roomserver/storage/sqlite3/previous_events_table.go +++ b/roomserver/storage/sqlite3/previous_events_table.go @@ -19,7 +19,7 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -74,7 +74,7 @@ func (s *previousEventStatements) insertPreviousEvent( previousEventReferenceSHA256 []byte, eventNID types.EventNID, ) error { - stmt := common.TxStmt(txn, s.insertPreviousEventStmt) + stmt := internal.TxStmt(txn, s.insertPreviousEventStmt) _, err := stmt.ExecContext( ctx, previousEventID, previousEventReferenceSHA256, int64(eventNID), ) @@ -87,6 +87,6 @@ func (s *previousEventStatements) selectPreviousEventExists( ctx context.Context, txn *sql.Tx, eventID string, eventReferenceSHA256 []byte, ) error { var ok int64 - stmt := common.TxStmt(txn, s.selectPreviousEventExistsStmt) + stmt := internal.TxStmt(txn, s.selectPreviousEventExistsStmt) return stmt.QueryRowContext(ctx, eventID, eventReferenceSHA256).Scan(&ok) } diff --git a/roomserver/storage/sqlite3/room_aliases_table.go b/roomserver/storage/sqlite3/room_aliases_table.go index d29833918..592ef9782 100644 --- a/roomserver/storage/sqlite3/room_aliases_table.go +++ b/roomserver/storage/sqlite3/room_aliases_table.go @@ -19,7 +19,7 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" ) const roomAliasesSchema = ` @@ -77,7 +77,7 @@ func (s *roomAliasesStatements) prepare(db *sql.DB) (err error) { func (s *roomAliasesStatements) insertRoomAlias( ctx context.Context, txn *sql.Tx, alias string, roomID string, creatorUserID string, ) (err error) { - insertStmt := common.TxStmt(txn, s.insertRoomAliasStmt) + insertStmt := internal.TxStmt(txn, s.insertRoomAliasStmt) _, err = insertStmt.ExecContext(ctx, alias, roomID, creatorUserID) return } @@ -85,7 +85,7 @@ func (s *roomAliasesStatements) insertRoomAlias( func (s *roomAliasesStatements) selectRoomIDFromAlias( ctx context.Context, txn *sql.Tx, alias string, ) (roomID string, err error) { - selectStmt := common.TxStmt(txn, s.selectRoomIDFromAliasStmt) + selectStmt := internal.TxStmt(txn, s.selectRoomIDFromAliasStmt) err = selectStmt.QueryRowContext(ctx, alias).Scan(&roomID) if err == sql.ErrNoRows { return "", nil @@ -97,13 +97,13 @@ func (s *roomAliasesStatements) selectAliasesFromRoomID( ctx context.Context, txn *sql.Tx, roomID string, ) (aliases []string, err error) { aliases = []string{} - selectStmt := common.TxStmt(txn, s.selectAliasesFromRoomIDStmt) + selectStmt := internal.TxStmt(txn, s.selectAliasesFromRoomIDStmt) rows, err := selectStmt.QueryContext(ctx, roomID) if err != nil { return } - defer common.CloseAndLogIfError(ctx, rows, "selectAliasesFromRoomID: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "selectAliasesFromRoomID: rows.close() failed") for rows.Next() { var alias string @@ -120,7 +120,7 @@ func (s *roomAliasesStatements) selectAliasesFromRoomID( func (s *roomAliasesStatements) selectCreatorIDFromAlias( ctx context.Context, txn *sql.Tx, alias string, ) (creatorID string, err error) { - selectStmt := common.TxStmt(txn, s.selectCreatorIDFromAliasStmt) + selectStmt := internal.TxStmt(txn, s.selectCreatorIDFromAliasStmt) err = selectStmt.QueryRowContext(ctx, alias).Scan(&creatorID) if err == sql.ErrNoRows { return "", nil @@ -131,7 +131,7 @@ func (s *roomAliasesStatements) selectCreatorIDFromAlias( func (s *roomAliasesStatements) deleteRoomAlias( ctx context.Context, txn *sql.Tx, alias string, ) (err error) { - deleteStmt := common.TxStmt(txn, s.deleteRoomAliasStmt) + deleteStmt := internal.TxStmt(txn, s.deleteRoomAliasStmt) _, err = deleteStmt.ExecContext(ctx, alias) return } diff --git a/roomserver/storage/sqlite3/rooms_table.go b/roomserver/storage/sqlite3/rooms_table.go index 427eeeb70..ea949d1e9 100644 --- a/roomserver/storage/sqlite3/rooms_table.go +++ b/roomserver/storage/sqlite3/rooms_table.go @@ -21,7 +21,7 @@ import ( "encoding/json" "errors" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -92,7 +92,7 @@ func (s *roomStatements) insertRoomNID( roomID string, roomVersion gomatrixserverlib.RoomVersion, ) (types.RoomNID, error) { var err error - insertStmt := common.TxStmt(txn, s.insertRoomNIDStmt) + insertStmt := internal.TxStmt(txn, s.insertRoomNIDStmt) if _, err = insertStmt.ExecContext(ctx, roomID, roomVersion); err == nil { return s.selectRoomNID(ctx, txn, roomID) } else { @@ -104,7 +104,7 @@ func (s *roomStatements) selectRoomNID( ctx context.Context, txn *sql.Tx, roomID string, ) (types.RoomNID, error) { var roomNID int64 - stmt := common.TxStmt(txn, s.selectRoomNIDStmt) + stmt := internal.TxStmt(txn, s.selectRoomNIDStmt) err := stmt.QueryRowContext(ctx, roomID).Scan(&roomNID) return types.RoomNID(roomNID), err } @@ -115,7 +115,7 @@ func (s *roomStatements) selectLatestEventNIDs( var eventNIDs []types.EventNID var nidsJSON string var stateSnapshotNID int64 - stmt := common.TxStmt(txn, s.selectLatestEventNIDsStmt) + stmt := internal.TxStmt(txn, s.selectLatestEventNIDsStmt) err := stmt.QueryRowContext(ctx, int64(roomNID)).Scan(&nidsJSON, &stateSnapshotNID) if err != nil { return nil, 0, err @@ -133,7 +133,7 @@ func (s *roomStatements) selectLatestEventsNIDsForUpdate( var nidsJSON string var lastEventSentNID int64 var stateSnapshotNID int64 - stmt := common.TxStmt(txn, s.selectLatestEventNIDsForUpdateStmt) + stmt := internal.TxStmt(txn, s.selectLatestEventNIDsForUpdateStmt) err := stmt.QueryRowContext(ctx, int64(roomNID)).Scan(&nidsJSON, &lastEventSentNID, &stateSnapshotNID) if err != nil { return nil, 0, 0, err @@ -152,7 +152,7 @@ func (s *roomStatements) updateLatestEventNIDs( lastEventSentNID types.EventNID, stateSnapshotNID types.StateSnapshotNID, ) error { - stmt := common.TxStmt(txn, s.updateLatestEventNIDsStmt) + stmt := internal.TxStmt(txn, s.updateLatestEventNIDsStmt) _, err := stmt.ExecContext( ctx, eventNIDsAsArray(eventNIDs), @@ -167,7 +167,7 @@ func (s *roomStatements) selectRoomVersionForRoomID( ctx context.Context, txn *sql.Tx, roomID string, ) (gomatrixserverlib.RoomVersion, error) { var roomVersion gomatrixserverlib.RoomVersion - stmt := common.TxStmt(txn, s.selectRoomVersionForRoomIDStmt) + stmt := internal.TxStmt(txn, s.selectRoomVersionForRoomIDStmt) err := stmt.QueryRowContext(ctx, roomID).Scan(&roomVersion) if err == sql.ErrNoRows { return roomVersion, errors.New("room not found") @@ -179,7 +179,7 @@ func (s *roomStatements) selectRoomVersionForRoomNID( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, ) (gomatrixserverlib.RoomVersion, error) { var roomVersion gomatrixserverlib.RoomVersion - stmt := common.TxStmt(txn, s.selectRoomVersionForRoomNIDStmt) + stmt := internal.TxStmt(txn, s.selectRoomVersionForRoomNIDStmt) err := stmt.QueryRowContext(ctx, roomNID).Scan(&roomVersion) if err == sql.ErrNoRows { return roomVersion, errors.New("room not found") diff --git a/roomserver/storage/sqlite3/state_block_table.go b/roomserver/storage/sqlite3/state_block_table.go index cc7c75733..861d76cf9 100644 --- a/roomserver/storage/sqlite3/state_block_table.go +++ b/roomserver/storage/sqlite3/state_block_table.go @@ -22,7 +22,7 @@ import ( "sort" "strings" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/util" ) @@ -127,17 +127,17 @@ func (s *stateBlockStatements) bulkSelectStateBlockEntries( for k, v := range stateBlockNIDs { nids[k] = v } - selectOrig := strings.Replace(bulkSelectStateBlockEntriesSQL, "($1)", common.QueryVariadic(len(nids)), 1) + selectOrig := strings.Replace(bulkSelectStateBlockEntriesSQL, "($1)", internal.QueryVariadic(len(nids)), 1) selectPrep, err := s.db.Prepare(selectOrig) if err != nil { return nil, err } - selectStmt := common.TxStmt(txn, selectPrep) + selectStmt := internal.TxStmt(txn, selectPrep) rows, err := selectStmt.QueryContext(ctx, nids...) if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "bulkSelectStateBlockEntries: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectStateBlockEntries: rows.close() failed") results := make([]types.StateEntryList, len(stateBlockNIDs)) // current is a pointer to the StateEntryList to append the state entries to. @@ -184,9 +184,9 @@ func (s *stateBlockStatements) bulkSelectFilteredStateBlockEntries( sort.Sort(tuples) eventTypeNIDArray, eventStateKeyNIDArray := tuples.typesAndStateKeysAsArrays() - sqlStatement := strings.Replace(bulkSelectFilteredStateBlockEntriesSQL, "($1)", common.QueryVariadic(len(stateBlockNIDs)), 1) - sqlStatement = strings.Replace(sqlStatement, "($2)", common.QueryVariadicOffset(len(eventTypeNIDArray), len(stateBlockNIDs)), 1) - sqlStatement = strings.Replace(sqlStatement, "($3)", common.QueryVariadicOffset(len(eventStateKeyNIDArray), len(stateBlockNIDs)+len(eventTypeNIDArray)), 1) + sqlStatement := strings.Replace(bulkSelectFilteredStateBlockEntriesSQL, "($1)", internal.QueryVariadic(len(stateBlockNIDs)), 1) + sqlStatement = strings.Replace(sqlStatement, "($2)", internal.QueryVariadicOffset(len(eventTypeNIDArray), len(stateBlockNIDs)), 1) + sqlStatement = strings.Replace(sqlStatement, "($3)", internal.QueryVariadicOffset(len(eventStateKeyNIDArray), len(stateBlockNIDs)+len(eventTypeNIDArray)), 1) var params []interface{} for _, val := range stateBlockNIDs { @@ -207,7 +207,7 @@ func (s *stateBlockStatements) bulkSelectFilteredStateBlockEntries( if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "bulkSelectFilteredStateBlockEntries: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectFilteredStateBlockEntries: rows.close() failed") var results []types.StateEntryList var current types.StateEntryList diff --git a/roomserver/storage/sqlite3/state_snapshot_table.go b/roomserver/storage/sqlite3/state_snapshot_table.go index f367a779b..8682627a0 100644 --- a/roomserver/storage/sqlite3/state_snapshot_table.go +++ b/roomserver/storage/sqlite3/state_snapshot_table.go @@ -22,7 +22,7 @@ import ( "fmt" "strings" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -89,7 +89,7 @@ func (s *stateSnapshotStatements) bulkSelectStateBlockNIDs( for k, v := range stateNIDs { nids[k] = v } - selectOrig := strings.Replace(bulkSelectStateBlockNIDsSQL, "($1)", common.QueryVariadic(len(nids)), 1) + selectOrig := strings.Replace(bulkSelectStateBlockNIDsSQL, "($1)", internal.QueryVariadic(len(nids)), 1) selectStmt, err := txn.Prepare(selectOrig) if err != nil { return nil, err @@ -99,7 +99,7 @@ func (s *stateSnapshotStatements) bulkSelectStateBlockNIDs( if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "bulkSelectStateBlockNIDs: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "bulkSelectStateBlockNIDs: rows.close() failed") results := make([]types.StateBlockNIDList, len(stateNIDs)) i := 0 for ; rows.Next(); i++ { diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index 209922fa2..bb38f800f 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -24,7 +24,7 @@ import ( "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" @@ -52,7 +52,7 @@ func Open(dataSourceName string) (*Database, error) { } else { return nil, errors.New("no filename or path in connect string") } - if d.db, err = sqlutil.Open(common.SQLiteDriverName(), cs, nil); err != nil { + if d.db, err = sqlutil.Open(internal.SQLiteDriverName(), cs, nil); err != nil { return nil, err } //d.db.Exec("PRAGMA journal_mode=WAL;") @@ -83,7 +83,7 @@ func (d *Database) StoreEvent( err error ) - err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { if txnAndSessionID != nil { if err = d.statements.insertTransaction( ctx, txn, txnAndSessionID.TransactionID, @@ -242,7 +242,7 @@ func (d *Database) assignStateKeyNID( func (d *Database) StateEntriesForEventIDs( ctx context.Context, eventIDs []string, ) (se []types.StateEntry, err error) { - err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { se, err = d.statements.bulkSelectStateEventByID(ctx, txn, eventIDs) return err }) @@ -253,7 +253,7 @@ func (d *Database) StateEntriesForEventIDs( func (d *Database) EventTypeNIDs( ctx context.Context, eventTypes []string, ) (etnids map[string]types.EventTypeNID, err error) { - err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { etnids, err = d.statements.bulkSelectEventTypeNID(ctx, txn, eventTypes) return err }) @@ -264,7 +264,7 @@ func (d *Database) EventTypeNIDs( func (d *Database) EventStateKeyNIDs( ctx context.Context, eventStateKeys []string, ) (esknids map[string]types.EventStateKeyNID, err error) { - err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { esknids, err = d.statements.bulkSelectEventStateKeyNID(ctx, txn, eventStateKeys) return err }) @@ -275,7 +275,7 @@ func (d *Database) EventStateKeyNIDs( func (d *Database) EventStateKeys( ctx context.Context, eventStateKeyNIDs []types.EventStateKeyNID, ) (out map[types.EventStateKeyNID]string, err error) { - err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { out, err = d.statements.bulkSelectEventStateKey(ctx, txn, eventStateKeyNIDs) return err }) @@ -286,7 +286,7 @@ func (d *Database) EventStateKeys( func (d *Database) EventNIDs( ctx context.Context, eventIDs []string, ) (out map[string]types.EventNID, err error) { - err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { out, err = d.statements.bulkSelectEventNID(ctx, txn, eventIDs) return err }) @@ -300,7 +300,7 @@ func (d *Database) Events( var eventJSONs []eventJSONPair var err error var results []types.Event - err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { eventJSONs, err = d.statements.bulkSelectEventJSON(ctx, txn, eventNIDs) if err != nil || len(eventJSONs) == 0 { return nil @@ -341,7 +341,7 @@ func (d *Database) AddState( stateBlockNIDs []types.StateBlockNID, state []types.StateEntry, ) (stateNID types.StateSnapshotNID, err error) { - err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { if len(state) > 0 { var stateBlockNID types.StateBlockNID stateBlockNID, err = d.statements.bulkInsertStateData(ctx, txn, state) @@ -363,7 +363,7 @@ func (d *Database) AddState( func (d *Database) SetState( ctx context.Context, eventNID types.EventNID, stateNID types.StateSnapshotNID, ) error { - e := common.WithTransaction(d.db, func(txn *sql.Tx) error { + e := internal.WithTransaction(d.db, func(txn *sql.Tx) error { return d.statements.updateEventState(ctx, txn, eventNID, stateNID) }) return e @@ -373,7 +373,7 @@ func (d *Database) SetState( func (d *Database) StateAtEventIDs( ctx context.Context, eventIDs []string, ) (se []types.StateAtEvent, err error) { - err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { se, err = d.statements.bulkSelectStateAtEventByID(ctx, txn, eventIDs) return err }) @@ -384,7 +384,7 @@ func (d *Database) StateAtEventIDs( func (d *Database) StateBlockNIDs( ctx context.Context, stateNIDs []types.StateSnapshotNID, ) (sl []types.StateBlockNIDList, err error) { - err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { sl, err = d.statements.bulkSelectStateBlockNIDs(ctx, txn, stateNIDs) return err }) @@ -395,7 +395,7 @@ func (d *Database) StateBlockNIDs( func (d *Database) StateEntries( ctx context.Context, stateBlockNIDs []types.StateBlockNID, ) (sel []types.StateEntryList, err error) { - err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { sel, err = d.statements.bulkSelectStateBlockEntries(ctx, txn, stateBlockNIDs) return err }) @@ -406,7 +406,7 @@ func (d *Database) StateEntries( func (d *Database) SnapshotNIDFromEventID( ctx context.Context, eventID string, ) (stateNID types.StateSnapshotNID, err error) { - err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { _, stateNID, err = d.statements.selectEvent(ctx, txn, eventID) return err }) @@ -417,7 +417,7 @@ func (d *Database) SnapshotNIDFromEventID( func (d *Database) EventIDs( ctx context.Context, eventNIDs []types.EventNID, ) (out map[types.EventNID]string, err error) { - err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { out, err = d.statements.bulkSelectEventID(ctx, txn, eventNIDs) return err }) @@ -509,7 +509,7 @@ func (u *roomRecentEventsUpdater) CurrentStateSnapshotNID() types.StateSnapshotN // StorePreviousEvents implements types.RoomRecentEventsUpdater func (u *roomRecentEventsUpdater) StorePreviousEvents(eventNID types.EventNID, previousEventReferences []gomatrixserverlib.EventReference) error { - err := common.WithTransaction(u.d.db, func(txn *sql.Tx) error { + err := internal.WithTransaction(u.d.db, func(txn *sql.Tx) error { for _, ref := range previousEventReferences { if err := u.d.statements.insertPreviousEvent(u.ctx, txn, ref.EventID, ref.EventSHA256, eventNID); err != nil { return err @@ -522,7 +522,7 @@ func (u *roomRecentEventsUpdater) StorePreviousEvents(eventNID types.EventNID, p // IsReferenced implements types.RoomRecentEventsUpdater func (u *roomRecentEventsUpdater) IsReferenced(eventReference gomatrixserverlib.EventReference) (res bool, err error) { - err = common.WithTransaction(u.d.db, func(txn *sql.Tx) error { + err = internal.WithTransaction(u.d.db, func(txn *sql.Tx) error { err := u.d.statements.selectPreviousEventExists(u.ctx, txn, eventReference.EventID, eventReference.EventSHA256) if err == nil { res = true @@ -542,7 +542,7 @@ func (u *roomRecentEventsUpdater) SetLatestEvents( roomNID types.RoomNID, latest []types.StateAtEventAndReference, lastEventNIDSent types.EventNID, currentStateSnapshotNID types.StateSnapshotNID, ) error { - err := common.WithTransaction(u.d.db, func(txn *sql.Tx) error { + err := internal.WithTransaction(u.d.db, func(txn *sql.Tx) error { eventNIDs := make([]types.EventNID, len(latest)) for i := range latest { eventNIDs[i] = latest[i].EventNID @@ -554,7 +554,7 @@ func (u *roomRecentEventsUpdater) SetLatestEvents( // HasEventBeenSent implements types.RoomRecentEventsUpdater func (u *roomRecentEventsUpdater) HasEventBeenSent(eventNID types.EventNID) (res bool, err error) { - err = common.WithTransaction(u.d.db, func(txn *sql.Tx) error { + err = internal.WithTransaction(u.d.db, func(txn *sql.Tx) error { res, err = u.d.statements.selectEventSentToOutput(u.ctx, txn, eventNID) return err }) @@ -563,14 +563,14 @@ func (u *roomRecentEventsUpdater) HasEventBeenSent(eventNID types.EventNID) (res // MarkEventAsSent implements types.RoomRecentEventsUpdater func (u *roomRecentEventsUpdater) MarkEventAsSent(eventNID types.EventNID) error { - err := common.WithTransaction(u.d.db, func(txn *sql.Tx) error { + err := internal.WithTransaction(u.d.db, func(txn *sql.Tx) error { return u.d.statements.updateEventSentToOutput(u.ctx, txn, eventNID) }) return err } func (u *roomRecentEventsUpdater) MembershipUpdater(targetUserNID types.EventStateKeyNID, targetLocal bool) (mu types.MembershipUpdater, err error) { - err = common.WithTransaction(u.d.db, func(txn *sql.Tx) error { + err = internal.WithTransaction(u.d.db, func(txn *sql.Tx) error { mu, err = u.d.membershipUpdaterTxn(u.ctx, txn, u.roomNID, targetUserNID, targetLocal) return err }) @@ -579,7 +579,7 @@ func (u *roomRecentEventsUpdater) MembershipUpdater(targetUserNID types.EventSta // RoomNID implements query.RoomserverQueryAPIDB func (d *Database) RoomNID(ctx context.Context, roomID string) (roomNID types.RoomNID, err error) { - err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { roomNID, err = d.statements.selectRoomNID(ctx, txn, roomID) if err == sql.ErrNoRows { roomNID = 0 @@ -611,7 +611,7 @@ func (d *Database) RoomNIDExcludingStubs(ctx context.Context, roomID string) (ro func (d *Database) LatestEventIDs( ctx context.Context, roomNID types.RoomNID, ) (references []gomatrixserverlib.EventReference, currentStateSnapshotNID types.StateSnapshotNID, depth int64, err error) { - err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { var eventNIDs []types.EventNID eventNIDs, currentStateSnapshotNID, err = d.statements.selectLatestEventNIDs(ctx, txn, roomNID) if err != nil { @@ -773,7 +773,7 @@ func (u *membershipUpdater) IsLeave() bool { // SetToInvite implements types.MembershipUpdater func (u *membershipUpdater) SetToInvite(event gomatrixserverlib.Event) (inserted bool, err error) { - err = common.WithTransaction(u.d.db, func(txn *sql.Tx) error { + err = internal.WithTransaction(u.d.db, func(txn *sql.Tx) error { senderUserNID, err := u.d.assignStateKeyNID(u.ctx, txn, event.Sender()) if err != nil { return err @@ -798,7 +798,7 @@ func (u *membershipUpdater) SetToInvite(event gomatrixserverlib.Event) (inserted // SetToJoin implements types.MembershipUpdater func (u *membershipUpdater) SetToJoin(senderUserID string, eventID string, isUpdate bool) (inviteEventIDs []string, err error) { - err = common.WithTransaction(u.d.db, func(txn *sql.Tx) error { + err = internal.WithTransaction(u.d.db, func(txn *sql.Tx) error { senderUserNID, err := u.d.assignStateKeyNID(u.ctx, txn, senderUserID) if err != nil { return err @@ -836,7 +836,7 @@ func (u *membershipUpdater) SetToJoin(senderUserID string, eventID string, isUpd // SetToLeave implements types.MembershipUpdater func (u *membershipUpdater) SetToLeave(senderUserID string, eventID string) (inviteEventIDs []string, err error) { - err = common.WithTransaction(u.d.db, func(txn *sql.Tx) error { + err = internal.WithTransaction(u.d.db, func(txn *sql.Tx) error { senderUserNID, err := u.d.assignStateKeyNID(u.ctx, txn, senderUserID) if err != nil { return err @@ -871,7 +871,7 @@ func (u *membershipUpdater) SetToLeave(senderUserID string, eventID string) (inv func (d *Database) GetMembership( ctx context.Context, roomNID types.RoomNID, requestSenderUserID string, ) (membershipEventNID types.EventNID, stillInRoom bool, err error) { - err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { requestSenderUserNID, err := d.assignStateKeyNID(ctx, txn, requestSenderUserID) if err != nil { return err @@ -899,7 +899,7 @@ func (d *Database) GetMembership( func (d *Database) GetMembershipEventNIDsForRoom( ctx context.Context, roomNID types.RoomNID, joinOnly bool, localOnly bool, ) (eventNIDs []types.EventNID, err error) { - err = common.WithTransaction(d.db, func(txn *sql.Tx) error { + err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { if joinOnly { eventNIDs, err = d.statements.selectMembershipsFromRoomAndMembership( ctx, txn, roomNID, membershipStateJoin, localOnly, diff --git a/roomserver/storage/sqlite3/transactions_table.go b/roomserver/storage/sqlite3/transactions_table.go index 7740e5f07..d22c73845 100644 --- a/roomserver/storage/sqlite3/transactions_table.go +++ b/roomserver/storage/sqlite3/transactions_table.go @@ -19,7 +19,7 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" ) const transactionsSchema = ` @@ -65,7 +65,7 @@ func (s *transactionStatements) insertTransaction( userID string, eventID string, ) (err error) { - stmt := common.TxStmt(txn, s.insertTransactionStmt) + stmt := internal.TxStmt(txn, s.insertTransactionStmt) _, err = stmt.ExecContext( ctx, transactionID, sessionID, userID, eventID, ) @@ -78,7 +78,7 @@ func (s *transactionStatements) selectTransactionEventID( sessionID int64, userID string, ) (eventID string, err error) { - stmt := common.TxStmt(txn, s.selectTransactionEventIDStmt) + stmt := internal.TxStmt(txn, s.selectTransactionEventIDStmt) err = stmt.QueryRowContext( ctx, transactionID, sessionID, userID, ).Scan(&eventID) diff --git a/roomserver/storage/storage.go b/roomserver/storage/storage.go index 99e99a008..071e2cf1d 100644 --- a/roomserver/storage/storage.go +++ b/roomserver/storage/storage.go @@ -19,13 +19,13 @@ package storage import ( "net/url" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/roomserver/storage/postgres" "github.com/matrix-org/dendrite/roomserver/storage/sqlite3" ) // Open opens a database connection. -func Open(dataSourceName string, dbProperties common.DbProperties) (Database, error) { +func Open(dataSourceName string, dbProperties internal.DbProperties) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return postgres.Open(dataSourceName, dbProperties) diff --git a/roomserver/storage/storage_wasm.go b/roomserver/storage/storage_wasm.go index 5fa48bc95..ef30ca593 100644 --- a/roomserver/storage/storage_wasm.go +++ b/roomserver/storage/storage_wasm.go @@ -18,14 +18,14 @@ import ( "fmt" "net/url" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/roomserver/storage/sqlite3" ) // NewPublicRoomsServerDatabase opens a database connection. func Open( dataSourceName string, - dbProperties common.DbProperties, // nolint:unparam + dbProperties internal.DbProperties, // nolint:unparam ) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { diff --git a/roomserver/types/types.go b/roomserver/types/types.go index 74e6b0784..87a4f935c 100644 --- a/roomserver/types/types.go +++ b/roomserver/types/types.go @@ -16,7 +16,7 @@ package types import ( - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/gomatrixserverlib" ) @@ -174,7 +174,7 @@ type RoomRecentEventsUpdater interface { // It will share the same transaction as this updater. MembershipUpdater(targetUserNID EventStateKeyNID, isTargetLocalUser bool) (MembershipUpdater, error) // Implements Transaction so it can be committed or rolledback - common.Transaction + internal.Transaction } // A MembershipUpdater is used to update the membership of a user in a room. @@ -199,7 +199,7 @@ type MembershipUpdater interface { // Returns a list of invite event IDs that this state change retired. SetToLeave(senderUserID string, eventID string) (inviteEventIDs []string, err error) // Implements Transaction so it can be committed or rolledback. - common.Transaction + internal.Transaction } // A MissingEventError is an error that happened because the roomserver was diff --git a/syncapi/api/query.go b/syncapi/api/query.go index 2993829e0..6585bc093 100644 --- a/syncapi/api/query.go +++ b/syncapi/api/query.go @@ -18,7 +18,7 @@ import ( "context" "net/http" - commonHTTP "github.com/matrix-org/dendrite/common/http" + internalHTTP "github.com/matrix-org/dendrite/internal/http" "github.com/matrix-org/util" opentracing "github.com/opentracing/opentracing-go" ) @@ -80,7 +80,7 @@ func (h *httpSyncQueryAPI) QuerySync( defer span.Finish() apiURL := h.syncapiURL + SyncAPIQuerySyncPath - return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // QueryStateAfterEvents implements SyncQueryAPI @@ -93,7 +93,7 @@ func (h *httpSyncQueryAPI) QueryState( defer span.Finish() apiURL := h.syncapiURL + SyncAPIQueryStatePath - return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // QueryEventsByID implements SyncQueryAPI @@ -106,7 +106,7 @@ func (h *httpSyncQueryAPI) QueryStateType( defer span.Finish() apiURL := h.syncapiURL + SyncAPIQueryStateTypePath - return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // QueryMembershipForUser implements SyncQueryAPI @@ -119,5 +119,5 @@ func (h *httpSyncQueryAPI) QueryMessages( defer span.Finish() apiURL := h.syncapiURL + SyncAPIQueryMessagesPath - return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } diff --git a/syncapi/consumers/clientapi.go b/syncapi/consumers/clientapi.go index b65d01a04..4d43e8111 100644 --- a/syncapi/consumers/clientapi.go +++ b/syncapi/consumers/clientapi.go @@ -19,8 +19,8 @@ import ( "encoding/json" "github.com/Shopify/sarama" - "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/sync" "github.com/matrix-org/dendrite/syncapi/types" @@ -29,7 +29,7 @@ import ( // OutputClientDataConsumer consumes events that originated in the client API server. type OutputClientDataConsumer struct { - clientAPIConsumer *common.ContinualConsumer + clientAPIConsumer *internal.ContinualConsumer db storage.Database notifier *sync.Notifier } @@ -42,7 +42,7 @@ func NewOutputClientDataConsumer( store storage.Database, ) *OutputClientDataConsumer { - consumer := common.ContinualConsumer{ + consumer := internal.ContinualConsumer{ Topic: string(cfg.Kafka.Topics.OutputClientData), Consumer: kafkaConsumer, PartitionStore: store, @@ -67,7 +67,7 @@ func (s *OutputClientDataConsumer) Start() error { // sync stream position may race and be incorrectly calculated. func (s *OutputClientDataConsumer) onMessage(msg *sarama.ConsumerMessage) error { // Parse out the event JSON - var output common.AccountData + var output internal.AccountData if err := json.Unmarshal(msg.Value, &output); err != nil { // If the message was invalid, log it and move on to the next message in the stream log.WithError(err).Errorf("client API server output log: message parse failure") diff --git a/syncapi/consumers/eduserver.go b/syncapi/consumers/eduserver.go index ece999d59..12b1efbc0 100644 --- a/syncapi/consumers/eduserver.go +++ b/syncapi/consumers/eduserver.go @@ -18,9 +18,9 @@ import ( "encoding/json" "github.com/Shopify/sarama" - "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/common/config" "github.com/matrix-org/dendrite/eduserver/api" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/sync" "github.com/matrix-org/dendrite/syncapi/types" @@ -29,7 +29,7 @@ import ( // OutputTypingEventConsumer consumes events that originated in the EDU server. type OutputTypingEventConsumer struct { - typingConsumer *common.ContinualConsumer + typingConsumer *internal.ContinualConsumer db storage.Database notifier *sync.Notifier } @@ -43,7 +43,7 @@ func NewOutputTypingEventConsumer( store storage.Database, ) *OutputTypingEventConsumer { - consumer := common.ContinualConsumer{ + consumer := internal.ContinualConsumer{ Topic: string(cfg.Kafka.Topics.OutputTypingEvent), Consumer: kafkaConsumer, PartitionStore: store, diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index 368420a6c..055f76603 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -20,8 +20,8 @@ import ( "fmt" "github.com/Shopify/sarama" - "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/sync" @@ -33,7 +33,7 @@ import ( // OutputRoomEventConsumer consumes events that originated in the room server. type OutputRoomEventConsumer struct { rsAPI api.RoomserverInternalAPI - rsConsumer *common.ContinualConsumer + rsConsumer *internal.ContinualConsumer db storage.Database notifier *sync.Notifier } @@ -47,7 +47,7 @@ func NewOutputRoomEventConsumer( rsAPI api.RoomserverInternalAPI, ) *OutputRoomEventConsumer { - consumer := common.ContinualConsumer{ + consumer := internal.ContinualConsumer{ Topic: string(cfg.Kafka.Topics.OutputRoomEvent), Consumer: kafkaConsumer, PartitionStore: store, diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index 88e16fe57..8c8976345 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -22,7 +22,7 @@ import ( "strconv" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/types" diff --git a/syncapi/routing/routing.go b/syncapi/routing/routing.go index 5a36a279f..6300b17bb 100644 --- a/syncapi/routing/routing.go +++ b/syncapi/routing/routing.go @@ -21,8 +21,8 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" - "github.com/matrix-org/dendrite/common" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/sync" @@ -52,12 +52,12 @@ func Setup( } // TODO: Add AS support for all handlers below. - r0mux.Handle("/sync", common.MakeAuthAPI("sync", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + r0mux.Handle("/sync", internal.MakeAuthAPI("sync", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return srp.OnIncomingSyncRequest(req, device) })).Methods(http.MethodGet, http.MethodOptions) - r0mux.Handle("/rooms/{roomID}/messages", common.MakeAuthAPI("room_messages", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := common.URLDecodeMapValues(mux.Vars(req)) + r0mux.Handle("/rooms/{roomID}/messages", internal.MakeAuthAPI("room_messages", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := internal.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index 121e94c71..3e9ff0533 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -19,15 +19,15 @@ import ( "time" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/eduserver/cache" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" ) type Database interface { - common.PartitionStorer + internal.PartitionStorer // AllJoinedUsersInRooms returns a map of room ID to a list of all joined user IDs. AllJoinedUsersInRooms(ctx context.Context) (map[string][]string, error) // Events lookups a list of event by their event ID. diff --git a/syncapi/storage/postgres/account_data_table.go b/syncapi/storage/postgres/account_data_table.go index a5e0c1214..e0b87bd6d 100644 --- a/syncapi/storage/postgres/account_data_table.go +++ b/syncapi/storage/postgres/account_data_table.go @@ -20,7 +20,7 @@ import ( "database/sql" "github.com/lib/pq" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -113,7 +113,7 @@ func (s *accountDataStatements) SelectAccountDataInRange( if err != nil { return } - defer common.CloseAndLogIfError(ctx, rows, "selectAccountDataInRange: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "selectAccountDataInRange: rows.close() failed") for rows.Next() { var dataType string @@ -136,7 +136,7 @@ func (s *accountDataStatements) SelectMaxAccountDataID( ctx context.Context, txn *sql.Tx, ) (id int64, err error) { var nullableID sql.NullInt64 - stmt := common.TxStmt(txn, s.selectMaxAccountDataIDStmt) + stmt := internal.TxStmt(txn, s.selectMaxAccountDataIDStmt) err = stmt.QueryRowContext(ctx).Scan(&nullableID) if nullableID.Valid { id = nullableID.Int64 diff --git a/syncapi/storage/postgres/backwards_extremities_table.go b/syncapi/storage/postgres/backwards_extremities_table.go index fa498a401..71569a108 100644 --- a/syncapi/storage/postgres/backwards_extremities_table.go +++ b/syncapi/storage/postgres/backwards_extremities_table.go @@ -18,7 +18,7 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/syncapi/storage/tables" ) @@ -84,7 +84,7 @@ func (s *backwardExtremitiesStatements) SelectBackwardExtremitiesForRoom( if err != nil { return } - defer common.CloseAndLogIfError(ctx, rows, "selectBackwardExtremitiesForRoom: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "selectBackwardExtremitiesForRoom: rows.close() failed") bwExtrems = make(map[string][]string) for rows.Next() { diff --git a/syncapi/storage/postgres/current_room_state_table.go b/syncapi/storage/postgres/current_room_state_table.go index b6ab72215..cc0fb6b1e 100644 --- a/syncapi/storage/postgres/current_room_state_table.go +++ b/syncapi/storage/postgres/current_room_state_table.go @@ -21,7 +21,7 @@ import ( "encoding/json" "github.com/lib/pq" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -142,7 +142,7 @@ func (s *currentRoomStateStatements) SelectJoinedUsers( if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "selectJoinedUsers: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "selectJoinedUsers: rows.close() failed") result := make(map[string][]string) for rows.Next() { @@ -165,12 +165,12 @@ func (s *currentRoomStateStatements) SelectRoomIDsWithMembership( userID string, membership string, // nolint: unparam ) ([]string, error) { - stmt := common.TxStmt(txn, s.selectRoomIDsWithMembershipStmt) + stmt := internal.TxStmt(txn, s.selectRoomIDsWithMembershipStmt) rows, err := stmt.QueryContext(ctx, userID, membership) if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "selectRoomIDsWithMembership: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "selectRoomIDsWithMembership: rows.close() failed") var result []string for rows.Next() { @@ -188,7 +188,7 @@ func (s *currentRoomStateStatements) SelectCurrentState( ctx context.Context, txn *sql.Tx, roomID string, stateFilter *gomatrixserverlib.StateFilter, ) ([]gomatrixserverlib.HeaderedEvent, error) { - stmt := common.TxStmt(txn, s.selectCurrentStateStmt) + stmt := internal.TxStmt(txn, s.selectCurrentStateStmt) rows, err := stmt.QueryContext(ctx, roomID, pq.StringArray(stateFilter.Senders), pq.StringArray(stateFilter.NotSenders), @@ -200,7 +200,7 @@ func (s *currentRoomStateStatements) SelectCurrentState( if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "selectCurrentState: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "selectCurrentState: rows.close() failed") return rowsToEvents(rows) } @@ -208,7 +208,7 @@ func (s *currentRoomStateStatements) SelectCurrentState( func (s *currentRoomStateStatements) DeleteRoomStateByEventID( ctx context.Context, txn *sql.Tx, eventID string, ) error { - stmt := common.TxStmt(txn, s.deleteRoomStateByEventIDStmt) + stmt := internal.TxStmt(txn, s.deleteRoomStateByEventIDStmt) _, err := stmt.ExecContext(ctx, eventID) return err } @@ -231,7 +231,7 @@ func (s *currentRoomStateStatements) UpsertRoomState( } // upsert state event - stmt := common.TxStmt(txn, s.upsertRoomStateStmt) + stmt := internal.TxStmt(txn, s.upsertRoomStateStmt) _, err = stmt.ExecContext( ctx, event.RoomID(), @@ -250,12 +250,12 @@ func (s *currentRoomStateStatements) UpsertRoomState( func (s *currentRoomStateStatements) SelectEventsWithEventIDs( ctx context.Context, txn *sql.Tx, eventIDs []string, ) ([]types.StreamEvent, error) { - stmt := common.TxStmt(txn, s.selectEventsWithEventIDsStmt) + stmt := internal.TxStmt(txn, s.selectEventsWithEventIDsStmt) rows, err := stmt.QueryContext(ctx, pq.StringArray(eventIDs)) if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "selectEventsWithEventIDs: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "selectEventsWithEventIDs: rows.close() failed") return rowsToStreamEvents(rows) } diff --git a/syncapi/storage/postgres/invites_table.go b/syncapi/storage/postgres/invites_table.go index 01f2e7f44..caa12d2f8 100644 --- a/syncapi/storage/postgres/invites_table.go +++ b/syncapi/storage/postgres/invites_table.go @@ -20,7 +20,7 @@ import ( "database/sql" "encoding/json" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -119,12 +119,12 @@ func (s *inviteEventsStatements) DeleteInviteEvent( func (s *inviteEventsStatements) SelectInviteEventsInRange( ctx context.Context, txn *sql.Tx, targetUserID string, r types.Range, ) (map[string]gomatrixserverlib.HeaderedEvent, error) { - stmt := common.TxStmt(txn, s.selectInviteEventsInRangeStmt) + stmt := internal.TxStmt(txn, s.selectInviteEventsInRangeStmt) rows, err := stmt.QueryContext(ctx, targetUserID, r.Low(), r.High()) if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "selectInviteEventsInRange: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "selectInviteEventsInRange: rows.close() failed") result := map[string]gomatrixserverlib.HeaderedEvent{} for rows.Next() { var ( @@ -149,7 +149,7 @@ func (s *inviteEventsStatements) SelectMaxInviteID( ctx context.Context, txn *sql.Tx, ) (id int64, err error) { var nullableID sql.NullInt64 - stmt := common.TxStmt(txn, s.selectMaxInviteIDStmt) + stmt := internal.TxStmt(txn, s.selectMaxInviteIDStmt) err = stmt.QueryRowContext(ctx).Scan(&nullableID) if nullableID.Valid { id = nullableID.Int64 diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go index 5020d1e7d..ae6dfb381 100644 --- a/syncapi/storage/postgres/output_room_events_table.go +++ b/syncapi/storage/postgres/output_room_events_table.go @@ -26,7 +26,7 @@ import ( "github.com/matrix-org/dendrite/syncapi/types" "github.com/lib/pq" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/gomatrixserverlib" log "github.com/sirupsen/logrus" ) @@ -158,7 +158,7 @@ func (s *outputRoomEventsStatements) SelectStateInRange( ctx context.Context, txn *sql.Tx, r types.Range, stateFilter *gomatrixserverlib.StateFilter, ) (map[string]map[string]bool, map[string]types.StreamEvent, error) { - stmt := common.TxStmt(txn, s.selectStateInRangeStmt) + stmt := internal.TxStmt(txn, s.selectStateInRangeStmt) rows, err := stmt.QueryContext( ctx, r.Low(), r.High(), @@ -172,7 +172,7 @@ func (s *outputRoomEventsStatements) SelectStateInRange( if err != nil { return nil, nil, err } - defer common.CloseAndLogIfError(ctx, rows, "selectStateInRange: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "selectStateInRange: rows.close() failed") // Fetch all the state change events for all rooms between the two positions then loop each event and: // - Keep a cache of the event by ID (99% of state change events are for the event itself) // - For each room ID, build up an array of event IDs which represents cumulative adds/removes @@ -239,7 +239,7 @@ func (s *outputRoomEventsStatements) SelectMaxEventID( ctx context.Context, txn *sql.Tx, ) (id int64, err error) { var nullableID sql.NullInt64 - stmt := common.TxStmt(txn, s.selectMaxEventIDStmt) + stmt := internal.TxStmt(txn, s.selectMaxEventIDStmt) err = stmt.QueryRowContext(ctx).Scan(&nullableID) if nullableID.Valid { id = nullableID.Int64 @@ -275,7 +275,7 @@ func (s *outputRoomEventsStatements) InsertEvent( return } - stmt := common.TxStmt(txn, s.insertEventStmt) + stmt := internal.TxStmt(txn, s.insertEventStmt) err = stmt.QueryRowContext( ctx, event.RoomID(), @@ -303,15 +303,15 @@ func (s *outputRoomEventsStatements) SelectRecentEvents( ) ([]types.StreamEvent, error) { var stmt *sql.Stmt if onlySyncEvents { - stmt = common.TxStmt(txn, s.selectRecentEventsForSyncStmt) + stmt = internal.TxStmt(txn, s.selectRecentEventsForSyncStmt) } else { - stmt = common.TxStmt(txn, s.selectRecentEventsStmt) + stmt = internal.TxStmt(txn, s.selectRecentEventsStmt) } rows, err := stmt.QueryContext(ctx, roomID, r.Low(), r.High(), limit) if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "selectRecentEvents: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "selectRecentEvents: rows.close() failed") events, err := rowsToStreamEvents(rows) if err != nil { return nil, err @@ -333,12 +333,12 @@ func (s *outputRoomEventsStatements) SelectEarlyEvents( ctx context.Context, txn *sql.Tx, roomID string, r types.Range, limit int, ) ([]types.StreamEvent, error) { - stmt := common.TxStmt(txn, s.selectEarlyEventsStmt) + stmt := internal.TxStmt(txn, s.selectEarlyEventsStmt) rows, err := stmt.QueryContext(ctx, roomID, r.Low(), r.High(), limit) if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "selectEarlyEvents: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "selectEarlyEvents: rows.close() failed") events, err := rowsToStreamEvents(rows) if err != nil { return nil, err @@ -357,12 +357,12 @@ func (s *outputRoomEventsStatements) SelectEarlyEvents( func (s *outputRoomEventsStatements) SelectEvents( ctx context.Context, txn *sql.Tx, eventIDs []string, ) ([]types.StreamEvent, error) { - stmt := common.TxStmt(txn, s.selectEventsStmt) + stmt := internal.TxStmt(txn, s.selectEventsStmt) rows, err := stmt.QueryContext(ctx, pq.StringArray(eventIDs)) if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "selectEvents: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "selectEvents: rows.close() failed") return rowsToStreamEvents(rows) } diff --git a/syncapi/storage/postgres/output_room_events_topology_table.go b/syncapi/storage/postgres/output_room_events_topology_table.go index 9fc5f5194..ffbeece30 100644 --- a/syncapi/storage/postgres/output_room_events_topology_table.go +++ b/syncapi/storage/postgres/output_room_events_topology_table.go @@ -18,7 +18,7 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" @@ -139,7 +139,7 @@ func (s *outputRoomEventsTopologyStatements) SelectEventIDsInRange( } else if err != nil { return } - defer common.CloseAndLogIfError(ctx, rows, "selectEventIDsInRange: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "selectEventIDsInRange: rows.close() failed") // Return the IDs. var eventID string diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go index d44f8b7e6..dc73350a2 100644 --- a/syncapi/storage/postgres/syncserver.go +++ b/syncapi/storage/postgres/syncserver.go @@ -22,8 +22,8 @@ import ( // Import the postgres database driver. _ "github.com/lib/pq" - "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/eduserver/cache" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/syncapi/storage/shared" ) @@ -32,11 +32,11 @@ import ( type SyncServerDatasource struct { shared.Database db *sql.DB - common.PartitionOffsetStatements + internal.PartitionOffsetStatements } // NewDatabase creates a new sync server database -func NewDatabase(dbDataSourceName string, dbProperties common.DbProperties) (*SyncServerDatasource, error) { +func NewDatabase(dbDataSourceName string, dbProperties internal.DbProperties) (*SyncServerDatasource, error) { var d SyncServerDatasource var err error if d.db, err = sqlutil.Open("postgres", dbDataSourceName, dbProperties); err != nil { diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go index 543e5b4a3..145a3dbf3 100644 --- a/syncapi/storage/shared/syncserver.go +++ b/syncapi/storage/shared/syncserver.go @@ -8,8 +8,8 @@ import ( "time" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/eduserver/cache" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" @@ -106,7 +106,7 @@ func (d *Database) GetStateEvent( func (d *Database) GetStateEventsForRoom( ctx context.Context, roomID string, stateFilter *gomatrixserverlib.StateFilter, ) (stateEvents []gomatrixserverlib.HeaderedEvent, err error) { - err = common.WithTransaction(d.DB, func(txn *sql.Tx) error { + err = internal.WithTransaction(d.DB, func(txn *sql.Tx) error { stateEvents, err = d.CurrentRoomState.SelectCurrentState(ctx, txn, roomID, stateFilter) return err }) @@ -116,7 +116,7 @@ func (d *Database) GetStateEventsForRoom( func (d *Database) SyncStreamPosition(ctx context.Context) (types.StreamPosition, error) { var maxID int64 var err error - err = common.WithTransaction(d.DB, func(txn *sql.Tx) error { + err = internal.WithTransaction(d.DB, func(txn *sql.Tx) error { maxID, err = d.OutputEvents.SelectMaxEventID(ctx, txn) if err != nil { return err @@ -148,7 +148,7 @@ func (d *Database) SyncStreamPosition(ctx context.Context) (types.StreamPosition func (d *Database) AddInviteEvent( ctx context.Context, inviteEvent gomatrixserverlib.HeaderedEvent, ) (sp types.StreamPosition, err error) { - err = common.WithTransaction(d.DB, func(txn *sql.Tx) error { + err = internal.WithTransaction(d.DB, func(txn *sql.Tx) error { sp, err = d.Invites.InsertInviteEvent(ctx, txn, inviteEvent) return err }) @@ -187,7 +187,7 @@ func (d *Database) GetAccountDataInRange( func (d *Database) UpsertAccountData( ctx context.Context, userID, roomID, dataType string, ) (sp types.StreamPosition, err error) { - err = common.WithTransaction(d.DB, func(txn *sql.Tx) error { + err = internal.WithTransaction(d.DB, func(txn *sql.Tx) error { sp, err = d.AccountData.InsertAccountData(ctx, txn, userID, roomID, dataType) return err }) @@ -255,7 +255,7 @@ func (d *Database) WriteEvent( addStateEventIDs, removeStateEventIDs []string, transactionID *api.TransactionID, excludeFromSync bool, ) (pduPosition types.StreamPosition, returnErr error) { - returnErr = common.WithTransaction(d.DB, func(txn *sql.Tx) error { + returnErr = internal.WithTransaction(d.DB, func(txn *sql.Tx) error { var err error pos, err := d.OutputEvents.InsertEvent( ctx, txn, ev, addStateEventIDs, removeStateEventIDs, transactionID, excludeFromSync, @@ -355,7 +355,7 @@ func (d *Database) GetEventsInTopologicalRange( } func (d *Database) SyncPosition(ctx context.Context) (tok types.StreamingToken, err error) { - err = common.WithTransaction(d.DB, func(txn *sql.Tx) error { + err = internal.WithTransaction(d.DB, func(txn *sql.Tx) error { pos, err := d.syncPositionTx(ctx, txn) if err != nil { return err @@ -434,7 +434,7 @@ func (d *Database) addPDUDeltaToResponse( } var succeeded bool defer func() { - txerr := common.EndTransaction(txn, &succeeded) + txerr := internal.EndTransaction(txn, &succeeded) if err == nil && txerr != nil { err = txerr } @@ -588,7 +588,7 @@ func (d *Database) getResponseWithPDUsForCompleteSync( } var succeeded bool defer func() { - txerr := common.EndTransaction(txn, &succeeded) + txerr := internal.EndTransaction(txn, &succeeded) if err == nil && txerr != nil { err = txerr } diff --git a/syncapi/storage/sqlite3/account_data_table.go b/syncapi/storage/sqlite3/account_data_table.go index 435a399c3..a3c797342 100644 --- a/syncapi/storage/sqlite3/account_data_table.go +++ b/syncapi/storage/sqlite3/account_data_table.go @@ -19,7 +19,7 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" @@ -100,7 +100,7 @@ func (s *accountDataStatements) SelectAccountDataInRange( if err != nil { return } - defer common.CloseAndLogIfError(ctx, rows, "selectAccountDataInRange: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "selectAccountDataInRange: rows.close() failed") var entries int diff --git a/syncapi/storage/sqlite3/backwards_extremities_table.go b/syncapi/storage/sqlite3/backwards_extremities_table.go index c5d9cae59..e16e54a6f 100644 --- a/syncapi/storage/sqlite3/backwards_extremities_table.go +++ b/syncapi/storage/sqlite3/backwards_extremities_table.go @@ -18,7 +18,7 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/syncapi/storage/tables" ) @@ -84,7 +84,7 @@ func (s *backwardExtremitiesStatements) SelectBackwardExtremitiesForRoom( if err != nil { return } - defer common.CloseAndLogIfError(ctx, rows, "selectBackwardExtremitiesForRoom: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "selectBackwardExtremitiesForRoom: rows.close() failed") bwExtrems = make(map[string][]string) for rows.Next() { diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go index b540fbc69..b0cf1297c 100644 --- a/syncapi/storage/sqlite3/current_room_state_table.go +++ b/syncapi/storage/sqlite3/current_room_state_table.go @@ -21,7 +21,7 @@ import ( "encoding/json" "strings" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -129,7 +129,7 @@ func (s *currentRoomStateStatements) SelectJoinedUsers( if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "selectJoinedUsers: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "selectJoinedUsers: rows.close() failed") result := make(map[string][]string) for rows.Next() { @@ -152,12 +152,12 @@ func (s *currentRoomStateStatements) SelectRoomIDsWithMembership( userID string, membership string, // nolint: unparam ) ([]string, error) { - stmt := common.TxStmt(txn, s.selectRoomIDsWithMembershipStmt) + stmt := internal.TxStmt(txn, s.selectRoomIDsWithMembershipStmt) rows, err := stmt.QueryContext(ctx, userID, membership) if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "selectRoomIDsWithMembership: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "selectRoomIDsWithMembership: rows.close() failed") var result []string for rows.Next() { @@ -175,7 +175,7 @@ func (s *currentRoomStateStatements) SelectCurrentState( ctx context.Context, txn *sql.Tx, roomID string, stateFilterPart *gomatrixserverlib.StateFilter, ) ([]gomatrixserverlib.HeaderedEvent, error) { - stmt := common.TxStmt(txn, s.selectCurrentStateStmt) + stmt := internal.TxStmt(txn, s.selectCurrentStateStmt) rows, err := stmt.QueryContext(ctx, roomID, nil, // FIXME: pq.StringArray(stateFilterPart.Senders), nil, // FIXME: pq.StringArray(stateFilterPart.NotSenders), @@ -187,7 +187,7 @@ func (s *currentRoomStateStatements) SelectCurrentState( if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "selectCurrentState: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "selectCurrentState: rows.close() failed") return rowsToEvents(rows) } @@ -195,7 +195,7 @@ func (s *currentRoomStateStatements) SelectCurrentState( func (s *currentRoomStateStatements) DeleteRoomStateByEventID( ctx context.Context, txn *sql.Tx, eventID string, ) error { - stmt := common.TxStmt(txn, s.deleteRoomStateByEventIDStmt) + stmt := internal.TxStmt(txn, s.deleteRoomStateByEventIDStmt) _, err := stmt.ExecContext(ctx, eventID) return err } @@ -218,7 +218,7 @@ func (s *currentRoomStateStatements) UpsertRoomState( } // upsert state event - stmt := common.TxStmt(txn, s.upsertRoomStateStmt) + stmt := internal.TxStmt(txn, s.upsertRoomStateStmt) _, err = stmt.ExecContext( ctx, event.RoomID(), @@ -241,12 +241,12 @@ func (s *currentRoomStateStatements) SelectEventsWithEventIDs( for k, v := range eventIDs { iEventIDs[k] = v } - query := strings.Replace(selectEventsWithEventIDsSQL, "($1)", common.QueryVariadic(len(iEventIDs)), 1) + query := strings.Replace(selectEventsWithEventIDsSQL, "($1)", internal.QueryVariadic(len(iEventIDs)), 1) rows, err := txn.QueryContext(ctx, query, iEventIDs...) if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "selectEventsWithEventIDs: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "selectEventsWithEventIDs: rows.close() failed") return rowsToStreamEvents(rows) } diff --git a/syncapi/storage/sqlite3/invites_table.go b/syncapi/storage/sqlite3/invites_table.go index 9cb184f20..64b523394 100644 --- a/syncapi/storage/sqlite3/invites_table.go +++ b/syncapi/storage/sqlite3/invites_table.go @@ -20,7 +20,7 @@ import ( "database/sql" "encoding/json" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -123,12 +123,12 @@ func (s *inviteEventsStatements) DeleteInviteEvent( func (s *inviteEventsStatements) SelectInviteEventsInRange( ctx context.Context, txn *sql.Tx, targetUserID string, r types.Range, ) (map[string]gomatrixserverlib.HeaderedEvent, error) { - stmt := common.TxStmt(txn, s.selectInviteEventsInRangeStmt) + stmt := internal.TxStmt(txn, s.selectInviteEventsInRangeStmt) rows, err := stmt.QueryContext(ctx, targetUserID, r.Low(), r.High()) if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "selectInviteEventsInRange: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "selectInviteEventsInRange: rows.close() failed") result := map[string]gomatrixserverlib.HeaderedEvent{} for rows.Next() { var ( @@ -153,7 +153,7 @@ func (s *inviteEventsStatements) SelectMaxInviteID( ctx context.Context, txn *sql.Tx, ) (id int64, err error) { var nullableID sql.NullInt64 - stmt := common.TxStmt(txn, s.selectMaxInviteIDStmt) + stmt := internal.TxStmt(txn, s.selectMaxInviteIDStmt) err = stmt.QueryRowContext(ctx).Scan(&nullableID) if nullableID.Valid { id = nullableID.Int64 diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index c89e007e6..4a84ecc6c 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -25,7 +25,7 @@ import ( "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/gomatrixserverlib" log "github.com/sirupsen/logrus" ) @@ -149,7 +149,7 @@ func (s *outputRoomEventsStatements) SelectStateInRange( ctx context.Context, txn *sql.Tx, r types.Range, stateFilterPart *gomatrixserverlib.StateFilter, ) (map[string]map[string]bool, map[string]types.StreamEvent, error) { - stmt := common.TxStmt(txn, s.selectStateInRangeStmt) + stmt := internal.TxStmt(txn, s.selectStateInRangeStmt) rows, err := stmt.QueryContext( ctx, r.Low(), r.High(), @@ -236,7 +236,7 @@ func (s *outputRoomEventsStatements) SelectMaxEventID( ctx context.Context, txn *sql.Tx, ) (id int64, err error) { var nullableID sql.NullInt64 - stmt := common.TxStmt(txn, s.selectMaxEventIDStmt) + stmt := internal.TxStmt(txn, s.selectMaxEventIDStmt) err = stmt.QueryRowContext(ctx).Scan(&nullableID) if nullableID.Valid { id = nullableID.Int64 @@ -286,7 +286,7 @@ func (s *outputRoomEventsStatements) InsertEvent( return } - insertStmt := common.TxStmt(txn, s.insertEventStmt) + insertStmt := internal.TxStmt(txn, s.insertEventStmt) _, err = insertStmt.ExecContext( ctx, streamPos, @@ -313,16 +313,16 @@ func (s *outputRoomEventsStatements) SelectRecentEvents( ) ([]types.StreamEvent, error) { var stmt *sql.Stmt if onlySyncEvents { - stmt = common.TxStmt(txn, s.selectRecentEventsForSyncStmt) + stmt = internal.TxStmt(txn, s.selectRecentEventsForSyncStmt) } else { - stmt = common.TxStmt(txn, s.selectRecentEventsStmt) + stmt = internal.TxStmt(txn, s.selectRecentEventsStmt) } rows, err := stmt.QueryContext(ctx, roomID, r.Low(), r.High(), limit) if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "selectRecentEvents: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "selectRecentEvents: rows.close() failed") events, err := rowsToStreamEvents(rows) if err != nil { return nil, err @@ -342,12 +342,12 @@ func (s *outputRoomEventsStatements) SelectEarlyEvents( ctx context.Context, txn *sql.Tx, roomID string, r types.Range, limit int, ) ([]types.StreamEvent, error) { - stmt := common.TxStmt(txn, s.selectEarlyEventsStmt) + stmt := internal.TxStmt(txn, s.selectEarlyEventsStmt) rows, err := stmt.QueryContext(ctx, roomID, r.Low(), r.High(), limit) if err != nil { return nil, err } - defer common.CloseAndLogIfError(ctx, rows, "selectEarlyEvents: rows.close() failed") + defer internal.CloseAndLogIfError(ctx, rows, "selectEarlyEvents: rows.close() failed") events, err := rowsToStreamEvents(rows) if err != nil { return nil, err @@ -367,7 +367,7 @@ func (s *outputRoomEventsStatements) SelectEvents( ctx context.Context, txn *sql.Tx, eventIDs []string, ) ([]types.StreamEvent, error) { var returnEvents []types.StreamEvent - stmt := common.TxStmt(txn, s.selectEventsStmt) + stmt := internal.TxStmt(txn, s.selectEventsStmt) for _, eventID := range eventIDs { rows, err := stmt.QueryContext(ctx, eventID) if err != nil { @@ -376,7 +376,7 @@ func (s *outputRoomEventsStatements) SelectEvents( if streamEvents, err := rowsToStreamEvents(rows); err == nil { returnEvents = append(returnEvents, streamEvents...) } - common.CloseAndLogIfError(ctx, rows, "selectEvents: rows.close() failed") + internal.CloseAndLogIfError(ctx, rows, "selectEvents: rows.close() failed") } return returnEvents, nil } diff --git a/syncapi/storage/sqlite3/output_room_events_topology_table.go b/syncapi/storage/sqlite3/output_room_events_topology_table.go index 22779238c..0d727cb18 100644 --- a/syncapi/storage/sqlite3/output_room_events_topology_table.go +++ b/syncapi/storage/sqlite3/output_room_events_topology_table.go @@ -18,7 +18,7 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -102,7 +102,7 @@ func NewSqliteTopologyTable(db *sql.DB) (tables.Topology, error) { func (s *outputRoomEventsTopologyStatements) InsertEventInTopology( ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.HeaderedEvent, pos types.StreamPosition, ) (err error) { - stmt := common.TxStmt(txn, s.insertEventInTopologyStmt) + stmt := internal.TxStmt(txn, s.insertEventInTopologyStmt) _, err = stmt.ExecContext( ctx, event.EventID(), event.Depth(), event.RoomID(), pos, ) @@ -118,9 +118,9 @@ func (s *outputRoomEventsTopologyStatements) SelectEventIDsInRange( // is requested or not. var stmt *sql.Stmt if chronologicalOrder { - stmt = common.TxStmt(txn, s.selectEventIDsInRangeASCStmt) + stmt = internal.TxStmt(txn, s.selectEventIDsInRangeASCStmt) } else { - stmt = common.TxStmt(txn, s.selectEventIDsInRangeDESCStmt) + stmt = internal.TxStmt(txn, s.selectEventIDsInRangeDESCStmt) } // Query the event IDs. @@ -149,7 +149,7 @@ func (s *outputRoomEventsTopologyStatements) SelectEventIDsInRange( func (s *outputRoomEventsTopologyStatements) SelectPositionInTopology( ctx context.Context, txn *sql.Tx, eventID string, ) (pos types.StreamPosition, spos types.StreamPosition, err error) { - stmt := common.TxStmt(txn, s.selectPositionInTopologyStmt) + stmt := internal.TxStmt(txn, s.selectPositionInTopologyStmt) err = stmt.QueryRowContext(ctx, eventID).Scan(&pos, &spos) return } @@ -157,7 +157,7 @@ func (s *outputRoomEventsTopologyStatements) SelectPositionInTopology( func (s *outputRoomEventsTopologyStatements) SelectMaxPositionInTopology( ctx context.Context, txn *sql.Tx, roomID string, ) (pos types.StreamPosition, spos types.StreamPosition, err error) { - stmt := common.TxStmt(txn, s.selectMaxPositionInTopologyStmt) + stmt := internal.TxStmt(txn, s.selectMaxPositionInTopologyStmt) err = stmt.QueryRowContext(ctx, roomID).Scan(&pos, &spos) return } diff --git a/syncapi/storage/sqlite3/stream_id_table.go b/syncapi/storage/sqlite3/stream_id_table.go index 260f7a95d..3bfe22dd0 100644 --- a/syncapi/storage/sqlite3/stream_id_table.go +++ b/syncapi/storage/sqlite3/stream_id_table.go @@ -4,7 +4,7 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/syncapi/types" ) @@ -46,8 +46,8 @@ func (s *streamIDStatements) prepare(db *sql.DB) (err error) { } func (s *streamIDStatements) nextStreamID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error) { - increaseStmt := common.TxStmt(txn, s.increaseStreamIDStmt) - selectStmt := common.TxStmt(txn, s.selectStreamIDStmt) + increaseStmt := internal.TxStmt(txn, s.increaseStreamIDStmt) + selectStmt := internal.TxStmt(txn, s.selectStreamIDStmt) if _, err = increaseStmt.ExecContext(ctx, "global"); err != nil { return } diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index e23aeca4e..8ab1d4040 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -25,8 +25,8 @@ import ( // Import the sqlite3 package _ "github.com/mattn/go-sqlite3" - "github.com/matrix-org/dendrite/common" "github.com/matrix-org/dendrite/eduserver/cache" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/syncapi/storage/shared" ) @@ -35,7 +35,7 @@ import ( type SyncServerDatasource struct { shared.Database db *sql.DB - common.PartitionOffsetStatements + internal.PartitionOffsetStatements streamID streamIDStatements } @@ -55,7 +55,7 @@ func NewDatabase(dataSourceName string) (*SyncServerDatasource, error) { } else { return nil, errors.New("no filename or path in connect string") } - if d.db, err = sqlutil.Open(common.SQLiteDriverName(), cs, nil); err != nil { + if d.db, err = sqlutil.Open(internal.SQLiteDriverName(), cs, nil); err != nil { return nil, err } if err = d.prepare(); err != nil { diff --git a/syncapi/storage/storage.go b/syncapi/storage/storage.go index 76a4b7a4e..a4275da0c 100644 --- a/syncapi/storage/storage.go +++ b/syncapi/storage/storage.go @@ -19,13 +19,13 @@ package storage import ( "net/url" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/syncapi/storage/postgres" "github.com/matrix-org/dendrite/syncapi/storage/sqlite3" ) // NewSyncServerDatasource opens a database connection. -func NewSyncServerDatasource(dataSourceName string, dbProperties common.DbProperties) (Database, error) { +func NewSyncServerDatasource(dataSourceName string, dbProperties internal.DbProperties) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return postgres.NewDatabase(dataSourceName, dbProperties) diff --git a/syncapi/storage/storage_wasm.go b/syncapi/storage/storage_wasm.go index 666179afb..a2b269a14 100644 --- a/syncapi/storage/storage_wasm.go +++ b/syncapi/storage/storage_wasm.go @@ -18,14 +18,14 @@ import ( "fmt" "net/url" - "github.com/matrix-org/dendrite/common" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/syncapi/storage/sqlite3" ) // NewPublicRoomsServerDatabase opens a database connection. func NewSyncServerDatasource( dataSourceName string, - dbProperties common.DbProperties, // nolint:unparam + dbProperties internal.DbProperties, // nolint:unparam ) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go index 4219d5603..022ed7bab 100644 --- a/syncapi/syncapi.go +++ b/syncapi/syncapi.go @@ -20,8 +20,8 @@ import ( "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" - "github.com/matrix-org/dendrite/common/basecomponent" - "github.com/matrix-org/dendrite/common/config" + "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" From f223da2f35e690e80c6e3d8c1050f0984ab33a2f Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 21 May 2020 16:22:25 +0100 Subject: [PATCH 083/200] Fix URL in compose file --- build/docker/hub/docker-compose.polylith.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/docker/hub/docker-compose.polylith.yml b/build/docker/hub/docker-compose.polylith.yml index f39670788..d88e5bfb3 100644 --- a/build/docker/hub/docker-compose.polylith.yml +++ b/build/docker/hub/docker-compose.polylith.yml @@ -94,7 +94,7 @@ services: image: matrixdotorg/dendrite:federationproxy command: [ "--bind-address=:8448", - "--federation-api-url=http://federation_api_server:7772", + "--federation-api-url=http://federation_api:7772", "--media-api-server-url=http://media_api:7774" ] volumes: From fe82e1f7255c05e0bc7a7872a53cf2a1a78ffaa0 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 22 May 2020 11:43:17 +0100 Subject: [PATCH 084/200] Separate muxes for public and internal APIs (#1056) * Separate muxes for public and internal APIs * Update client-api-proxy and federation-api-proxy so they don't add /api to the path * Tidy up * Consistent HTTP setup * Set up prefixes properly --- appservice/appservice.go | 2 +- appservice/routing/routing.go | 2 +- clientapi/clientapi.go | 2 +- clientapi/routing/routing.go | 16 +++++------ cmd/client-api-proxy/main.go | 1 - cmd/dendrite-demo-libp2p/main.go | 14 ++++----- cmd/dendrite-monolith-server/main.go | 16 +++++------ cmd/dendritejs/main.go | 10 +++++-- cmd/federation-api-proxy/main.go | 1 - federationapi/federationapi.go | 2 +- federationapi/routing/routing.go | 14 ++++----- federationsender/federationsender.go | 6 +--- federationsender/internal/api.go | 11 +++---- internal/basecomponent/base.go | 17 ++++++++--- internal/httpapi.go | 13 +++++++-- keyserver/keyserver.go | 2 +- keyserver/routing/routing.go | 6 ++-- mediaapi/mediaapi.go | 2 +- mediaapi/routing/routing.go | 6 ++-- publicroomsapi/publicroomsapi.go | 2 +- publicroomsapi/routing/routing.go | 8 +++--- roomserver/api/alias.go | 10 +++---- roomserver/api/input.go | 2 +- roomserver/api/perform.go | 4 +-- roomserver/api/query.go | 24 ++++++++-------- roomserver/internal/api.go | 43 ++++++++++++++-------------- roomserver/roomserver.go | 6 +--- syncapi/routing/routing.go | 6 ++-- syncapi/syncapi.go | 2 +- 29 files changed, 131 insertions(+), 119 deletions(-) diff --git a/appservice/appservice.go b/appservice/appservice.go index 3c67ce9c1..76181d2a3 100644 --- a/appservice/appservice.go +++ b/appservice/appservice.go @@ -101,7 +101,7 @@ func SetupAppServiceAPIComponent( // Set up HTTP Endpoints routing.Setup( - base.APIMux, base.Cfg, rsAPI, + base.PublicAPIMux, base.Cfg, rsAPI, accountsDB, federation, transactionsCache, ) diff --git a/appservice/routing/routing.go b/appservice/routing/routing.go index ac1ff32ea..e5ed7ab40 100644 --- a/appservice/routing/routing.go +++ b/appservice/routing/routing.go @@ -27,7 +27,7 @@ import ( "github.com/matrix-org/util" ) -const pathPrefixApp = "/_matrix/app/v1" +const pathPrefixApp = "/app/v1" // Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client // to clients which need to make outbound HTTP requests. diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go index 66317e92e..0ce44c211 100644 --- a/clientapi/clientapi.go +++ b/clientapi/clientapi.go @@ -65,7 +65,7 @@ func SetupClientAPIComponent( } routing.Setup( - base.APIMux, base.Cfg, roomserverProducer, rsAPI, asAPI, + base.PublicAPIMux, base.Cfg, roomserverProducer, rsAPI, asAPI, accountsDB, deviceDB, federation, *keyRing, userUpdateProducer, syncProducer, eduProducer, transactionsCache, fsAPI, ) diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 377881cb6..934d9f065 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -36,9 +36,9 @@ import ( "github.com/matrix-org/util" ) -const pathPrefixV1 = "/_matrix/client/api/v1" -const pathPrefixR0 = "/_matrix/client/r0" -const pathPrefixUnstable = "/_matrix/client/unstable" +const pathPrefixV1 = "/client/api/v1" +const pathPrefixR0 = "/client/r0" +const pathPrefixUnstable = "/client/unstable" // Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client // to clients which need to make outbound HTTP requests. @@ -47,7 +47,7 @@ const pathPrefixUnstable = "/_matrix/client/unstable" // applied: // nolint: gocyclo func Setup( - apiMux *mux.Router, cfg *config.Dendrite, + publicAPIMux *mux.Router, cfg *config.Dendrite, producer *producers.RoomserverProducer, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, @@ -62,7 +62,7 @@ func Setup( federationSender federationSenderAPI.FederationSenderInternalAPI, ) { - apiMux.Handle("/_matrix/client/versions", + publicAPIMux.Handle("/client/versions", internal.MakeExternalAPI("versions", func(req *http.Request) util.JSONResponse { return util.JSONResponse{ Code: http.StatusOK, @@ -78,9 +78,9 @@ func Setup( }), ).Methods(http.MethodGet, http.MethodOptions) - r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter() - v1mux := apiMux.PathPrefix(pathPrefixV1).Subrouter() - unstableMux := apiMux.PathPrefix(pathPrefixUnstable).Subrouter() + r0mux := publicAPIMux.PathPrefix(pathPrefixR0).Subrouter() + v1mux := publicAPIMux.PathPrefix(pathPrefixV1).Subrouter() + unstableMux := publicAPIMux.PathPrefix(pathPrefixUnstable).Subrouter() authData := auth.Data{ AccountDB: accountDB, diff --git a/cmd/client-api-proxy/main.go b/cmd/client-api-proxy/main.go index 27991c109..979b0b042 100644 --- a/cmd/client-api-proxy/main.go +++ b/cmd/client-api-proxy/main.go @@ -75,7 +75,6 @@ func makeProxy(targetURL string) (*httputil.ReverseProxy, error) { // Pratically this means that any distinction between '%2F' and '/' // in the URL will be lost by the time it reaches the target. path := req.URL.Path - path = "api" + path log.WithFields(log.Fields{ "path": path, "url": targetURL, diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index a04a28e06..3f53cb429 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -47,7 +47,6 @@ import ( "github.com/matrix-org/dendrite/eduserver/cache" - "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" ) @@ -178,12 +177,13 @@ func main() { publicroomsapi.SetupPublicRoomsAPIComponent(&base.Base, deviceDB, publicRoomsDB, rsAPI, federation, nil) // Check this later syncapi.SetupSyncAPIComponent(&base.Base, deviceDB, accountDB, rsAPI, federation, &cfg) - httpHandler := internal.WrapHandlerInCORS(base.Base.APIMux) - - // Set up the API endpoints we handle. /metrics is for prometheus, and is - // not wrapped by CORS, while everything else is - http.Handle("/metrics", promhttp.Handler()) - http.Handle("/", httpHandler) + internal.SetupHTTPAPI( + http.DefaultServeMux, + base.Base.PublicAPIMux, + base.Base.InternalAPIMux, + &cfg, + base.Base.EnableHTTPAPIs, + ) // Expose the matrix APIs directly rather than putting them under a /api path. go func() { diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 6d3a479ac..0042a9b9c 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -35,7 +35,6 @@ import ( "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/syncapi" - "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" ) @@ -91,14 +90,13 @@ func main() { publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, rsAPI, federation, nil) syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, rsAPI, federation, cfg) - httpHandler := internal.WrapHandlerInCORS(base.APIMux) - - // Set up the API endpoints we handle. /metrics is for prometheus, and is - // not wrapped by CORS, while everything else is - if cfg.Metrics.Enabled { - http.Handle("/metrics", internal.WrapHandlerInBasicAuth(promhttp.Handler(), cfg.Metrics.BasicAuth)) - } - http.Handle("/", httpHandler) + internal.SetupHTTPAPI( + http.DefaultServeMux, + base.PublicAPIMux, + base.InternalAPIMux, + cfg, + base.EnableHTTPAPIs, + ) // Expose the matrix APIs directly rather than putting them under a /api path. go func() { diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 2609c74cd..1398bc6da 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -227,9 +227,13 @@ func main() { publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, rsAPI, federation, p2pPublicRoomProvider) syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, rsAPI, federation, cfg) - httpHandler := internal.WrapHandlerInCORS(base.APIMux) - - http.Handle("/", httpHandler) + internal.SetupHTTPAPI( + http.DefaultServeMux, + base.PublicAPIMux, + base.InternalAPIMux, + cfg, + base.EnableHTTPAPIs, + ) // Expose the matrix APIs via libp2p-js - for federation traffic if node != nil { diff --git a/cmd/federation-api-proxy/main.go b/cmd/federation-api-proxy/main.go index fa90482d5..7324de148 100644 --- a/cmd/federation-api-proxy/main.go +++ b/cmd/federation-api-proxy/main.go @@ -73,7 +73,6 @@ func makeProxy(targetURL string) (*httputil.ReverseProxy, error) { // Pratically this means that any distinction between '%2F' and '/' // in the URL will be lost by the time it reaches the target. path := req.URL.Path - path = "api" + path log.WithFields(log.Fields{ "path": path, "url": targetURL, diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go index 9e3411403..baeaa36bc 100644 --- a/federationapi/federationapi.go +++ b/federationapi/federationapi.go @@ -44,7 +44,7 @@ func SetupFederationAPIComponent( roomserverProducer := producers.NewRoomserverProducer(rsAPI) routing.Setup( - base.APIMux, base.Cfg, rsAPI, asAPI, roomserverProducer, + base.PublicAPIMux, base.Cfg, rsAPI, asAPI, roomserverProducer, eduProducer, federationSenderAPI, *keyRing, federation, accountsDB, deviceDB, ) diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index c368c16c4..86d3192a2 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -31,9 +31,9 @@ import ( ) const ( - pathPrefixV2Keys = "/_matrix/key/v2" - pathPrefixV1Federation = "/_matrix/federation/v1" - pathPrefixV2Federation = "/_matrix/federation/v2" + pathPrefixV2Keys = "/key/v2" + pathPrefixV1Federation = "/federation/v1" + pathPrefixV2Federation = "/federation/v2" ) // Setup registers HTTP handlers with the given ServeMux. @@ -42,7 +42,7 @@ const ( // applied: // nolint: gocyclo func Setup( - apiMux *mux.Router, + publicAPIMux *mux.Router, cfg *config.Dendrite, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, @@ -54,9 +54,9 @@ func Setup( accountDB accounts.Database, deviceDB devices.Database, ) { - v2keysmux := apiMux.PathPrefix(pathPrefixV2Keys).Subrouter() - v1fedmux := apiMux.PathPrefix(pathPrefixV1Federation).Subrouter() - v2fedmux := apiMux.PathPrefix(pathPrefixV2Federation).Subrouter() + v2keysmux := publicAPIMux.PathPrefix(pathPrefixV2Keys).Subrouter() + v1fedmux := publicAPIMux.PathPrefix(pathPrefixV1Federation).Subrouter() + v2fedmux := publicAPIMux.PathPrefix(pathPrefixV2Federation).Subrouter() localKeys := internal.MakeExternalAPI("localkeys", func(req *http.Request) util.JSONResponse { return LocalKeys(cfg) diff --git a/federationsender/federationsender.go b/federationsender/federationsender.go index 446f920c2..cca847a51 100644 --- a/federationsender/federationsender.go +++ b/federationsender/federationsender.go @@ -15,8 +15,6 @@ package federationsender import ( - "net/http" - "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/federationsender/consumers" "github.com/matrix-org/dendrite/federationsender/internal" @@ -72,9 +70,7 @@ func SetupFederationSenderComponent( statistics, ) - if base.EnableHTTPAPIs { - queryAPI.SetupHTTP(http.DefaultServeMux) - } + queryAPI.SetupHTTP(base.InternalAPIMux) return queryAPI } diff --git a/federationsender/internal/api.go b/federationsender/internal/api.go index 1f6d857f4..4805e7050 100644 --- a/federationsender/internal/api.go +++ b/federationsender/internal/api.go @@ -4,6 +4,7 @@ import ( "encoding/json" "net/http" + "github.com/gorilla/mux" "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/federationsender/producers" "github.com/matrix-org/dendrite/federationsender/storage" @@ -43,8 +44,8 @@ func NewFederationSenderInternalAPI( } // SetupHTTP adds the FederationSenderInternalAPI handlers to the http.ServeMux. -func (f *FederationSenderInternalAPI) SetupHTTP(servMux *http.ServeMux) { - servMux.Handle( +func (f *FederationSenderInternalAPI) SetupHTTP(internalAPIMux *mux.Router) { + internalAPIMux.Handle( api.FederationSenderQueryJoinedHostsInRoomPath, internal.MakeInternalAPI("QueryJoinedHostsInRoom", func(req *http.Request) util.JSONResponse { var request api.QueryJoinedHostsInRoomRequest @@ -58,7 +59,7 @@ func (f *FederationSenderInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle( + internalAPIMux.Handle( api.FederationSenderQueryJoinedHostServerNamesInRoomPath, internal.MakeInternalAPI("QueryJoinedHostServerNamesInRoom", func(req *http.Request) util.JSONResponse { var request api.QueryJoinedHostServerNamesInRoomRequest @@ -72,7 +73,7 @@ func (f *FederationSenderInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle(api.FederationSenderPerformJoinRequestPath, + internalAPIMux.Handle(api.FederationSenderPerformJoinRequestPath, internal.MakeInternalAPI("PerformJoinRequest", func(req *http.Request) util.JSONResponse { var request api.PerformJoinRequest var response api.PerformJoinResponse @@ -85,7 +86,7 @@ func (f *FederationSenderInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle(api.FederationSenderPerformLeaveRequestPath, + internalAPIMux.Handle(api.FederationSenderPerformLeaveRequestPath, internal.MakeInternalAPI("PerformLeaveRequest", func(req *http.Request) util.JSONResponse { var request api.PerformLeaveRequest var response api.PerformLeaveResponse diff --git a/internal/basecomponent/base.go b/internal/basecomponent/base.go index e9a375a70..0682ae0ea 100644 --- a/internal/basecomponent/base.go +++ b/internal/basecomponent/base.go @@ -56,8 +56,9 @@ type BaseDendrite struct { componentName string tracerCloser io.Closer - // APIMux should be used to register new public matrix api endpoints - APIMux *mux.Router + // PublicAPIMux should be used to register new public matrix api endpoints + PublicAPIMux *mux.Router + InternalAPIMux *mux.Router EnableHTTPAPIs bool httpClient *http.Client Cfg *config.Dendrite @@ -95,13 +96,15 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, enableHTTPAPIs logrus.WithError(err).Warnf("Failed to create cache") } + httpmux := mux.NewRouter() return &BaseDendrite{ componentName: componentName, EnableHTTPAPIs: enableHTTPAPIs, tracerCloser: closer, Cfg: cfg, ImmutableCache: cache, - APIMux: mux.NewRouter().UseEncodedPath(), + PublicAPIMux: httpmux.PathPrefix(internal.HTTPPublicPathPrefix).Subrouter().UseEncodedPath(), + InternalAPIMux: httpmux.PathPrefix(internal.HTTPInternalPathPrefix).Subrouter().UseEncodedPath(), httpClient: &http.Client{Timeout: HTTPClientTimeout}, KafkaConsumer: kafkaConsumer, KafkaProducer: kafkaProducer, @@ -221,7 +224,13 @@ func (b *BaseDendrite) SetupAndServeHTTP(bindaddr string, listenaddr string) { WriteTimeout: HTTPServerTimeout, } - internal.SetupHTTPAPI(http.DefaultServeMux, internal.WrapHandlerInCORS(b.APIMux), b.Cfg) + internal.SetupHTTPAPI( + http.DefaultServeMux, + b.PublicAPIMux, + b.InternalAPIMux, + b.Cfg, + b.EnableHTTPAPIs, + ) logrus.Infof("Starting %s server on %s", b.componentName, serv.Addr) err := serv.ListenAndServe() diff --git a/internal/httpapi.go b/internal/httpapi.go index b4c53f58d..526ff8a2d 100644 --- a/internal/httpapi.go +++ b/internal/httpapi.go @@ -9,6 +9,7 @@ import ( "strings" "time" + "github.com/gorilla/mux" "github.com/matrix-org/dendrite/clientapi/auth" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/internal/config" @@ -22,6 +23,11 @@ import ( "github.com/sirupsen/logrus" ) +const ( + HTTPPublicPathPrefix = "/_matrix/" + HTTPInternalPathPrefix = "/api/" +) + // BasicAuth is used for authorization on /metrics handlers type BasicAuth struct { Username string `yaml:"username"` @@ -184,11 +190,14 @@ func MakeFedAPI( // SetupHTTPAPI registers an HTTP API mux under /api and sets up a metrics // listener. -func SetupHTTPAPI(servMux *http.ServeMux, apiMux http.Handler, cfg *config.Dendrite) { +func SetupHTTPAPI(servMux *http.ServeMux, publicApiMux *mux.Router, internalApiMux *mux.Router, cfg *config.Dendrite, enableHTTPAPIs bool) { if cfg.Metrics.Enabled { servMux.Handle("/metrics", WrapHandlerInBasicAuth(promhttp.Handler(), cfg.Metrics.BasicAuth)) } - servMux.Handle("/api/", http.StripPrefix("/api", apiMux)) + if enableHTTPAPIs { + servMux.Handle(HTTPInternalPathPrefix, internalApiMux) + } + servMux.Handle(HTTPPublicPathPrefix, WrapHandlerInCORS(publicApiMux)) } // WrapHandlerInBasicAuth adds basic auth to a handler. Only used for /metrics diff --git a/keyserver/keyserver.go b/keyserver/keyserver.go index 0f450165f..4e785cbc7 100644 --- a/keyserver/keyserver.go +++ b/keyserver/keyserver.go @@ -28,5 +28,5 @@ func SetupKeyServerComponent( deviceDB devices.Database, accountsDB accounts.Database, ) { - routing.Setup(base.APIMux, base.Cfg, accountsDB, deviceDB) + routing.Setup(base.PublicAPIMux, base.Cfg, accountsDB, deviceDB) } diff --git a/keyserver/routing/routing.go b/keyserver/routing/routing.go index 3ca68ade4..2d1122c62 100644 --- a/keyserver/routing/routing.go +++ b/keyserver/routing/routing.go @@ -27,7 +27,7 @@ import ( "github.com/matrix-org/util" ) -const pathPrefixR0 = "/_matrix/client/r0" +const pathPrefixR0 = "/client/r0" // Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client // to clients which need to make outbound HTTP requests. @@ -36,11 +36,11 @@ const pathPrefixR0 = "/_matrix/client/r0" // applied: // nolint: gocyclo func Setup( - apiMux *mux.Router, cfg *config.Dendrite, + publicAPIMux *mux.Router, cfg *config.Dendrite, accountDB accounts.Database, deviceDB devices.Database, ) { - r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter() + r0mux := publicAPIMux.PathPrefix(pathPrefixR0).Subrouter() authData := auth.Data{ AccountDB: accountDB, diff --git a/mediaapi/mediaapi.go b/mediaapi/mediaapi.go index 38572daf8..b5bec3907 100644 --- a/mediaapi/mediaapi.go +++ b/mediaapi/mediaapi.go @@ -35,6 +35,6 @@ func SetupMediaAPIComponent( } routing.Setup( - base.APIMux, base.Cfg, mediaDB, deviceDB, gomatrixserverlib.NewClient(), + base.PublicAPIMux, base.Cfg, mediaDB, deviceDB, gomatrixserverlib.NewClient(), ) } diff --git a/mediaapi/routing/routing.go b/mediaapi/routing/routing.go index 4f765d173..11e4ecad5 100644 --- a/mediaapi/routing/routing.go +++ b/mediaapi/routing/routing.go @@ -33,7 +33,7 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" ) -const pathPrefixR0 = "/_matrix/media/r0" +const pathPrefixR0 = "/media/r0" // Setup registers the media API HTTP handlers // @@ -41,13 +41,13 @@ const pathPrefixR0 = "/_matrix/media/r0" // applied: // nolint: gocyclo func Setup( - apiMux *mux.Router, + publicAPIMux *mux.Router, cfg *config.Dendrite, db storage.Database, deviceDB devices.Database, client *gomatrixserverlib.Client, ) { - r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter() + r0mux := publicAPIMux.PathPrefix(pathPrefixR0).Subrouter() activeThumbnailGeneration := &types.ActiveThumbnailGeneration{ PathToResult: map[string]*types.ThumbnailGenerationResult{}, diff --git a/publicroomsapi/publicroomsapi.go b/publicroomsapi/publicroomsapi.go index f767a5303..b53351ffd 100644 --- a/publicroomsapi/publicroomsapi.go +++ b/publicroomsapi/publicroomsapi.go @@ -43,5 +43,5 @@ func SetupPublicRoomsAPIComponent( logrus.WithError(err).Panic("failed to start public rooms server consumer") } - routing.Setup(base.APIMux, deviceDB, publicRoomsDB, rsAPI, fedClient, extRoomsProvider) + routing.Setup(base.PublicAPIMux, deviceDB, publicRoomsDB, rsAPI, fedClient, extRoomsProvider) } diff --git a/publicroomsapi/routing/routing.go b/publicroomsapi/routing/routing.go index 39a9dcca1..ee1434392 100644 --- a/publicroomsapi/routing/routing.go +++ b/publicroomsapi/routing/routing.go @@ -31,7 +31,7 @@ import ( "github.com/matrix-org/util" ) -const pathPrefixR0 = "/_matrix/client/r0" +const pathPrefixR0 = "/client/r0" // Setup configures the given mux with publicroomsapi server listeners // @@ -39,10 +39,10 @@ const pathPrefixR0 = "/_matrix/client/r0" // applied: // nolint: gocyclo func Setup( - apiMux *mux.Router, deviceDB devices.Database, publicRoomsDB storage.Database, rsAPI api.RoomserverInternalAPI, + publicAPIMux *mux.Router, deviceDB devices.Database, publicRoomsDB storage.Database, rsAPI api.RoomserverInternalAPI, fedClient *gomatrixserverlib.FederationClient, extRoomsProvider types.ExternalPublicRoomsProvider, ) { - r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter() + r0mux := publicAPIMux.PathPrefix(pathPrefixR0).Subrouter() authData := auth.Data{ AccountDB: nil, @@ -79,7 +79,7 @@ func Setup( ).Methods(http.MethodGet, http.MethodPost, http.MethodOptions) // Federation - TODO: should this live here or in federation API? It's sure easier if it's here so here it is. - apiMux.Handle("/_matrix/federation/v1/publicRooms", + publicAPIMux.Handle("/federation/v1/publicRooms", internal.MakeExternalAPI("federation_public_rooms", func(req *http.Request) util.JSONResponse { return directory.GetPostPublicRooms(req, publicRoomsDB) }), diff --git a/roomserver/api/alias.go b/roomserver/api/alias.go index 93975d36c..54d2c633b 100644 --- a/roomserver/api/alias.go +++ b/roomserver/api/alias.go @@ -85,19 +85,19 @@ type RemoveRoomAliasRequest struct { type RemoveRoomAliasResponse struct{} // RoomserverSetRoomAliasPath is the HTTP path for the SetRoomAlias API. -const RoomserverSetRoomAliasPath = "/api/roomserver/setRoomAlias" +const RoomserverSetRoomAliasPath = "/roomserver/setRoomAlias" // RoomserverGetRoomIDForAliasPath is the HTTP path for the GetRoomIDForAlias API. -const RoomserverGetRoomIDForAliasPath = "/api/roomserver/GetRoomIDForAlias" +const RoomserverGetRoomIDForAliasPath = "/roomserver/GetRoomIDForAlias" // RoomserverGetAliasesForRoomIDPath is the HTTP path for the GetAliasesForRoomID API. -const RoomserverGetAliasesForRoomIDPath = "/api/roomserver/GetAliasesForRoomID" +const RoomserverGetAliasesForRoomIDPath = "/roomserver/GetAliasesForRoomID" // RoomserverGetCreatorIDForAliasPath is the HTTP path for the GetCreatorIDForAlias API. -const RoomserverGetCreatorIDForAliasPath = "/api/roomserver/GetCreatorIDForAlias" +const RoomserverGetCreatorIDForAliasPath = "/roomserver/GetCreatorIDForAlias" // RoomserverRemoveRoomAliasPath is the HTTP path for the RemoveRoomAlias API. -const RoomserverRemoveRoomAliasPath = "/api/roomserver/removeRoomAlias" +const RoomserverRemoveRoomAliasPath = "/roomserver/removeRoomAlias" // SetRoomAlias implements RoomserverAliasAPI func (h *httpRoomserverInternalAPI) SetRoomAlias( diff --git a/roomserver/api/input.go b/roomserver/api/input.go index f1eec75a8..d35ead764 100644 --- a/roomserver/api/input.go +++ b/roomserver/api/input.go @@ -103,7 +103,7 @@ type InputRoomEventsResponse struct { } // RoomserverInputRoomEventsPath is the HTTP path for the InputRoomEvents API. -const RoomserverInputRoomEventsPath = "/api/roomserver/inputRoomEvents" +const RoomserverInputRoomEventsPath = "/roomserver/inputRoomEvents" // InputRoomEvents implements RoomserverInputAPI func (h *httpRoomserverInternalAPI) InputRoomEvents( diff --git a/roomserver/api/perform.go b/roomserver/api/perform.go index 93f36f451..c98f91fb4 100644 --- a/roomserver/api/perform.go +++ b/roomserver/api/perform.go @@ -10,10 +10,10 @@ import ( const ( // RoomserverPerformJoinPath is the HTTP path for the PerformJoin API. - RoomserverPerformJoinPath = "/api/roomserver/performJoin" + RoomserverPerformJoinPath = "/roomserver/performJoin" // RoomserverPerformLeavePath is the HTTP path for the PerformLeave API. - RoomserverPerformLeavePath = "/api/roomserver/performLeave" + RoomserverPerformLeavePath = "/roomserver/performLeave" ) type PerformJoinRequest struct { diff --git a/roomserver/api/query.go b/roomserver/api/query.go index 98f46c771..916ecb362 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -273,40 +273,40 @@ type QueryRoomVersionForRoomResponse struct { } // RoomserverQueryLatestEventsAndStatePath is the HTTP path for the QueryLatestEventsAndState API. -const RoomserverQueryLatestEventsAndStatePath = "/api/roomserver/queryLatestEventsAndState" +const RoomserverQueryLatestEventsAndStatePath = "/roomserver/queryLatestEventsAndState" // RoomserverQueryStateAfterEventsPath is the HTTP path for the QueryStateAfterEvents API. -const RoomserverQueryStateAfterEventsPath = "/api/roomserver/queryStateAfterEvents" +const RoomserverQueryStateAfterEventsPath = "/roomserver/queryStateAfterEvents" // RoomserverQueryEventsByIDPath is the HTTP path for the QueryEventsByID API. -const RoomserverQueryEventsByIDPath = "/api/roomserver/queryEventsByID" +const RoomserverQueryEventsByIDPath = "/roomserver/queryEventsByID" // RoomserverQueryMembershipForUserPath is the HTTP path for the QueryMembershipForUser API. -const RoomserverQueryMembershipForUserPath = "/api/roomserver/queryMembershipForUser" +const RoomserverQueryMembershipForUserPath = "/roomserver/queryMembershipForUser" // RoomserverQueryMembershipsForRoomPath is the HTTP path for the QueryMembershipsForRoom API -const RoomserverQueryMembershipsForRoomPath = "/api/roomserver/queryMembershipsForRoom" +const RoomserverQueryMembershipsForRoomPath = "/roomserver/queryMembershipsForRoom" // RoomserverQueryInvitesForUserPath is the HTTP path for the QueryInvitesForUser API -const RoomserverQueryInvitesForUserPath = "/api/roomserver/queryInvitesForUser" +const RoomserverQueryInvitesForUserPath = "/roomserver/queryInvitesForUser" // RoomserverQueryServerAllowedToSeeEventPath is the HTTP path for the QueryServerAllowedToSeeEvent API -const RoomserverQueryServerAllowedToSeeEventPath = "/api/roomserver/queryServerAllowedToSeeEvent" +const RoomserverQueryServerAllowedToSeeEventPath = "/roomserver/queryServerAllowedToSeeEvent" // RoomserverQueryMissingEventsPath is the HTTP path for the QueryMissingEvents API -const RoomserverQueryMissingEventsPath = "/api/roomserver/queryMissingEvents" +const RoomserverQueryMissingEventsPath = "/roomserver/queryMissingEvents" // RoomserverQueryStateAndAuthChainPath is the HTTP path for the QueryStateAndAuthChain API -const RoomserverQueryStateAndAuthChainPath = "/api/roomserver/queryStateAndAuthChain" +const RoomserverQueryStateAndAuthChainPath = "/roomserver/queryStateAndAuthChain" // RoomserverQueryBackfillPath is the HTTP path for the QueryBackfillPath API -const RoomserverQueryBackfillPath = "/api/roomserver/queryBackfill" +const RoomserverQueryBackfillPath = "/roomserver/queryBackfill" // RoomserverQueryRoomVersionCapabilitiesPath is the HTTP path for the QueryRoomVersionCapabilities API -const RoomserverQueryRoomVersionCapabilitiesPath = "/api/roomserver/queryRoomVersionCapabilities" +const RoomserverQueryRoomVersionCapabilitiesPath = "/roomserver/queryRoomVersionCapabilities" // RoomserverQueryRoomVersionForRoomPath is the HTTP path for the QueryRoomVersionForRoom API -const RoomserverQueryRoomVersionForRoomPath = "/api/roomserver/queryRoomVersionForRoom" +const RoomserverQueryRoomVersionForRoomPath = "/roomserver/queryRoomVersionForRoom" // QueryLatestEventsAndState implements RoomserverQueryAPI func (h *httpRoomserverInternalAPI) QueryLatestEventsAndState( diff --git a/roomserver/internal/api.go b/roomserver/internal/api.go index 4d223cf36..248e457d1 100644 --- a/roomserver/internal/api.go +++ b/roomserver/internal/api.go @@ -6,6 +6,7 @@ import ( "sync" "github.com/Shopify/sarama" + "github.com/gorilla/mux" fsAPI "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/caching" @@ -32,8 +33,8 @@ type RoomserverInternalAPI struct { // SetupHTTP adds the RoomserverInternalAPI handlers to the http.ServeMux. // nolint: gocyclo -func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { - servMux.Handle(api.RoomserverInputRoomEventsPath, +func (r *RoomserverInternalAPI) SetupHTTP(internalAPIMux *mux.Router) { + internalAPIMux.Handle(api.RoomserverInputRoomEventsPath, internal.MakeInternalAPI("inputRoomEvents", func(req *http.Request) util.JSONResponse { var request api.InputRoomEventsRequest var response api.InputRoomEventsResponse @@ -46,7 +47,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle(api.RoomserverPerformJoinPath, + internalAPIMux.Handle(api.RoomserverPerformJoinPath, internal.MakeInternalAPI("performJoin", func(req *http.Request) util.JSONResponse { var request api.PerformJoinRequest var response api.PerformJoinResponse @@ -59,7 +60,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle(api.RoomserverPerformLeavePath, + internalAPIMux.Handle(api.RoomserverPerformLeavePath, internal.MakeInternalAPI("performLeave", func(req *http.Request) util.JSONResponse { var request api.PerformLeaveRequest var response api.PerformLeaveResponse @@ -72,7 +73,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle( + internalAPIMux.Handle( api.RoomserverQueryLatestEventsAndStatePath, internal.MakeInternalAPI("queryLatestEventsAndState", func(req *http.Request) util.JSONResponse { var request api.QueryLatestEventsAndStateRequest @@ -86,7 +87,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle( + internalAPIMux.Handle( api.RoomserverQueryStateAfterEventsPath, internal.MakeInternalAPI("queryStateAfterEvents", func(req *http.Request) util.JSONResponse { var request api.QueryStateAfterEventsRequest @@ -100,7 +101,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle( + internalAPIMux.Handle( api.RoomserverQueryEventsByIDPath, internal.MakeInternalAPI("queryEventsByID", func(req *http.Request) util.JSONResponse { var request api.QueryEventsByIDRequest @@ -114,7 +115,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle( + internalAPIMux.Handle( api.RoomserverQueryMembershipForUserPath, internal.MakeInternalAPI("QueryMembershipForUser", func(req *http.Request) util.JSONResponse { var request api.QueryMembershipForUserRequest @@ -128,7 +129,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle( + internalAPIMux.Handle( api.RoomserverQueryMembershipsForRoomPath, internal.MakeInternalAPI("queryMembershipsForRoom", func(req *http.Request) util.JSONResponse { var request api.QueryMembershipsForRoomRequest @@ -142,7 +143,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle( + internalAPIMux.Handle( api.RoomserverQueryInvitesForUserPath, internal.MakeInternalAPI("queryInvitesForUser", func(req *http.Request) util.JSONResponse { var request api.QueryInvitesForUserRequest @@ -156,7 +157,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle( + internalAPIMux.Handle( api.RoomserverQueryServerAllowedToSeeEventPath, internal.MakeInternalAPI("queryServerAllowedToSeeEvent", func(req *http.Request) util.JSONResponse { var request api.QueryServerAllowedToSeeEventRequest @@ -170,7 +171,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle( + internalAPIMux.Handle( api.RoomserverQueryMissingEventsPath, internal.MakeInternalAPI("queryMissingEvents", func(req *http.Request) util.JSONResponse { var request api.QueryMissingEventsRequest @@ -184,7 +185,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle( + internalAPIMux.Handle( api.RoomserverQueryStateAndAuthChainPath, internal.MakeInternalAPI("queryStateAndAuthChain", func(req *http.Request) util.JSONResponse { var request api.QueryStateAndAuthChainRequest @@ -198,7 +199,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle( + internalAPIMux.Handle( api.RoomserverQueryBackfillPath, internal.MakeInternalAPI("QueryBackfill", func(req *http.Request) util.JSONResponse { var request api.QueryBackfillRequest @@ -212,7 +213,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle( + internalAPIMux.Handle( api.RoomserverQueryRoomVersionCapabilitiesPath, internal.MakeInternalAPI("QueryRoomVersionCapabilities", func(req *http.Request) util.JSONResponse { var request api.QueryRoomVersionCapabilitiesRequest @@ -226,7 +227,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle( + internalAPIMux.Handle( api.RoomserverQueryRoomVersionForRoomPath, internal.MakeInternalAPI("QueryRoomVersionForRoom", func(req *http.Request) util.JSONResponse { var request api.QueryRoomVersionForRoomRequest @@ -240,7 +241,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle( + internalAPIMux.Handle( api.RoomserverSetRoomAliasPath, internal.MakeInternalAPI("setRoomAlias", func(req *http.Request) util.JSONResponse { var request api.SetRoomAliasRequest @@ -254,7 +255,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle( + internalAPIMux.Handle( api.RoomserverGetRoomIDForAliasPath, internal.MakeInternalAPI("GetRoomIDForAlias", func(req *http.Request) util.JSONResponse { var request api.GetRoomIDForAliasRequest @@ -268,7 +269,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle( + internalAPIMux.Handle( api.RoomserverGetCreatorIDForAliasPath, internal.MakeInternalAPI("GetCreatorIDForAlias", func(req *http.Request) util.JSONResponse { var request api.GetCreatorIDForAliasRequest @@ -282,7 +283,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle( + internalAPIMux.Handle( api.RoomserverGetAliasesForRoomIDPath, internal.MakeInternalAPI("getAliasesForRoomID", func(req *http.Request) util.JSONResponse { var request api.GetAliasesForRoomIDRequest @@ -296,7 +297,7 @@ func (r *RoomserverInternalAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle( + internalAPIMux.Handle( api.RoomserverRemoveRoomAliasPath, internal.MakeInternalAPI("removeRoomAlias", func(req *http.Request) util.JSONResponse { var request api.RemoveRoomAliasRequest diff --git a/roomserver/roomserver.go b/roomserver/roomserver.go index 07250e7ce..82934d50d 100644 --- a/roomserver/roomserver.go +++ b/roomserver/roomserver.go @@ -15,8 +15,6 @@ package roomserver import ( - "net/http" - "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" @@ -51,9 +49,7 @@ func SetupRoomServerComponent( KeyRing: keyRing, } - if base.EnableHTTPAPIs { - internalAPI.SetupHTTP(http.DefaultServeMux) - } + internalAPI.SetupHTTP(base.InternalAPIMux) return &internalAPI } diff --git a/syncapi/routing/routing.go b/syncapi/routing/routing.go index 6300b17bb..c876164d7 100644 --- a/syncapi/routing/routing.go +++ b/syncapi/routing/routing.go @@ -30,7 +30,7 @@ import ( "github.com/matrix-org/util" ) -const pathPrefixR0 = "/_matrix/client/r0" +const pathPrefixR0 = "/client/r0" // Setup configures the given mux with sync-server listeners // @@ -38,12 +38,12 @@ const pathPrefixR0 = "/_matrix/client/r0" // applied: // nolint: gocyclo func Setup( - apiMux *mux.Router, srp *sync.RequestPool, syncDB storage.Database, + publicAPIMux *mux.Router, srp *sync.RequestPool, syncDB storage.Database, deviceDB devices.Database, federation *gomatrixserverlib.FederationClient, rsAPI api.RoomserverInternalAPI, cfg *config.Dendrite, ) { - r0mux := apiMux.PathPrefix(pathPrefixR0).Subrouter() + r0mux := publicAPIMux.PathPrefix(pathPrefixR0).Subrouter() authData := auth.Data{ AccountDB: nil, diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go index 022ed7bab..9251f6182 100644 --- a/syncapi/syncapi.go +++ b/syncapi/syncapi.go @@ -81,5 +81,5 @@ func SetupSyncAPIComponent( logrus.WithError(err).Panicf("failed to start typing server consumer") } - routing.Setup(base.APIMux, requestPool, syncDB, deviceDB, federation, rsAPI, cfg) + routing.Setup(base.PublicAPIMux, requestPool, syncDB, deviceDB, federation, rsAPI, cfg) } From fbdcfdd25674eb73c4804be42031397e415797a9 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 22 May 2020 12:28:36 +0100 Subject: [PATCH 085/200] Use HTTP APIs when -api specified (#1057) --- cmd/dendrite-monolith-server/main.go | 31 +++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 0042a9b9c..ded0934ae 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -27,6 +27,7 @@ import ( "github.com/matrix-org/dendrite/federationsender" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/keydb" "github.com/matrix-org/dendrite/internal/transactions" "github.com/matrix-org/dendrite/keyserver" @@ -44,11 +45,23 @@ var ( httpsBindAddr = flag.String("https-bind-address", ":8448", "The HTTPS listening port for the server") certFile = flag.String("tls-cert", "", "The PEM formatted X509 certificate to use for TLS") keyFile = flag.String("tls-key", "", "The PEM private key to use for TLS") - enableHTTPAPIs = flag.Bool("api", false, "Expose internal HTTP APIs in monolith mode") + enableHTTPAPIs = flag.Bool("api", false, "Use HTTP APIs instead of short-circuiting (warning: exposes API endpoints!)") ) func main() { cfg := basecomponent.ParseMonolithFlags() + if *enableHTTPAPIs { + // If the HTTP APIs are enabled then we need to update the Listen + // statements in the configuration so that we know where to find + // the API endpoints. They'll listen on the same port as the monolith + // itself. + addr := config.Address(*httpBindAddr) + cfg.Listen.RoomServer = addr + cfg.Listen.EDUServer = addr + cfg.Listen.AppServiceAPI = addr + cfg.Listen.FederationSender = addr + } + base := basecomponent.NewBaseDendrite(cfg, "Monolith", *enableHTTPAPIs) defer base.Close() // nolint: errcheck @@ -61,15 +74,30 @@ func main() { rsAPI := roomserver.SetupRoomServerComponent( base, keyRing, federation, ) + if base.EnableHTTPAPIs { + rsAPI = base.CreateHTTPRoomserverAPIs() + } + eduInputAPI := eduserver.SetupEDUServerComponent( base, cache.New(), ) + if base.EnableHTTPAPIs { + eduInputAPI = base.CreateHTTPEDUServerAPIs() + } + asAPI := appservice.SetupAppServiceAPIComponent( base, accountDB, deviceDB, federation, rsAPI, transactions.New(), ) + if base.EnableHTTPAPIs { + asAPI = base.CreateHTTPAppServiceAPIs() + } + fsAPI := federationsender.SetupFederationSenderComponent( base, federation, rsAPI, &keyRing, ) + if base.EnableHTTPAPIs { + fsAPI = base.CreateHTTPFederationSenderAPIs() + } rsAPI.SetFederationSenderAPI(fsAPI) clientapi.SetupClientAPIComponent( @@ -77,6 +105,7 @@ func main() { federation, &keyRing, rsAPI, eduInputAPI, asAPI, transactions.New(), fsAPI, ) + keyserver.SetupKeyServerComponent( base, deviceDB, accountDB, ) From 3daa2327edc90e1ff70bcbc578b793ae5ba5b69f Mon Sep 17 00:00:00 2001 From: Kegsay Date: Fri, 22 May 2020 12:28:48 +0100 Subject: [PATCH 086/200] dendritejs tweaks for persisting sqlite DBs (#1058) * Use uri.path so we don't have file: in the filename * New go-sqlite-js version --- appservice/storage/storage_wasm.go | 2 +- .../auth/storage/accounts/storage_wasm.go | 2 +- .../auth/storage/devices/storage_wasm.go | 2 +- cmd/dendritejs/main.go | 20 +++++++++---------- federationsender/storage/storage_wasm.go | 2 +- go.mod | 2 +- go.sum | 2 ++ internal/basecomponent/base.go | 2 +- internal/keydb/keydb_wasm.go | 2 +- mediaapi/storage/storage_wasm.go | 2 +- publicroomsapi/storage/storage_wasm.go | 2 +- 11 files changed, 21 insertions(+), 19 deletions(-) diff --git a/appservice/storage/storage_wasm.go b/appservice/storage/storage_wasm.go index a6144b435..de1acf92f 100644 --- a/appservice/storage/storage_wasm.go +++ b/appservice/storage/storage_wasm.go @@ -34,7 +34,7 @@ func NewDatabase( case "postgres": return nil, fmt.Errorf("Cannot use postgres implementation") case "file": - return sqlite3.NewDatabase(dataSourceName) + return sqlite3.NewDatabase(uri.Path) default: return nil, fmt.Errorf("Cannot use postgres implementation") } diff --git a/clientapi/auth/storage/accounts/storage_wasm.go b/clientapi/auth/storage/accounts/storage_wasm.go index 81e77cf79..7cf50de79 100644 --- a/clientapi/auth/storage/accounts/storage_wasm.go +++ b/clientapi/auth/storage/accounts/storage_wasm.go @@ -36,7 +36,7 @@ func NewDatabase( case "postgres": return nil, fmt.Errorf("Cannot use postgres implementation") case "file": - return sqlite3.NewDatabase(dataSourceName, serverName) + return sqlite3.NewDatabase(uri.Path, serverName) default: return nil, fmt.Errorf("Cannot use postgres implementation") } diff --git a/clientapi/auth/storage/devices/storage_wasm.go b/clientapi/auth/storage/devices/storage_wasm.go index 14c19e74b..c89ad887b 100644 --- a/clientapi/auth/storage/devices/storage_wasm.go +++ b/clientapi/auth/storage/devices/storage_wasm.go @@ -36,7 +36,7 @@ func NewDatabase( case "postgres": return nil, fmt.Errorf("Cannot use postgres implementation") case "file": - return sqlite3.NewDatabase(dataSourceName, serverName) + return sqlite3.NewDatabase(uri.Path, serverName) default: return nil, fmt.Errorf("Cannot use postgres implementation") } diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 1398bc6da..73d381492 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -163,16 +163,16 @@ func main() { cfg := &config.Dendrite{} cfg.SetDefaults() cfg.Kafka.UseNaffka = true - cfg.Database.Account = "file:dendritejs_account.db" - cfg.Database.AppService = "file:dendritejs_appservice.db" - cfg.Database.Device = "file:dendritejs_device.db" - cfg.Database.FederationSender = "file:dendritejs_fedsender.db" - cfg.Database.MediaAPI = "file:dendritejs_mediaapi.db" - cfg.Database.Naffka = "file:dendritejs_naffka.db" - cfg.Database.PublicRoomsAPI = "file:dendritejs_publicrooms.db" - cfg.Database.RoomServer = "file:dendritejs_roomserver.db" - cfg.Database.ServerKey = "file:dendritejs_serverkey.db" - cfg.Database.SyncAPI = "file:dendritejs_syncapi.db" + cfg.Database.Account = "file:/idb/dendritejs_account.db" + cfg.Database.AppService = "file:/idb/dendritejs_appservice.db" + cfg.Database.Device = "file:/idb/dendritejs_device.db" + cfg.Database.FederationSender = "file:/idb/dendritejs_fedsender.db" + cfg.Database.MediaAPI = "file:/idb/dendritejs_mediaapi.db" + cfg.Database.Naffka = "file:/idb/dendritejs_naffka.db" + cfg.Database.PublicRoomsAPI = "file:/idb/dendritejs_publicrooms.db" + cfg.Database.RoomServer = "file:/idb/dendritejs_roomserver.db" + cfg.Database.ServerKey = "file:/idb/dendritejs_serverkey.db" + cfg.Database.SyncAPI = "file:/idb/dendritejs_syncapi.db" cfg.Kafka.Topics.UserUpdates = "user_updates" cfg.Kafka.Topics.OutputTypingEvent = "output_typing_event" cfg.Kafka.Topics.OutputClientData = "output_client_data" diff --git a/federationsender/storage/storage_wasm.go b/federationsender/storage/storage_wasm.go index f593fd448..3d071bfef 100644 --- a/federationsender/storage/storage_wasm.go +++ b/federationsender/storage/storage_wasm.go @@ -33,7 +33,7 @@ func NewDatabase( } switch uri.Scheme { case "file": - return sqlite3.NewDatabase(dataSourceName) + return sqlite3.NewDatabase(uri.Path) case "postgres": return nil, fmt.Errorf("Cannot use postgres implementation") default: diff --git a/go.mod b/go.mod index bbac53eca..151fee27a 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/libp2p/go-libp2p-record v0.1.2 github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 - github.com/matrix-org/go-sqlite3-js v0.0.0-20200326102434-98eda28055bd + github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 github.com/matrix-org/gomatrixserverlib v0.0.0-20200511154227-5cc71d36632b github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f diff --git a/go.sum b/go.sum index d487f42a3..15822b532 100644 --- a/go.sum +++ b/go.sum @@ -354,6 +354,8 @@ github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 h1:eq github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4/go.mod h1:3WluEZ9QXSwU30tWYqktnpC1x9mwZKx1r8uAv8Iq+a4= github.com/matrix-org/go-sqlite3-js v0.0.0-20200326102434-98eda28055bd h1:C1FV4dRKF1uuGK8UH01+IoW6zZpfsTV1MvQimZvt418= github.com/matrix-org/go-sqlite3-js v0.0.0-20200326102434-98eda28055bd/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= +github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 h1:Yb+Wlf/iHhWlLWd+kCgG+Fsg4Dc+xBl7hptfK7lD0zY= +github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bhrnp3Ky1qgx/fzCtCALOoGYylh2tpS9K4= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrixserverlib v0.0.0-20200511154227-5cc71d36632b h1:nAmSc1KvQOumoRTz/LD68KyrB6Q5/6q7CmQ5Bswc2nM= diff --git a/internal/basecomponent/base.go b/internal/basecomponent/base.go index 0682ae0ea..45aa454a1 100644 --- a/internal/basecomponent/base.go +++ b/internal/basecomponent/base.go @@ -264,7 +264,7 @@ func setupNaffka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) { uri, err := url.Parse(string(cfg.Database.Naffka)) if err != nil || uri.Scheme == "file" { - db, err = sqlutil.Open(internal.SQLiteDriverName(), string(cfg.Database.Naffka), nil) + db, err = sqlutil.Open(internal.SQLiteDriverName(), string(uri.Path), nil) if err != nil { logrus.WithError(err).Panic("Failed to open naffka database") } diff --git a/internal/keydb/keydb_wasm.go b/internal/keydb/keydb_wasm.go index 349381ee7..c80329027 100644 --- a/internal/keydb/keydb_wasm.go +++ b/internal/keydb/keydb_wasm.go @@ -41,7 +41,7 @@ func NewDatabase( case "postgres": return nil, fmt.Errorf("Cannot use postgres implementation") case "file": - return sqlite3.NewDatabase(dataSourceName, serverName, serverKey, serverKeyID) + return sqlite3.NewDatabase(uri.Path, serverName, serverKey, serverKeyID) default: return nil, fmt.Errorf("Cannot use postgres implementation") } diff --git a/mediaapi/storage/storage_wasm.go b/mediaapi/storage/storage_wasm.go index 78de2cb82..aa188997f 100644 --- a/mediaapi/storage/storage_wasm.go +++ b/mediaapi/storage/storage_wasm.go @@ -35,7 +35,7 @@ func Open( case "postgres": return nil, fmt.Errorf("Cannot use postgres implementation") case "file": - return sqlite3.Open(dataSourceName) + return sqlite3.Open(uri.Path) default: return nil, fmt.Errorf("Cannot use postgres implementation") } diff --git a/publicroomsapi/storage/storage_wasm.go b/publicroomsapi/storage/storage_wasm.go index d00c339d8..2157ce110 100644 --- a/publicroomsapi/storage/storage_wasm.go +++ b/publicroomsapi/storage/storage_wasm.go @@ -31,7 +31,7 @@ func NewPublicRoomsServerDatabase(dataSourceName string) (Database, error) { case "postgres": return nil, fmt.Errorf("Cannot use postgres implementation") case "file": - return sqlite3.NewPublicRoomsServerDatabase(dataSourceName) + return sqlite3.NewPublicRoomsServerDatabase(uri.Path) default: return nil, fmt.Errorf("Cannot use postgres implementation") } From 3d06fe91f278413bf883d853575239e322402b2e Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 22 May 2020 13:54:04 +0100 Subject: [PATCH 087/200] Fix internal HTTP API calls --- internal/http/http.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/internal/http/http.go b/internal/http/http.go index 3c6475443..d0b4d6c50 100644 --- a/internal/http/http.go +++ b/internal/http/http.go @@ -6,6 +6,8 @@ import ( "encoding/json" "fmt" "net/http" + "net/url" + "strings" opentracing "github.com/opentracing/opentracing-go" "github.com/opentracing/opentracing-go/ext" @@ -21,6 +23,14 @@ func PostJSON( return err } + parsedAPIURL, err := url.Parse(apiURL) + if err != nil { + return err + } + + parsedAPIURL.Path = "/api/" + strings.TrimLeft(parsedAPIURL.Path, "/") + apiURL = parsedAPIURL.String() + req, err := http.NewRequest(http.MethodPost, apiURL, bytes.NewReader(jsonBytes)) if err != nil { return err @@ -48,10 +58,10 @@ func PostJSON( var errorBody struct { Message string `json:"message"` } - if err = json.NewDecoder(res.Body).Decode(&errorBody); err != nil { - return err + if msgerr := json.NewDecoder(res.Body).Decode(&errorBody); msgerr == nil { + return fmt.Errorf("api: %d from %s: %s", res.StatusCode, apiURL, errorBody.Message) } - return fmt.Errorf("api: %d: %s", res.StatusCode, errorBody.Message) + return fmt.Errorf("api: %d from %s", res.StatusCode, apiURL) } return json.NewDecoder(res.Body).Decode(response) } From 0978630b552b0f88bf5efc1c59860cbeb706606d Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 22 May 2020 14:11:06 +0100 Subject: [PATCH 088/200] Fix monolith room server-federation sender connection --- cmd/dendrite-monolith-server/main.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index ded0934ae..f9993774f 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -71,9 +71,10 @@ func main() { federation := base.CreateFederationClient() keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) - rsAPI := roomserver.SetupRoomServerComponent( + rsComponent := roomserver.SetupRoomServerComponent( base, keyRing, federation, ) + rsAPI := rsComponent if base.EnableHTTPAPIs { rsAPI = base.CreateHTTPRoomserverAPIs() } @@ -98,7 +99,7 @@ func main() { if base.EnableHTTPAPIs { fsAPI = base.CreateHTTPFederationSenderAPIs() } - rsAPI.SetFederationSenderAPI(fsAPI) + rsComponent.SetFederationSenderAPI(fsAPI) clientapi.SetupClientAPIComponent( base, deviceDB, accountDB, From 06d5f1e6dcd9f3e1db92880c7c3f4068cc92ab99 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 22 May 2020 14:14:39 +0100 Subject: [PATCH 089/200] Fix API paths --- appservice/api/query.go | 4 ++-- eduserver/api/input.go | 2 +- federationsender/api/perform.go | 6 +++--- federationsender/api/query.go | 4 ++-- internal/http/http.go | 10 ---------- syncapi/api/query.go | 8 ++++---- 6 files changed, 12 insertions(+), 22 deletions(-) diff --git a/appservice/api/query.go b/appservice/api/query.go index 4a4e31a99..d36e138cb 100644 --- a/appservice/api/query.go +++ b/appservice/api/query.go @@ -84,10 +84,10 @@ type AppServiceQueryAPI interface { } // AppServiceRoomAliasExistsPath is the HTTP path for the RoomAliasExists API -const AppServiceRoomAliasExistsPath = "/api/appservice/RoomAliasExists" +const AppServiceRoomAliasExistsPath = "/appservice/RoomAliasExists" // AppServiceUserIDExistsPath is the HTTP path for the UserIDExists API -const AppServiceUserIDExistsPath = "/api/appservice/UserIDExists" +const AppServiceUserIDExistsPath = "/appservice/UserIDExists" // httpAppServiceQueryAPI contains the URL to an appservice query API and a // reference to a httpClient used to reach it diff --git a/eduserver/api/input.go b/eduserver/api/input.go index 999a5b41a..8b5b6d76a 100644 --- a/eduserver/api/input.go +++ b/eduserver/api/input.go @@ -55,7 +55,7 @@ type EDUServerInputAPI interface { } // EDUServerInputTypingEventPath is the HTTP path for the InputTypingEvent API. -const EDUServerInputTypingEventPath = "/api/eduserver/input" +const EDUServerInputTypingEventPath = "/eduserver/input" // NewEDUServerInputAPIHTTP creates a EDUServerInputAPI implemented by talking to a HTTP POST API. func NewEDUServerInputAPIHTTP(eduServerURL string, httpClient *http.Client) (EDUServerInputAPI, error) { diff --git a/federationsender/api/perform.go b/federationsender/api/perform.go index 427e70683..a73ba0475 100644 --- a/federationsender/api/perform.go +++ b/federationsender/api/perform.go @@ -11,13 +11,13 @@ import ( const ( // FederationSenderPerformJoinRequestPath is the HTTP path for the PerformJoinRequest API. - FederationSenderPerformDirectoryLookupRequestPath = "/api/federationsender/performDirectoryLookup" + FederationSenderPerformDirectoryLookupRequestPath = "/federationsender/performDirectoryLookup" // FederationSenderPerformJoinRequestPath is the HTTP path for the PerformJoinRequest API. - FederationSenderPerformJoinRequestPath = "/api/federationsender/performJoinRequest" + FederationSenderPerformJoinRequestPath = "/federationsender/performJoinRequest" // FederationSenderPerformLeaveRequestPath is the HTTP path for the PerformLeaveRequest API. - FederationSenderPerformLeaveRequestPath = "/api/federationsender/performLeaveRequest" + FederationSenderPerformLeaveRequestPath = "/federationsender/performLeaveRequest" ) type PerformDirectoryLookupRequest struct { diff --git a/federationsender/api/query.go b/federationsender/api/query.go index bf58d5cc9..4c0f757b2 100644 --- a/federationsender/api/query.go +++ b/federationsender/api/query.go @@ -11,10 +11,10 @@ import ( ) // FederationSenderQueryJoinedHostsInRoomPath is the HTTP path for the QueryJoinedHostsInRoom API. -const FederationSenderQueryJoinedHostsInRoomPath = "/api/federationsender/queryJoinedHostsInRoom" +const FederationSenderQueryJoinedHostsInRoomPath = "/federationsender/queryJoinedHostsInRoom" // FederationSenderQueryJoinedHostServerNamesInRoomPath is the HTTP path for the QueryJoinedHostServerNamesInRoom API. -const FederationSenderQueryJoinedHostServerNamesInRoomPath = "/api/federationsender/queryJoinedHostServerNamesInRoom" +const FederationSenderQueryJoinedHostServerNamesInRoomPath = "/federationsender/queryJoinedHostServerNamesInRoom" // QueryJoinedHostsInRoomRequest is a request to QueryJoinedHostsInRoom type QueryJoinedHostsInRoomRequest struct { diff --git a/internal/http/http.go b/internal/http/http.go index d0b4d6c50..77896a53f 100644 --- a/internal/http/http.go +++ b/internal/http/http.go @@ -6,8 +6,6 @@ import ( "encoding/json" "fmt" "net/http" - "net/url" - "strings" opentracing "github.com/opentracing/opentracing-go" "github.com/opentracing/opentracing-go/ext" @@ -23,14 +21,6 @@ func PostJSON( return err } - parsedAPIURL, err := url.Parse(apiURL) - if err != nil { - return err - } - - parsedAPIURL.Path = "/api/" + strings.TrimLeft(parsedAPIURL.Path, "/") - apiURL = parsedAPIURL.String() - req, err := http.NewRequest(http.MethodPost, apiURL, bytes.NewReader(jsonBytes)) if err != nil { return err diff --git a/syncapi/api/query.go b/syncapi/api/query.go index 6585bc093..1da05fcc8 100644 --- a/syncapi/api/query.go +++ b/syncapi/api/query.go @@ -24,10 +24,10 @@ import ( ) const ( - SyncAPIQuerySyncPath = "/api/syncapi/querySync" - SyncAPIQueryStatePath = "/api/syncapi/queryState" - SyncAPIQueryStateTypePath = "/api/syncapi/queryStateType" - SyncAPIQueryMessagesPath = "/api/syncapi/queryMessages" + SyncAPIQuerySyncPath = "/syncapi/querySync" + SyncAPIQueryStatePath = "/syncapi/queryState" + SyncAPIQueryStateTypePath = "/syncapi/queryStateType" + SyncAPIQueryMessagesPath = "/syncapi/queryMessages" ) func NewSyncQueryAPIHTTP(syncapiURL string, httpClient *http.Client) SyncQueryAPI { From 3c3e014901fd03ea46ff4c5dfa125c6aa148d5ca Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 22 May 2020 14:18:41 +0100 Subject: [PATCH 090/200] Define path prefixes in a package that doesn't create import cycles --- internal/basecomponent/base.go | 5 +++-- internal/http/http.go | 11 +++++++++++ internal/httpapi.go | 10 +++------- internal/httpapis/paths.go | 6 ++++++ 4 files changed, 23 insertions(+), 9 deletions(-) create mode 100644 internal/httpapis/paths.go diff --git a/internal/basecomponent/base.go b/internal/basecomponent/base.go index 45aa454a1..3781cea48 100644 --- a/internal/basecomponent/base.go +++ b/internal/basecomponent/base.go @@ -24,6 +24,7 @@ import ( "golang.org/x/crypto/ed25519" "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/httpapis" "github.com/matrix-org/dendrite/internal/keydb" "github.com/matrix-org/dendrite/internal/keydb/cache" "github.com/matrix-org/dendrite/internal/sqlutil" @@ -103,8 +104,8 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, enableHTTPAPIs tracerCloser: closer, Cfg: cfg, ImmutableCache: cache, - PublicAPIMux: httpmux.PathPrefix(internal.HTTPPublicPathPrefix).Subrouter().UseEncodedPath(), - InternalAPIMux: httpmux.PathPrefix(internal.HTTPInternalPathPrefix).Subrouter().UseEncodedPath(), + PublicAPIMux: httpmux.PathPrefix(httpapis.PublicPathPrefix).Subrouter().UseEncodedPath(), + InternalAPIMux: httpmux.PathPrefix(httpapis.InternalPathPrefix).Subrouter().UseEncodedPath(), httpClient: &http.Client{Timeout: HTTPClientTimeout}, KafkaConsumer: kafkaConsumer, KafkaProducer: kafkaProducer, diff --git a/internal/http/http.go b/internal/http/http.go index 77896a53f..da90b3751 100644 --- a/internal/http/http.go +++ b/internal/http/http.go @@ -6,7 +6,10 @@ import ( "encoding/json" "fmt" "net/http" + "net/url" + "strings" + "github.com/matrix-org/dendrite/internal/httpapis" opentracing "github.com/opentracing/opentracing-go" "github.com/opentracing/opentracing-go/ext" ) @@ -21,6 +24,14 @@ func PostJSON( return err } + parsedAPIURL, err := url.Parse(apiURL) + if err != nil { + return err + } + + parsedAPIURL.Path = httpapis.InternalPathPrefix + strings.TrimLeft(parsedAPIURL.Path, "/") + apiURL = parsedAPIURL.String() + req, err := http.NewRequest(http.MethodPost, apiURL, bytes.NewReader(jsonBytes)) if err != nil { return err diff --git a/internal/httpapi.go b/internal/httpapi.go index 526ff8a2d..07bbacdd6 100644 --- a/internal/httpapi.go +++ b/internal/httpapi.go @@ -13,6 +13,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/httpapis" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" opentracing "github.com/opentracing/opentracing-go" @@ -23,11 +24,6 @@ import ( "github.com/sirupsen/logrus" ) -const ( - HTTPPublicPathPrefix = "/_matrix/" - HTTPInternalPathPrefix = "/api/" -) - // BasicAuth is used for authorization on /metrics handlers type BasicAuth struct { Username string `yaml:"username"` @@ -195,9 +191,9 @@ func SetupHTTPAPI(servMux *http.ServeMux, publicApiMux *mux.Router, internalApiM servMux.Handle("/metrics", WrapHandlerInBasicAuth(promhttp.Handler(), cfg.Metrics.BasicAuth)) } if enableHTTPAPIs { - servMux.Handle(HTTPInternalPathPrefix, internalApiMux) + servMux.Handle(httpapis.InternalPathPrefix, internalApiMux) } - servMux.Handle(HTTPPublicPathPrefix, WrapHandlerInCORS(publicApiMux)) + servMux.Handle(httpapis.PublicPathPrefix, WrapHandlerInCORS(publicApiMux)) } // WrapHandlerInBasicAuth adds basic auth to a handler. Only used for /metrics diff --git a/internal/httpapis/paths.go b/internal/httpapis/paths.go new file mode 100644 index 000000000..8adec2dff --- /dev/null +++ b/internal/httpapis/paths.go @@ -0,0 +1,6 @@ +package httpapis + +const ( + PublicPathPrefix = "/_matrix/" + InternalPathPrefix = "/api/" +) From 492af0f2ec3850d41dd02c9c512de8361d50d271 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 25 May 2020 15:29:49 +0100 Subject: [PATCH 091/200] Use Opaque in addition to Path to set naffka DB names --- internal/basecomponent/base.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/internal/basecomponent/base.go b/internal/basecomponent/base.go index 3781cea48..e86d91025 100644 --- a/internal/basecomponent/base.go +++ b/internal/basecomponent/base.go @@ -265,7 +265,15 @@ func setupNaffka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) { uri, err := url.Parse(string(cfg.Database.Naffka)) if err != nil || uri.Scheme == "file" { - db, err = sqlutil.Open(internal.SQLiteDriverName(), string(uri.Path), nil) + var cs string + if uri.Opaque != "" { // file:filename.db + cs = uri.Opaque + } else if uri.Path != "" { // file:///path/to/filename.db + cs = uri.Path + } else { + logrus.Panic("file uri has no filename") + } + db, err = sqlutil.Open(internal.SQLiteDriverName(), cs, nil) if err != nil { logrus.WithError(err).Panic("Failed to open naffka database") } From 6d50212f29f2ce3231097833de7686e3237359b4 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 26 May 2020 14:41:16 +0100 Subject: [PATCH 092/200] Miscellaneous fixes (#1060) * Add missing routing for PerformDirectoryLookupRequest * Tweak output * Fix some bugs in devices * Don't default to federated room joins in response to invite * Update sytest-whitelist * Update comments * Return correct room ID from PerformJoin * Fix appservice and EDU server API setup, update sytest-whitelist * Update sytest-whitelist --- appservice/appservice.go | 4 +-- appservice/query/query.go | 7 ++-- .../storage/devices/postgres/devices_table.go | 12 +++++-- .../storage/devices/sqlite3/devices_table.go | 12 +++++-- clientapi/routing/joinroom.go | 2 +- eduserver/eduserver.go | 6 +--- eduserver/input/input.go | 5 +-- federationsender/internal/api.go | 13 +++++++ internal/http/http.go | 4 +-- roomserver/api/perform.go | 1 + roomserver/internal/perform_join.go | 36 +++++++++++++------ sytest-whitelist | 1 + 12 files changed, 71 insertions(+), 32 deletions(-) diff --git a/appservice/appservice.go b/appservice/appservice.go index 76181d2a3..be5b30e2b 100644 --- a/appservice/appservice.go +++ b/appservice/appservice.go @@ -82,9 +82,7 @@ func SetupAppServiceAPIComponent( Cfg: base.Cfg, } - if base.EnableHTTPAPIs { - appserviceQueryAPI.SetupHTTP(http.DefaultServeMux) - } + appserviceQueryAPI.SetupHTTP(base.InternalAPIMux) consumer := consumers.NewOutputRoomEventConsumer( base.Cfg, base.KafkaConsumer, accountsDB, appserviceDB, diff --git a/appservice/query/query.go b/appservice/query/query.go index a61997b42..812ca9f49 100644 --- a/appservice/query/query.go +++ b/appservice/query/query.go @@ -23,6 +23,7 @@ import ( "net/url" "time" + "github.com/gorilla/mux" "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" @@ -182,8 +183,8 @@ func makeHTTPClient() *http.Client { // SetupHTTP adds the AppServiceQueryPAI handlers to the http.ServeMux. This // handles and muxes incoming api requests the to internal AppServiceQueryAPI. -func (a *AppServiceQueryAPI) SetupHTTP(servMux *http.ServeMux) { - servMux.Handle( +func (a *AppServiceQueryAPI) SetupHTTP(internalAPIMux *mux.Router) { + internalAPIMux.Handle( api.AppServiceRoomAliasExistsPath, internal.MakeInternalAPI("appserviceRoomAliasExists", func(req *http.Request) util.JSONResponse { var request api.RoomAliasExistsRequest @@ -197,7 +198,7 @@ func (a *AppServiceQueryAPI) SetupHTTP(servMux *http.ServeMux) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - servMux.Handle( + internalAPIMux.Handle( api.AppServiceUserIDExistsPath, internal.MakeInternalAPI("appserviceUserIDExists", func(req *http.Request) util.JSONResponse { var request api.UserIDExistsRequest diff --git a/clientapi/auth/storage/devices/postgres/devices_table.go b/clientapi/auth/storage/devices/postgres/devices_table.go index 67d573f95..c84e83d3d 100644 --- a/clientapi/auth/storage/devices/postgres/devices_table.go +++ b/clientapi/auth/storage/devices/postgres/devices_table.go @@ -206,9 +206,8 @@ func (s *devicesStatements) selectDeviceByID( ctx context.Context, localpart, deviceID string, ) (*authtypes.Device, error) { var dev authtypes.Device - var created sql.NullInt64 stmt := s.selectDeviceByIDStmt - err := stmt.QueryRowContext(ctx, localpart, deviceID).Scan(&created) + err := stmt.QueryRowContext(ctx, localpart, deviceID).Scan(&dev.DisplayName) if err == nil { dev.ID = deviceID dev.UserID = userutil.MakeUserID(localpart, s.serverName) @@ -230,10 +229,17 @@ func (s *devicesStatements) selectDevicesByLocalpart( for rows.Next() { var dev authtypes.Device - err = rows.Scan(&dev.ID, &dev.DisplayName) + var id, displayname sql.NullString + err = rows.Scan(&id, &displayname) if err != nil { return devices, err } + if id.Valid { + dev.ID = id.String + } + if displayname.Valid { + dev.DisplayName = displayname.String + } dev.UserID = userutil.MakeUserID(localpart, s.serverName) devices = append(devices, dev) } diff --git a/clientapi/auth/storage/devices/sqlite3/devices_table.go b/clientapi/auth/storage/devices/sqlite3/devices_table.go index 6ffc1646f..49f3eaed7 100644 --- a/clientapi/auth/storage/devices/sqlite3/devices_table.go +++ b/clientapi/auth/storage/devices/sqlite3/devices_table.go @@ -208,9 +208,8 @@ func (s *devicesStatements) selectDeviceByID( ctx context.Context, localpart, deviceID string, ) (*authtypes.Device, error) { var dev authtypes.Device - var created sql.NullInt64 stmt := s.selectDeviceByIDStmt - err := stmt.QueryRowContext(ctx, localpart, deviceID).Scan(&created) + err := stmt.QueryRowContext(ctx, localpart, deviceID).Scan(&dev.DisplayName) if err == nil { dev.ID = deviceID dev.UserID = userutil.MakeUserID(localpart, s.serverName) @@ -231,10 +230,17 @@ func (s *devicesStatements) selectDevicesByLocalpart( for rows.Next() { var dev authtypes.Device - err = rows.Scan(&dev.ID, &dev.DisplayName) + var id, displayname sql.NullString + err = rows.Scan(&id, &displayname) if err != nil { return devices, err } + if id.Valid { + dev.ID = id.String + } + if displayname.Valid { + dev.DisplayName = displayname.String + } dev.UserID = userutil.MakeUserID(localpart, s.serverName) devices = append(devices, dev) } diff --git a/clientapi/routing/joinroom.go b/clientapi/routing/joinroom.go index 500df337a..a3d676532 100644 --- a/clientapi/routing/joinroom.go +++ b/clientapi/routing/joinroom.go @@ -75,6 +75,6 @@ func JoinRoomByIDOrAlias( // TODO: Put the response struct somewhere internal. JSON: struct { RoomID string `json:"room_id"` - }{joinReq.RoomIDOrAlias}, + }{joinRes.RoomID}, } } diff --git a/eduserver/eduserver.go b/eduserver/eduserver.go index cba60b8ea..14fbd3328 100644 --- a/eduserver/eduserver.go +++ b/eduserver/eduserver.go @@ -13,8 +13,6 @@ package eduserver import ( - "net/http" - "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/eduserver/input" @@ -35,9 +33,7 @@ func SetupEDUServerComponent( OutputTypingEventTopic: string(base.Cfg.Kafka.Topics.OutputTypingEvent), } - if base.EnableHTTPAPIs { - inputAPI.SetupHTTP(http.DefaultServeMux) - } + inputAPI.SetupHTTP(base.InternalAPIMux) return inputAPI } diff --git a/eduserver/input/input.go b/eduserver/input/input.go index 50837154a..73777e323 100644 --- a/eduserver/input/input.go +++ b/eduserver/input/input.go @@ -19,6 +19,7 @@ import ( "time" "github.com/Shopify/sarama" + "github.com/gorilla/mux" "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/internal" @@ -90,8 +91,8 @@ func (t *EDUServerInputAPI) sendEvent(ite *api.InputTypingEvent) error { } // SetupHTTP adds the EDUServerInputAPI handlers to the http.ServeMux. -func (t *EDUServerInputAPI) SetupHTTP(servMux *http.ServeMux) { - servMux.Handle(api.EDUServerInputTypingEventPath, +func (t *EDUServerInputAPI) SetupHTTP(internalAPIMux *mux.Router) { + internalAPIMux.Handle(api.EDUServerInputTypingEventPath, internal.MakeInternalAPI("inputTypingEvents", func(req *http.Request) util.JSONResponse { var request api.InputTypingEventRequest var response api.InputTypingEventResponse diff --git a/federationsender/internal/api.go b/federationsender/internal/api.go index 4805e7050..dd3942583 100644 --- a/federationsender/internal/api.go +++ b/federationsender/internal/api.go @@ -99,4 +99,17 @@ func (f *FederationSenderInternalAPI) SetupHTTP(internalAPIMux *mux.Router) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) + internalAPIMux.Handle(api.FederationSenderPerformDirectoryLookupRequestPath, + internal.MakeInternalAPI("PerformDirectoryLookupRequest", func(req *http.Request) util.JSONResponse { + var request api.PerformDirectoryLookupRequest + var response api.PerformDirectoryLookupResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := f.PerformDirectoryLookup(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) } diff --git a/internal/http/http.go b/internal/http/http.go index da90b3751..2b1891403 100644 --- a/internal/http/http.go +++ b/internal/http/http.go @@ -60,9 +60,9 @@ func PostJSON( Message string `json:"message"` } if msgerr := json.NewDecoder(res.Body).Decode(&errorBody); msgerr == nil { - return fmt.Errorf("api: %d from %s: %s", res.StatusCode, apiURL, errorBody.Message) + return fmt.Errorf("Internal API: %d from %s: %s", res.StatusCode, apiURL, errorBody.Message) } - return fmt.Errorf("api: %d from %s", res.StatusCode, apiURL) + return fmt.Errorf("Internal API: %d from %s", res.StatusCode, apiURL) } return json.NewDecoder(res.Body).Decode(response) } diff --git a/roomserver/api/perform.go b/roomserver/api/perform.go index c98f91fb4..f5afd67bf 100644 --- a/roomserver/api/perform.go +++ b/roomserver/api/perform.go @@ -24,6 +24,7 @@ type PerformJoinRequest struct { } type PerformJoinResponse struct { + RoomID string `json:"room_id"` } func (h *httpRoomserverInternalAPI) PerformJoin( diff --git a/roomserver/internal/perform_join.go b/roomserver/internal/perform_join.go index dc786b296..75dfdd19c 100644 --- a/roomserver/internal/perform_join.go +++ b/roomserver/internal/perform_join.go @@ -90,6 +90,12 @@ func (r *RoomserverInternalAPI) performJoinRoomByID( req *api.PerformJoinRequest, res *api.PerformJoinResponse, // nolint:unparam ) error { + // By this point, if req.RoomIDOrAlias contained an alias, then + // it will have been overwritten with a room ID by performJoinRoomByAlias. + // We should now include this in the response so that the CS API can + // return the right room ID. + res.RoomID = req.RoomIDOrAlias + // Get the domain part of the room ID. _, domain, err := gomatrixserverlib.SplitID('!', req.RoomIDOrAlias) if err != nil { @@ -121,20 +127,30 @@ func (r *RoomserverInternalAPI) performJoinRoomByID( return fmt.Errorf("eb.SetContent: %w", err) } - // First work out if this is in response to an existing invite. - // If it is then we avoid the situation where we might think we - // know about a room in the following section but don't know the - // latest state as all of our users have left. + // First work out if this is in response to an existing invite + // from a federated server. If it is then we avoid the situation + // where we might think we know about a room in the following + // section but don't know the latest state as all of our users + // have left. isInvitePending, inviteSender, err := r.isInvitePending(ctx, req.RoomIDOrAlias, req.UserID) if err == nil && isInvitePending { - // Add the server of the person who invited us to the server list, - // as they should be a fairly good bet. - if _, inviterDomain, ierr := gomatrixserverlib.SplitID('@', inviteSender); ierr == nil { - req.ServerNames = append(req.ServerNames, inviterDomain) + // Check if there's an invite pending. + _, inviterDomain, ierr := gomatrixserverlib.SplitID('@', inviteSender) + if ierr != nil { + return fmt.Errorf("gomatrixserverlib.SplitID: %w", err) } - // Perform a federated room join. - return r.performFederatedJoinRoomByID(ctx, req, res) + // Check that the domain isn't ours. If it's local then we don't + // need to do anything as our own copy of the room state will be + // up-to-date. + if inviterDomain != r.Cfg.Matrix.ServerName { + // Add the server of the person who invited us to the server list, + // as they should be a fairly good bet. + req.ServerNames = append(req.ServerNames, inviterDomain) + + // Perform a federated room join. + return r.performFederatedJoinRoomByID(ctx, req, res) + } } // Try to construct an actual join event from the template. diff --git a/sytest-whitelist b/sytest-whitelist index 1b12aba1b..d4e6be9a4 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -288,3 +288,4 @@ New room members see their own join event Existing members see new members' join events Inbound federation can receive events Inbound federation can receive redacted events +Can logout current device From 737c83e0ae496449327ef596811e984c2752e39b Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 26 May 2020 15:42:42 +0100 Subject: [PATCH 093/200] roomserver: start refactoring storage layer (#1061) * Begin refactoring out roomserver/storage/*/storage.go * Convert event_types table --- .../postgres/event_state_keys_table.go | 18 ++--- .../storage/postgres/event_types_table.go | 24 ++++--- roomserver/storage/postgres/sql.go | 2 - roomserver/storage/postgres/storage.go | 69 +++++++++---------- roomserver/storage/shared/storage.go | 34 +++++++++ .../storage/sqlite3/event_state_keys_table.go | 26 +++---- .../storage/sqlite3/event_types_table.go | 21 +++--- roomserver/storage/sqlite3/sql.go | 2 - roomserver/storage/sqlite3/storage.go | 66 +++++++----------- roomserver/storage/tables/interface.go | 21 ++++++ 10 files changed, 162 insertions(+), 121 deletions(-) create mode 100644 roomserver/storage/shared/storage.go create mode 100644 roomserver/storage/tables/interface.go diff --git a/roomserver/storage/postgres/event_state_keys_table.go b/roomserver/storage/postgres/event_state_keys_table.go index b213e057b..81b9b06e8 100644 --- a/roomserver/storage/postgres/event_state_keys_table.go +++ b/roomserver/storage/postgres/event_state_keys_table.go @@ -21,6 +21,7 @@ import ( "github.com/lib/pq" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -74,12 +75,13 @@ type eventStateKeyStatements struct { bulkSelectEventStateKeyStmt *sql.Stmt } -func (s *eventStateKeyStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(eventStateKeysSchema) +func NewPostgresEventStateKeysTable(db *sql.DB) (tables.EventStateKeys, error) { + s := &eventStateKeyStatements{} + _, err := db.Exec(eventStateKeysSchema) if err != nil { - return + return nil, err } - return statementList{ + return s, statementList{ {&s.insertEventStateKeyNIDStmt, insertEventStateKeyNIDSQL}, {&s.selectEventStateKeyNIDStmt, selectEventStateKeyNIDSQL}, {&s.bulkSelectEventStateKeyNIDStmt, bulkSelectEventStateKeyNIDSQL}, @@ -87,7 +89,7 @@ func (s *eventStateKeyStatements) prepare(db *sql.DB) (err error) { }.prepare(db) } -func (s *eventStateKeyStatements) insertEventStateKeyNID( +func (s *eventStateKeyStatements) InsertEventStateKeyNID( ctx context.Context, txn *sql.Tx, eventStateKey string, ) (types.EventStateKeyNID, error) { var eventStateKeyNID int64 @@ -96,7 +98,7 @@ func (s *eventStateKeyStatements) insertEventStateKeyNID( return types.EventStateKeyNID(eventStateKeyNID), err } -func (s *eventStateKeyStatements) selectEventStateKeyNID( +func (s *eventStateKeyStatements) SelectEventStateKeyNID( ctx context.Context, txn *sql.Tx, eventStateKey string, ) (types.EventStateKeyNID, error) { var eventStateKeyNID int64 @@ -105,7 +107,7 @@ func (s *eventStateKeyStatements) selectEventStateKeyNID( return types.EventStateKeyNID(eventStateKeyNID), err } -func (s *eventStateKeyStatements) bulkSelectEventStateKeyNID( +func (s *eventStateKeyStatements) BulkSelectEventStateKeyNID( ctx context.Context, eventStateKeys []string, ) (map[string]types.EventStateKeyNID, error) { rows, err := s.bulkSelectEventStateKeyNIDStmt.QueryContext( @@ -128,7 +130,7 @@ func (s *eventStateKeyStatements) bulkSelectEventStateKeyNID( return result, rows.Err() } -func (s *eventStateKeyStatements) bulkSelectEventStateKey( +func (s *eventStateKeyStatements) BulkSelectEventStateKey( ctx context.Context, eventStateKeyNIDs []types.EventStateKeyNID, ) (map[types.EventStateKeyNID]string, error) { nIDs := make(pq.Int64Array, len(eventStateKeyNIDs)) diff --git a/roomserver/storage/postgres/event_types_table.go b/roomserver/storage/postgres/event_types_table.go index 2b0910e71..aaba614a4 100644 --- a/roomserver/storage/postgres/event_types_table.go +++ b/roomserver/storage/postgres/event_types_table.go @@ -22,6 +22,7 @@ import ( "github.com/matrix-org/dendrite/internal" "github.com/lib/pq" + "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -98,36 +99,37 @@ type eventTypeStatements struct { bulkSelectEventTypeNIDStmt *sql.Stmt } -func (s *eventTypeStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(eventTypesSchema) +func NewPostgresEventTypesTable(db *sql.DB) (tables.EventTypes, error) { + s := &eventTypeStatements{} + _, err := db.Exec(eventTypesSchema) if err != nil { - return + return nil, err } - return statementList{ + return s, statementList{ {&s.insertEventTypeNIDStmt, insertEventTypeNIDSQL}, {&s.selectEventTypeNIDStmt, selectEventTypeNIDSQL}, {&s.bulkSelectEventTypeNIDStmt, bulkSelectEventTypeNIDSQL}, }.prepare(db) } -func (s *eventTypeStatements) insertEventTypeNID( - ctx context.Context, eventType string, +func (s *eventTypeStatements) InsertEventTypeNID( + ctx context.Context, txn *sql.Tx, eventType string, ) (types.EventTypeNID, error) { var eventTypeNID int64 - err := s.insertEventTypeNIDStmt.QueryRowContext(ctx, eventType).Scan(&eventTypeNID) + err := txn.Stmt(s.insertEventTypeNIDStmt).QueryRowContext(ctx, eventType).Scan(&eventTypeNID) return types.EventTypeNID(eventTypeNID), err } -func (s *eventTypeStatements) selectEventTypeNID( - ctx context.Context, eventType string, +func (s *eventTypeStatements) SelectEventTypeNID( + ctx context.Context, txn *sql.Tx, eventType string, ) (types.EventTypeNID, error) { var eventTypeNID int64 - err := s.selectEventTypeNIDStmt.QueryRowContext(ctx, eventType).Scan(&eventTypeNID) + err := txn.Stmt(s.selectEventTypeNIDStmt).QueryRowContext(ctx, eventType).Scan(&eventTypeNID) return types.EventTypeNID(eventTypeNID), err } -func (s *eventTypeStatements) bulkSelectEventTypeNID( +func (s *eventTypeStatements) BulkSelectEventTypeNID( ctx context.Context, eventTypes []string, ) (map[string]types.EventTypeNID, error) { rows, err := s.bulkSelectEventTypeNIDStmt.QueryContext(ctx, pq.StringArray(eventTypes)) diff --git a/roomserver/storage/postgres/sql.go b/roomserver/storage/postgres/sql.go index 5956886ce..e41c5a398 100644 --- a/roomserver/storage/postgres/sql.go +++ b/roomserver/storage/postgres/sql.go @@ -38,8 +38,6 @@ func (s *statements) prepare(db *sql.DB) error { var err error for _, prepare := range []func(db *sql.DB) error{ - s.eventTypeStatements.prepare, - s.eventStateKeyStatements.prepare, s.roomStatements.prepare, s.eventStatements.prepare, s.eventJSONStatements.prepare, diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go index 4d1d603e3..6fcceced5 100644 --- a/roomserver/storage/postgres/storage.go +++ b/roomserver/storage/postgres/storage.go @@ -26,14 +26,19 @@ import ( // Import the postgres database driver. _ "github.com/lib/pq" "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/storage/shared" + "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" ) // A Database is used to store room events and stream offsets. type Database struct { - statements statements - db *sql.DB + shared.Database + statements statements + eventTypes tables.EventTypes + eventStateKeys tables.EventStateKeys + db *sql.DB } // Open a postgres database. @@ -46,6 +51,18 @@ func Open(dataSourceName string, dbProperties internal.DbProperties) (*Database, if err = d.statements.prepare(d.db); err != nil { return nil, err } + d.eventStateKeys, err = NewPostgresEventStateKeysTable(d.db) + if err != nil { + return nil, err + } + d.eventTypes, err = NewPostgresEventTypesTable(d.db) + if err != nil { + return nil, err + } + d.Database = shared.Database{ + EventTypesTable: d.eventTypes, + EventStateKeysTable: d.eventStateKeys, + } return &d, nil } @@ -180,17 +197,20 @@ func (d *Database) assignRoomNID( func (d *Database) assignEventTypeNID( ctx context.Context, eventType string, -) (types.EventTypeNID, error) { - // Check if we already have a numeric ID in the database. - eventTypeNID, err := d.statements.selectEventTypeNID(ctx, eventType) - if err == sql.ErrNoRows { - // We don't have a numeric ID so insert one into the database. - eventTypeNID, err = d.statements.insertEventTypeNID(ctx, eventType) +) (eventTypeNID types.EventTypeNID, err error) { + err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { + // Check if we already have a numeric ID in the database. + eventTypeNID, err = d.eventTypes.SelectEventTypeNID(ctx, txn, eventType) if err == sql.ErrNoRows { - // We raced with another insert so run the select again. - eventTypeNID, err = d.statements.selectEventTypeNID(ctx, eventType) + // We don't have a numeric ID so insert one into the database. + eventTypeNID, err = d.eventTypes.InsertEventTypeNID(ctx, txn, eventType) + if err == sql.ErrNoRows { + // We raced with another insert so run the select again. + eventTypeNID, err = d.eventTypes.SelectEventTypeNID(ctx, txn, eventType) + } } - } + return err + }) return eventTypeNID, err } @@ -198,13 +218,13 @@ func (d *Database) assignStateKeyNID( ctx context.Context, txn *sql.Tx, eventStateKey string, ) (types.EventStateKeyNID, error) { // Check if we already have a numeric ID in the database. - eventStateKeyNID, err := d.statements.selectEventStateKeyNID(ctx, txn, eventStateKey) + eventStateKeyNID, err := d.eventStateKeys.SelectEventStateKeyNID(ctx, txn, eventStateKey) if err == sql.ErrNoRows { // We don't have a numeric ID so insert one into the database. - eventStateKeyNID, err = d.statements.insertEventStateKeyNID(ctx, txn, eventStateKey) + eventStateKeyNID, err = d.eventStateKeys.InsertEventStateKeyNID(ctx, txn, eventStateKey) if err == sql.ErrNoRows { // We raced with another insert so run the select again. - eventStateKeyNID, err = d.statements.selectEventStateKeyNID(ctx, txn, eventStateKey) + eventStateKeyNID, err = d.eventStateKeys.SelectEventStateKeyNID(ctx, txn, eventStateKey) } } return eventStateKeyNID, err @@ -217,27 +237,6 @@ func (d *Database) StateEntriesForEventIDs( return d.statements.bulkSelectStateEventByID(ctx, eventIDs) } -// EventTypeNIDs implements state.RoomStateDatabase -func (d *Database) EventTypeNIDs( - ctx context.Context, eventTypes []string, -) (map[string]types.EventTypeNID, error) { - return d.statements.bulkSelectEventTypeNID(ctx, eventTypes) -} - -// EventStateKeyNIDs implements state.RoomStateDatabase -func (d *Database) EventStateKeyNIDs( - ctx context.Context, eventStateKeys []string, -) (map[string]types.EventStateKeyNID, error) { - return d.statements.bulkSelectEventStateKeyNID(ctx, eventStateKeys) -} - -// EventStateKeys implements query.RoomserverQueryAPIDatabase -func (d *Database) EventStateKeys( - ctx context.Context, eventStateKeyNIDs []types.EventStateKeyNID, -) (map[types.EventStateKeyNID]string, error) { - return d.statements.bulkSelectEventStateKey(ctx, eventStateKeyNIDs) -} - // EventNIDs implements query.RoomserverQueryAPIDatabase func (d *Database) EventNIDs( ctx context.Context, eventIDs []string, diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go new file mode 100644 index 000000000..7a8da8658 --- /dev/null +++ b/roomserver/storage/shared/storage.go @@ -0,0 +1,34 @@ +package shared + +import ( + "context" + + "github.com/matrix-org/dendrite/roomserver/storage/tables" + "github.com/matrix-org/dendrite/roomserver/types" +) + +type Database struct { + EventTypesTable tables.EventTypes + EventStateKeysTable tables.EventStateKeys +} + +// EventTypeNIDs implements state.RoomStateDatabase +func (d *Database) EventTypeNIDs( + ctx context.Context, eventTypes []string, +) (map[string]types.EventTypeNID, error) { + return d.EventTypesTable.BulkSelectEventTypeNID(ctx, eventTypes) +} + +// EventStateKeys implements query.RoomserverQueryAPIDatabase +func (d *Database) EventStateKeys( + ctx context.Context, eventStateKeyNIDs []types.EventStateKeyNID, +) (map[types.EventStateKeyNID]string, error) { + return d.EventStateKeysTable.BulkSelectEventStateKey(ctx, eventStateKeyNIDs) +} + +// EventStateKeyNIDs implements state.RoomStateDatabase +func (d *Database) EventStateKeyNIDs( + ctx context.Context, eventStateKeys []string, +) (map[string]types.EventStateKeyNID, error) { + return d.EventStateKeysTable.BulkSelectEventStateKeyNID(ctx, eventStateKeys) +} diff --git a/roomserver/storage/sqlite3/event_state_keys_table.go b/roomserver/storage/sqlite3/event_state_keys_table.go index f49ebf554..0d3d323fb 100644 --- a/roomserver/storage/sqlite3/event_state_keys_table.go +++ b/roomserver/storage/sqlite3/event_state_keys_table.go @@ -21,6 +21,7 @@ import ( "strings" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -67,13 +68,14 @@ type eventStateKeyStatements struct { bulkSelectEventStateKeyStmt *sql.Stmt } -func (s *eventStateKeyStatements) prepare(db *sql.DB) (err error) { +func NewSqliteEventStateKeysTable(db *sql.DB) (tables.EventStateKeys, error) { + s := &eventStateKeyStatements{} s.db = db - _, err = db.Exec(eventStateKeysSchema) + _, err := db.Exec(eventStateKeysSchema) if err != nil { - return + return nil, err } - return statementList{ + return s, statementList{ {&s.insertEventStateKeyNIDStmt, insertEventStateKeyNIDSQL}, {&s.selectEventStateKeyNIDStmt, selectEventStateKeyNIDSQL}, {&s.bulkSelectEventStateKeyNIDStmt, bulkSelectEventStateKeyNIDSQL}, @@ -81,7 +83,7 @@ func (s *eventStateKeyStatements) prepare(db *sql.DB) (err error) { }.prepare(db) } -func (s *eventStateKeyStatements) insertEventStateKeyNID( +func (s *eventStateKeyStatements) InsertEventStateKeyNID( ctx context.Context, txn *sql.Tx, eventStateKey string, ) (types.EventStateKeyNID, error) { var eventStateKeyNID int64 @@ -94,7 +96,7 @@ func (s *eventStateKeyStatements) insertEventStateKeyNID( return types.EventStateKeyNID(eventStateKeyNID), err } -func (s *eventStateKeyStatements) selectEventStateKeyNID( +func (s *eventStateKeyStatements) SelectEventStateKeyNID( ctx context.Context, txn *sql.Tx, eventStateKey string, ) (types.EventStateKeyNID, error) { var eventStateKeyNID int64 @@ -103,8 +105,8 @@ func (s *eventStateKeyStatements) selectEventStateKeyNID( return types.EventStateKeyNID(eventStateKeyNID), err } -func (s *eventStateKeyStatements) bulkSelectEventStateKeyNID( - ctx context.Context, txn *sql.Tx, eventStateKeys []string, +func (s *eventStateKeyStatements) BulkSelectEventStateKeyNID( + ctx context.Context, eventStateKeys []string, ) (map[string]types.EventStateKeyNID, error) { iEventStateKeys := make([]interface{}, len(eventStateKeys)) for k, v := range eventStateKeys { @@ -112,7 +114,7 @@ func (s *eventStateKeyStatements) bulkSelectEventStateKeyNID( } selectOrig := strings.Replace(bulkSelectEventStateKeySQL, "($1)", internal.QueryVariadic(len(eventStateKeys)), 1) - rows, err := txn.QueryContext(ctx, selectOrig, iEventStateKeys...) + rows, err := s.db.QueryContext(ctx, selectOrig, iEventStateKeys...) if err != nil { return nil, err } @@ -129,8 +131,8 @@ func (s *eventStateKeyStatements) bulkSelectEventStateKeyNID( return result, nil } -func (s *eventStateKeyStatements) bulkSelectEventStateKey( - ctx context.Context, txn *sql.Tx, eventStateKeyNIDs []types.EventStateKeyNID, +func (s *eventStateKeyStatements) BulkSelectEventStateKey( + ctx context.Context, eventStateKeyNIDs []types.EventStateKeyNID, ) (map[types.EventStateKeyNID]string, error) { iEventStateKeyNIDs := make([]interface{}, len(eventStateKeyNIDs)) for k, v := range eventStateKeyNIDs { @@ -138,7 +140,7 @@ func (s *eventStateKeyStatements) bulkSelectEventStateKey( } selectOrig := strings.Replace(bulkSelectEventStateKeyNIDSQL, "($1)", internal.QueryVariadic(len(eventStateKeyNIDs)), 1) - rows, err := txn.QueryContext(ctx, selectOrig, iEventStateKeyNIDs...) + rows, err := s.db.QueryContext(ctx, selectOrig, iEventStateKeyNIDs...) if err != nil { return nil, err } diff --git a/roomserver/storage/sqlite3/event_types_table.go b/roomserver/storage/sqlite3/event_types_table.go index 13abcd4df..d47be5453 100644 --- a/roomserver/storage/sqlite3/event_types_table.go +++ b/roomserver/storage/sqlite3/event_types_table.go @@ -21,6 +21,7 @@ import ( "strings" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -81,14 +82,15 @@ type eventTypeStatements struct { bulkSelectEventTypeNIDStmt *sql.Stmt } -func (s *eventTypeStatements) prepare(db *sql.DB) (err error) { +func NewSqliteEventTypesTable(db *sql.DB) (tables.EventTypes, error) { + s := &eventTypeStatements{} s.db = db - _, err = db.Exec(eventTypesSchema) + _, err := db.Exec(eventTypesSchema) if err != nil { - return + return nil, err } - return statementList{ + return s, statementList{ {&s.insertEventTypeNIDStmt, insertEventTypeNIDSQL}, {&s.insertEventTypeNIDResultStmt, insertEventTypeNIDResultSQL}, {&s.selectEventTypeNIDStmt, selectEventTypeNIDSQL}, @@ -96,7 +98,7 @@ func (s *eventTypeStatements) prepare(db *sql.DB) (err error) { }.prepare(db) } -func (s *eventTypeStatements) insertEventTypeNID( +func (s *eventTypeStatements) InsertEventTypeNID( ctx context.Context, tx *sql.Tx, eventType string, ) (types.EventTypeNID, error) { var eventTypeNID int64 @@ -109,7 +111,7 @@ func (s *eventTypeStatements) insertEventTypeNID( return types.EventTypeNID(eventTypeNID), err } -func (s *eventTypeStatements) selectEventTypeNID( +func (s *eventTypeStatements) SelectEventTypeNID( ctx context.Context, tx *sql.Tx, eventType string, ) (types.EventTypeNID, error) { var eventTypeNID int64 @@ -118,8 +120,8 @@ func (s *eventTypeStatements) selectEventTypeNID( return types.EventTypeNID(eventTypeNID), err } -func (s *eventTypeStatements) bulkSelectEventTypeNID( - ctx context.Context, tx *sql.Tx, eventTypes []string, +func (s *eventTypeStatements) BulkSelectEventTypeNID( + ctx context.Context, eventTypes []string, ) (map[string]types.EventTypeNID, error) { /////////////// iEventTypes := make([]interface{}, len(eventTypes)) @@ -133,8 +135,7 @@ func (s *eventTypeStatements) bulkSelectEventTypeNID( } /////////////// - selectStmt := internal.TxStmt(tx, selectPrep) - rows, err := selectStmt.QueryContext(ctx, iEventTypes...) + rows, err := selectPrep.QueryContext(ctx, iEventTypes...) if err != nil { return nil, err } diff --git a/roomserver/storage/sqlite3/sql.go b/roomserver/storage/sqlite3/sql.go index 0d49432b8..bb3318b2d 100644 --- a/roomserver/storage/sqlite3/sql.go +++ b/roomserver/storage/sqlite3/sql.go @@ -38,8 +38,6 @@ func (s *statements) prepare(db *sql.DB) error { var err error for _, prepare := range []func(db *sql.DB) error{ - s.eventTypeStatements.prepare, - s.eventStateKeyStatements.prepare, s.roomStatements.prepare, s.eventStatements.prepare, s.eventJSONStatements.prepare, diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index bb38f800f..b9157e3a5 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -26,6 +26,8 @@ import ( "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/storage/shared" + "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" _ "github.com/mattn/go-sqlite3" @@ -33,8 +35,11 @@ import ( // A Database is used to store room events and stream offsets. type Database struct { - statements statements - db *sql.DB + shared.Database + statements statements + eventTypes tables.EventTypes + eventStateKeys tables.EventStateKeys + db *sql.DB } // Open a sqlite database. @@ -66,6 +71,18 @@ func Open(dataSourceName string) (*Database, error) { if err = d.statements.prepare(d.db); err != nil { return nil, err } + d.eventStateKeys, err = NewSqliteEventStateKeysTable(d.db) + if err != nil { + return nil, err + } + d.eventTypes, err = NewSqliteEventTypesTable(d.db) + if err != nil { + return nil, err + } + d.Database = shared.Database{ + EventTypesTable: d.eventTypes, + EventStateKeysTable: d.eventStateKeys, + } return &d, nil } @@ -210,13 +227,13 @@ func (d *Database) assignEventTypeNID( ctx context.Context, txn *sql.Tx, eventType string, ) (eventTypeNID types.EventTypeNID, err error) { // Check if we already have a numeric ID in the database. - eventTypeNID, err = d.statements.selectEventTypeNID(ctx, txn, eventType) + eventTypeNID, err = d.eventTypes.SelectEventTypeNID(ctx, txn, eventType) if err == sql.ErrNoRows { // We don't have a numeric ID so insert one into the database. - eventTypeNID, err = d.statements.insertEventTypeNID(ctx, txn, eventType) + eventTypeNID, err = d.eventTypes.InsertEventTypeNID(ctx, txn, eventType) if err == sql.ErrNoRows { // We raced with another insert so run the select again. - eventTypeNID, err = d.statements.selectEventTypeNID(ctx, txn, eventType) + eventTypeNID, err = d.eventTypes.SelectEventTypeNID(ctx, txn, eventType) } } return @@ -226,13 +243,13 @@ func (d *Database) assignStateKeyNID( ctx context.Context, txn *sql.Tx, eventStateKey string, ) (eventStateKeyNID types.EventStateKeyNID, err error) { // Check if we already have a numeric ID in the database. - eventStateKeyNID, err = d.statements.selectEventStateKeyNID(ctx, txn, eventStateKey) + eventStateKeyNID, err = d.eventStateKeys.SelectEventStateKeyNID(ctx, txn, eventStateKey) if err == sql.ErrNoRows { // We don't have a numeric ID so insert one into the database. - eventStateKeyNID, err = d.statements.insertEventStateKeyNID(ctx, txn, eventStateKey) + eventStateKeyNID, err = d.eventStateKeys.InsertEventStateKeyNID(ctx, txn, eventStateKey) if err == sql.ErrNoRows { // We raced with another insert so run the select again. - eventStateKeyNID, err = d.statements.selectEventStateKeyNID(ctx, txn, eventStateKey) + eventStateKeyNID, err = d.eventStateKeys.SelectEventStateKeyNID(ctx, txn, eventStateKey) } } return @@ -249,39 +266,6 @@ func (d *Database) StateEntriesForEventIDs( return } -// EventTypeNIDs implements state.RoomStateDatabase -func (d *Database) EventTypeNIDs( - ctx context.Context, eventTypes []string, -) (etnids map[string]types.EventTypeNID, err error) { - err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { - etnids, err = d.statements.bulkSelectEventTypeNID(ctx, txn, eventTypes) - return err - }) - return -} - -// EventStateKeyNIDs implements state.RoomStateDatabase -func (d *Database) EventStateKeyNIDs( - ctx context.Context, eventStateKeys []string, -) (esknids map[string]types.EventStateKeyNID, err error) { - err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { - esknids, err = d.statements.bulkSelectEventStateKeyNID(ctx, txn, eventStateKeys) - return err - }) - return -} - -// EventStateKeys implements query.RoomserverQueryAPIDatabase -func (d *Database) EventStateKeys( - ctx context.Context, eventStateKeyNIDs []types.EventStateKeyNID, -) (out map[types.EventStateKeyNID]string, err error) { - err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { - out, err = d.statements.bulkSelectEventStateKey(ctx, txn, eventStateKeyNIDs) - return err - }) - return -} - // EventNIDs implements query.RoomserverQueryAPIDatabase func (d *Database) EventNIDs( ctx context.Context, eventIDs []string, diff --git a/roomserver/storage/tables/interface.go b/roomserver/storage/tables/interface.go new file mode 100644 index 000000000..d607865dc --- /dev/null +++ b/roomserver/storage/tables/interface.go @@ -0,0 +1,21 @@ +package tables + +import ( + "context" + "database/sql" + + "github.com/matrix-org/dendrite/roomserver/types" +) + +type EventTypes interface { + InsertEventTypeNID(ctx context.Context, tx *sql.Tx, eventType string) (types.EventTypeNID, error) + SelectEventTypeNID(ctx context.Context, tx *sql.Tx, eventType string) (types.EventTypeNID, error) + BulkSelectEventTypeNID(ctx context.Context, eventTypes []string) (map[string]types.EventTypeNID, error) +} + +type EventStateKeys interface { + InsertEventStateKeyNID(ctx context.Context, txn *sql.Tx, eventStateKey string) (types.EventStateKeyNID, error) + SelectEventStateKeyNID(ctx context.Context, txn *sql.Tx, eventStateKey string) (types.EventStateKeyNID, error) + BulkSelectEventStateKeyNID(ctx context.Context, eventStateKeys []string) (map[string]types.EventStateKeyNID, error) + BulkSelectEventStateKey(ctx context.Context, eventStateKeyNIDs []types.EventStateKeyNID) (map[types.EventStateKeyNID]string, error) +} From 803af87dc49f6db57019892215c6c6cf049b5c50 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 26 May 2020 16:45:28 +0100 Subject: [PATCH 094/200] Convert events/event_json tables to share code (#1062) * Convert event_json table * Convert the events table --- .../storage/postgres/event_json_table.go | 25 ++-- roomserver/storage/postgres/events_table.go | 43 +++---- roomserver/storage/postgres/sql.go | 2 - roomserver/storage/postgres/storage.go | 106 +++++++---------- roomserver/storage/shared/storage.go | 45 ++++++++ .../storage/sqlite3/event_json_table.go | 27 ++--- roomserver/storage/sqlite3/events_table.go | 69 ++++++----- roomserver/storage/sqlite3/sql.go | 2 - roomserver/storage/sqlite3/storage.go | 107 +++++------------- roomserver/storage/tables/interface.go | 36 ++++++ 10 files changed, 226 insertions(+), 236 deletions(-) diff --git a/roomserver/storage/postgres/event_json_table.go b/roomserver/storage/postgres/event_json_table.go index 661c44721..a32629260 100644 --- a/roomserver/storage/postgres/event_json_table.go +++ b/roomserver/storage/postgres/event_json_table.go @@ -21,6 +21,7 @@ import ( "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -58,32 +59,28 @@ type eventJSONStatements struct { bulkSelectEventJSONStmt *sql.Stmt } -func (s *eventJSONStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(eventJSONSchema) +func NewPostgresEventJSONTable(db *sql.DB) (tables.EventJSON, error) { + s := &eventJSONStatements{} + _, err := db.Exec(eventJSONSchema) if err != nil { - return + return nil, err } - return statementList{ + return s, statementList{ {&s.insertEventJSONStmt, insertEventJSONSQL}, {&s.bulkSelectEventJSONStmt, bulkSelectEventJSONSQL}, }.prepare(db) } -func (s *eventJSONStatements) insertEventJSON( - ctx context.Context, eventNID types.EventNID, eventJSON []byte, +func (s *eventJSONStatements) InsertEventJSON( + ctx context.Context, txn *sql.Tx, eventNID types.EventNID, eventJSON []byte, ) error { _, err := s.insertEventJSONStmt.ExecContext(ctx, int64(eventNID), eventJSON) return err } -type eventJSONPair struct { - EventNID types.EventNID - EventJSON []byte -} - -func (s *eventJSONStatements) bulkSelectEventJSON( +func (s *eventJSONStatements) BulkSelectEventJSON( ctx context.Context, eventNIDs []types.EventNID, -) ([]eventJSONPair, error) { +) ([]tables.EventJSONPair, error) { rows, err := s.bulkSelectEventJSONStmt.QueryContext(ctx, eventNIDsAsArray(eventNIDs)) if err != nil { return nil, err @@ -94,7 +91,7 @@ func (s *eventJSONStatements) bulkSelectEventJSON( // because of the unique constraint on event NIDs. // So we can allocate an array of the correct size now. // We might get fewer results than NIDs so we adjust the length of the slice before returning it. - results := make([]eventJSONPair, len(eventNIDs)) + results := make([]tables.EventJSONPair, len(eventNIDs)) i := 0 for ; rows.Next(); i++ { result := &results[i] diff --git a/roomserver/storage/postgres/events_table.go b/roomserver/storage/postgres/events_table.go index c28fa8e6b..9c4649468 100644 --- a/roomserver/storage/postgres/events_table.go +++ b/roomserver/storage/postgres/events_table.go @@ -22,6 +22,7 @@ import ( "github.com/lib/pq" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -136,13 +137,14 @@ type eventStatements struct { selectRoomNIDForEventNIDStmt *sql.Stmt } -func (s *eventStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(eventsSchema) +func NewPostgresEventsTable(db *sql.DB) (tables.Events, error) { + s := &eventStatements{} + _, err := db.Exec(eventsSchema) if err != nil { - return + return nil, err } - return statementList{ + return s, statementList{ {&s.insertEventStmt, insertEventSQL}, {&s.selectEventStmt, selectEventSQL}, {&s.bulkSelectStateEventByIDStmt, bulkSelectStateEventByIDSQL}, @@ -160,8 +162,9 @@ func (s *eventStatements) prepare(db *sql.DB) (err error) { }.prepare(db) } -func (s *eventStatements) insertEvent( +func (s *eventStatements) InsertEvent( ctx context.Context, + txn *sql.Tx, roomNID types.RoomNID, eventTypeNID types.EventTypeNID, eventStateKeyNID types.EventStateKeyNID, @@ -179,8 +182,8 @@ func (s *eventStatements) insertEvent( return types.EventNID(eventNID), types.StateSnapshotNID(stateNID), err } -func (s *eventStatements) selectEvent( - ctx context.Context, eventID string, +func (s *eventStatements) SelectEvent( + ctx context.Context, txn *sql.Tx, eventID string, ) (types.EventNID, types.StateSnapshotNID, error) { var eventNID int64 var stateNID int64 @@ -190,7 +193,7 @@ func (s *eventStatements) selectEvent( // bulkSelectStateEventByID lookups a list of state events by event ID. // If any of the requested events are missing from the database it returns a types.MissingEventError -func (s *eventStatements) bulkSelectStateEventByID( +func (s *eventStatements) BulkSelectStateEventByID( ctx context.Context, eventIDs []string, ) ([]types.StateEntry, error) { rows, err := s.bulkSelectStateEventByIDStmt.QueryContext(ctx, pq.StringArray(eventIDs)) @@ -233,7 +236,7 @@ func (s *eventStatements) bulkSelectStateEventByID( // bulkSelectStateAtEventByID lookups the state at a list of events by event ID. // If any of the requested events are missing from the database it returns a types.MissingEventError. // If we do not have the state for any of the requested events it returns a types.MissingEventError. -func (s *eventStatements) bulkSelectStateAtEventByID( +func (s *eventStatements) BulkSelectStateAtEventByID( ctx context.Context, eventIDs []string, ) ([]types.StateAtEvent, error) { rows, err := s.bulkSelectStateAtEventByIDStmt.QueryContext(ctx, pq.StringArray(eventIDs)) @@ -270,14 +273,14 @@ func (s *eventStatements) bulkSelectStateAtEventByID( return results, nil } -func (s *eventStatements) updateEventState( +func (s *eventStatements) UpdateEventState( ctx context.Context, eventNID types.EventNID, stateNID types.StateSnapshotNID, ) error { _, err := s.updateEventStateStmt.ExecContext(ctx, int64(eventNID), int64(stateNID)) return err } -func (s *eventStatements) selectEventSentToOutput( +func (s *eventStatements) SelectEventSentToOutput( ctx context.Context, txn *sql.Tx, eventNID types.EventNID, ) (sentToOutput bool, err error) { stmt := internal.TxStmt(txn, s.selectEventSentToOutputStmt) @@ -285,13 +288,13 @@ func (s *eventStatements) selectEventSentToOutput( return } -func (s *eventStatements) updateEventSentToOutput(ctx context.Context, txn *sql.Tx, eventNID types.EventNID) error { +func (s *eventStatements) UpdateEventSentToOutput(ctx context.Context, txn *sql.Tx, eventNID types.EventNID) error { stmt := internal.TxStmt(txn, s.updateEventSentToOutputStmt) _, err := stmt.ExecContext(ctx, int64(eventNID)) return err } -func (s *eventStatements) selectEventID( +func (s *eventStatements) SelectEventID( ctx context.Context, txn *sql.Tx, eventNID types.EventNID, ) (eventID string, err error) { stmt := internal.TxStmt(txn, s.selectEventIDStmt) @@ -299,7 +302,7 @@ func (s *eventStatements) selectEventID( return } -func (s *eventStatements) bulkSelectStateAtEventAndReference( +func (s *eventStatements) BulkSelectStateAtEventAndReference( ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID, ) ([]types.StateAtEventAndReference, error) { stmt := internal.TxStmt(txn, s.bulkSelectStateAtEventAndReferenceStmt) @@ -341,8 +344,8 @@ func (s *eventStatements) bulkSelectStateAtEventAndReference( return results, nil } -func (s *eventStatements) bulkSelectEventReference( - ctx context.Context, eventNIDs []types.EventNID, +func (s *eventStatements) BulkSelectEventReference( + ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID, ) ([]gomatrixserverlib.EventReference, error) { rows, err := s.bulkSelectEventReferenceStmt.QueryContext(ctx, eventNIDsAsArray(eventNIDs)) if err != nil { @@ -367,7 +370,7 @@ func (s *eventStatements) bulkSelectEventReference( } // bulkSelectEventID returns a map from numeric event ID to string event ID. -func (s *eventStatements) bulkSelectEventID(ctx context.Context, eventNIDs []types.EventNID) (map[types.EventNID]string, error) { +func (s *eventStatements) BulkSelectEventID(ctx context.Context, eventNIDs []types.EventNID) (map[types.EventNID]string, error) { rows, err := s.bulkSelectEventIDStmt.QueryContext(ctx, eventNIDsAsArray(eventNIDs)) if err != nil { return nil, err @@ -394,7 +397,7 @@ func (s *eventStatements) bulkSelectEventID(ctx context.Context, eventNIDs []typ // bulkSelectEventNIDs returns a map from string event ID to numeric event ID. // If an event ID is not in the database then it is omitted from the map. -func (s *eventStatements) bulkSelectEventNID(ctx context.Context, eventIDs []string) (map[string]types.EventNID, error) { +func (s *eventStatements) BulkSelectEventNID(ctx context.Context, eventIDs []string) (map[string]types.EventNID, error) { rows, err := s.bulkSelectEventNIDStmt.QueryContext(ctx, pq.StringArray(eventIDs)) if err != nil { return nil, err @@ -412,7 +415,7 @@ func (s *eventStatements) bulkSelectEventNID(ctx context.Context, eventIDs []str return results, rows.Err() } -func (s *eventStatements) selectMaxEventDepth(ctx context.Context, eventNIDs []types.EventNID) (int64, error) { +func (s *eventStatements) SelectMaxEventDepth(ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID) (int64, error) { var result int64 stmt := s.selectMaxEventDepthStmt err := stmt.QueryRowContext(ctx, eventNIDsAsArray(eventNIDs)).Scan(&result) @@ -422,7 +425,7 @@ func (s *eventStatements) selectMaxEventDepth(ctx context.Context, eventNIDs []t return result, nil } -func (s *eventStatements) selectRoomNIDForEventNID( +func (s *eventStatements) SelectRoomNIDForEventNID( ctx context.Context, txn *sql.Tx, eventNID types.EventNID, ) (roomNID types.RoomNID, err error) { selectStmt := internal.TxStmt(txn, s.selectRoomNIDForEventNIDStmt) diff --git a/roomserver/storage/postgres/sql.go b/roomserver/storage/postgres/sql.go index e41c5a398..964dabbb2 100644 --- a/roomserver/storage/postgres/sql.go +++ b/roomserver/storage/postgres/sql.go @@ -39,8 +39,6 @@ func (s *statements) prepare(db *sql.DB) error { for _, prepare := range []func(db *sql.DB) error{ s.roomStatements.prepare, - s.eventStatements.prepare, - s.eventJSONStatements.prepare, s.stateSnapshotStatements.prepare, s.stateBlockStatements.prepare, s.previousEventStatements.prepare, diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go index 6fcceced5..0022c617e 100644 --- a/roomserver/storage/postgres/storage.go +++ b/roomserver/storage/postgres/storage.go @@ -36,8 +36,10 @@ import ( type Database struct { shared.Database statements statements + events tables.Events eventTypes tables.EventTypes eventStateKeys tables.EventStateKeys + eventJSON tables.EventJSON db *sql.DB } @@ -59,9 +61,19 @@ func Open(dataSourceName string, dbProperties internal.DbProperties) (*Database, if err != nil { return nil, err } + d.eventJSON, err = NewPostgresEventJSONTable(d.db) + if err != nil { + return nil, err + } + d.events, err = NewPostgresEventsTable(d.db) + if err != nil { + return nil, err + } d.Database = shared.Database{ EventTypesTable: d.eventTypes, EventStateKeysTable: d.eventStateKeys, + EventJSONTable: d.eventJSON, + EventsTable: d.events, } return &d, nil } @@ -120,8 +132,9 @@ func (d *Database) StoreEvent( } } - if eventNID, stateNID, err = d.statements.insertEvent( + if eventNID, stateNID, err = d.events.InsertEvent( ctx, + nil, roomNID, eventTypeNID, eventStateKeyNID, @@ -132,14 +145,14 @@ func (d *Database) StoreEvent( ); err != nil { if err == sql.ErrNoRows { // We've already inserted the event so select the numeric event ID - eventNID, stateNID, err = d.statements.selectEvent(ctx, event.EventID()) + eventNID, stateNID, err = d.events.SelectEvent(ctx, nil, event.EventID()) } if err != nil { return 0, types.StateAtEvent{}, err } } - if err = d.statements.insertEventJSON(ctx, eventNID, event.JSON()); err != nil { + if err = d.eventJSON.InsertEventJSON(ctx, nil, eventNID, event.JSON()); err != nil { return 0, types.StateAtEvent{}, err } @@ -230,25 +243,11 @@ func (d *Database) assignStateKeyNID( return eventStateKeyNID, err } -// StateEntriesForEventIDs implements input.EventDatabase -func (d *Database) StateEntriesForEventIDs( - ctx context.Context, eventIDs []string, -) ([]types.StateEntry, error) { - return d.statements.bulkSelectStateEventByID(ctx, eventIDs) -} - -// EventNIDs implements query.RoomserverQueryAPIDatabase -func (d *Database) EventNIDs( - ctx context.Context, eventIDs []string, -) (map[string]types.EventNID, error) { - return d.statements.bulkSelectEventNID(ctx, eventIDs) -} - // Events implements input.EventDatabase func (d *Database) Events( ctx context.Context, eventNIDs []types.EventNID, ) ([]types.Event, error) { - eventJSONs, err := d.statements.bulkSelectEventJSON(ctx, eventNIDs) + eventJSONs, err := d.eventJSON.BulkSelectEventJSON(ctx, eventNIDs) if err != nil { return nil, err } @@ -258,7 +257,7 @@ func (d *Database) Events( var roomVersion gomatrixserverlib.RoomVersion result := &results[i] result.EventNID = eventJSON.EventNID - roomNID, err = d.statements.selectRoomNIDForEventNID(ctx, nil, eventJSON.EventNID) + roomNID, err = d.events.SelectRoomNIDForEventNID(ctx, nil, eventJSON.EventNID) if err != nil { return nil, err } @@ -297,20 +296,6 @@ func (d *Database) AddState( return d.statements.insertState(ctx, roomNID, stateBlockNIDs) } -// SetState implements input.EventDatabase -func (d *Database) SetState( - ctx context.Context, eventNID types.EventNID, stateNID types.StateSnapshotNID, -) error { - return d.statements.updateEventState(ctx, eventNID, stateNID) -} - -// StateAtEventIDs implements input.EventDatabase -func (d *Database) StateAtEventIDs( - ctx context.Context, eventIDs []string, -) ([]types.StateAtEvent, error) { - return d.statements.bulkSelectStateAtEventByID(ctx, eventIDs) -} - // StateBlockNIDs implements state.RoomStateDatabase func (d *Database) StateBlockNIDs( ctx context.Context, stateNIDs []types.StateSnapshotNID, @@ -325,21 +310,6 @@ func (d *Database) StateEntries( return d.statements.bulkSelectStateBlockEntries(ctx, stateBlockNIDs) } -// SnapshotNIDFromEventID implements state.RoomStateDatabase -func (d *Database) SnapshotNIDFromEventID( - ctx context.Context, eventID string, -) (types.StateSnapshotNID, error) { - _, stateNID, err := d.statements.selectEvent(ctx, eventID) - return stateNID, err -} - -// EventIDs implements input.RoomEventDatabase -func (d *Database) EventIDs( - ctx context.Context, eventNIDs []types.EventNID, -) (map[types.EventNID]string, error) { - return d.statements.bulkSelectEventID(ctx, eventNIDs) -} - // GetLatestEventsForUpdate implements input.EventDatabase func (d *Database) GetLatestEventsForUpdate( ctx context.Context, roomNID types.RoomNID, @@ -354,14 +324,14 @@ func (d *Database) GetLatestEventsForUpdate( txn.Rollback() // nolint: errcheck return nil, err } - stateAndRefs, err := d.statements.bulkSelectStateAtEventAndReference(ctx, txn, eventNIDs) + stateAndRefs, err := d.events.BulkSelectStateAtEventAndReference(ctx, txn, eventNIDs) if err != nil { txn.Rollback() // nolint: errcheck return nil, err } var lastEventIDSent string if lastEventNIDSent != 0 { - lastEventIDSent, err = d.statements.selectEventID(ctx, txn, lastEventNIDSent) + lastEventIDSent, err = d.events.SelectEventID(ctx, txn, lastEventNIDSent) if err != nil { txn.Rollback() // nolint: errcheck return nil, err @@ -450,12 +420,12 @@ func (u *roomRecentEventsUpdater) SetLatestEvents( // HasEventBeenSent implements types.RoomRecentEventsUpdater func (u *roomRecentEventsUpdater) HasEventBeenSent(eventNID types.EventNID) (bool, error) { - return u.d.statements.selectEventSentToOutput(u.ctx, u.txn, eventNID) + return u.d.events.SelectEventSentToOutput(u.ctx, u.txn, eventNID) } // MarkEventAsSent implements types.RoomRecentEventsUpdater func (u *roomRecentEventsUpdater) MarkEventAsSent(eventNID types.EventNID) error { - return u.d.statements.updateEventSentToOutput(u.ctx, u.txn, eventNID) + return u.d.events.UpdateEventSentToOutput(u.ctx, u.txn, eventNID) } func (u *roomRecentEventsUpdater) MembershipUpdater(targetUserNID types.EventStateKeyNID, targetLocal bool) (types.MembershipUpdater, error) { @@ -491,20 +461,24 @@ func (d *Database) RoomNIDExcludingStubs(ctx context.Context, roomID string) (ro // LatestEventIDs implements query.RoomserverQueryAPIDatabase func (d *Database) LatestEventIDs( ctx context.Context, roomNID types.RoomNID, -) ([]gomatrixserverlib.EventReference, types.StateSnapshotNID, int64, error) { - eventNIDs, currentStateSnapshotNID, err := d.statements.selectLatestEventNIDs(ctx, roomNID) - if err != nil { - return nil, 0, 0, err - } - references, err := d.statements.bulkSelectEventReference(ctx, eventNIDs) - if err != nil { - return nil, 0, 0, err - } - depth, err := d.statements.selectMaxEventDepth(ctx, eventNIDs) - if err != nil { - return nil, 0, 0, err - } - return references, currentStateSnapshotNID, depth, nil +) (references []gomatrixserverlib.EventReference, currentStateSnapshotNID types.StateSnapshotNID, depth int64, err error) { + err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { + var eventNIDs []types.EventNID + eventNIDs, currentStateSnapshotNID, err = d.statements.selectLatestEventNIDs(ctx, roomNID) + if err != nil { + return err + } + references, err = d.events.BulkSelectEventReference(ctx, txn, eventNIDs) + if err != nil { + return err + } + depth, err = d.events.SelectMaxEventDepth(ctx, txn, eventNIDs) + if err != nil { + return err + } + return nil + }) + return } // GetInvitesForUser implements query.RoomserverQueryAPIDatabase diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index 7a8da8658..a3b2c2e28 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -8,6 +8,8 @@ import ( ) type Database struct { + EventsTable tables.Events + EventJSONTable tables.EventJSON EventTypesTable tables.EventTypes EventStateKeysTable tables.EventStateKeys } @@ -32,3 +34,46 @@ func (d *Database) EventStateKeyNIDs( ) (map[string]types.EventStateKeyNID, error) { return d.EventStateKeysTable.BulkSelectEventStateKeyNID(ctx, eventStateKeys) } + +// StateEntriesForEventIDs implements input.EventDatabase +func (d *Database) StateEntriesForEventIDs( + ctx context.Context, eventIDs []string, +) ([]types.StateEntry, error) { + return d.EventsTable.BulkSelectStateEventByID(ctx, eventIDs) +} + +// EventNIDs implements query.RoomserverQueryAPIDatabase +func (d *Database) EventNIDs( + ctx context.Context, eventIDs []string, +) (map[string]types.EventNID, error) { + return d.EventsTable.BulkSelectEventNID(ctx, eventIDs) +} + +// SetState implements input.EventDatabase +func (d *Database) SetState( + ctx context.Context, eventNID types.EventNID, stateNID types.StateSnapshotNID, +) error { + return d.EventsTable.UpdateEventState(ctx, eventNID, stateNID) +} + +// StateAtEventIDs implements input.EventDatabase +func (d *Database) StateAtEventIDs( + ctx context.Context, eventIDs []string, +) ([]types.StateAtEvent, error) { + return d.EventsTable.BulkSelectStateAtEventByID(ctx, eventIDs) +} + +// SnapshotNIDFromEventID implements state.RoomStateDatabase +func (d *Database) SnapshotNIDFromEventID( + ctx context.Context, eventID string, +) (types.StateSnapshotNID, error) { + _, stateNID, err := d.EventsTable.SelectEvent(ctx, nil, eventID) + return stateNID, err +} + +// EventIDs implements input.RoomEventDatabase +func (d *Database) EventIDs( + ctx context.Context, eventNIDs []types.EventNID, +) (map[types.EventNID]string, error) { + return d.EventsTable.BulkSelectEventID(ctx, eventNIDs) +} diff --git a/roomserver/storage/sqlite3/event_json_table.go b/roomserver/storage/sqlite3/event_json_table.go index fbf35e711..34b067cb0 100644 --- a/roomserver/storage/sqlite3/event_json_table.go +++ b/roomserver/storage/sqlite3/event_json_table.go @@ -21,6 +21,7 @@ import ( "strings" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -51,40 +52,36 @@ type eventJSONStatements struct { bulkSelectEventJSONStmt *sql.Stmt } -func (s *eventJSONStatements) prepare(db *sql.DB) (err error) { +func NewSqliteEventJSONTable(db *sql.DB) (tables.EventJSON, error) { + s := &eventJSONStatements{} s.db = db - _, err = db.Exec(eventJSONSchema) + _, err := db.Exec(eventJSONSchema) if err != nil { - return + return nil, err } - return statementList{ + return s, statementList{ {&s.insertEventJSONStmt, insertEventJSONSQL}, {&s.bulkSelectEventJSONStmt, bulkSelectEventJSONSQL}, }.prepare(db) } -func (s *eventJSONStatements) insertEventJSON( +func (s *eventJSONStatements) InsertEventJSON( ctx context.Context, txn *sql.Tx, eventNID types.EventNID, eventJSON []byte, ) error { _, err := internal.TxStmt(txn, s.insertEventJSONStmt).ExecContext(ctx, int64(eventNID), eventJSON) return err } -type eventJSONPair struct { - EventNID types.EventNID - EventJSON []byte -} - -func (s *eventJSONStatements) bulkSelectEventJSON( - ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID, -) ([]eventJSONPair, error) { +func (s *eventJSONStatements) BulkSelectEventJSON( + ctx context.Context, eventNIDs []types.EventNID, +) ([]tables.EventJSONPair, error) { iEventNIDs := make([]interface{}, len(eventNIDs)) for k, v := range eventNIDs { iEventNIDs[k] = v } selectOrig := strings.Replace(bulkSelectEventJSONSQL, "($1)", internal.QueryVariadic(len(iEventNIDs)), 1) - rows, err := txn.QueryContext(ctx, selectOrig, iEventNIDs...) + rows, err := s.db.QueryContext(ctx, selectOrig, iEventNIDs...) if err != nil { return nil, err } @@ -94,7 +91,7 @@ func (s *eventJSONStatements) bulkSelectEventJSON( // because of the unique constraint on event NIDs. // So we can allocate an array of the correct size now. // We might get fewer results than NIDs so we adjust the length of the slice before returning it. - results := make([]eventJSONPair, len(eventNIDs)) + results := make([]tables.EventJSONPair, len(eventNIDs)) i := 0 for ; rows.Next(); i++ { result := &results[i] diff --git a/roomserver/storage/sqlite3/events_table.go b/roomserver/storage/sqlite3/events_table.go index 551134950..a41a8737f 100644 --- a/roomserver/storage/sqlite3/events_table.go +++ b/roomserver/storage/sqlite3/events_table.go @@ -23,6 +23,7 @@ import ( "strings" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -111,14 +112,15 @@ type eventStatements struct { selectRoomNIDForEventNIDStmt *sql.Stmt } -func (s *eventStatements) prepare(db *sql.DB) (err error) { +func NewSqliteEventsTable(db *sql.DB) (tables.Events, error) { + s := &eventStatements{} s.db = db - _, err = db.Exec(eventsSchema) + _, err := db.Exec(eventsSchema) if err != nil { - return + return nil, err } - return statementList{ + return s, statementList{ {&s.insertEventStmt, insertEventSQL}, {&s.selectEventStmt, selectEventSQL}, {&s.bulkSelectStateEventByIDStmt, bulkSelectStateEventByIDSQL}, @@ -135,7 +137,7 @@ func (s *eventStatements) prepare(db *sql.DB) (err error) { }.prepare(db) } -func (s *eventStatements) insertEvent( +func (s *eventStatements) InsertEvent( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, @@ -145,7 +147,7 @@ func (s *eventStatements) insertEvent( referenceSHA256 []byte, authEventNIDs []types.EventNID, depth int64, -) (types.EventNID, error) { +) (types.EventNID, types.StateSnapshotNID, error) { // attempt to insert: the last_row_id is the event NID insertStmt := internal.TxStmt(txn, s.insertEventStmt) result, err := insertStmt.ExecContext( @@ -153,17 +155,17 @@ func (s *eventStatements) insertEvent( eventID, referenceSHA256, eventNIDsAsArray(authEventNIDs), depth, ) if err != nil { - return 0, err + return 0, 0, err } modified, err := result.RowsAffected() if modified == 0 && err == nil { - return 0, sql.ErrNoRows + return 0, 0, sql.ErrNoRows } eventNID, err := result.LastInsertId() - return types.EventNID(eventNID), err + return types.EventNID(eventNID), 0, err } -func (s *eventStatements) selectEvent( +func (s *eventStatements) SelectEvent( ctx context.Context, txn *sql.Tx, eventID string, ) (types.EventNID, types.StateSnapshotNID, error) { var eventNID int64 @@ -175,8 +177,8 @@ func (s *eventStatements) selectEvent( // bulkSelectStateEventByID lookups a list of state events by event ID. // If any of the requested events are missing from the database it returns a types.MissingEventError -func (s *eventStatements) bulkSelectStateEventByID( - ctx context.Context, txn *sql.Tx, eventIDs []string, +func (s *eventStatements) BulkSelectStateEventByID( + ctx context.Context, eventIDs []string, ) ([]types.StateEntry, error) { /////////////// iEventIDs := make([]interface{}, len(eventIDs)) @@ -184,13 +186,12 @@ func (s *eventStatements) bulkSelectStateEventByID( iEventIDs[k] = v } selectOrig := strings.Replace(bulkSelectStateEventByIDSQL, "($1)", internal.QueryVariadic(len(iEventIDs)), 1) - selectPrep, err := txn.Prepare(selectOrig) + selectStmt, err := s.db.Prepare(selectOrig) if err != nil { return nil, err } /////////////// - selectStmt := internal.TxStmt(txn, selectPrep) rows, err := selectStmt.QueryContext(ctx, iEventIDs...) if err != nil { return nil, err @@ -228,8 +229,8 @@ func (s *eventStatements) bulkSelectStateEventByID( // bulkSelectStateAtEventByID lookups the state at a list of events by event ID. // If any of the requested events are missing from the database it returns a types.MissingEventError. // If we do not have the state for any of the requested events it returns a types.MissingEventError. -func (s *eventStatements) bulkSelectStateAtEventByID( - ctx context.Context, txn *sql.Tx, eventIDs []string, +func (s *eventStatements) BulkSelectStateAtEventByID( + ctx context.Context, eventIDs []string, ) ([]types.StateAtEvent, error) { /////////////// iEventIDs := make([]interface{}, len(eventIDs)) @@ -237,13 +238,11 @@ func (s *eventStatements) bulkSelectStateAtEventByID( iEventIDs[k] = v } selectOrig := strings.Replace(bulkSelectStateAtEventByIDSQL, "($1)", internal.QueryVariadic(len(iEventIDs)), 1) - selectPrep, err := txn.Prepare(selectOrig) + selectStmt, err := s.db.Prepare(selectOrig) if err != nil { return nil, err } /////////////// - - selectStmt := internal.TxStmt(txn, selectPrep) rows, err := selectStmt.QueryContext(ctx, iEventIDs...) if err != nil { return nil, err @@ -275,15 +274,14 @@ func (s *eventStatements) bulkSelectStateAtEventByID( return results, err } -func (s *eventStatements) updateEventState( - ctx context.Context, txn *sql.Tx, eventNID types.EventNID, stateNID types.StateSnapshotNID, +func (s *eventStatements) UpdateEventState( + ctx context.Context, eventNID types.EventNID, stateNID types.StateSnapshotNID, ) error { - updateStmt := internal.TxStmt(txn, s.updateEventStateStmt) - _, err := updateStmt.ExecContext(ctx, int64(stateNID), int64(eventNID)) + _, err := s.updateEventStateStmt.ExecContext(ctx, int64(stateNID), int64(eventNID)) return err } -func (s *eventStatements) selectEventSentToOutput( +func (s *eventStatements) SelectEventSentToOutput( ctx context.Context, txn *sql.Tx, eventNID types.EventNID, ) (sentToOutput bool, err error) { selectStmt := internal.TxStmt(txn, s.selectEventSentToOutputStmt) @@ -294,14 +292,14 @@ func (s *eventStatements) selectEventSentToOutput( return } -func (s *eventStatements) updateEventSentToOutput(ctx context.Context, txn *sql.Tx, eventNID types.EventNID) error { +func (s *eventStatements) UpdateEventSentToOutput(ctx context.Context, txn *sql.Tx, eventNID types.EventNID) error { updateStmt := internal.TxStmt(txn, s.updateEventSentToOutputStmt) _, err := updateStmt.ExecContext(ctx, int64(eventNID)) //_, err := s.updateEventSentToOutputStmt.ExecContext(ctx, int64(eventNID)) return err } -func (s *eventStatements) selectEventID( +func (s *eventStatements) SelectEventID( ctx context.Context, txn *sql.Tx, eventNID types.EventNID, ) (eventID string, err error) { selectStmt := internal.TxStmt(txn, s.selectEventIDStmt) @@ -309,7 +307,7 @@ func (s *eventStatements) selectEventID( return } -func (s *eventStatements) bulkSelectStateAtEventAndReference( +func (s *eventStatements) BulkSelectStateAtEventAndReference( ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID, ) ([]types.StateAtEventAndReference, error) { /////////////// @@ -355,7 +353,7 @@ func (s *eventStatements) bulkSelectStateAtEventAndReference( return results, nil } -func (s *eventStatements) bulkSelectEventReference( +func (s *eventStatements) BulkSelectEventReference( ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID, ) ([]gomatrixserverlib.EventReference, error) { /////////////// @@ -391,20 +389,19 @@ func (s *eventStatements) bulkSelectEventReference( } // bulkSelectEventID returns a map from numeric event ID to string event ID. -func (s *eventStatements) bulkSelectEventID(ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID) (map[types.EventNID]string, error) { +func (s *eventStatements) BulkSelectEventID(ctx context.Context, eventNIDs []types.EventNID) (map[types.EventNID]string, error) { /////////////// iEventNIDs := make([]interface{}, len(eventNIDs)) for k, v := range eventNIDs { iEventNIDs[k] = v } selectOrig := strings.Replace(bulkSelectEventIDSQL, "($1)", internal.QueryVariadic(len(iEventNIDs)), 1) - selectPrep, err := txn.Prepare(selectOrig) + selectStmt, err := s.db.Prepare(selectOrig) if err != nil { return nil, err } /////////////// - selectStmt := internal.TxStmt(txn, selectPrep) rows, err := selectStmt.QueryContext(ctx, iEventNIDs...) if err != nil { return nil, err @@ -428,20 +425,18 @@ func (s *eventStatements) bulkSelectEventID(ctx context.Context, txn *sql.Tx, ev // bulkSelectEventNIDs returns a map from string event ID to numeric event ID. // If an event ID is not in the database then it is omitted from the map. -func (s *eventStatements) bulkSelectEventNID(ctx context.Context, txn *sql.Tx, eventIDs []string) (map[string]types.EventNID, error) { +func (s *eventStatements) BulkSelectEventNID(ctx context.Context, eventIDs []string) (map[string]types.EventNID, error) { /////////////// iEventIDs := make([]interface{}, len(eventIDs)) for k, v := range eventIDs { iEventIDs[k] = v } selectOrig := strings.Replace(bulkSelectEventNIDSQL, "($1)", internal.QueryVariadic(len(iEventIDs)), 1) - selectPrep, err := txn.Prepare(selectOrig) + selectStmt, err := s.db.Prepare(selectOrig) if err != nil { return nil, err } /////////////// - - selectStmt := internal.TxStmt(txn, selectPrep) rows, err := selectStmt.QueryContext(ctx, iEventIDs...) if err != nil { return nil, err @@ -459,7 +454,7 @@ func (s *eventStatements) bulkSelectEventNID(ctx context.Context, txn *sql.Tx, e return results, nil } -func (s *eventStatements) selectMaxEventDepth(ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID) (int64, error) { +func (s *eventStatements) SelectMaxEventDepth(ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID) (int64, error) { var result int64 iEventIDs := make([]interface{}, len(eventNIDs)) for i, v := range eventNIDs { @@ -473,7 +468,7 @@ func (s *eventStatements) selectMaxEventDepth(ctx context.Context, txn *sql.Tx, return result, nil } -func (s *eventStatements) selectRoomNIDForEventNID( +func (s *eventStatements) SelectRoomNIDForEventNID( ctx context.Context, txn *sql.Tx, eventNID types.EventNID, ) (roomNID types.RoomNID, err error) { selectStmt := internal.TxStmt(txn, s.selectRoomNIDForEventNIDStmt) diff --git a/roomserver/storage/sqlite3/sql.go b/roomserver/storage/sqlite3/sql.go index bb3318b2d..ac44b398d 100644 --- a/roomserver/storage/sqlite3/sql.go +++ b/roomserver/storage/sqlite3/sql.go @@ -39,8 +39,6 @@ func (s *statements) prepare(db *sql.DB) error { for _, prepare := range []func(db *sql.DB) error{ s.roomStatements.prepare, - s.eventStatements.prepare, - s.eventJSONStatements.prepare, s.stateSnapshotStatements.prepare, s.stateBlockStatements.prepare, s.previousEventStatements.prepare, diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index b9157e3a5..05bad7fb9 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -37,6 +37,8 @@ import ( type Database struct { shared.Database statements statements + events tables.Events + eventJSON tables.EventJSON eventTypes tables.EventTypes eventStateKeys tables.EventStateKeys db *sql.DB @@ -79,9 +81,19 @@ func Open(dataSourceName string) (*Database, error) { if err != nil { return nil, err } + d.eventJSON, err = NewSqliteEventJSONTable(d.db) + if err != nil { + return nil, err + } + d.events, err = NewSqliteEventsTable(d.db) + if err != nil { + return nil, err + } d.Database = shared.Database{ + EventsTable: d.events, EventTypesTable: d.eventTypes, EventStateKeysTable: d.eventStateKeys, + EventJSONTable: d.eventJSON, } return &d, nil } @@ -141,7 +153,7 @@ func (d *Database) StoreEvent( } } - if eventNID, err = d.statements.insertEvent( + if eventNID, stateNID, err = d.events.InsertEvent( ctx, txn, roomNID, @@ -154,14 +166,14 @@ func (d *Database) StoreEvent( ); err != nil { if err == sql.ErrNoRows { // We've already inserted the event so select the numeric event ID - eventNID, stateNID, err = d.statements.selectEvent(ctx, txn, event.EventID()) + eventNID, stateNID, err = d.events.SelectEvent(ctx, txn, event.EventID()) } if err != nil { return err } } - if err = d.statements.insertEventJSON(ctx, txn, eventNID, event.JSON()); err != nil { + if err = d.eventJSON.InsertEventJSON(ctx, txn, eventNID, event.JSON()); err != nil { return err } @@ -255,47 +267,25 @@ func (d *Database) assignStateKeyNID( return } -// StateEntriesForEventIDs implements input.EventDatabase -func (d *Database) StateEntriesForEventIDs( - ctx context.Context, eventIDs []string, -) (se []types.StateEntry, err error) { - err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { - se, err = d.statements.bulkSelectStateEventByID(ctx, txn, eventIDs) - return err - }) - return -} - -// EventNIDs implements query.RoomserverQueryAPIDatabase -func (d *Database) EventNIDs( - ctx context.Context, eventIDs []string, -) (out map[string]types.EventNID, err error) { - err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { - out, err = d.statements.bulkSelectEventNID(ctx, txn, eventIDs) - return err - }) - return -} - // Events implements input.EventDatabase func (d *Database) Events( ctx context.Context, eventNIDs []types.EventNID, ) ([]types.Event, error) { - var eventJSONs []eventJSONPair + var eventJSONs []tables.EventJSONPair var err error var results []types.Event + eventJSONs, err = d.eventJSON.BulkSelectEventJSON(ctx, eventNIDs) + if err != nil || len(eventJSONs) == 0 { + return nil, nil + } err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { - eventJSONs, err = d.statements.bulkSelectEventJSON(ctx, txn, eventNIDs) - if err != nil || len(eventJSONs) == 0 { - return nil - } results = make([]types.Event, len(eventJSONs)) for i, eventJSON := range eventJSONs { var roomNID types.RoomNID var roomVersion gomatrixserverlib.RoomVersion result := &results[i] result.EventNID = eventJSON.EventNID - roomNID, err = d.statements.selectRoomNIDForEventNID(ctx, txn, eventJSON.EventNID) + roomNID, err = d.events.SelectRoomNIDForEventNID(ctx, txn, eventJSON.EventNID) if err != nil { return err } @@ -343,27 +333,6 @@ func (d *Database) AddState( return } -// SetState implements input.EventDatabase -func (d *Database) SetState( - ctx context.Context, eventNID types.EventNID, stateNID types.StateSnapshotNID, -) error { - e := internal.WithTransaction(d.db, func(txn *sql.Tx) error { - return d.statements.updateEventState(ctx, txn, eventNID, stateNID) - }) - return e -} - -// StateAtEventIDs implements input.EventDatabase -func (d *Database) StateAtEventIDs( - ctx context.Context, eventIDs []string, -) (se []types.StateAtEvent, err error) { - err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { - se, err = d.statements.bulkSelectStateAtEventByID(ctx, txn, eventIDs) - return err - }) - return -} - // StateBlockNIDs implements state.RoomStateDatabase func (d *Database) StateBlockNIDs( ctx context.Context, stateNIDs []types.StateSnapshotNID, @@ -386,28 +355,6 @@ func (d *Database) StateEntries( return } -// SnapshotNIDFromEventID implements state.RoomStateDatabase -func (d *Database) SnapshotNIDFromEventID( - ctx context.Context, eventID string, -) (stateNID types.StateSnapshotNID, err error) { - err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { - _, stateNID, err = d.statements.selectEvent(ctx, txn, eventID) - return err - }) - return -} - -// EventIDs implements input.RoomEventDatabase -func (d *Database) EventIDs( - ctx context.Context, eventNIDs []types.EventNID, -) (out map[types.EventNID]string, err error) { - err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { - out, err = d.statements.bulkSelectEventID(ctx, txn, eventNIDs) - return err - }) - return -} - // GetLatestEventsForUpdate implements input.EventDatabase func (d *Database) GetLatestEventsForUpdate( ctx context.Context, roomNID types.RoomNID, @@ -422,14 +369,14 @@ func (d *Database) GetLatestEventsForUpdate( txn.Rollback() // nolint: errcheck return nil, err } - stateAndRefs, err := d.statements.bulkSelectStateAtEventAndReference(ctx, txn, eventNIDs) + stateAndRefs, err := d.events.BulkSelectStateAtEventAndReference(ctx, txn, eventNIDs) if err != nil { txn.Rollback() // nolint: errcheck return nil, err } var lastEventIDSent string if lastEventNIDSent != 0 { - lastEventIDSent, err = d.statements.selectEventID(ctx, txn, lastEventNIDSent) + lastEventIDSent, err = d.events.SelectEventID(ctx, txn, lastEventNIDSent) if err != nil { txn.Rollback() // nolint: errcheck return nil, err @@ -539,7 +486,7 @@ func (u *roomRecentEventsUpdater) SetLatestEvents( // HasEventBeenSent implements types.RoomRecentEventsUpdater func (u *roomRecentEventsUpdater) HasEventBeenSent(eventNID types.EventNID) (res bool, err error) { err = internal.WithTransaction(u.d.db, func(txn *sql.Tx) error { - res, err = u.d.statements.selectEventSentToOutput(u.ctx, txn, eventNID) + res, err = u.d.events.SelectEventSentToOutput(u.ctx, txn, eventNID) return err }) return @@ -548,7 +495,7 @@ func (u *roomRecentEventsUpdater) HasEventBeenSent(eventNID types.EventNID) (res // MarkEventAsSent implements types.RoomRecentEventsUpdater func (u *roomRecentEventsUpdater) MarkEventAsSent(eventNID types.EventNID) error { err := internal.WithTransaction(u.d.db, func(txn *sql.Tx) error { - return u.d.statements.updateEventSentToOutput(u.ctx, txn, eventNID) + return u.d.events.UpdateEventSentToOutput(u.ctx, txn, eventNID) }) return err } @@ -601,11 +548,11 @@ func (d *Database) LatestEventIDs( if err != nil { return err } - references, err = d.statements.bulkSelectEventReference(ctx, txn, eventNIDs) + references, err = d.events.BulkSelectEventReference(ctx, txn, eventNIDs) if err != nil { return err } - depth, err = d.statements.selectMaxEventDepth(ctx, txn, eventNIDs) + depth, err = d.events.SelectMaxEventDepth(ctx, txn, eventNIDs) if err != nil { return err } diff --git a/roomserver/storage/tables/interface.go b/roomserver/storage/tables/interface.go index d607865dc..78ddc5fea 100644 --- a/roomserver/storage/tables/interface.go +++ b/roomserver/storage/tables/interface.go @@ -5,8 +5,19 @@ import ( "database/sql" "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/gomatrixserverlib" ) +type EventJSONPair struct { + EventNID types.EventNID + EventJSON []byte +} + +type EventJSON interface { + InsertEventJSON(ctx context.Context, tx *sql.Tx, eventNID types.EventNID, eventJSON []byte) error + BulkSelectEventJSON(ctx context.Context, eventNIDs []types.EventNID) ([]EventJSONPair, error) +} + type EventTypes interface { InsertEventTypeNID(ctx context.Context, tx *sql.Tx, eventType string) (types.EventTypeNID, error) SelectEventTypeNID(ctx context.Context, tx *sql.Tx, eventType string) (types.EventTypeNID, error) @@ -19,3 +30,28 @@ type EventStateKeys interface { BulkSelectEventStateKeyNID(ctx context.Context, eventStateKeys []string) (map[string]types.EventStateKeyNID, error) BulkSelectEventStateKey(ctx context.Context, eventStateKeyNIDs []types.EventStateKeyNID) (map[types.EventStateKeyNID]string, error) } + +type Events interface { + InsertEvent(c context.Context, txn *sql.Tx, i types.RoomNID, j types.EventTypeNID, k types.EventStateKeyNID, eventID string, referenceSHA256 []byte, authEventNIDs []types.EventNID, depth int64) (types.EventNID, types.StateSnapshotNID, error) + SelectEvent(ctx context.Context, txn *sql.Tx, eventID string) (types.EventNID, types.StateSnapshotNID, error) + // bulkSelectStateEventByID lookups a list of state events by event ID. + // If any of the requested events are missing from the database it returns a types.MissingEventError + BulkSelectStateEventByID(ctx context.Context, eventIDs []string) ([]types.StateEntry, error) + // BulkSelectStateAtEventByID lookups the state at a list of events by event ID. + // If any of the requested events are missing from the database it returns a types.MissingEventError. + // If we do not have the state for any of the requested events it returns a types.MissingEventError. + BulkSelectStateAtEventByID(ctx context.Context, eventIDs []string) ([]types.StateAtEvent, error) + UpdateEventState(ctx context.Context, eventNID types.EventNID, stateNID types.StateSnapshotNID) error + SelectEventSentToOutput(ctx context.Context, txn *sql.Tx, eventNID types.EventNID) (sentToOutput bool, err error) + UpdateEventSentToOutput(ctx context.Context, txn *sql.Tx, eventNID types.EventNID) error + SelectEventID(ctx context.Context, txn *sql.Tx, eventNID types.EventNID) (eventID string, err error) + BulkSelectStateAtEventAndReference(ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID) ([]types.StateAtEventAndReference, error) + BulkSelectEventReference(ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID) ([]gomatrixserverlib.EventReference, error) + // BulkSelectEventID returns a map from numeric event ID to string event ID. + BulkSelectEventID(ctx context.Context, eventNIDs []types.EventNID) (map[types.EventNID]string, error) + // BulkSelectEventNIDs returns a map from string event ID to numeric event ID. + // If an event ID is not in the database then it is omitted from the map. + BulkSelectEventNID(ctx context.Context, eventIDs []string) (map[string]types.EventNID, error) + SelectMaxEventDepth(ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID) (int64, error) + SelectRoomNIDForEventNID(ctx context.Context, txn *sql.Tx, eventNID types.EventNID) (roomNID types.RoomNID, err error) +} From 19aa44ecaef70a2be7294e8ad738467da41d1f2e Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 26 May 2020 18:23:39 +0100 Subject: [PATCH 095/200] Convert transactions/rooms table to share more code (#1063) * Convert rooms table * Convert transactions table * Convert rooms table and factor out lots of functions * I think you'll be needing this.. --- roomserver/storage/postgres/events_table.go | 5 +- roomserver/storage/postgres/rooms_table.go | 31 +- roomserver/storage/postgres/sql.go | 2 - roomserver/storage/postgres/storage.go | 281 ++-------------- .../storage/postgres/transactions_table.go | 17 +- roomserver/storage/shared/storage.go | 301 ++++++++++++++++++ roomserver/storage/sqlite3/events_table.go | 5 +- roomserver/storage/sqlite3/rooms_table.go | 31 +- roomserver/storage/sqlite3/sql.go | 2 - roomserver/storage/sqlite3/storage.go | 298 ++--------------- .../storage/sqlite3/transactions_table.go | 19 +- roomserver/storage/tables/interface.go | 17 +- 12 files changed, 409 insertions(+), 600 deletions(-) diff --git a/roomserver/storage/postgres/events_table.go b/roomserver/storage/postgres/events_table.go index 9c4649468..5a567bf23 100644 --- a/roomserver/storage/postgres/events_table.go +++ b/roomserver/storage/postgres/events_table.go @@ -426,10 +426,9 @@ func (s *eventStatements) SelectMaxEventDepth(ctx context.Context, txn *sql.Tx, } func (s *eventStatements) SelectRoomNIDForEventNID( - ctx context.Context, txn *sql.Tx, eventNID types.EventNID, + ctx context.Context, eventNID types.EventNID, ) (roomNID types.RoomNID, err error) { - selectStmt := internal.TxStmt(txn, s.selectRoomNIDForEventNIDStmt) - err = selectStmt.QueryRowContext(ctx, int64(eventNID)).Scan(&roomNID) + err = s.selectRoomNIDForEventNIDStmt.QueryRowContext(ctx, int64(eventNID)).Scan(&roomNID) return } diff --git a/roomserver/storage/postgres/rooms_table.go b/roomserver/storage/postgres/rooms_table.go index fc64489de..98881390b 100644 --- a/roomserver/storage/postgres/rooms_table.go +++ b/roomserver/storage/postgres/rooms_table.go @@ -22,6 +22,7 @@ import ( "github.com/lib/pq" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -82,12 +83,13 @@ type roomStatements struct { selectRoomVersionForRoomNIDStmt *sql.Stmt } -func (s *roomStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(roomsSchema) +func NewPostgresRoomsTable(db *sql.DB) (tables.Rooms, error) { + s := &roomStatements{} + _, err := db.Exec(roomsSchema) if err != nil { - return + return nil, err } - return statementList{ + return s, statementList{ {&s.insertRoomNIDStmt, insertRoomNIDSQL}, {&s.selectRoomNIDStmt, selectRoomNIDSQL}, {&s.selectLatestEventNIDsStmt, selectLatestEventNIDsSQL}, @@ -98,7 +100,7 @@ func (s *roomStatements) prepare(db *sql.DB) (err error) { }.prepare(db) } -func (s *roomStatements) insertRoomNID( +func (s *roomStatements) InsertRoomNID( ctx context.Context, txn *sql.Tx, roomID string, roomVersion gomatrixserverlib.RoomVersion, ) (types.RoomNID, error) { @@ -108,7 +110,7 @@ func (s *roomStatements) insertRoomNID( return types.RoomNID(roomNID), err } -func (s *roomStatements) selectRoomNID( +func (s *roomStatements) SelectRoomNID( ctx context.Context, txn *sql.Tx, roomID string, ) (types.RoomNID, error) { var roomNID int64 @@ -117,8 +119,8 @@ func (s *roomStatements) selectRoomNID( return types.RoomNID(roomNID), err } -func (s *roomStatements) selectLatestEventNIDs( - ctx context.Context, roomNID types.RoomNID, +func (s *roomStatements) SelectLatestEventNIDs( + ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, ) ([]types.EventNID, types.StateSnapshotNID, error) { var nids pq.Int64Array var stateSnapshotNID int64 @@ -134,7 +136,7 @@ func (s *roomStatements) selectLatestEventNIDs( return eventNIDs, types.StateSnapshotNID(stateSnapshotNID), nil } -func (s *roomStatements) selectLatestEventsNIDsForUpdate( +func (s *roomStatements) SelectLatestEventsNIDsForUpdate( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, ) ([]types.EventNID, types.EventNID, types.StateSnapshotNID, error) { var nids pq.Int64Array @@ -152,7 +154,7 @@ func (s *roomStatements) selectLatestEventsNIDsForUpdate( return eventNIDs, types.EventNID(lastEventSentNID), types.StateSnapshotNID(stateSnapshotNID), nil } -func (s *roomStatements) updateLatestEventNIDs( +func (s *roomStatements) UpdateLatestEventNIDs( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, @@ -171,7 +173,7 @@ func (s *roomStatements) updateLatestEventNIDs( return err } -func (s *roomStatements) selectRoomVersionForRoomID( +func (s *roomStatements) SelectRoomVersionForRoomID( ctx context.Context, txn *sql.Tx, roomID string, ) (gomatrixserverlib.RoomVersion, error) { var roomVersion gomatrixserverlib.RoomVersion @@ -183,12 +185,11 @@ func (s *roomStatements) selectRoomVersionForRoomID( return roomVersion, err } -func (s *roomStatements) selectRoomVersionForRoomNID( - ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, +func (s *roomStatements) SelectRoomVersionForRoomNID( + ctx context.Context, roomNID types.RoomNID, ) (gomatrixserverlib.RoomVersion, error) { var roomVersion gomatrixserverlib.RoomVersion - stmt := internal.TxStmt(txn, s.selectRoomVersionForRoomNIDStmt) - err := stmt.QueryRowContext(ctx, roomNID).Scan(&roomVersion) + err := s.selectRoomVersionForRoomNIDStmt.QueryRowContext(ctx, roomNID).Scan(&roomVersion) if err == sql.ErrNoRows { return roomVersion, errors.New("room not found") } diff --git a/roomserver/storage/postgres/sql.go b/roomserver/storage/postgres/sql.go index 964dabbb2..914f269c5 100644 --- a/roomserver/storage/postgres/sql.go +++ b/roomserver/storage/postgres/sql.go @@ -38,14 +38,12 @@ func (s *statements) prepare(db *sql.DB) error { var err error for _, prepare := range []func(db *sql.DB) error{ - s.roomStatements.prepare, s.stateSnapshotStatements.prepare, s.stateBlockStatements.prepare, s.previousEventStatements.prepare, s.roomAliasesStatements.prepare, s.inviteStatements.prepare, s.membershipStatements.prepare, - s.transactionStatements.prepare, } { if err = prepare(db); err != nil { return err diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go index 0022c617e..d44da8585 100644 --- a/roomserver/storage/postgres/storage.go +++ b/roomserver/storage/postgres/storage.go @@ -18,14 +18,12 @@ package postgres import ( "context" "database/sql" - "encoding/json" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" // Import the postgres database driver. _ "github.com/lib/pq" - "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" @@ -40,6 +38,8 @@ type Database struct { eventTypes tables.EventTypes eventStateKeys tables.EventStateKeys eventJSON tables.EventJSON + rooms tables.Rooms + transactions tables.Transactions db *sql.DB } @@ -69,164 +69,43 @@ func Open(dataSourceName string, dbProperties internal.DbProperties) (*Database, if err != nil { return nil, err } + d.rooms, err = NewPostgresRoomsTable(d.db) + if err != nil { + return nil, err + } + d.transactions, err = NewPostgresTransactionsTable(d.db) + if err != nil { + return nil, err + } d.Database = shared.Database{ + DB: d.db, EventTypesTable: d.eventTypes, EventStateKeysTable: d.eventStateKeys, EventJSONTable: d.eventJSON, EventsTable: d.events, + RoomsTable: d.rooms, + TransactionsTable: d.transactions, } return &d, nil } -// StoreEvent implements input.EventDatabase -func (d *Database) StoreEvent( - ctx context.Context, event gomatrixserverlib.Event, - txnAndSessionID *api.TransactionID, authEventNIDs []types.EventNID, -) (types.RoomNID, types.StateAtEvent, error) { - var ( - roomNID types.RoomNID - eventTypeNID types.EventTypeNID - eventStateKeyNID types.EventStateKeyNID - eventNID types.EventNID - stateNID types.StateSnapshotNID - err error - ) - - if txnAndSessionID != nil { - if err = d.statements.insertTransaction( - ctx, txnAndSessionID.TransactionID, - txnAndSessionID.SessionID, event.Sender(), event.EventID(), - ); err != nil { - return 0, types.StateAtEvent{}, err - } - } - - // TODO: Here we should aim to have two different code paths for new rooms - // vs existing ones. - - // Get the default room version. If the client doesn't supply a room_version - // then we will use our configured default to create the room. - // https://matrix.org/docs/spec/client_server/r0.6.0#post-matrix-client-r0-createroom - // Note that the below logic depends on the m.room.create event being the - // first event that is persisted to the database when creating or joining a - // room. - var roomVersion gomatrixserverlib.RoomVersion - if roomVersion, err = extractRoomVersionFromCreateEvent(event); err != nil { - return 0, types.StateAtEvent{}, err - } - - if roomNID, err = d.assignRoomNID(ctx, nil, event.RoomID(), roomVersion); err != nil { - return 0, types.StateAtEvent{}, err - } - - if eventTypeNID, err = d.assignEventTypeNID(ctx, event.Type()); err != nil { - return 0, types.StateAtEvent{}, err - } - - eventStateKey := event.StateKey() - // Assigned a numeric ID for the state_key if there is one present. - // Otherwise set the numeric ID for the state_key to 0. - if eventStateKey != nil { - if eventStateKeyNID, err = d.assignStateKeyNID(ctx, nil, *eventStateKey); err != nil { - return 0, types.StateAtEvent{}, err - } - } - - if eventNID, stateNID, err = d.events.InsertEvent( - ctx, - nil, - roomNID, - eventTypeNID, - eventStateKeyNID, - event.EventID(), - event.EventReference().EventSHA256, - authEventNIDs, - event.Depth(), - ); err != nil { - if err == sql.ErrNoRows { - // We've already inserted the event so select the numeric event ID - eventNID, stateNID, err = d.events.SelectEvent(ctx, nil, event.EventID()) - } - if err != nil { - return 0, types.StateAtEvent{}, err - } - } - - if err = d.eventJSON.InsertEventJSON(ctx, nil, eventNID, event.JSON()); err != nil { - return 0, types.StateAtEvent{}, err - } - - return roomNID, types.StateAtEvent{ - BeforeStateSnapshotNID: stateNID, - StateEntry: types.StateEntry{ - StateKeyTuple: types.StateKeyTuple{ - EventTypeNID: eventTypeNID, - EventStateKeyNID: eventStateKeyNID, - }, - EventNID: eventNID, - }, - }, nil -} - -func extractRoomVersionFromCreateEvent(event gomatrixserverlib.Event) ( - gomatrixserverlib.RoomVersion, error, -) { - var err error - var roomVersion gomatrixserverlib.RoomVersion - // Look for m.room.create events. - if event.Type() != gomatrixserverlib.MRoomCreate { - return gomatrixserverlib.RoomVersion(""), nil - } - roomVersion = gomatrixserverlib.RoomVersionV1 - var createContent gomatrixserverlib.CreateContent - // The m.room.create event contains an optional "room_version" key in - // the event content, so we need to unmarshal that first. - if err = json.Unmarshal(event.Content(), &createContent); err != nil { - return gomatrixserverlib.RoomVersion(""), err - } - // A room version was specified in the event content? - if createContent.RoomVersion != nil { - roomVersion = gomatrixserverlib.RoomVersion(*createContent.RoomVersion) - } - return roomVersion, err -} - func (d *Database) assignRoomNID( ctx context.Context, txn *sql.Tx, roomID string, roomVersion gomatrixserverlib.RoomVersion, ) (types.RoomNID, error) { // Check if we already have a numeric ID in the database. - roomNID, err := d.statements.selectRoomNID(ctx, txn, roomID) + roomNID, err := d.rooms.SelectRoomNID(ctx, txn, roomID) if err == sql.ErrNoRows { // We don't have a numeric ID so insert one into the database. - roomNID, err = d.statements.insertRoomNID(ctx, txn, roomID, roomVersion) + roomNID, err = d.rooms.InsertRoomNID(ctx, txn, roomID, roomVersion) if err == sql.ErrNoRows { // We raced with another insert so run the select again. - roomNID, err = d.statements.selectRoomNID(ctx, txn, roomID) + roomNID, err = d.rooms.SelectRoomNID(ctx, txn, roomID) } } return roomNID, err } -func (d *Database) assignEventTypeNID( - ctx context.Context, eventType string, -) (eventTypeNID types.EventTypeNID, err error) { - err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { - // Check if we already have a numeric ID in the database. - eventTypeNID, err = d.eventTypes.SelectEventTypeNID(ctx, txn, eventType) - if err == sql.ErrNoRows { - // We don't have a numeric ID so insert one into the database. - eventTypeNID, err = d.eventTypes.InsertEventTypeNID(ctx, txn, eventType) - if err == sql.ErrNoRows { - // We raced with another insert so run the select again. - eventTypeNID, err = d.eventTypes.SelectEventTypeNID(ctx, txn, eventType) - } - } - return err - }) - return eventTypeNID, err -} - func (d *Database) assignStateKeyNID( ctx context.Context, txn *sql.Tx, eventStateKey string, ) (types.EventStateKeyNID, error) { @@ -243,38 +122,6 @@ func (d *Database) assignStateKeyNID( return eventStateKeyNID, err } -// Events implements input.EventDatabase -func (d *Database) Events( - ctx context.Context, eventNIDs []types.EventNID, -) ([]types.Event, error) { - eventJSONs, err := d.eventJSON.BulkSelectEventJSON(ctx, eventNIDs) - if err != nil { - return nil, err - } - results := make([]types.Event, len(eventJSONs)) - for i, eventJSON := range eventJSONs { - var roomNID types.RoomNID - var roomVersion gomatrixserverlib.RoomVersion - result := &results[i] - result.EventNID = eventJSON.EventNID - roomNID, err = d.events.SelectRoomNIDForEventNID(ctx, nil, eventJSON.EventNID) - if err != nil { - return nil, err - } - roomVersion, err = d.statements.selectRoomVersionForRoomNID(ctx, nil, roomNID) - if err != nil { - return nil, err - } - result.Event, err = gomatrixserverlib.NewEventFromTrustedJSON( - eventJSON.EventJSON, false, roomVersion, - ) - if err != nil { - return nil, err - } - } - return results, nil -} - // AddState implements input.EventDatabase func (d *Database) AddState( ctx context.Context, @@ -319,7 +166,7 @@ func (d *Database) GetLatestEventsForUpdate( return nil, err } eventNIDs, lastEventNIDSent, currentStateSnapshotNID, err := - d.statements.selectLatestEventsNIDsForUpdate(ctx, txn, roomNID) + d.rooms.SelectLatestEventsNIDsForUpdate(ctx, txn, roomNID) if err != nil { txn.Rollback() // nolint: errcheck return nil, err @@ -342,18 +189,6 @@ func (d *Database) GetLatestEventsForUpdate( }, nil } -// GetTransactionEventID implements input.EventDatabase -func (d *Database) GetTransactionEventID( - ctx context.Context, transactionID string, - sessionID int64, userID string, -) (string, error) { - eventID, err := d.statements.selectTransactionEventID(ctx, transactionID, sessionID, userID) - if err == sql.ErrNoRows { - return "", nil - } - return eventID, err -} - type roomRecentEventsUpdater struct { transaction d *Database @@ -415,7 +250,7 @@ func (u *roomRecentEventsUpdater) SetLatestEvents( for i := range latest { eventNIDs[i] = latest[i].EventNID } - return u.d.statements.updateLatestEventNIDs(u.ctx, u.txn, roomNID, eventNIDs, lastEventNIDSent, currentStateSnapshotNID) + return u.d.rooms.UpdateLatestEventNIDs(u.ctx, u.txn, roomNID, eventNIDs, lastEventNIDSent, currentStateSnapshotNID) } // HasEventBeenSent implements types.RoomRecentEventsUpdater @@ -432,55 +267,6 @@ func (u *roomRecentEventsUpdater) MembershipUpdater(targetUserNID types.EventSta return u.d.membershipUpdaterTxn(u.ctx, u.txn, u.roomNID, targetUserNID, targetLocal) } -// RoomNID implements query.RoomserverQueryAPIDB -func (d *Database) RoomNID(ctx context.Context, roomID string) (types.RoomNID, error) { - roomNID, err := d.statements.selectRoomNID(ctx, nil, roomID) - if err == sql.ErrNoRows { - return 0, nil - } - return roomNID, err -} - -// RoomNIDExcludingStubs implements query.RoomserverQueryAPIDB -func (d *Database) RoomNIDExcludingStubs(ctx context.Context, roomID string) (roomNID types.RoomNID, err error) { - roomNID, err = d.RoomNID(ctx, roomID) - if err != nil { - return - } - latestEvents, _, err := d.statements.selectLatestEventNIDs(ctx, roomNID) - if err != nil { - return - } - if len(latestEvents) == 0 { - roomNID = 0 - return - } - return -} - -// LatestEventIDs implements query.RoomserverQueryAPIDatabase -func (d *Database) LatestEventIDs( - ctx context.Context, roomNID types.RoomNID, -) (references []gomatrixserverlib.EventReference, currentStateSnapshotNID types.StateSnapshotNID, depth int64, err error) { - err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { - var eventNIDs []types.EventNID - eventNIDs, currentStateSnapshotNID, err = d.statements.selectLatestEventNIDs(ctx, roomNID) - if err != nil { - return err - } - references, err = d.events.BulkSelectEventReference(ctx, txn, eventNIDs) - if err != nil { - return err - } - depth, err = d.events.SelectMaxEventDepth(ctx, txn, eventNIDs) - if err != nil { - return err - } - return nil - }) - return -} - // GetInvitesForUser implements query.RoomserverQueryAPIDatabase func (d *Database) GetInvitesForUser( ctx context.Context, @@ -733,37 +519,6 @@ func (d *Database) GetMembershipEventNIDsForRoom( return d.statements.selectMembershipsFromRoom(ctx, roomNID, localOnly) } -// EventsFromIDs implements query.RoomserverQueryAPIEventDB -func (d *Database) EventsFromIDs(ctx context.Context, eventIDs []string) ([]types.Event, error) { - nidMap, err := d.EventNIDs(ctx, eventIDs) - if err != nil { - return nil, err - } - - var nids []types.EventNID - for _, nid := range nidMap { - nids = append(nids, nid) - } - - return d.Events(ctx, nids) -} - -func (d *Database) GetRoomVersionForRoom( - ctx context.Context, roomID string, -) (gomatrixserverlib.RoomVersion, error) { - return d.statements.selectRoomVersionForRoomID( - ctx, nil, roomID, - ) -} - -func (d *Database) GetRoomVersionForRoomNID( - ctx context.Context, roomNID types.RoomNID, -) (gomatrixserverlib.RoomVersion, error) { - return d.statements.selectRoomVersionForRoomNID( - ctx, nil, roomNID, - ) -} - type transaction struct { ctx context.Context txn *sql.Tx diff --git a/roomserver/storage/postgres/transactions_table.go b/roomserver/storage/postgres/transactions_table.go index 87c1cacae..7f7ef76ac 100644 --- a/roomserver/storage/postgres/transactions_table.go +++ b/roomserver/storage/postgres/transactions_table.go @@ -18,6 +18,8 @@ package postgres import ( "context" "database/sql" + + "github.com/matrix-org/dendrite/roomserver/storage/tables" ) const transactionsSchema = ` @@ -51,20 +53,21 @@ type transactionStatements struct { selectTransactionEventIDStmt *sql.Stmt } -func (s *transactionStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(transactionsSchema) +func NewPostgresTransactionsTable(db *sql.DB) (tables.Transactions, error) { + s := &transactionStatements{} + _, err := db.Exec(transactionsSchema) if err != nil { - return + return nil, err } - return statementList{ + return s, statementList{ {&s.insertTransactionStmt, insertTransactionSQL}, {&s.selectTransactionEventIDStmt, selectTransactionEventIDSQL}, }.prepare(db) } -func (s *transactionStatements) insertTransaction( - ctx context.Context, +func (s *transactionStatements) InsertTransaction( + ctx context.Context, txn *sql.Tx, transactionID string, sessionID int64, userID string, @@ -76,7 +79,7 @@ func (s *transactionStatements) insertTransaction( return } -func (s *transactionStatements) selectTransactionEventID( +func (s *transactionStatements) SelectTransactionEventID( ctx context.Context, transactionID string, sessionID int64, diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index a3b2c2e28..814b6e812 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -2,16 +2,24 @@ package shared import ( "context" + "database/sql" + "encoding/json" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/gomatrixserverlib" ) type Database struct { + DB *sql.DB EventsTable tables.Events EventJSONTable tables.EventJSON EventTypesTable tables.EventTypes EventStateKeysTable tables.EventStateKeys + RoomsTable tables.Rooms + TransactionsTable tables.Transactions } // EventTypeNIDs implements state.RoomStateDatabase @@ -77,3 +85,296 @@ func (d *Database) EventIDs( ) (map[types.EventNID]string, error) { return d.EventsTable.BulkSelectEventID(ctx, eventNIDs) } + +// EventsFromIDs implements query.RoomserverQueryAPIEventDB +func (d *Database) EventsFromIDs(ctx context.Context, eventIDs []string) ([]types.Event, error) { + nidMap, err := d.EventNIDs(ctx, eventIDs) + if err != nil { + return nil, err + } + + var nids []types.EventNID + for _, nid := range nidMap { + nids = append(nids, nid) + } + + return d.Events(ctx, nids) +} + +// RoomNID implements query.RoomserverQueryAPIDB +func (d *Database) RoomNID(ctx context.Context, roomID string) (types.RoomNID, error) { + roomNID, err := d.RoomsTable.SelectRoomNID(ctx, nil, roomID) + if err == sql.ErrNoRows { + return 0, nil + } + return roomNID, err +} + +// RoomNIDExcludingStubs implements query.RoomserverQueryAPIDB +func (d *Database) RoomNIDExcludingStubs(ctx context.Context, roomID string) (roomNID types.RoomNID, err error) { + roomNID, err = d.RoomNID(ctx, roomID) + if err != nil { + return + } + latestEvents, _, err := d.RoomsTable.SelectLatestEventNIDs(ctx, nil, roomNID) + if err != nil { + return + } + if len(latestEvents) == 0 { + roomNID = 0 + return + } + return +} + +// LatestEventIDs implements query.RoomserverQueryAPIDatabase +func (d *Database) LatestEventIDs( + ctx context.Context, roomNID types.RoomNID, +) (references []gomatrixserverlib.EventReference, currentStateSnapshotNID types.StateSnapshotNID, depth int64, err error) { + err = internal.WithTransaction(d.DB, func(txn *sql.Tx) error { + var eventNIDs []types.EventNID + eventNIDs, currentStateSnapshotNID, err = d.RoomsTable.SelectLatestEventNIDs(ctx, txn, roomNID) + if err != nil { + return err + } + references, err = d.EventsTable.BulkSelectEventReference(ctx, txn, eventNIDs) + if err != nil { + return err + } + depth, err = d.EventsTable.SelectMaxEventDepth(ctx, txn, eventNIDs) + if err != nil { + return err + } + return nil + }) + return +} + +func (d *Database) GetRoomVersionForRoom( + ctx context.Context, roomID string, +) (gomatrixserverlib.RoomVersion, error) { + return d.RoomsTable.SelectRoomVersionForRoomID( + ctx, nil, roomID, + ) +} + +func (d *Database) GetRoomVersionForRoomNID( + ctx context.Context, roomNID types.RoomNID, +) (gomatrixserverlib.RoomVersion, error) { + return d.RoomsTable.SelectRoomVersionForRoomNID( + ctx, roomNID, + ) +} + +// Events implements input.EventDatabase +func (d *Database) Events( + ctx context.Context, eventNIDs []types.EventNID, +) ([]types.Event, error) { + eventJSONs, err := d.EventJSONTable.BulkSelectEventJSON(ctx, eventNIDs) + if err != nil { + return nil, err + } + results := make([]types.Event, len(eventJSONs)) + for i, eventJSON := range eventJSONs { + var roomNID types.RoomNID + var roomVersion gomatrixserverlib.RoomVersion + result := &results[i] + result.EventNID = eventJSON.EventNID + roomNID, err = d.EventsTable.SelectRoomNIDForEventNID(ctx, eventJSON.EventNID) + if err != nil { + return nil, err + } + roomVersion, err = d.RoomsTable.SelectRoomVersionForRoomNID(ctx, roomNID) + if err != nil { + return nil, err + } + result.Event, err = gomatrixserverlib.NewEventFromTrustedJSON( + eventJSON.EventJSON, false, roomVersion, + ) + if err != nil { + return nil, err + } + } + return results, nil +} + +// GetTransactionEventID implements input.EventDatabase +func (d *Database) GetTransactionEventID( + ctx context.Context, transactionID string, + sessionID int64, userID string, +) (string, error) { + eventID, err := d.TransactionsTable.SelectTransactionEventID(ctx, transactionID, sessionID, userID) + if err == sql.ErrNoRows { + return "", nil + } + return eventID, err +} + +// StoreEvent implements input.EventDatabase +func (d *Database) StoreEvent( + ctx context.Context, event gomatrixserverlib.Event, + txnAndSessionID *api.TransactionID, authEventNIDs []types.EventNID, +) (types.RoomNID, types.StateAtEvent, error) { + var ( + roomNID types.RoomNID + eventTypeNID types.EventTypeNID + eventStateKeyNID types.EventStateKeyNID + eventNID types.EventNID + stateNID types.StateSnapshotNID + err error + ) + + err = internal.WithTransaction(d.DB, func(txn *sql.Tx) error { + if txnAndSessionID != nil { + if err = d.TransactionsTable.InsertTransaction( + ctx, txn, txnAndSessionID.TransactionID, + txnAndSessionID.SessionID, event.Sender(), event.EventID(), + ); err != nil { + return err + } + } + + // TODO: Here we should aim to have two different code paths for new rooms + // vs existing ones. + + // Get the default room version. If the client doesn't supply a room_version + // then we will use our configured default to create the room. + // https://matrix.org/docs/spec/client_server/r0.6.0#post-matrix-client-r0-createroom + // Note that the below logic depends on the m.room.create event being the + // first event that is persisted to the database when creating or joining a + // room. + var roomVersion gomatrixserverlib.RoomVersion + if roomVersion, err = extractRoomVersionFromCreateEvent(event); err != nil { + return err + } + + if roomNID, err = d.assignRoomNID(ctx, txn, event.RoomID(), roomVersion); err != nil { + return err + } + + if eventTypeNID, err = d.assignEventTypeNID(ctx, txn, event.Type()); err != nil { + return err + } + + eventStateKey := event.StateKey() + // Assigned a numeric ID for the state_key if there is one present. + // Otherwise set the numeric ID for the state_key to 0. + if eventStateKey != nil { + if eventStateKeyNID, err = d.assignStateKeyNID(ctx, txn, *eventStateKey); err != nil { + return err + } + } + + if eventNID, stateNID, err = d.EventsTable.InsertEvent( + ctx, + txn, + roomNID, + eventTypeNID, + eventStateKeyNID, + event.EventID(), + event.EventReference().EventSHA256, + authEventNIDs, + event.Depth(), + ); err != nil { + if err == sql.ErrNoRows { + // We've already inserted the event so select the numeric event ID + eventNID, stateNID, err = d.EventsTable.SelectEvent(ctx, txn, event.EventID()) + } + if err != nil { + return err + } + } + + if err = d.EventJSONTable.InsertEventJSON(ctx, txn, eventNID, event.JSON()); err != nil { + return err + } + + return nil + }) + if err != nil { + return 0, types.StateAtEvent{}, err + } + + return roomNID, types.StateAtEvent{ + BeforeStateSnapshotNID: stateNID, + StateEntry: types.StateEntry{ + StateKeyTuple: types.StateKeyTuple{ + EventTypeNID: eventTypeNID, + EventStateKeyNID: eventStateKeyNID, + }, + EventNID: eventNID, + }, + }, nil +} + +func (d *Database) assignRoomNID( + ctx context.Context, txn *sql.Tx, + roomID string, roomVersion gomatrixserverlib.RoomVersion, +) (types.RoomNID, error) { + // Check if we already have a numeric ID in the database. + roomNID, err := d.RoomsTable.SelectRoomNID(ctx, txn, roomID) + if err == sql.ErrNoRows { + // We don't have a numeric ID so insert one into the database. + roomNID, err = d.RoomsTable.InsertRoomNID(ctx, txn, roomID, roomVersion) + if err == sql.ErrNoRows { + // We raced with another insert so run the select again. + roomNID, err = d.RoomsTable.SelectRoomNID(ctx, txn, roomID) + } + } + return roomNID, err +} + +func (d *Database) assignEventTypeNID( + ctx context.Context, txn *sql.Tx, eventType string, +) (eventTypeNID types.EventTypeNID, err error) { + // Check if we already have a numeric ID in the database. + eventTypeNID, err = d.EventTypesTable.SelectEventTypeNID(ctx, txn, eventType) + if err == sql.ErrNoRows { + // We don't have a numeric ID so insert one into the database. + eventTypeNID, err = d.EventTypesTable.InsertEventTypeNID(ctx, txn, eventType) + if err == sql.ErrNoRows { + // We raced with another insert so run the select again. + eventTypeNID, err = d.EventTypesTable.SelectEventTypeNID(ctx, txn, eventType) + } + } + return +} + +func (d *Database) assignStateKeyNID( + ctx context.Context, txn *sql.Tx, eventStateKey string, +) (types.EventStateKeyNID, error) { + // Check if we already have a numeric ID in the database. + eventStateKeyNID, err := d.EventStateKeysTable.SelectEventStateKeyNID(ctx, txn, eventStateKey) + if err == sql.ErrNoRows { + // We don't have a numeric ID so insert one into the database. + eventStateKeyNID, err = d.EventStateKeysTable.InsertEventStateKeyNID(ctx, txn, eventStateKey) + if err == sql.ErrNoRows { + // We raced with another insert so run the select again. + eventStateKeyNID, err = d.EventStateKeysTable.SelectEventStateKeyNID(ctx, txn, eventStateKey) + } + } + return eventStateKeyNID, err +} + +func extractRoomVersionFromCreateEvent(event gomatrixserverlib.Event) ( + gomatrixserverlib.RoomVersion, error, +) { + var err error + var roomVersion gomatrixserverlib.RoomVersion + // Look for m.room.create events. + if event.Type() != gomatrixserverlib.MRoomCreate { + return gomatrixserverlib.RoomVersion(""), nil + } + roomVersion = gomatrixserverlib.RoomVersionV1 + var createContent gomatrixserverlib.CreateContent + // The m.room.create event contains an optional "room_version" key in + // the event content, so we need to unmarshal that first. + if err = json.Unmarshal(event.Content(), &createContent); err != nil { + return gomatrixserverlib.RoomVersion(""), err + } + // A room version was specified in the event content? + if createContent.RoomVersion != nil { + roomVersion = gomatrixserverlib.RoomVersion(*createContent.RoomVersion) + } + return roomVersion, err +} diff --git a/roomserver/storage/sqlite3/events_table.go b/roomserver/storage/sqlite3/events_table.go index a41a8737f..247faa68f 100644 --- a/roomserver/storage/sqlite3/events_table.go +++ b/roomserver/storage/sqlite3/events_table.go @@ -469,10 +469,9 @@ func (s *eventStatements) SelectMaxEventDepth(ctx context.Context, txn *sql.Tx, } func (s *eventStatements) SelectRoomNIDForEventNID( - ctx context.Context, txn *sql.Tx, eventNID types.EventNID, + ctx context.Context, eventNID types.EventNID, ) (roomNID types.RoomNID, err error) { - selectStmt := internal.TxStmt(txn, s.selectRoomNIDForEventNIDStmt) - err = selectStmt.QueryRowContext(ctx, int64(eventNID)).Scan(&roomNID) + err = s.selectRoomNIDForEventNIDStmt.QueryRowContext(ctx, int64(eventNID)).Scan(&roomNID) return } diff --git a/roomserver/storage/sqlite3/rooms_table.go b/roomserver/storage/sqlite3/rooms_table.go index ea949d1e9..75b8fec90 100644 --- a/roomserver/storage/sqlite3/rooms_table.go +++ b/roomserver/storage/sqlite3/rooms_table.go @@ -22,6 +22,7 @@ import ( "errors" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" ) @@ -71,12 +72,13 @@ type roomStatements struct { selectRoomVersionForRoomNIDStmt *sql.Stmt } -func (s *roomStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(roomsSchema) +func NewSqliteRoomsTable(db *sql.DB) (tables.Rooms, error) { + s := &roomStatements{} + _, err := db.Exec(roomsSchema) if err != nil { - return + return nil, err } - return statementList{ + return s, statementList{ {&s.insertRoomNIDStmt, insertRoomNIDSQL}, {&s.selectRoomNIDStmt, selectRoomNIDSQL}, {&s.selectLatestEventNIDsStmt, selectLatestEventNIDsSQL}, @@ -87,20 +89,20 @@ func (s *roomStatements) prepare(db *sql.DB) (err error) { }.prepare(db) } -func (s *roomStatements) insertRoomNID( +func (s *roomStatements) InsertRoomNID( ctx context.Context, txn *sql.Tx, roomID string, roomVersion gomatrixserverlib.RoomVersion, ) (types.RoomNID, error) { var err error insertStmt := internal.TxStmt(txn, s.insertRoomNIDStmt) if _, err = insertStmt.ExecContext(ctx, roomID, roomVersion); err == nil { - return s.selectRoomNID(ctx, txn, roomID) + return s.SelectRoomNID(ctx, txn, roomID) } else { return types.RoomNID(0), err } } -func (s *roomStatements) selectRoomNID( +func (s *roomStatements) SelectRoomNID( ctx context.Context, txn *sql.Tx, roomID string, ) (types.RoomNID, error) { var roomNID int64 @@ -109,7 +111,7 @@ func (s *roomStatements) selectRoomNID( return types.RoomNID(roomNID), err } -func (s *roomStatements) selectLatestEventNIDs( +func (s *roomStatements) SelectLatestEventNIDs( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, ) ([]types.EventNID, types.StateSnapshotNID, error) { var eventNIDs []types.EventNID @@ -126,7 +128,7 @@ func (s *roomStatements) selectLatestEventNIDs( return eventNIDs, types.StateSnapshotNID(stateSnapshotNID), nil } -func (s *roomStatements) selectLatestEventsNIDsForUpdate( +func (s *roomStatements) SelectLatestEventsNIDsForUpdate( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, ) ([]types.EventNID, types.EventNID, types.StateSnapshotNID, error) { var eventNIDs []types.EventNID @@ -144,7 +146,7 @@ func (s *roomStatements) selectLatestEventsNIDsForUpdate( return eventNIDs, types.EventNID(lastEventSentNID), types.StateSnapshotNID(stateSnapshotNID), nil } -func (s *roomStatements) updateLatestEventNIDs( +func (s *roomStatements) UpdateLatestEventNIDs( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, @@ -163,7 +165,7 @@ func (s *roomStatements) updateLatestEventNIDs( return err } -func (s *roomStatements) selectRoomVersionForRoomID( +func (s *roomStatements) SelectRoomVersionForRoomID( ctx context.Context, txn *sql.Tx, roomID string, ) (gomatrixserverlib.RoomVersion, error) { var roomVersion gomatrixserverlib.RoomVersion @@ -175,12 +177,11 @@ func (s *roomStatements) selectRoomVersionForRoomID( return roomVersion, err } -func (s *roomStatements) selectRoomVersionForRoomNID( - ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, +func (s *roomStatements) SelectRoomVersionForRoomNID( + ctx context.Context, roomNID types.RoomNID, ) (gomatrixserverlib.RoomVersion, error) { var roomVersion gomatrixserverlib.RoomVersion - stmt := internal.TxStmt(txn, s.selectRoomVersionForRoomNIDStmt) - err := stmt.QueryRowContext(ctx, roomNID).Scan(&roomVersion) + err := s.selectRoomVersionForRoomNIDStmt.QueryRowContext(ctx, roomNID).Scan(&roomVersion) if err == sql.ErrNoRows { return roomVersion, errors.New("room not found") } diff --git a/roomserver/storage/sqlite3/sql.go b/roomserver/storage/sqlite3/sql.go index ac44b398d..fe899174a 100644 --- a/roomserver/storage/sqlite3/sql.go +++ b/roomserver/storage/sqlite3/sql.go @@ -38,14 +38,12 @@ func (s *statements) prepare(db *sql.DB) error { var err error for _, prepare := range []func(db *sql.DB) error{ - s.roomStatements.prepare, s.stateSnapshotStatements.prepare, s.stateBlockStatements.prepare, s.previousEventStatements.prepare, s.roomAliasesStatements.prepare, s.inviteStatements.prepare, s.membershipStatements.prepare, - s.transactionStatements.prepare, } { if err = prepare(db); err != nil { return err diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index 05bad7fb9..a0a1b5684 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -18,14 +18,12 @@ package sqlite3 import ( "context" "database/sql" - "encoding/json" "errors" "net/url" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" @@ -41,6 +39,8 @@ type Database struct { eventJSON tables.EventJSON eventTypes tables.EventTypes eventStateKeys tables.EventStateKeys + rooms tables.Rooms + transactions tables.Transactions db *sql.DB } @@ -89,163 +89,38 @@ func Open(dataSourceName string) (*Database, error) { if err != nil { return nil, err } + d.rooms, err = NewSqliteRoomsTable(d.db) + if err != nil { + return nil, err + } + d.transactions, err = NewSqliteTransactionsTable(d.db) + if err != nil { + return nil, err + } d.Database = shared.Database{ + DB: d.db, EventsTable: d.events, EventTypesTable: d.eventTypes, EventStateKeysTable: d.eventStateKeys, EventJSONTable: d.eventJSON, + RoomsTable: d.rooms, + TransactionsTable: d.transactions, } return &d, nil } -// StoreEvent implements input.EventDatabase -func (d *Database) StoreEvent( - ctx context.Context, event gomatrixserverlib.Event, - txnAndSessionID *api.TransactionID, authEventNIDs []types.EventNID, -) (types.RoomNID, types.StateAtEvent, error) { - var ( - roomNID types.RoomNID - eventTypeNID types.EventTypeNID - eventStateKeyNID types.EventStateKeyNID - eventNID types.EventNID - stateNID types.StateSnapshotNID - err error - ) - - err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { - if txnAndSessionID != nil { - if err = d.statements.insertTransaction( - ctx, txn, txnAndSessionID.TransactionID, - txnAndSessionID.SessionID, event.Sender(), event.EventID(), - ); err != nil { - return err - } - } - - // TODO: Here we should aim to have two different code paths for new rooms - // vs existing ones. - - // Get the default room version. If the client doesn't supply a room_version - // then we will use our configured default to create the room. - // https://matrix.org/docs/spec/client_server/r0.6.0#post-matrix-client-r0-createroom - // Note that the below logic depends on the m.room.create event being the - // first event that is persisted to the database when creating or joining a - // room. - var roomVersion gomatrixserverlib.RoomVersion - if roomVersion, err = extractRoomVersionFromCreateEvent(event); err != nil { - return err - } - - if roomNID, err = d.assignRoomNID(ctx, txn, event.RoomID(), roomVersion); err != nil { - return err - } - - if eventTypeNID, err = d.assignEventTypeNID(ctx, txn, event.Type()); err != nil { - return err - } - - eventStateKey := event.StateKey() - // Assigned a numeric ID for the state_key if there is one present. - // Otherwise set the numeric ID for the state_key to 0. - if eventStateKey != nil { - if eventStateKeyNID, err = d.assignStateKeyNID(ctx, txn, *eventStateKey); err != nil { - return err - } - } - - if eventNID, stateNID, err = d.events.InsertEvent( - ctx, - txn, - roomNID, - eventTypeNID, - eventStateKeyNID, - event.EventID(), - event.EventReference().EventSHA256, - authEventNIDs, - event.Depth(), - ); err != nil { - if err == sql.ErrNoRows { - // We've already inserted the event so select the numeric event ID - eventNID, stateNID, err = d.events.SelectEvent(ctx, txn, event.EventID()) - } - if err != nil { - return err - } - } - - if err = d.eventJSON.InsertEventJSON(ctx, txn, eventNID, event.JSON()); err != nil { - return err - } - - return nil - }) - if err != nil { - return 0, types.StateAtEvent{}, err - } - - return roomNID, types.StateAtEvent{ - BeforeStateSnapshotNID: stateNID, - StateEntry: types.StateEntry{ - StateKeyTuple: types.StateKeyTuple{ - EventTypeNID: eventTypeNID, - EventStateKeyNID: eventStateKeyNID, - }, - EventNID: eventNID, - }, - }, nil -} - -func extractRoomVersionFromCreateEvent(event gomatrixserverlib.Event) ( - gomatrixserverlib.RoomVersion, error, -) { - var err error - var roomVersion gomatrixserverlib.RoomVersion - // Look for m.room.create events. - if event.Type() != gomatrixserverlib.MRoomCreate { - return gomatrixserverlib.RoomVersion(""), nil - } - roomVersion = gomatrixserverlib.RoomVersionV1 - var createContent gomatrixserverlib.CreateContent - // The m.room.create event contains an optional "room_version" key in - // the event content, so we need to unmarshal that first. - if err = json.Unmarshal(event.Content(), &createContent); err != nil { - return gomatrixserverlib.RoomVersion(""), err - } - // A room version was specified in the event content? - if createContent.RoomVersion != nil { - roomVersion = gomatrixserverlib.RoomVersion(*createContent.RoomVersion) - } - return roomVersion, err -} - func (d *Database) assignRoomNID( ctx context.Context, txn *sql.Tx, roomID string, roomVersion gomatrixserverlib.RoomVersion, ) (roomNID types.RoomNID, err error) { // Check if we already have a numeric ID in the database. - roomNID, err = d.statements.selectRoomNID(ctx, txn, roomID) + roomNID, err = d.rooms.SelectRoomNID(ctx, txn, roomID) if err == sql.ErrNoRows { // We don't have a numeric ID so insert one into the database. - roomNID, err = d.statements.insertRoomNID(ctx, txn, roomID, roomVersion) + roomNID, err = d.rooms.InsertRoomNID(ctx, txn, roomID, roomVersion) if err == nil { // Now get the numeric ID back out of the database - roomNID, err = d.statements.selectRoomNID(ctx, txn, roomID) - } - } - return -} - -func (d *Database) assignEventTypeNID( - ctx context.Context, txn *sql.Tx, eventType string, -) (eventTypeNID types.EventTypeNID, err error) { - // Check if we already have a numeric ID in the database. - eventTypeNID, err = d.eventTypes.SelectEventTypeNID(ctx, txn, eventType) - if err == sql.ErrNoRows { - // We don't have a numeric ID so insert one into the database. - eventTypeNID, err = d.eventTypes.InsertEventTypeNID(ctx, txn, eventType) - if err == sql.ErrNoRows { - // We raced with another insert so run the select again. - eventTypeNID, err = d.eventTypes.SelectEventTypeNID(ctx, txn, eventType) + roomNID, err = d.rooms.SelectRoomNID(ctx, txn, roomID) } } return @@ -267,47 +142,6 @@ func (d *Database) assignStateKeyNID( return } -// Events implements input.EventDatabase -func (d *Database) Events( - ctx context.Context, eventNIDs []types.EventNID, -) ([]types.Event, error) { - var eventJSONs []tables.EventJSONPair - var err error - var results []types.Event - eventJSONs, err = d.eventJSON.BulkSelectEventJSON(ctx, eventNIDs) - if err != nil || len(eventJSONs) == 0 { - return nil, nil - } - err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { - results = make([]types.Event, len(eventJSONs)) - for i, eventJSON := range eventJSONs { - var roomNID types.RoomNID - var roomVersion gomatrixserverlib.RoomVersion - result := &results[i] - result.EventNID = eventJSON.EventNID - roomNID, err = d.events.SelectRoomNIDForEventNID(ctx, txn, eventJSON.EventNID) - if err != nil { - return err - } - roomVersion, err = d.statements.selectRoomVersionForRoomNID(ctx, txn, roomNID) - if err != nil { - return err - } - result.Event, err = gomatrixserverlib.NewEventFromTrustedJSON( - eventJSON.EventJSON, false, roomVersion, - ) - if err != nil { - return nil - } - } - return nil - }) - if err != nil { - return []types.Event{}, err - } - return results, nil -} - // AddState implements input.EventDatabase func (d *Database) AddState( ctx context.Context, @@ -364,7 +198,7 @@ func (d *Database) GetLatestEventsForUpdate( return nil, err } eventNIDs, lastEventNIDSent, currentStateSnapshotNID, err := - d.statements.selectLatestEventsNIDsForUpdate(ctx, txn, roomNID) + d.rooms.SelectLatestEventsNIDsForUpdate(ctx, txn, roomNID) if err != nil { txn.Rollback() // nolint: errcheck return nil, err @@ -396,18 +230,6 @@ func (d *Database) GetLatestEventsForUpdate( }, nil } -// GetTransactionEventID implements input.EventDatabase -func (d *Database) GetTransactionEventID( - ctx context.Context, transactionID string, - sessionID int64, userID string, -) (string, error) { - eventID, err := d.statements.selectTransactionEventID(ctx, nil, transactionID, sessionID, userID) - if err == sql.ErrNoRows { - return "", nil - } - return eventID, err -} - type roomRecentEventsUpdater struct { transaction d *Database @@ -478,7 +300,7 @@ func (u *roomRecentEventsUpdater) SetLatestEvents( for i := range latest { eventNIDs[i] = latest[i].EventNID } - return u.d.statements.updateLatestEventNIDs(u.ctx, txn, roomNID, eventNIDs, lastEventNIDSent, currentStateSnapshotNID) + return u.d.rooms.UpdateLatestEventNIDs(u.ctx, txn, roomNID, eventNIDs, lastEventNIDSent, currentStateSnapshotNID) }) return err } @@ -508,59 +330,6 @@ func (u *roomRecentEventsUpdater) MembershipUpdater(targetUserNID types.EventSta return } -// RoomNID implements query.RoomserverQueryAPIDB -func (d *Database) RoomNID(ctx context.Context, roomID string) (roomNID types.RoomNID, err error) { - err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { - roomNID, err = d.statements.selectRoomNID(ctx, txn, roomID) - if err == sql.ErrNoRows { - roomNID = 0 - err = nil - } - return err - }) - return -} - -// RoomNIDExcludingStubs implements query.RoomserverQueryAPIDB -func (d *Database) RoomNIDExcludingStubs(ctx context.Context, roomID string) (roomNID types.RoomNID, err error) { - roomNID, err = d.RoomNID(ctx, roomID) - if err != nil { - return - } - latestEvents, _, err := d.statements.selectLatestEventNIDs(ctx, nil, roomNID) - if err != nil { - return - } - if len(latestEvents) == 0 { - roomNID = 0 - return - } - return -} - -// LatestEventIDs implements query.RoomserverQueryAPIDatabase -func (d *Database) LatestEventIDs( - ctx context.Context, roomNID types.RoomNID, -) (references []gomatrixserverlib.EventReference, currentStateSnapshotNID types.StateSnapshotNID, depth int64, err error) { - err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { - var eventNIDs []types.EventNID - eventNIDs, currentStateSnapshotNID, err = d.statements.selectLatestEventNIDs(ctx, txn, roomNID) - if err != nil { - return err - } - references, err = d.events.BulkSelectEventReference(ctx, txn, eventNIDs) - if err != nil { - return err - } - depth, err = d.events.SelectMaxEventDepth(ctx, txn, eventNIDs) - if err != nil { - return err - } - return nil - }) - return -} - // GetInvitesForUser implements query.RoomserverQueryAPIDatabase func (d *Database) GetInvitesForUser( ctx context.Context, @@ -844,37 +613,6 @@ func (d *Database) GetMembershipEventNIDsForRoom( return } -// EventsFromIDs implements query.RoomserverQueryAPIEventDB -func (d *Database) EventsFromIDs(ctx context.Context, eventIDs []string) ([]types.Event, error) { - nidMap, err := d.EventNIDs(ctx, eventIDs) - if err != nil { - return nil, err - } - - var nids []types.EventNID - for _, nid := range nidMap { - nids = append(nids, nid) - } - - return d.Events(ctx, nids) -} - -func (d *Database) GetRoomVersionForRoom( - ctx context.Context, roomID string, -) (gomatrixserverlib.RoomVersion, error) { - return d.statements.selectRoomVersionForRoomID( - ctx, nil, roomID, - ) -} - -func (d *Database) GetRoomVersionForRoomNID( - ctx context.Context, roomNID types.RoomNID, -) (gomatrixserverlib.RoomVersion, error) { - return d.statements.selectRoomVersionForRoomNID( - ctx, nil, roomNID, - ) -} - type transaction struct { ctx context.Context txn *sql.Tx diff --git a/roomserver/storage/sqlite3/transactions_table.go b/roomserver/storage/sqlite3/transactions_table.go index d22c73845..37ea15c0f 100644 --- a/roomserver/storage/sqlite3/transactions_table.go +++ b/roomserver/storage/sqlite3/transactions_table.go @@ -20,6 +20,7 @@ import ( "database/sql" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/roomserver/storage/tables" ) const transactionsSchema = ` @@ -46,19 +47,20 @@ type transactionStatements struct { selectTransactionEventIDStmt *sql.Stmt } -func (s *transactionStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(transactionsSchema) +func NewSqliteTransactionsTable(db *sql.DB) (tables.Transactions, error) { + s := &transactionStatements{} + _, err := db.Exec(transactionsSchema) if err != nil { - return + return nil, err } - return statementList{ + return s, statementList{ {&s.insertTransactionStmt, insertTransactionSQL}, {&s.selectTransactionEventIDStmt, selectTransactionEventIDSQL}, }.prepare(db) } -func (s *transactionStatements) insertTransaction( +func (s *transactionStatements) InsertTransaction( ctx context.Context, txn *sql.Tx, transactionID string, sessionID int64, @@ -72,14 +74,13 @@ func (s *transactionStatements) insertTransaction( return } -func (s *transactionStatements) selectTransactionEventID( - ctx context.Context, txn *sql.Tx, +func (s *transactionStatements) SelectTransactionEventID( + ctx context.Context, transactionID string, sessionID int64, userID string, ) (eventID string, err error) { - stmt := internal.TxStmt(txn, s.selectTransactionEventIDStmt) - err = stmt.QueryRowContext( + err = s.selectTransactionEventIDStmt.QueryRowContext( ctx, transactionID, sessionID, userID, ).Scan(&eventID) return diff --git a/roomserver/storage/tables/interface.go b/roomserver/storage/tables/interface.go index 78ddc5fea..e913de6b4 100644 --- a/roomserver/storage/tables/interface.go +++ b/roomserver/storage/tables/interface.go @@ -53,5 +53,20 @@ type Events interface { // If an event ID is not in the database then it is omitted from the map. BulkSelectEventNID(ctx context.Context, eventIDs []string) (map[string]types.EventNID, error) SelectMaxEventDepth(ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID) (int64, error) - SelectRoomNIDForEventNID(ctx context.Context, txn *sql.Tx, eventNID types.EventNID) (roomNID types.RoomNID, err error) + SelectRoomNIDForEventNID(ctx context.Context, eventNID types.EventNID) (roomNID types.RoomNID, err error) +} + +type Rooms interface { + InsertRoomNID(ctx context.Context, txn *sql.Tx, roomID string, roomVersion gomatrixserverlib.RoomVersion) (types.RoomNID, error) + SelectRoomNID(ctx context.Context, txn *sql.Tx, roomID string) (types.RoomNID, error) + SelectLatestEventNIDs(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID) ([]types.EventNID, types.StateSnapshotNID, error) + SelectLatestEventsNIDsForUpdate(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID) ([]types.EventNID, types.EventNID, types.StateSnapshotNID, error) + UpdateLatestEventNIDs(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, eventNIDs []types.EventNID, lastEventSentNID types.EventNID, stateSnapshotNID types.StateSnapshotNID) error + SelectRoomVersionForRoomID(ctx context.Context, txn *sql.Tx, roomID string) (gomatrixserverlib.RoomVersion, error) + SelectRoomVersionForRoomNID(ctx context.Context, roomNID types.RoomNID) (gomatrixserverlib.RoomVersion, error) +} + +type Transactions interface { + InsertTransaction(ctx context.Context, txn *sql.Tx, transactionID string, sessionID int64, userID string, eventID string) error + SelectTransactionEventID(ctx context.Context, transactionID string, sessionID int64, userID string) (eventID string, err error) } From c0c5d9452a24dcecb9378dd147f48098f18cce04 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Wed, 27 May 2020 09:36:09 +0100 Subject: [PATCH 096/200] Convert room_aliases previous_events state_block and state_snapshot tables (#1064) * Convert state_snapshot and state_block tables * Convert room_aliases and previous_events tables * Add missing table --- .../storage/postgres/previous_events_table.go | 14 ++- .../storage/postgres/room_aliases_table.go | 20 ++-- roomserver/storage/postgres/sql.go | 4 - .../storage/postgres/state_block_table.go | 28 +++-- .../storage/postgres/state_snapshot_table.go | 18 +-- roomserver/storage/postgres/storage.go | 99 ++++------------ roomserver/storage/shared/storage.go | 81 +++++++++++++ .../storage/sqlite3/previous_events_table.go | 14 ++- .../storage/sqlite3/room_aliases_table.go | 45 ++++--- roomserver/storage/sqlite3/sql.go | 4 - .../storage/sqlite3/state_block_table.go | 23 ++-- .../storage/sqlite3/state_snapshot_table.go | 18 +-- roomserver/storage/sqlite3/storage.go | 111 ++++-------------- roomserver/storage/tables/interface.go | 26 ++++ 14 files changed, 252 insertions(+), 253 deletions(-) diff --git a/roomserver/storage/postgres/previous_events_table.go b/roomserver/storage/postgres/previous_events_table.go index e3ad5dc80..b3d32c953 100644 --- a/roomserver/storage/postgres/previous_events_table.go +++ b/roomserver/storage/postgres/previous_events_table.go @@ -20,6 +20,7 @@ import ( "database/sql" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -63,19 +64,20 @@ type previousEventStatements struct { selectPreviousEventExistsStmt *sql.Stmt } -func (s *previousEventStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(previousEventSchema) +func NewPostgresPreviousEventsTable(db *sql.DB) (tables.PreviousEvents, error) { + s := &previousEventStatements{} + _, err := db.Exec(previousEventSchema) if err != nil { - return + return nil, err } - return statementList{ + return s, statementList{ {&s.insertPreviousEventStmt, insertPreviousEventSQL}, {&s.selectPreviousEventExistsStmt, selectPreviousEventExistsSQL}, }.prepare(db) } -func (s *previousEventStatements) insertPreviousEvent( +func (s *previousEventStatements) InsertPreviousEvent( ctx context.Context, txn *sql.Tx, previousEventID string, @@ -91,7 +93,7 @@ func (s *previousEventStatements) insertPreviousEvent( // Check if the event reference exists // Returns sql.ErrNoRows if the event reference doesn't exist. -func (s *previousEventStatements) selectPreviousEventExists( +func (s *previousEventStatements) SelectPreviousEventExists( ctx context.Context, txn *sql.Tx, eventID string, eventReferenceSHA256 []byte, ) error { var ok int64 diff --git a/roomserver/storage/postgres/room_aliases_table.go b/roomserver/storage/postgres/room_aliases_table.go index c77edd0e0..f869cf4fb 100644 --- a/roomserver/storage/postgres/room_aliases_table.go +++ b/roomserver/storage/postgres/room_aliases_table.go @@ -20,6 +20,7 @@ import ( "database/sql" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/roomserver/storage/tables" ) const roomAliasesSchema = ` @@ -59,12 +60,13 @@ type roomAliasesStatements struct { deleteRoomAliasStmt *sql.Stmt } -func (s *roomAliasesStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(roomAliasesSchema) +func NewPostgresRoomAliasesTable(db *sql.DB) (tables.RoomAliases, error) { + s := &roomAliasesStatements{} + _, err := db.Exec(roomAliasesSchema) if err != nil { - return + return nil, err } - return statementList{ + return s, statementList{ {&s.insertRoomAliasStmt, insertRoomAliasSQL}, {&s.selectRoomIDFromAliasStmt, selectRoomIDFromAliasSQL}, {&s.selectAliasesFromRoomIDStmt, selectAliasesFromRoomIDSQL}, @@ -73,14 +75,14 @@ func (s *roomAliasesStatements) prepare(db *sql.DB) (err error) { }.prepare(db) } -func (s *roomAliasesStatements) insertRoomAlias( +func (s *roomAliasesStatements) InsertRoomAlias( ctx context.Context, alias string, roomID string, creatorUserID string, ) (err error) { _, err = s.insertRoomAliasStmt.ExecContext(ctx, alias, roomID, creatorUserID) return } -func (s *roomAliasesStatements) selectRoomIDFromAlias( +func (s *roomAliasesStatements) SelectRoomIDFromAlias( ctx context.Context, alias string, ) (roomID string, err error) { err = s.selectRoomIDFromAliasStmt.QueryRowContext(ctx, alias).Scan(&roomID) @@ -90,7 +92,7 @@ func (s *roomAliasesStatements) selectRoomIDFromAlias( return } -func (s *roomAliasesStatements) selectAliasesFromRoomID( +func (s *roomAliasesStatements) SelectAliasesFromRoomID( ctx context.Context, roomID string, ) ([]string, error) { rows, err := s.selectAliasesFromRoomIDStmt.QueryContext(ctx, roomID) @@ -111,7 +113,7 @@ func (s *roomAliasesStatements) selectAliasesFromRoomID( return aliases, rows.Err() } -func (s *roomAliasesStatements) selectCreatorIDFromAlias( +func (s *roomAliasesStatements) SelectCreatorIDFromAlias( ctx context.Context, alias string, ) (creatorID string, err error) { err = s.selectCreatorIDFromAliasStmt.QueryRowContext(ctx, alias).Scan(&creatorID) @@ -121,7 +123,7 @@ func (s *roomAliasesStatements) selectCreatorIDFromAlias( return } -func (s *roomAliasesStatements) deleteRoomAlias( +func (s *roomAliasesStatements) DeleteRoomAlias( ctx context.Context, alias string, ) (err error) { _, err = s.deleteRoomAliasStmt.ExecContext(ctx, alias) diff --git a/roomserver/storage/postgres/sql.go b/roomserver/storage/postgres/sql.go index 914f269c5..eb626dd88 100644 --- a/roomserver/storage/postgres/sql.go +++ b/roomserver/storage/postgres/sql.go @@ -38,10 +38,6 @@ func (s *statements) prepare(db *sql.DB) error { var err error for _, prepare := range []func(db *sql.DB) error{ - s.stateSnapshotStatements.prepare, - s.stateBlockStatements.prepare, - s.previousEventStatements.prepare, - s.roomAliasesStatements.prepare, s.inviteStatements.prepare, s.membershipStatements.prepare, } { diff --git a/roomserver/storage/postgres/state_block_table.go b/roomserver/storage/postgres/state_block_table.go index 38334fa9e..d1aaaa003 100644 --- a/roomserver/storage/postgres/state_block_table.go +++ b/roomserver/storage/postgres/state_block_table.go @@ -24,6 +24,7 @@ import ( "github.com/matrix-org/dendrite/internal" "github.com/lib/pq" + "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/util" ) @@ -87,13 +88,14 @@ type stateBlockStatements struct { bulkSelectFilteredStateBlockEntriesStmt *sql.Stmt } -func (s *stateBlockStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(stateDataSchema) +func NewPostgresStateBlockTable(db *sql.DB) (tables.StateBlock, error) { + s := &stateBlockStatements{} + _, err := db.Exec(stateDataSchema) if err != nil { - return + return nil, err } - return statementList{ + return s, statementList{ {&s.insertStateDataStmt, insertStateDataSQL}, {&s.selectNextStateBlockNIDStmt, selectNextStateBlockNIDSQL}, {&s.bulkSelectStateBlockEntriesStmt, bulkSelectStateBlockEntriesSQL}, @@ -101,11 +103,15 @@ func (s *stateBlockStatements) prepare(db *sql.DB) (err error) { }.prepare(db) } -func (s *stateBlockStatements) bulkInsertStateData( +func (s *stateBlockStatements) BulkInsertStateData( ctx context.Context, - stateBlockNID types.StateBlockNID, + txn *sql.Tx, entries []types.StateEntry, -) error { +) (types.StateBlockNID, error) { + stateBlockNID, err := s.selectNextStateBlockNID(ctx) + if err != nil { + return 0, err + } for _, entry := range entries { _, err := s.insertStateDataStmt.ExecContext( ctx, @@ -115,10 +121,10 @@ func (s *stateBlockStatements) bulkInsertStateData( int64(entry.EventNID), ) if err != nil { - return err + return 0, err } } - return nil + return stateBlockNID, nil } func (s *stateBlockStatements) selectNextStateBlockNID( @@ -129,7 +135,7 @@ func (s *stateBlockStatements) selectNextStateBlockNID( return types.StateBlockNID(stateBlockNID), err } -func (s *stateBlockStatements) bulkSelectStateBlockEntries( +func (s *stateBlockStatements) BulkSelectStateBlockEntries( ctx context.Context, stateBlockNIDs []types.StateBlockNID, ) ([]types.StateEntryList, error) { nids := make([]int64, len(stateBlockNIDs)) @@ -180,7 +186,7 @@ func (s *stateBlockStatements) bulkSelectStateBlockEntries( return results, err } -func (s *stateBlockStatements) bulkSelectFilteredStateBlockEntries( +func (s *stateBlockStatements) BulkSelectFilteredStateBlockEntries( ctx context.Context, stateBlockNIDs []types.StateBlockNID, stateKeyTuples []types.StateKeyTuple, diff --git a/roomserver/storage/postgres/state_snapshot_table.go b/roomserver/storage/postgres/state_snapshot_table.go index a1f26e228..8971292fc 100644 --- a/roomserver/storage/postgres/state_snapshot_table.go +++ b/roomserver/storage/postgres/state_snapshot_table.go @@ -21,6 +21,7 @@ import ( "fmt" "github.com/lib/pq" + "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -64,30 +65,31 @@ type stateSnapshotStatements struct { bulkSelectStateBlockNIDsStmt *sql.Stmt } -func (s *stateSnapshotStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(stateSnapshotSchema) +func NewPostgresStateSnapshotTable(db *sql.DB) (tables.StateSnapshot, error) { + s := &stateSnapshotStatements{} + _, err := db.Exec(stateSnapshotSchema) if err != nil { - return + return nil, err } - return statementList{ + return s, statementList{ {&s.insertStateStmt, insertStateSQL}, {&s.bulkSelectStateBlockNIDsStmt, bulkSelectStateBlockNIDsSQL}, }.prepare(db) } -func (s *stateSnapshotStatements) insertState( - ctx context.Context, roomNID types.RoomNID, stateBlockNIDs []types.StateBlockNID, +func (s *stateSnapshotStatements) InsertState( + ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, stateBlockNIDs []types.StateBlockNID, ) (stateNID types.StateSnapshotNID, err error) { nids := make([]int64, len(stateBlockNIDs)) for i := range stateBlockNIDs { nids[i] = int64(stateBlockNIDs[i]) } - err = s.insertStateStmt.QueryRowContext(ctx, int64(roomNID), pq.Int64Array(nids)).Scan(&stateNID) + err = txn.Stmt(s.insertStateStmt).QueryRowContext(ctx, int64(roomNID), pq.Int64Array(nids)).Scan(&stateNID) return } -func (s *stateSnapshotStatements) bulkSelectStateBlockNIDs( +func (s *stateSnapshotStatements) BulkSelectStateBlockNIDs( ctx context.Context, stateNIDs []types.StateSnapshotNID, ) ([]types.StateBlockNIDList, error) { nids := make([]int64, len(stateNIDs)) diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go index d44da8585..03cfb7f0e 100644 --- a/roomserver/storage/postgres/storage.go +++ b/roomserver/storage/postgres/storage.go @@ -40,10 +40,12 @@ type Database struct { eventJSON tables.EventJSON rooms tables.Rooms transactions tables.Transactions + prevEvents tables.PreviousEvents db *sql.DB } // Open a postgres database. +// nolint: gocyclo func Open(dataSourceName string, dbProperties internal.DbProperties) (*Database, error) { var d Database var err error @@ -77,6 +79,22 @@ func Open(dataSourceName string, dbProperties internal.DbProperties) (*Database, if err != nil { return nil, err } + stateBlock, err := NewPostgresStateBlockTable(d.db) + if err != nil { + return nil, err + } + stateSnapshot, err := NewPostgresStateSnapshotTable(d.db) + if err != nil { + return nil, err + } + roomAliases, err := NewPostgresRoomAliasesTable(d.db) + if err != nil { + return nil, err + } + d.prevEvents, err = NewPostgresPreviousEventsTable(d.db) + if err != nil { + return nil, err + } d.Database = shared.Database{ DB: d.db, EventTypesTable: d.eventTypes, @@ -85,6 +103,10 @@ func Open(dataSourceName string, dbProperties internal.DbProperties) (*Database, EventsTable: d.events, RoomsTable: d.rooms, TransactionsTable: d.transactions, + StateBlockTable: stateBlock, + StateSnapshotTable: stateSnapshot, + PrevEventsTable: d.prevEvents, + RoomAliasesTable: roomAliases, } return &d, nil } @@ -122,41 +144,6 @@ func (d *Database) assignStateKeyNID( return eventStateKeyNID, err } -// AddState implements input.EventDatabase -func (d *Database) AddState( - ctx context.Context, - roomNID types.RoomNID, - stateBlockNIDs []types.StateBlockNID, - state []types.StateEntry, -) (types.StateSnapshotNID, error) { - if len(state) > 0 { - stateBlockNID, err := d.statements.selectNextStateBlockNID(ctx) - if err != nil { - return 0, err - } - if err = d.statements.bulkInsertStateData(ctx, stateBlockNID, state); err != nil { - return 0, err - } - stateBlockNIDs = append(stateBlockNIDs[:len(stateBlockNIDs):len(stateBlockNIDs)], stateBlockNID) - } - - return d.statements.insertState(ctx, roomNID, stateBlockNIDs) -} - -// StateBlockNIDs implements state.RoomStateDatabase -func (d *Database) StateBlockNIDs( - ctx context.Context, stateNIDs []types.StateSnapshotNID, -) ([]types.StateBlockNIDList, error) { - return d.statements.bulkSelectStateBlockNIDs(ctx, stateNIDs) -} - -// StateEntries implements state.RoomStateDatabase -func (d *Database) StateEntries( - ctx context.Context, stateBlockNIDs []types.StateBlockNID, -) ([]types.StateEntryList, error) { - return d.statements.bulkSelectStateBlockEntries(ctx, stateBlockNIDs) -} - // GetLatestEventsForUpdate implements input.EventDatabase func (d *Database) GetLatestEventsForUpdate( ctx context.Context, roomNID types.RoomNID, @@ -222,7 +209,7 @@ func (u *roomRecentEventsUpdater) CurrentStateSnapshotNID() types.StateSnapshotN // StorePreviousEvents implements types.RoomRecentEventsUpdater func (u *roomRecentEventsUpdater) StorePreviousEvents(eventNID types.EventNID, previousEventReferences []gomatrixserverlib.EventReference) error { for _, ref := range previousEventReferences { - if err := u.d.statements.insertPreviousEvent(u.ctx, u.txn, ref.EventID, ref.EventSHA256, eventNID); err != nil { + if err := u.d.prevEvents.InsertPreviousEvent(u.ctx, u.txn, ref.EventID, ref.EventSHA256, eventNID); err != nil { return err } } @@ -231,7 +218,7 @@ func (u *roomRecentEventsUpdater) StorePreviousEvents(eventNID types.EventNID, p // IsReferenced implements types.RoomRecentEventsUpdater func (u *roomRecentEventsUpdater) IsReferenced(eventReference gomatrixserverlib.EventReference) (bool, error) { - err := u.d.statements.selectPreviousEventExists(u.ctx, u.txn, eventReference.EventID, eventReference.EventSHA256) + err := u.d.prevEvents.SelectPreviousEventExists(u.ctx, u.txn, eventReference.EventID, eventReference.EventSHA256) if err == nil { return true, nil } @@ -276,44 +263,6 @@ func (d *Database) GetInvitesForUser( return d.statements.selectInviteActiveForUserInRoom(ctx, targetUserNID, roomNID) } -// SetRoomAlias implements alias.RoomserverAliasAPIDB -func (d *Database) SetRoomAlias(ctx context.Context, alias string, roomID string, creatorUserID string) error { - return d.statements.insertRoomAlias(ctx, alias, roomID, creatorUserID) -} - -// GetRoomIDForAlias implements alias.RoomserverAliasAPIDB -func (d *Database) GetRoomIDForAlias(ctx context.Context, alias string) (string, error) { - return d.statements.selectRoomIDFromAlias(ctx, alias) -} - -// GetAliasesForRoomID implements alias.RoomserverAliasAPIDB -func (d *Database) GetAliasesForRoomID(ctx context.Context, roomID string) ([]string, error) { - return d.statements.selectAliasesFromRoomID(ctx, roomID) -} - -// GetCreatorIDForAlias implements alias.RoomserverAliasAPIDB -func (d *Database) GetCreatorIDForAlias( - ctx context.Context, alias string, -) (string, error) { - return d.statements.selectCreatorIDFromAlias(ctx, alias) -} - -// RemoveRoomAlias implements alias.RoomserverAliasAPIDB -func (d *Database) RemoveRoomAlias(ctx context.Context, alias string) error { - return d.statements.deleteRoomAlias(ctx, alias) -} - -// StateEntriesForTuples implements state.RoomStateDatabase -func (d *Database) StateEntriesForTuples( - ctx context.Context, - stateBlockNIDs []types.StateBlockNID, - stateKeyTuples []types.StateKeyTuple, -) ([]types.StateEntryList, error) { - return d.statements.bulkSelectFilteredStateBlockEntries( - ctx, stateBlockNIDs, stateKeyTuples, - ) -} - // MembershipUpdater implements input.RoomEventDatabase func (d *Database) MembershipUpdater( ctx context.Context, roomID, targetUserID string, diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index 814b6e812..29c6f73eb 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -20,6 +20,10 @@ type Database struct { EventStateKeysTable tables.EventStateKeys RoomsTable tables.Rooms TransactionsTable tables.Transactions + StateSnapshotTable tables.StateSnapshot + StateBlockTable tables.StateBlock + RoomAliasesTable tables.RoomAliases + PrevEventsTable tables.PreviousEvents } // EventTypeNIDs implements state.RoomStateDatabase @@ -50,6 +54,42 @@ func (d *Database) StateEntriesForEventIDs( return d.EventsTable.BulkSelectStateEventByID(ctx, eventIDs) } +// StateEntriesForTuples implements state.RoomStateDatabase +func (d *Database) StateEntriesForTuples( + ctx context.Context, + stateBlockNIDs []types.StateBlockNID, + stateKeyTuples []types.StateKeyTuple, +) ([]types.StateEntryList, error) { + return d.StateBlockTable.BulkSelectFilteredStateBlockEntries( + ctx, stateBlockNIDs, stateKeyTuples, + ) +} + +// AddState implements input.EventDatabase +func (d *Database) AddState( + ctx context.Context, + roomNID types.RoomNID, + stateBlockNIDs []types.StateBlockNID, + state []types.StateEntry, +) (stateNID types.StateSnapshotNID, err error) { + err = internal.WithTransaction(d.DB, func(txn *sql.Tx) error { + if len(state) > 0 { + var stateBlockNID types.StateBlockNID + stateBlockNID, err = d.StateBlockTable.BulkInsertStateData(ctx, txn, state) + if err != nil { + return err + } + stateBlockNIDs = append(stateBlockNIDs[:len(stateBlockNIDs):len(stateBlockNIDs)], stateBlockNID) + } + stateNID, err = d.StateSnapshotTable.InsertState(ctx, txn, roomNID, stateBlockNIDs) + return err + }) + if err != nil { + return 0, err + } + return +} + // EventNIDs implements query.RoomserverQueryAPIDatabase func (d *Database) EventNIDs( ctx context.Context, eventIDs []string, @@ -150,6 +190,20 @@ func (d *Database) LatestEventIDs( return } +// StateBlockNIDs implements state.RoomStateDatabase +func (d *Database) StateBlockNIDs( + ctx context.Context, stateNIDs []types.StateSnapshotNID, +) ([]types.StateBlockNIDList, error) { + return d.StateSnapshotTable.BulkSelectStateBlockNIDs(ctx, stateNIDs) +} + +// StateEntries implements state.RoomStateDatabase +func (d *Database) StateEntries( + ctx context.Context, stateBlockNIDs []types.StateBlockNID, +) ([]types.StateEntryList, error) { + return d.StateBlockTable.BulkSelectStateBlockEntries(ctx, stateBlockNIDs) +} + func (d *Database) GetRoomVersionForRoom( ctx context.Context, roomID string, ) (gomatrixserverlib.RoomVersion, error) { @@ -166,6 +220,33 @@ func (d *Database) GetRoomVersionForRoomNID( ) } +// SetRoomAlias implements alias.RoomserverAliasAPIDB +func (d *Database) SetRoomAlias(ctx context.Context, alias string, roomID string, creatorUserID string) error { + return d.RoomAliasesTable.InsertRoomAlias(ctx, alias, roomID, creatorUserID) +} + +// GetRoomIDForAlias implements alias.RoomserverAliasAPIDB +func (d *Database) GetRoomIDForAlias(ctx context.Context, alias string) (string, error) { + return d.RoomAliasesTable.SelectRoomIDFromAlias(ctx, alias) +} + +// GetAliasesForRoomID implements alias.RoomserverAliasAPIDB +func (d *Database) GetAliasesForRoomID(ctx context.Context, roomID string) ([]string, error) { + return d.RoomAliasesTable.SelectAliasesFromRoomID(ctx, roomID) +} + +// GetCreatorIDForAlias implements alias.RoomserverAliasAPIDB +func (d *Database) GetCreatorIDForAlias( + ctx context.Context, alias string, +) (string, error) { + return d.RoomAliasesTable.SelectCreatorIDFromAlias(ctx, alias) +} + +// RemoveRoomAlias implements alias.RoomserverAliasAPIDB +func (d *Database) RemoveRoomAlias(ctx context.Context, alias string) error { + return d.RoomAliasesTable.DeleteRoomAlias(ctx, alias) +} + // Events implements input.EventDatabase func (d *Database) Events( ctx context.Context, eventNIDs []types.EventNID, diff --git a/roomserver/storage/sqlite3/previous_events_table.go b/roomserver/storage/sqlite3/previous_events_table.go index f344bda95..6b758ccc6 100644 --- a/roomserver/storage/sqlite3/previous_events_table.go +++ b/roomserver/storage/sqlite3/previous_events_table.go @@ -20,6 +20,7 @@ import ( "database/sql" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -55,19 +56,20 @@ type previousEventStatements struct { selectPreviousEventExistsStmt *sql.Stmt } -func (s *previousEventStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(previousEventSchema) +func NewSqlitePrevEventsTable(db *sql.DB) (tables.PreviousEvents, error) { + s := &previousEventStatements{} + _, err := db.Exec(previousEventSchema) if err != nil { - return + return nil, err } - return statementList{ + return s, statementList{ {&s.insertPreviousEventStmt, insertPreviousEventSQL}, {&s.selectPreviousEventExistsStmt, selectPreviousEventExistsSQL}, }.prepare(db) } -func (s *previousEventStatements) insertPreviousEvent( +func (s *previousEventStatements) InsertPreviousEvent( ctx context.Context, txn *sql.Tx, previousEventID string, @@ -83,7 +85,7 @@ func (s *previousEventStatements) insertPreviousEvent( // Check if the event reference exists // Returns sql.ErrNoRows if the event reference doesn't exist. -func (s *previousEventStatements) selectPreviousEventExists( +func (s *previousEventStatements) SelectPreviousEventExists( ctx context.Context, txn *sql.Tx, eventID string, eventReferenceSHA256 []byte, ) error { var ok int64 diff --git a/roomserver/storage/sqlite3/room_aliases_table.go b/roomserver/storage/sqlite3/room_aliases_table.go index 592ef9782..686391f68 100644 --- a/roomserver/storage/sqlite3/room_aliases_table.go +++ b/roomserver/storage/sqlite3/room_aliases_table.go @@ -20,6 +20,7 @@ import ( "database/sql" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/roomserver/storage/tables" ) const roomAliasesSchema = ` @@ -60,12 +61,13 @@ type roomAliasesStatements struct { deleteRoomAliasStmt *sql.Stmt } -func (s *roomAliasesStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(roomAliasesSchema) +func NewSqliteRoomAliasesTable(db *sql.DB) (tables.RoomAliases, error) { + s := &roomAliasesStatements{} + _, err := db.Exec(roomAliasesSchema) if err != nil { - return + return nil, err } - return statementList{ + return s, statementList{ {&s.insertRoomAliasStmt, insertRoomAliasSQL}, {&s.selectRoomIDFromAliasStmt, selectRoomIDFromAliasSQL}, {&s.selectAliasesFromRoomIDStmt, selectAliasesFromRoomIDSQL}, @@ -74,31 +76,28 @@ func (s *roomAliasesStatements) prepare(db *sql.DB) (err error) { }.prepare(db) } -func (s *roomAliasesStatements) insertRoomAlias( - ctx context.Context, txn *sql.Tx, alias string, roomID string, creatorUserID string, +func (s *roomAliasesStatements) InsertRoomAlias( + ctx context.Context, alias string, roomID string, creatorUserID string, ) (err error) { - insertStmt := internal.TxStmt(txn, s.insertRoomAliasStmt) - _, err = insertStmt.ExecContext(ctx, alias, roomID, creatorUserID) + _, err = s.insertRoomAliasStmt.ExecContext(ctx, alias, roomID, creatorUserID) return } -func (s *roomAliasesStatements) selectRoomIDFromAlias( - ctx context.Context, txn *sql.Tx, alias string, +func (s *roomAliasesStatements) SelectRoomIDFromAlias( + ctx context.Context, alias string, ) (roomID string, err error) { - selectStmt := internal.TxStmt(txn, s.selectRoomIDFromAliasStmt) - err = selectStmt.QueryRowContext(ctx, alias).Scan(&roomID) + err = s.selectRoomIDFromAliasStmt.QueryRowContext(ctx, alias).Scan(&roomID) if err == sql.ErrNoRows { return "", nil } return } -func (s *roomAliasesStatements) selectAliasesFromRoomID( - ctx context.Context, txn *sql.Tx, roomID string, +func (s *roomAliasesStatements) SelectAliasesFromRoomID( + ctx context.Context, roomID string, ) (aliases []string, err error) { aliases = []string{} - selectStmt := internal.TxStmt(txn, s.selectAliasesFromRoomIDStmt) - rows, err := selectStmt.QueryContext(ctx, roomID) + rows, err := s.selectAliasesFromRoomIDStmt.QueryContext(ctx, roomID) if err != nil { return } @@ -117,21 +116,19 @@ func (s *roomAliasesStatements) selectAliasesFromRoomID( return } -func (s *roomAliasesStatements) selectCreatorIDFromAlias( - ctx context.Context, txn *sql.Tx, alias string, +func (s *roomAliasesStatements) SelectCreatorIDFromAlias( + ctx context.Context, alias string, ) (creatorID string, err error) { - selectStmt := internal.TxStmt(txn, s.selectCreatorIDFromAliasStmt) - err = selectStmt.QueryRowContext(ctx, alias).Scan(&creatorID) + err = s.selectCreatorIDFromAliasStmt.QueryRowContext(ctx, alias).Scan(&creatorID) if err == sql.ErrNoRows { return "", nil } return } -func (s *roomAliasesStatements) deleteRoomAlias( - ctx context.Context, txn *sql.Tx, alias string, +func (s *roomAliasesStatements) DeleteRoomAlias( + ctx context.Context, alias string, ) (err error) { - deleteStmt := internal.TxStmt(txn, s.deleteRoomAliasStmt) - _, err = deleteStmt.ExecContext(ctx, alias) + _, err = s.deleteRoomAliasStmt.ExecContext(ctx, alias) return } diff --git a/roomserver/storage/sqlite3/sql.go b/roomserver/storage/sqlite3/sql.go index fe899174a..df994d508 100644 --- a/roomserver/storage/sqlite3/sql.go +++ b/roomserver/storage/sqlite3/sql.go @@ -38,10 +38,6 @@ func (s *statements) prepare(db *sql.DB) error { var err error for _, prepare := range []func(db *sql.DB) error{ - s.stateSnapshotStatements.prepare, - s.stateBlockStatements.prepare, - s.previousEventStatements.prepare, - s.roomAliasesStatements.prepare, s.inviteStatements.prepare, s.membershipStatements.prepare, } { diff --git a/roomserver/storage/sqlite3/state_block_table.go b/roomserver/storage/sqlite3/state_block_table.go index 861d76cf9..e3ca0b3c8 100644 --- a/roomserver/storage/sqlite3/state_block_table.go +++ b/roomserver/storage/sqlite3/state_block_table.go @@ -23,6 +23,7 @@ import ( "strings" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/util" ) @@ -77,14 +78,15 @@ type stateBlockStatements struct { bulkSelectFilteredStateBlockEntriesStmt *sql.Stmt } -func (s *stateBlockStatements) prepare(db *sql.DB) (err error) { +func NewSqliteStateBlockTable(db *sql.DB) (tables.StateBlock, error) { + s := &stateBlockStatements{} s.db = db - _, err = db.Exec(stateDataSchema) + _, err := db.Exec(stateDataSchema) if err != nil { - return + return nil, err } - return statementList{ + return s, statementList{ {&s.insertStateDataStmt, insertStateDataSQL}, {&s.selectNextStateBlockNIDStmt, selectNextStateBlockNIDSQL}, {&s.bulkSelectStateBlockEntriesStmt, bulkSelectStateBlockEntriesSQL}, @@ -92,7 +94,7 @@ func (s *stateBlockStatements) prepare(db *sql.DB) (err error) { }.prepare(db) } -func (s *stateBlockStatements) bulkInsertStateData( +func (s *stateBlockStatements) BulkInsertStateData( ctx context.Context, txn *sql.Tx, entries []types.StateEntry, ) (types.StateBlockNID, error) { @@ -120,19 +122,18 @@ func (s *stateBlockStatements) bulkInsertStateData( return stateBlockNID, nil } -func (s *stateBlockStatements) bulkSelectStateBlockEntries( - ctx context.Context, txn *sql.Tx, stateBlockNIDs []types.StateBlockNID, +func (s *stateBlockStatements) BulkSelectStateBlockEntries( + ctx context.Context, stateBlockNIDs []types.StateBlockNID, ) ([]types.StateEntryList, error) { nids := make([]interface{}, len(stateBlockNIDs)) for k, v := range stateBlockNIDs { nids[k] = v } selectOrig := strings.Replace(bulkSelectStateBlockEntriesSQL, "($1)", internal.QueryVariadic(len(nids)), 1) - selectPrep, err := s.db.Prepare(selectOrig) + selectStmt, err := s.db.Prepare(selectOrig) if err != nil { return nil, err } - selectStmt := internal.TxStmt(txn, selectPrep) rows, err := selectStmt.QueryContext(ctx, nids...) if err != nil { return nil, err @@ -174,8 +175,8 @@ func (s *stateBlockStatements) bulkSelectStateBlockEntries( return results, nil } -func (s *stateBlockStatements) bulkSelectFilteredStateBlockEntries( - ctx context.Context, txn *sql.Tx, // nolint: unparam +func (s *stateBlockStatements) BulkSelectFilteredStateBlockEntries( + ctx context.Context, stateBlockNIDs []types.StateBlockNID, stateKeyTuples []types.StateKeyTuple, ) ([]types.StateEntryList, error) { diff --git a/roomserver/storage/sqlite3/state_snapshot_table.go b/roomserver/storage/sqlite3/state_snapshot_table.go index 8682627a0..42eae1a7f 100644 --- a/roomserver/storage/sqlite3/state_snapshot_table.go +++ b/roomserver/storage/sqlite3/state_snapshot_table.go @@ -23,6 +23,7 @@ import ( "strings" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -51,20 +52,21 @@ type stateSnapshotStatements struct { bulkSelectStateBlockNIDsStmt *sql.Stmt } -func (s *stateSnapshotStatements) prepare(db *sql.DB) (err error) { +func NewSqliteStateSnapshotTable(db *sql.DB) (tables.StateSnapshot, error) { + s := &stateSnapshotStatements{} s.db = db - _, err = db.Exec(stateSnapshotSchema) + _, err := db.Exec(stateSnapshotSchema) if err != nil { - return + return nil, err } - return statementList{ + return s, statementList{ {&s.insertStateStmt, insertStateSQL}, {&s.bulkSelectStateBlockNIDsStmt, bulkSelectStateBlockNIDsSQL}, }.prepare(db) } -func (s *stateSnapshotStatements) insertState( +func (s *stateSnapshotStatements) InsertState( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, stateBlockNIDs []types.StateBlockNID, ) (stateNID types.StateSnapshotNID, err error) { stateBlockNIDsJSON, err := json.Marshal(stateBlockNIDs) @@ -82,15 +84,15 @@ func (s *stateSnapshotStatements) insertState( return } -func (s *stateSnapshotStatements) bulkSelectStateBlockNIDs( - ctx context.Context, txn *sql.Tx, stateNIDs []types.StateSnapshotNID, +func (s *stateSnapshotStatements) BulkSelectStateBlockNIDs( + ctx context.Context, stateNIDs []types.StateSnapshotNID, ) ([]types.StateBlockNIDList, error) { nids := make([]interface{}, len(stateNIDs)) for k, v := range stateNIDs { nids[k] = v } selectOrig := strings.Replace(bulkSelectStateBlockNIDsSQL, "($1)", internal.QueryVariadic(len(nids)), 1) - selectStmt, err := txn.Prepare(selectOrig) + selectStmt, err := s.db.Prepare(selectOrig) if err != nil { return nil, err } diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index a0a1b5684..16697f1b9 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -41,10 +41,12 @@ type Database struct { eventStateKeys tables.EventStateKeys rooms tables.Rooms transactions tables.Transactions + prevEvents tables.PreviousEvents db *sql.DB } // Open a sqlite database. +// nolint: gocyclo func Open(dataSourceName string) (*Database, error) { var d Database uri, err := url.Parse(dataSourceName) @@ -97,6 +99,22 @@ func Open(dataSourceName string) (*Database, error) { if err != nil { return nil, err } + stateBlock, err := NewSqliteStateBlockTable(d.db) + if err != nil { + return nil, err + } + stateSnapshot, err := NewSqliteStateSnapshotTable(d.db) + if err != nil { + return nil, err + } + d.prevEvents, err = NewSqlitePrevEventsTable(d.db) + if err != nil { + return nil, err + } + roomAliases, err := NewSqliteRoomAliasesTable(d.db) + if err != nil { + return nil, err + } d.Database = shared.Database{ DB: d.db, EventsTable: d.events, @@ -105,6 +123,10 @@ func Open(dataSourceName string) (*Database, error) { EventJSONTable: d.eventJSON, RoomsTable: d.rooms, TransactionsTable: d.transactions, + StateBlockTable: stateBlock, + StateSnapshotTable: stateSnapshot, + PrevEventsTable: d.prevEvents, + RoomAliasesTable: roomAliases, } return &d, nil } @@ -142,53 +164,6 @@ func (d *Database) assignStateKeyNID( return } -// AddState implements input.EventDatabase -func (d *Database) AddState( - ctx context.Context, - roomNID types.RoomNID, - stateBlockNIDs []types.StateBlockNID, - state []types.StateEntry, -) (stateNID types.StateSnapshotNID, err error) { - err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { - if len(state) > 0 { - var stateBlockNID types.StateBlockNID - stateBlockNID, err = d.statements.bulkInsertStateData(ctx, txn, state) - if err != nil { - return err - } - stateBlockNIDs = append(stateBlockNIDs[:len(stateBlockNIDs):len(stateBlockNIDs)], stateBlockNID) - } - stateNID, err = d.statements.insertState(ctx, txn, roomNID, stateBlockNIDs) - return err - }) - if err != nil { - return 0, err - } - return -} - -// StateBlockNIDs implements state.RoomStateDatabase -func (d *Database) StateBlockNIDs( - ctx context.Context, stateNIDs []types.StateSnapshotNID, -) (sl []types.StateBlockNIDList, err error) { - err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { - sl, err = d.statements.bulkSelectStateBlockNIDs(ctx, txn, stateNIDs) - return err - }) - return -} - -// StateEntries implements state.RoomStateDatabase -func (d *Database) StateEntries( - ctx context.Context, stateBlockNIDs []types.StateBlockNID, -) (sel []types.StateEntryList, err error) { - err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { - sel, err = d.statements.bulkSelectStateBlockEntries(ctx, txn, stateBlockNIDs) - return err - }) - return -} - // GetLatestEventsForUpdate implements input.EventDatabase func (d *Database) GetLatestEventsForUpdate( ctx context.Context, roomNID types.RoomNID, @@ -264,7 +239,7 @@ func (u *roomRecentEventsUpdater) CurrentStateSnapshotNID() types.StateSnapshotN func (u *roomRecentEventsUpdater) StorePreviousEvents(eventNID types.EventNID, previousEventReferences []gomatrixserverlib.EventReference) error { err := internal.WithTransaction(u.d.db, func(txn *sql.Tx) error { for _, ref := range previousEventReferences { - if err := u.d.statements.insertPreviousEvent(u.ctx, txn, ref.EventID, ref.EventSHA256, eventNID); err != nil { + if err := u.d.prevEvents.InsertPreviousEvent(u.ctx, txn, ref.EventID, ref.EventSHA256, eventNID); err != nil { return err } } @@ -276,7 +251,7 @@ func (u *roomRecentEventsUpdater) StorePreviousEvents(eventNID types.EventNID, p // IsReferenced implements types.RoomRecentEventsUpdater func (u *roomRecentEventsUpdater) IsReferenced(eventReference gomatrixserverlib.EventReference) (res bool, err error) { err = internal.WithTransaction(u.d.db, func(txn *sql.Tx) error { - err := u.d.statements.selectPreviousEventExists(u.ctx, txn, eventReference.EventID, eventReference.EventSHA256) + err := u.d.prevEvents.SelectPreviousEventExists(u.ctx, txn, eventReference.EventID, eventReference.EventSHA256) if err == nil { res = true err = nil @@ -339,44 +314,6 @@ func (d *Database) GetInvitesForUser( return d.statements.selectInviteActiveForUserInRoom(ctx, targetUserNID, roomNID) } -// SetRoomAlias implements alias.RoomserverAliasAPIDB -func (d *Database) SetRoomAlias(ctx context.Context, alias string, roomID string, creatorUserID string) error { - return d.statements.insertRoomAlias(ctx, nil, alias, roomID, creatorUserID) -} - -// GetRoomIDForAlias implements alias.RoomserverAliasAPIDB -func (d *Database) GetRoomIDForAlias(ctx context.Context, alias string) (string, error) { - return d.statements.selectRoomIDFromAlias(ctx, nil, alias) -} - -// GetAliasesForRoomID implements alias.RoomserverAliasAPIDB -func (d *Database) GetAliasesForRoomID(ctx context.Context, roomID string) ([]string, error) { - return d.statements.selectAliasesFromRoomID(ctx, nil, roomID) -} - -// GetCreatorIDForAlias implements alias.RoomserverAliasAPIDB -func (d *Database) GetCreatorIDForAlias( - ctx context.Context, alias string, -) (string, error) { - return d.statements.selectCreatorIDFromAlias(ctx, nil, alias) -} - -// RemoveRoomAlias implements alias.RoomserverAliasAPIDB -func (d *Database) RemoveRoomAlias(ctx context.Context, alias string) error { - return d.statements.deleteRoomAlias(ctx, nil, alias) -} - -// StateEntriesForTuples implements state.RoomStateDatabase -func (d *Database) StateEntriesForTuples( - ctx context.Context, - stateBlockNIDs []types.StateBlockNID, - stateKeyTuples []types.StateKeyTuple, -) ([]types.StateEntryList, error) { - return d.statements.bulkSelectFilteredStateBlockEntries( - ctx, nil, stateBlockNIDs, stateKeyTuples, - ) -} - // MembershipUpdater implements input.RoomEventDatabase func (d *Database) MembershipUpdater( ctx context.Context, roomID, targetUserID string, diff --git a/roomserver/storage/tables/interface.go b/roomserver/storage/tables/interface.go index e913de6b4..56ef5dd39 100644 --- a/roomserver/storage/tables/interface.go +++ b/roomserver/storage/tables/interface.go @@ -70,3 +70,29 @@ type Transactions interface { InsertTransaction(ctx context.Context, txn *sql.Tx, transactionID string, sessionID int64, userID string, eventID string) error SelectTransactionEventID(ctx context.Context, transactionID string, sessionID int64, userID string) (eventID string, err error) } + +type StateSnapshot interface { + InsertState(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, stateBlockNIDs []types.StateBlockNID) (stateNID types.StateSnapshotNID, err error) + BulkSelectStateBlockNIDs(ctx context.Context, stateNIDs []types.StateSnapshotNID) ([]types.StateBlockNIDList, error) +} + +type StateBlock interface { + BulkInsertStateData(ctx context.Context, txn *sql.Tx, entries []types.StateEntry) (types.StateBlockNID, error) + BulkSelectStateBlockEntries(ctx context.Context, stateBlockNIDs []types.StateBlockNID) ([]types.StateEntryList, error) + BulkSelectFilteredStateBlockEntries(ctx context.Context, stateBlockNIDs []types.StateBlockNID, stateKeyTuples []types.StateKeyTuple) ([]types.StateEntryList, error) +} + +type RoomAliases interface { + InsertRoomAlias(ctx context.Context, alias string, roomID string, creatorUserID string) (err error) + SelectRoomIDFromAlias(ctx context.Context, alias string) (roomID string, err error) + SelectAliasesFromRoomID(ctx context.Context, roomID string) ([]string, error) + SelectCreatorIDFromAlias(ctx context.Context, alias string) (creatorID string, err error) + DeleteRoomAlias(ctx context.Context, alias string) (err error) +} + +type PreviousEvents interface { + InsertPreviousEvent(ctx context.Context, txn *sql.Tx, previousEventID string, previousEventReferenceSHA256 []byte, eventNID types.EventNID) error + // Check if the event reference exists + // Returns sql.ErrNoRows if the event reference doesn't exist. + SelectPreviousEventExists(ctx context.Context, txn *sql.Tx, eventID string, eventReferenceSHA256 []byte) error +} From 267a4d1823fad787aef16318272710611e19d07e Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 27 May 2020 09:39:14 +0100 Subject: [PATCH 097/200] Update p2p docs --- docs/p2p.md | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/docs/p2p.md b/docs/p2p.md index 141aaa1fc..d69b47bea 100644 --- a/docs/p2p.md +++ b/docs/p2p.md @@ -1,6 +1,6 @@ ## Peer-to-peer Matrix -These are the instructions for setting up P2P Dendrite, current as of March 2020. There's both Go stuff and JS stuff to do to set this up. +These are the instructions for setting up P2P Dendrite, current as of May 2020. There's both Go stuff and JS stuff to do to set this up. ### Dendrite @@ -28,14 +28,13 @@ Then use `/ip4/127.0.0.1/tcp/9090/ws/p2p-websocket-star/`. ### Riot-web -You need to check out these repos: +You need to check out this repo: ``` $ git clone git@github.com:matrix-org/go-http-js-libp2p.git -$ git clone git@github.com:matrix-org/go-sqlite3-js.git ``` -Make sure to `yarn install` in both of these repos. Then: +Make sure to `yarn install` in the repo. Then: - `$ cp "$(go env GOROOT)/misc/wasm/wasm_exec.js" ./src/vector/` - Comment out the lines in `wasm_exec.js` which contains: @@ -49,7 +48,6 @@ if (!global.fs && global.require) { - Add the following symlinks: they HAVE to be symlinks as the diff in `webpack.config.js` references specific paths. ``` $ cd node_modules -$ ln -s ../../go-sqlite-js # NB: NOT go-sqlite3-js $ ln -s ../../go-http-js-libp2p ``` @@ -65,14 +63,7 @@ You need a Chrome and a Firefox running to test locally as service workers don't Assuming you've `yarn start`ed Riot-Web, go to `http://localhost:8080` and register with `http://localhost:8080` as your HS URL. -You can join rooms by room alias e.g `/join #foo:bar`. - -### Known issues - -- When registering you may be unable to find the server, it'll seem flakey. This happens because the SW, particularly in Firefox, - gets killed after 30s of inactivity. When you are not registered, you aren't doing `/sync` calls to keep the SW alive, so if you - don't register for a while and idle on the page, the HS will disappear. To fix, unregister the SW, and then refresh the page. - -- The libp2p layer has rate limits, so frequent Federation traffic may cause the connection to drop and messages to not be transferred. - I guess in other words, don't send too much traffic? - +You can: + - join rooms by room alias e.g `/join #foo:bar`. + - invite specific users to a room. + - explore the published room list. All members of the room can re-publish aliases (unlike Synapse). From 7d6461dd3c4429303148a115d72b1b3b0ebaba72 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 27 May 2020 10:19:24 +0100 Subject: [PATCH 098/200] Server key component (#1050) * Server key API (works for monolith but not for polylith yet) * Re-enable caching on server key API component * Groundwork for HTTP APIs for server key API * Hopefully implement HTTP for server key API * Simplify public key request marshalling from map keys * Update gomatrixserverlib * go mod tidy * Common -> internal * remove keyring.go * Update Docker Hub for server key API * YAML is funny about indentation * Wire in new server key API into hybrid monolith mode * Create maps * Route server key API endpoints on internal API mux * Fix server key API URLs * Add fetcher behaviour into server key API implementation * Return error if we failed to fetch some keys * Return results anyway * Move things about a bit * Remove unused code * Fix comments, don't use federation sender URL in polylith mode * Add server_key_api to sample config * Review comments * HTTP API to cache keys that have been requested * Overwrite server_key_api listen in monolith hybrid mode --- build/docker/hub/config/dendrite-config.yaml | 2 + build/docker/hub/docker-compose.polylith.yml | 13 +- build/docker/hub/images-build.sh | 25 ++-- cmd/dendrite-client-api-server/main.go | 8 +- cmd/dendrite-demo-libp2p/main.go | 32 +++-- cmd/dendrite-demo-libp2p/mdnslistener.go | 3 +- cmd/dendrite-federation-api-server/main.go | 9 +- cmd/dendrite-federation-sender-server/main.go | 9 +- cmd/dendrite-monolith-server/main.go | 19 ++- cmd/dendrite-room-server/main.go | 6 +- cmd/dendrite-server-key-api-server/main.go | 32 +++++ cmd/dendritejs/main.go | 8 +- dendrite-config.yaml | 3 +- go.mod | 2 +- go.sum | 4 +- internal/basecomponent/base.go | 40 +++---- internal/config/config.go | 12 ++ internal/keydb/keyring.go | 74 ------------ serverkeyapi/api/api.go | 113 ++++++++++++++++++ serverkeyapi/api/http.go | 57 +++++++++ serverkeyapi/internal/api.go | 76 ++++++++++++ serverkeyapi/internal/http.go | 60 ++++++++++ serverkeyapi/serverkeyapi.go | 83 +++++++++++++ .../storage}/cache/keydb.go | 5 +- .../storage}/interface.go | 2 +- .../keydb => serverkeyapi/storage}/keydb.go | 6 +- .../storage}/keydb_wasm.go | 8 +- .../storage}/postgres/keydb.go | 0 .../storage}/postgres/server_key_table.go | 0 .../storage}/sqlite3/keydb.go | 0 .../storage}/sqlite3/server_key_table.go | 0 31 files changed, 542 insertions(+), 169 deletions(-) create mode 100644 cmd/dendrite-server-key-api-server/main.go delete mode 100644 internal/keydb/keyring.go create mode 100644 serverkeyapi/api/api.go create mode 100644 serverkeyapi/api/http.go create mode 100644 serverkeyapi/internal/api.go create mode 100644 serverkeyapi/internal/http.go create mode 100644 serverkeyapi/serverkeyapi.go rename {internal/keydb => serverkeyapi/storage}/cache/keydb.go (91%) rename {internal/keydb => serverkeyapi/storage}/interface.go (96%) rename {internal/keydb => serverkeyapi/storage}/keydb.go (91%) rename {internal/keydb => serverkeyapi/storage}/keydb_wasm.go (88%) rename {internal/keydb => serverkeyapi/storage}/postgres/keydb.go (100%) rename {internal/keydb => serverkeyapi/storage}/postgres/server_key_table.go (100%) rename {internal/keydb => serverkeyapi/storage}/sqlite3/keydb.go (100%) rename {internal/keydb => serverkeyapi/storage}/sqlite3/server_key_table.go (100%) diff --git a/build/docker/hub/config/dendrite-config.yaml b/build/docker/hub/config/dendrite-config.yaml index 23d6479b5..26dc272ab 100644 --- a/build/docker/hub/config/dendrite-config.yaml +++ b/build/docker/hub/config/dendrite-config.yaml @@ -110,11 +110,13 @@ listen: room_server: "room_server:7770" client_api: "client_api:7771" federation_api: "federation_api:7772" + server_key_api: "server_key_api:7778" sync_api: "sync_api:7773" media_api: "media_api:7774" public_rooms_api: "public_rooms_api:7775" federation_sender: "federation_sender:7776" edu_server: "edu_server:7777" + key_server: "key_server:7779" # The configuration for tracing the dendrite components. tracing: diff --git a/build/docker/hub/docker-compose.polylith.yml b/build/docker/hub/docker-compose.polylith.yml index d88e5bfb3..178604093 100644 --- a/build/docker/hub/docker-compose.polylith.yml +++ b/build/docker/hub/docker-compose.polylith.yml @@ -131,7 +131,7 @@ services: - internal key_server: - hostname: key_serverde + hostname: key_server image: matrixdotorg/dendrite:keyserver command: [ "--config=dendrite.yaml" @@ -141,6 +141,17 @@ services: networks: - internal + server_key_api: + hostname: server_key_api + image: matrixdotorg/dendrite:serverkeyapi + command: [ + "--config=dendrite.yaml" + ] + volumes: + - ./config:/etc/dendrite + networks: + - internal + networks: internal: attachable: true diff --git a/build/docker/hub/images-build.sh b/build/docker/hub/images-build.sh index 0c6a0eb7a..b162a0639 100755 --- a/build/docker/hub/images-build.sh +++ b/build/docker/hub/images-build.sh @@ -2,16 +2,17 @@ cd $(git rev-parse --show-toplevel) -docker build -f docker/hub/Dockerfile -t matrixdotorg/dendrite:latest . +docker build -f build/docker/hub/Dockerfile -t matrixdotorg/dendrite:latest . -docker build -t matrixdotorg/dendrite:clientapi --build-arg component=dendrite-client-api-server -f docker/hub/Dockerfile.component . -docker build -t matrixdotorg/dendrite:clientproxy --build-arg component=client-api-proxy -f docker/hub/Dockerfile.component . -docker build -t matrixdotorg/dendrite:eduserver --build-arg component=dendrite-edu-server -f docker/hub/Dockerfile.component . -docker build -t matrixdotorg/dendrite:federationapi --build-arg component=dendrite-federation-api-server -f docker/hub/Dockerfile.component . -docker build -t matrixdotorg/dendrite:federationsender --build-arg component=dendrite-federation-sender-server -f docker/hub/Dockerfile.component . -docker build -t matrixdotorg/dendrite:federationproxy --build-arg component=federation-api-proxy -f docker/hub/Dockerfile.component . -docker build -t matrixdotorg/dendrite:keyserver --build-arg component=dendrite-key-server -f docker/hub/Dockerfile.component . -docker build -t matrixdotorg/dendrite:mediaapi --build-arg component=dendrite-media-api-server -f docker/hub/Dockerfile.component . -docker build -t matrixdotorg/dendrite:publicroomsapi --build-arg component=dendrite-public-rooms-api-server -f docker/hub/Dockerfile.component . -docker build -t matrixdotorg/dendrite:roomserver --build-arg component=dendrite-room-server -f docker/hub/Dockerfile.component . -docker build -t matrixdotorg/dendrite:syncapi --build-arg component=dendrite-sync-api-server -f docker/hub/Dockerfile.component . +docker build -t matrixdotorg/dendrite:clientapi --build-arg component=dendrite-client-api-server -f build/docker/hub/Dockerfile.component . +docker build -t matrixdotorg/dendrite:clientproxy --build-arg component=client-api-proxy -f build/docker/hub/Dockerfile.component . +docker build -t matrixdotorg/dendrite:eduserver --build-arg component=dendrite-edu-server -f build/docker/hub/Dockerfile.component . +docker build -t matrixdotorg/dendrite:federationapi --build-arg component=dendrite-federation-api-server -f build/docker/hub/Dockerfile.component . +docker build -t matrixdotorg/dendrite:federationsender --build-arg component=dendrite-federation-sender-server -f build/docker/hub/Dockerfile.component . +docker build -t matrixdotorg/dendrite:federationproxy --build-arg component=federation-api-proxy -f build/docker/hub/Dockerfile.component . +docker build -t matrixdotorg/dendrite:keyserver --build-arg component=dendrite-key-server -f build/docker/hub/Dockerfile.component . +docker build -t matrixdotorg/dendrite:mediaapi --build-arg component=dendrite-media-api-server -f build/docker/hub/Dockerfile.component . +docker build -t matrixdotorg/dendrite:publicroomsapi --build-arg component=dendrite-public-rooms-api-server -f build/docker/hub/Dockerfile.component . +docker build -t matrixdotorg/dendrite:roomserver --build-arg component=dendrite-room-server -f build/docker/hub/Dockerfile.component . +docker build -t matrixdotorg/dendrite:syncapi --build-arg component=dendrite-sync-api-server -f build/docker/hub/Dockerfile.component . +docker build -t matrixdotorg/dendrite:serverkeyapi --build-arg component=dendrite-server-key-api-server -f build/docker/hub/Dockerfile.component . diff --git a/cmd/dendrite-client-api-server/main.go b/cmd/dendrite-client-api-server/main.go index 0815be799..e06adf8ff 100644 --- a/cmd/dendrite-client-api-server/main.go +++ b/cmd/dendrite-client-api-server/main.go @@ -19,7 +19,6 @@ import ( "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/internal/basecomponent" - "github.com/matrix-org/dendrite/internal/keydb" "github.com/matrix-org/dendrite/internal/transactions" ) @@ -31,9 +30,10 @@ func main() { accountDB := base.CreateAccountsDB() deviceDB := base.CreateDeviceDB() - keyDB := base.CreateKeyDB() federation := base.CreateFederationClient() - keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) + + serverKeyAPI := base.CreateHTTPServerKeyAPIs() + keyRing := serverKeyAPI.KeyRing() asQuery := base.CreateHTTPAppServiceAPIs() rsAPI := base.CreateHTTPRoomserverAPIs() @@ -42,7 +42,7 @@ func main() { eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New()) clientapi.SetupClientAPIComponent( - base, deviceDB, accountDB, federation, &keyRing, + base, deviceDB, accountDB, federation, keyRing, rsAPI, eduInputAPI, asQuery, transactions.New(), fsAPI, ) diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index 3f53cb429..fc56b9bbf 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -37,11 +37,11 @@ import ( "github.com/matrix-org/dendrite/federationsender" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" - "github.com/matrix-org/dendrite/internal/keydb" "github.com/matrix-org/dendrite/internal/transactions" "github.com/matrix-org/dendrite/mediaapi" "github.com/matrix-org/dendrite/publicroomsapi" "github.com/matrix-org/dendrite/roomserver" + "github.com/matrix-org/dendrite/serverkeyapi" "github.com/matrix-org/dendrite/syncapi" "github.com/matrix-org/gomatrixserverlib" @@ -52,17 +52,8 @@ import ( func createKeyDB( base *P2PDendrite, -) keydb.Database { - db, err := keydb.NewDatabase( - string(base.Base.Cfg.Database.ServerKey), - base.Base.Cfg.DbProperties(), - base.Base.Cfg.Matrix.ServerName, - base.Base.Cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey), - base.Base.Cfg.Matrix.KeyID, - ) - if err != nil { - logrus.WithError(err).Panicf("failed to connect to keys db") - } + db gomatrixserverlib.KeyDatabase, +) { mdns := mDNSListener{ host: base.LibP2P, keydb: db, @@ -77,7 +68,6 @@ func createKeyDB( panic(err) } serv.RegisterNotifee(&mdns) - return db } func createFederationClient( @@ -144,9 +134,15 @@ func main() { accountDB := base.Base.CreateAccountsDB() deviceDB := base.Base.CreateDeviceDB() - keyDB := createKeyDB(base) federation := createFederationClient(base) - keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) + + serverKeyAPI := serverkeyapi.SetupServerKeyAPIComponent( + &base.Base, federation, + ) + keyRing := serverKeyAPI.KeyRing() + createKeyDB( + base, serverKeyAPI, + ) rsAPI := roomserver.SetupRoomServerComponent( &base.Base, keyRing, federation, @@ -158,17 +154,17 @@ func main() { &base.Base, accountDB, deviceDB, federation, rsAPI, transactions.New(), ) fsAPI := federationsender.SetupFederationSenderComponent( - &base.Base, federation, rsAPI, &keyRing, + &base.Base, federation, rsAPI, keyRing, ) rsAPI.SetFederationSenderAPI(fsAPI) clientapi.SetupClientAPIComponent( &base.Base, deviceDB, accountDB, - federation, &keyRing, rsAPI, + federation, keyRing, rsAPI, eduInputAPI, asAPI, transactions.New(), fsAPI, ) eduProducer := producers.NewEDUServerProducer(eduInputAPI) - federationapi.SetupFederationAPIComponent(&base.Base, accountDB, deviceDB, federation, &keyRing, rsAPI, asAPI, fsAPI, eduProducer) + federationapi.SetupFederationAPIComponent(&base.Base, accountDB, deviceDB, federation, keyRing, rsAPI, asAPI, fsAPI, eduProducer) mediaapi.SetupMediaAPIComponent(&base.Base, deviceDB) publicRoomsDB, err := storage.NewPublicRoomsServerDatabaseWithPubSub(string(base.Base.Cfg.Database.PublicRoomsAPI), base.LibP2PPubsub) if err != nil { diff --git a/cmd/dendrite-demo-libp2p/mdnslistener.go b/cmd/dendrite-demo-libp2p/mdnslistener.go index c05c94d59..01705cf39 100644 --- a/cmd/dendrite-demo-libp2p/mdnslistener.go +++ b/cmd/dendrite-demo-libp2p/mdnslistener.go @@ -21,12 +21,11 @@ import ( "github.com/libp2p/go-libp2p-core/host" "github.com/libp2p/go-libp2p-core/peer" - "github.com/matrix-org/dendrite/internal/keydb" "github.com/matrix-org/gomatrixserverlib" ) type mDNSListener struct { - keydb keydb.Database + keydb gomatrixserverlib.KeyDatabase host host.Host } diff --git a/cmd/dendrite-federation-api-server/main.go b/cmd/dendrite-federation-api-server/main.go index be97bc9a0..5425d117f 100644 --- a/cmd/dendrite-federation-api-server/main.go +++ b/cmd/dendrite-federation-api-server/main.go @@ -20,7 +20,6 @@ import ( "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/internal/basecomponent" - "github.com/matrix-org/dendrite/internal/keydb" ) func main() { @@ -30,10 +29,12 @@ func main() { accountDB := base.CreateAccountsDB() deviceDB := base.CreateDeviceDB() - keyDB := base.CreateKeyDB() federation := base.CreateFederationClient() + + serverKeyAPI := base.CreateHTTPServerKeyAPIs() + keyRing := serverKeyAPI.KeyRing() + fsAPI := base.CreateHTTPFederationSenderAPIs() - keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) rsAPI := base.CreateHTTPRoomserverAPIs() asAPI := base.CreateHTTPAppServiceAPIs() @@ -42,7 +43,7 @@ func main() { eduProducer := producers.NewEDUServerProducer(eduInputAPI) federationapi.SetupFederationAPIComponent( - base, accountDB, deviceDB, federation, &keyRing, + base, accountDB, deviceDB, federation, keyRing, rsAPI, asAPI, fsAPI, eduProducer, ) diff --git a/cmd/dendrite-federation-sender-server/main.go b/cmd/dendrite-federation-sender-server/main.go index b6af8deb7..2b1e0ae8f 100644 --- a/cmd/dendrite-federation-sender-server/main.go +++ b/cmd/dendrite-federation-sender-server/main.go @@ -17,7 +17,6 @@ package main import ( "github.com/matrix-org/dendrite/federationsender" "github.com/matrix-org/dendrite/internal/basecomponent" - "github.com/matrix-org/dendrite/internal/keydb" ) func main() { @@ -26,11 +25,13 @@ func main() { defer base.Close() // nolint: errcheck federation := base.CreateFederationClient() - keyDB := base.CreateKeyDB() - keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) + + serverKeyAPI := base.CreateHTTPServerKeyAPIs() + keyRing := serverKeyAPI.KeyRing() + rsAPI := base.CreateHTTPRoomserverAPIs() fsAPI := federationsender.SetupFederationSenderComponent( - base, federation, rsAPI, &keyRing, + base, federation, rsAPI, keyRing, ) rsAPI.SetFederationSenderAPI(fsAPI) diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index f9993774f..8367cd9da 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -28,13 +28,13 @@ import ( "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/basecomponent" "github.com/matrix-org/dendrite/internal/config" - "github.com/matrix-org/dendrite/internal/keydb" "github.com/matrix-org/dendrite/internal/transactions" "github.com/matrix-org/dendrite/keyserver" "github.com/matrix-org/dendrite/mediaapi" "github.com/matrix-org/dendrite/publicroomsapi" "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/roomserver" + "github.com/matrix-org/dendrite/serverkeyapi" "github.com/matrix-org/dendrite/syncapi" "github.com/sirupsen/logrus" @@ -60,6 +60,7 @@ func main() { cfg.Listen.EDUServer = addr cfg.Listen.AppServiceAPI = addr cfg.Listen.FederationSender = addr + cfg.Listen.ServerKeyAPI = addr } base := basecomponent.NewBaseDendrite(cfg, "Monolith", *enableHTTPAPIs) @@ -67,9 +68,15 @@ func main() { accountDB := base.CreateAccountsDB() deviceDB := base.CreateDeviceDB() - keyDB := base.CreateKeyDB() federation := base.CreateFederationClient() - keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) + + serverKeyAPI := serverkeyapi.SetupServerKeyAPIComponent( + base, federation, + ) + if base.EnableHTTPAPIs { + serverKeyAPI = base.CreateHTTPServerKeyAPIs() + } + keyRing := serverKeyAPI.KeyRing() rsComponent := roomserver.SetupRoomServerComponent( base, keyRing, federation, @@ -94,7 +101,7 @@ func main() { } fsAPI := federationsender.SetupFederationSenderComponent( - base, federation, rsAPI, &keyRing, + base, federation, rsAPI, keyRing, ) if base.EnableHTTPAPIs { fsAPI = base.CreateHTTPFederationSenderAPIs() @@ -103,7 +110,7 @@ func main() { clientapi.SetupClientAPIComponent( base, deviceDB, accountDB, - federation, &keyRing, rsAPI, + federation, keyRing, rsAPI, eduInputAPI, asAPI, transactions.New(), fsAPI, ) @@ -111,7 +118,7 @@ func main() { base, deviceDB, accountDB, ) eduProducer := producers.NewEDUServerProducer(eduInputAPI) - federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, rsAPI, asAPI, fsAPI, eduProducer) + federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, keyRing, rsAPI, asAPI, fsAPI, eduProducer) mediaapi.SetupMediaAPIComponent(base, deviceDB) publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties()) if err != nil { diff --git a/cmd/dendrite-room-server/main.go b/cmd/dendrite-room-server/main.go index 31cf4a042..b86b74972 100644 --- a/cmd/dendrite-room-server/main.go +++ b/cmd/dendrite-room-server/main.go @@ -16,7 +16,6 @@ package main import ( "github.com/matrix-org/dendrite/internal/basecomponent" - "github.com/matrix-org/dendrite/internal/keydb" "github.com/matrix-org/dendrite/roomserver" ) @@ -24,9 +23,10 @@ func main() { cfg := basecomponent.ParseFlags() base := basecomponent.NewBaseDendrite(cfg, "RoomServerAPI", true) defer base.Close() // nolint: errcheck - keyDB := base.CreateKeyDB() federation := base.CreateFederationClient() - keyRing := keydb.CreateKeyRing(federation.Client, keyDB, cfg.Matrix.KeyPerspectives) + + serverKeyAPI := base.CreateHTTPServerKeyAPIs() + keyRing := serverKeyAPI.KeyRing() fsAPI := base.CreateHTTPFederationSenderAPIs() rsAPI := roomserver.SetupRoomServerComponent(base, keyRing, federation) diff --git a/cmd/dendrite-server-key-api-server/main.go b/cmd/dendrite-server-key-api-server/main.go new file mode 100644 index 000000000..075734125 --- /dev/null +++ b/cmd/dendrite-server-key-api-server/main.go @@ -0,0 +1,32 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/serverkeyapi" +) + +func main() { + cfg := basecomponent.ParseFlags() + base := basecomponent.NewBaseDendrite(cfg, "ServerKeyAPI", true) + defer base.Close() // nolint: errcheck + + federation := base.CreateFederationClient() + + serverkeyapi.SetupServerKeyAPIComponent(base, federation) + + base.SetupAndServeHTTP(string(base.Cfg.Bind.ServerKeyAPI), string(base.Cfg.Listen.ServerKeyAPI)) +} diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 73d381492..b8517bf9d 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -37,6 +37,7 @@ import ( "github.com/matrix-org/dendrite/publicroomsapi" "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/roomserver" + "github.com/matrix-org/dendrite/serverkeyapi" "github.com/matrix-org/dendrite/syncapi" go_http_js_libp2p "github.com/matrix-org/go-http-js-libp2p" "github.com/matrix-org/gomatrixserverlib" @@ -194,13 +195,16 @@ func main() { accountDB := base.CreateAccountsDB() deviceDB := base.CreateDeviceDB() - keyDB := base.CreateKeyDB() federation := createFederationClient(cfg, node) + + serverKeyAPI := serverkeyapi.SetupServerKeyAPIComponent( + base, federation, + ) keyRing := gomatrixserverlib.KeyRing{ KeyFetchers: []gomatrixserverlib.KeyFetcher{ &libp2pKeyFetcher{}, }, - KeyDatabase: keyDB, + KeyDatabase: serverKeyAPI, } p2pPublicRoomProvider := NewLibP2PPublicRoomsProvider(node) diff --git a/dendrite-config.yaml b/dendrite-config.yaml index 2616d74db..1802b8b73 100644 --- a/dendrite-config.yaml +++ b/dendrite-config.yaml @@ -137,7 +137,8 @@ listen: federation_sender: "localhost:7776" appservice_api: "localhost:7777" edu_server: "localhost:7778" - key_server: "localhost:7779" + key_server: "localhost:7779" + server_key_api: "localhost:7780" # The configuration for tracing the dendrite components. tracing: diff --git a/go.mod b/go.mod index 151fee27a..fe2cff768 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200511154227-5cc71d36632b + github.com/matrix-org/gomatrixserverlib v0.0.0-20200521102632-2a81324a04ae github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible diff --git a/go.sum b/go.sum index 15822b532..aa0c8dd57 100644 --- a/go.sum +++ b/go.sum @@ -358,8 +358,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 h1:Yb+Wlf github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bhrnp3Ky1qgx/fzCtCALOoGYylh2tpS9K4= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200511154227-5cc71d36632b h1:nAmSc1KvQOumoRTz/LD68KyrB6Q5/6q7CmQ5Bswc2nM= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200511154227-5cc71d36632b/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200521102632-2a81324a04ae h1:kFMh2aU3pMCkVCUeH57rtgm05XImbxKOHFYeUp80RCk= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200521102632-2a81324a04ae/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f/go.mod h1:y0oDTjZDv5SM9a2rp3bl+CU+bvTRINQsdb7YlDql5Go= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= diff --git a/internal/basecomponent/base.go b/internal/basecomponent/base.go index e86d91025..a48323726 100644 --- a/internal/basecomponent/base.go +++ b/internal/basecomponent/base.go @@ -21,12 +21,8 @@ import ( "net/url" "time" - "golang.org/x/crypto/ed25519" - "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/httpapis" - "github.com/matrix-org/dendrite/internal/keydb" - "github.com/matrix-org/dendrite/internal/keydb/cache" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/naffka" @@ -43,6 +39,7 @@ import ( federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/internal/config" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + serverKeyAPI "github.com/matrix-org/dendrite/serverkeyapi/api" "github.com/sirupsen/logrus" _ "net/http/pprof" @@ -157,6 +154,20 @@ func (b *BaseDendrite) CreateHTTPFederationSenderAPIs() federationSenderAPI.Fede return f } +// CreateHTTPServerKeyAPIs returns ServerKeyInternalAPI for hitting the server key +// API over HTTP +func (b *BaseDendrite) CreateHTTPServerKeyAPIs() serverKeyAPI.ServerKeyInternalAPI { + f, err := serverKeyAPI.NewServerKeyInternalAPIHTTP( + b.Cfg.ServerKeyAPIURL(), + b.httpClient, + b.ImmutableCache, + ) + if err != nil { + logrus.WithError(err).Panic("NewServerKeyInternalAPIHTTP failed", b.httpClient) + } + return f +} + // CreateDeviceDB creates a new instance of the device database. Should only be // called once per component. func (b *BaseDendrite) CreateDeviceDB() devices.Database { @@ -179,27 +190,6 @@ func (b *BaseDendrite) CreateAccountsDB() accounts.Database { return db } -// CreateKeyDB creates a new instance of the key database. Should only be called -// once per component. -func (b *BaseDendrite) CreateKeyDB() keydb.Database { - db, err := keydb.NewDatabase( - string(b.Cfg.Database.ServerKey), - b.Cfg.DbProperties(), - b.Cfg.Matrix.ServerName, - b.Cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey), - b.Cfg.Matrix.KeyID, - ) - if err != nil { - logrus.WithError(err).Panicf("failed to connect to keys db") - } - - cachedDB, err := cache.NewKeyDatabase(db, b.ImmutableCache) - if err != nil { - logrus.WithError(err).Panicf("failed to create key cache wrapper") - } - return cachedDB -} - // CreateFederationClient creates a new federation client. Should only be called // once per component. func (b *BaseDendrite) CreateFederationClient() *gomatrixserverlib.FederationClient { diff --git a/internal/config/config.go b/internal/config/config.go index e1e96f9d5..2a95069a4 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -223,6 +223,7 @@ type Dendrite struct { MediaAPI Address `yaml:"media_api"` ClientAPI Address `yaml:"client_api"` FederationAPI Address `yaml:"federation_api"` + ServerKeyAPI Address `yaml:"server_key_api"` AppServiceAPI Address `yaml:"appservice_api"` SyncAPI Address `yaml:"sync_api"` RoomServer Address `yaml:"room_server"` @@ -237,6 +238,7 @@ type Dendrite struct { MediaAPI Address `yaml:"media_api"` ClientAPI Address `yaml:"client_api"` FederationAPI Address `yaml:"federation_api"` + ServerKeyAPI Address `yaml:"server_key_api"` AppServiceAPI Address `yaml:"appservice_api"` SyncAPI Address `yaml:"sync_api"` RoomServer Address `yaml:"room_server"` @@ -620,6 +622,7 @@ func (config *Dendrite) checkListen(configErrs *configErrors) { checkNotEmpty(configErrs, "listen.sync_api", string(config.Listen.SyncAPI)) checkNotEmpty(configErrs, "listen.room_server", string(config.Listen.RoomServer)) checkNotEmpty(configErrs, "listen.edu_server", string(config.Listen.EDUServer)) + checkNotEmpty(configErrs, "listen.server_key_api", string(config.Listen.EDUServer)) } // checkLogging verifies the parameters logging.* are valid. @@ -751,6 +754,15 @@ func (config *Dendrite) FederationSenderURL() string { return "http://" + string(config.Listen.FederationSender) } +// FederationSenderURL returns an HTTP URL for where the federation sender is listening. +func (config *Dendrite) ServerKeyAPIURL() string { + // Hard code the server key API server to talk HTTP for now. + // If we support HTTPS we need to think of a practical way to do certificate validation. + // People setting up servers shouldn't need to get a certificate valid for the public + // internet for an internal API. + return "http://" + string(config.Listen.ServerKeyAPI) +} + // SetupTracing configures the opentracing using the supplied configuration. func (config *Dendrite) SetupTracing(serviceName string) (closer io.Closer, err error) { if !config.Tracing.Enabled { diff --git a/internal/keydb/keyring.go b/internal/keydb/keyring.go deleted file mode 100644 index d0b1904ed..000000000 --- a/internal/keydb/keyring.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2017 New Vector Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package keydb - -import ( - "encoding/base64" - - "github.com/matrix-org/dendrite/internal/config" - "github.com/matrix-org/gomatrixserverlib" - "github.com/sirupsen/logrus" - "golang.org/x/crypto/ed25519" -) - -// CreateKeyRing creates and configures a KeyRing object. -// -// It creates the necessary key fetchers and collects them into a KeyRing -// backed by the given KeyDatabase. -func CreateKeyRing(client gomatrixserverlib.Client, - keyDB gomatrixserverlib.KeyDatabase, - cfg config.KeyPerspectives) gomatrixserverlib.KeyRing { - - fetchers := gomatrixserverlib.KeyRing{ - KeyFetchers: []gomatrixserverlib.KeyFetcher{ - &gomatrixserverlib.DirectKeyFetcher{ - Client: client, - }, - }, - KeyDatabase: keyDB, - } - - logrus.Info("Enabled direct key fetcher") - - var b64e = base64.StdEncoding.WithPadding(base64.NoPadding) - for _, ps := range cfg { - perspective := &gomatrixserverlib.PerspectiveKeyFetcher{ - PerspectiveServerName: ps.ServerName, - PerspectiveServerKeys: map[gomatrixserverlib.KeyID]ed25519.PublicKey{}, - Client: client, - } - - for _, key := range ps.Keys { - rawkey, err := b64e.DecodeString(key.PublicKey) - if err != nil { - logrus.WithError(err).WithFields(logrus.Fields{ - "server_name": ps.ServerName, - "public_key": key.PublicKey, - }).Warn("Couldn't parse perspective key") - continue - } - perspective.PerspectiveServerKeys[key.KeyID] = rawkey - } - - fetchers.KeyFetchers = append(fetchers.KeyFetchers, perspective) - - logrus.WithFields(logrus.Fields{ - "server_name": ps.ServerName, - "num_public_keys": len(ps.Keys), - }).Info("Enabled perspective key fetcher") - } - - return fetchers -} diff --git a/serverkeyapi/api/api.go b/serverkeyapi/api/api.go new file mode 100644 index 000000000..f108d437b --- /dev/null +++ b/serverkeyapi/api/api.go @@ -0,0 +1,113 @@ +package api + +import ( + "context" + "errors" + "net/http" + + "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/gomatrixserverlib" +) + +type ServerKeyInternalAPI interface { + gomatrixserverlib.KeyDatabase + + KeyRing() *gomatrixserverlib.KeyRing + + InputPublicKeys( + ctx context.Context, + request *InputPublicKeysRequest, + response *InputPublicKeysResponse, + ) error + + QueryPublicKeys( + ctx context.Context, + request *QueryPublicKeysRequest, + response *QueryPublicKeysResponse, + ) error +} + +// NewRoomserverInputAPIHTTP creates a RoomserverInputAPI implemented by talking to a HTTP POST API. +// If httpClient is nil an error is returned +func NewServerKeyInternalAPIHTTP( + serverKeyAPIURL string, + httpClient *http.Client, + immutableCache caching.ImmutableCache, +) (ServerKeyInternalAPI, error) { + if httpClient == nil { + return nil, errors.New("NewRoomserverInternalAPIHTTP: httpClient is ") + } + return &httpServerKeyInternalAPI{ + serverKeyAPIURL: serverKeyAPIURL, + httpClient: httpClient, + immutableCache: immutableCache, + }, nil +} + +type httpServerKeyInternalAPI struct { + ServerKeyInternalAPI + + serverKeyAPIURL string + httpClient *http.Client + immutableCache caching.ImmutableCache +} + +func (s *httpServerKeyInternalAPI) KeyRing() *gomatrixserverlib.KeyRing { + // This is a bit of a cheat - we tell gomatrixserverlib that this API is + // both the key database and the key fetcher. While this does have the + // rather unfortunate effect of preventing gomatrixserverlib from handling + // key fetchers directly, we can at least reimplement this behaviour on + // the other end of the API. + return &gomatrixserverlib.KeyRing{ + KeyDatabase: s, + KeyFetchers: []gomatrixserverlib.KeyFetcher{s}, + } +} + +func (s *httpServerKeyInternalAPI) FetcherName() string { + return "httpServerKeyInternalAPI" +} + +func (s *httpServerKeyInternalAPI) StoreKeys( + ctx context.Context, + results map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, +) error { + request := InputPublicKeysRequest{ + Keys: make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult), + } + response := InputPublicKeysResponse{} + for req, res := range results { + request.Keys[req] = res + s.immutableCache.StoreServerKey(req, res) + } + return s.InputPublicKeys(ctx, &request, &response) +} + +func (s *httpServerKeyInternalAPI) FetchKeys( + ctx context.Context, + requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp, +) (map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, error) { + result := make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult) + request := QueryPublicKeysRequest{ + Requests: make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp), + } + response := QueryPublicKeysResponse{ + Results: make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult), + } + for req, ts := range requests { + if res, ok := s.immutableCache.GetServerKey(req); ok { + result[req] = res + continue + } + request.Requests[req] = ts + } + err := s.QueryPublicKeys(ctx, &request, &response) + if err != nil { + return nil, err + } + for req, res := range response.Results { + result[req] = res + s.immutableCache.StoreServerKey(req, res) + } + return result, nil +} diff --git a/serverkeyapi/api/http.go b/serverkeyapi/api/http.go new file mode 100644 index 000000000..ab89f7bfc --- /dev/null +++ b/serverkeyapi/api/http.go @@ -0,0 +1,57 @@ +package api + +import ( + "context" + + commonHTTP "github.com/matrix-org/dendrite/internal/http" + "github.com/matrix-org/gomatrixserverlib" + + "github.com/opentracing/opentracing-go" +) + +const ( + // ServerKeyInputPublicKeyPath is the HTTP path for the InputPublicKeys API. + ServerKeyInputPublicKeyPath = "/serverkeyapi/inputPublicKey" + + // ServerKeyQueryPublicKeyPath is the HTTP path for the QueryPublicKeys API. + ServerKeyQueryPublicKeyPath = "/serverkeyapi/queryPublicKey" +) + +type InputPublicKeysRequest struct { + Keys map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult `json:"keys"` +} + +type InputPublicKeysResponse struct { +} + +func (h *httpServerKeyInternalAPI) InputPublicKeys( + ctx context.Context, + request *InputPublicKeysRequest, + response *InputPublicKeysResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "InputPublicKey") + defer span.Finish() + + apiURL := h.serverKeyAPIURL + ServerKeyInputPublicKeyPath + return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +type QueryPublicKeysRequest struct { + Requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp `json:"requests"` +} + +type QueryPublicKeysResponse struct { + Results map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult `json:"results"` +} + +func (h *httpServerKeyInternalAPI) QueryPublicKeys( + ctx context.Context, + request *QueryPublicKeysRequest, + response *QueryPublicKeysResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "QueryPublicKey") + defer span.Finish() + + apiURL := h.serverKeyAPIURL + ServerKeyQueryPublicKeyPath + return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} diff --git a/serverkeyapi/internal/api.go b/serverkeyapi/internal/api.go new file mode 100644 index 000000000..cbae59d9a --- /dev/null +++ b/serverkeyapi/internal/api.go @@ -0,0 +1,76 @@ +package internal + +import ( + "context" + "fmt" + + "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/serverkeyapi/api" + "github.com/matrix-org/gomatrixserverlib" +) + +type ServerKeyAPI struct { + api.ServerKeyInternalAPI + + ImmutableCache caching.ImmutableCache + OurKeyRing gomatrixserverlib.KeyRing + FedClient *gomatrixserverlib.FederationClient +} + +func (s *ServerKeyAPI) KeyRing() *gomatrixserverlib.KeyRing { + // Return a real keyring - one that has the real database and real + // fetchers. + return &s.OurKeyRing +} + +func (s *ServerKeyAPI) StoreKeys( + ctx context.Context, + results map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, +) error { + // Store any keys that we were given in our database. + return s.OurKeyRing.KeyDatabase.StoreKeys(ctx, results) +} + +func (s *ServerKeyAPI) FetchKeys( + ctx context.Context, + requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp, +) (map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, error) { + results := map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult{} + // First consult our local database and see if we have the requested + // keys. These might come from a cache, depending on the database + // implementation used. + if dbResults, err := s.OurKeyRing.KeyDatabase.FetchKeys(ctx, requests); err == nil { + // We successfully got some keys. Add them to the results and + // remove them from the request list. + for req, res := range dbResults { + results[req] = res + delete(requests, req) + } + } + // For any key requests that we still have outstanding, next try to + // fetch them directly. We'll go through each of the key fetchers to + // ask for the remaining keys. + for _, fetcher := range s.OurKeyRing.KeyFetchers { + if len(requests) == 0 { + break + } + if fetcherResults, err := fetcher.FetchKeys(ctx, requests); err == nil { + // We successfully got some keys. Add them to the results and + // remove them from the request list. + for req, res := range fetcherResults { + results[req] = res + delete(requests, req) + } + } + } + // If we failed to fetch any keys then we should report an error. + if len(requests) > 0 { + return results, fmt.Errorf("server key API failed to fetch %d keys", len(requests)) + } + // Return the keys. + return results, nil +} + +func (s *ServerKeyAPI) FetcherName() string { + return s.OurKeyRing.KeyDatabase.FetcherName() +} diff --git a/serverkeyapi/internal/http.go b/serverkeyapi/internal/http.go new file mode 100644 index 000000000..303275712 --- /dev/null +++ b/serverkeyapi/internal/http.go @@ -0,0 +1,60 @@ +package internal + +import ( + "encoding/json" + "net/http" + + "github.com/gorilla/mux" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/serverkeyapi/api" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" +) + +func (s *ServerKeyAPI) SetupHTTP(internalAPIMux *mux.Router) { + internalAPIMux.Handle(api.ServerKeyQueryPublicKeyPath, + internal.MakeInternalAPI("queryPublicKeys", func(req *http.Request) util.JSONResponse { + result := map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult{} + request := api.QueryPublicKeysRequest{} + response := api.QueryPublicKeysResponse{} + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + lookup := make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp) + for req, timestamp := range request.Requests { + if res, ok := s.ImmutableCache.GetServerKey(req); ok { + result[req] = res + continue + } + lookup[req] = timestamp + } + keys, err := s.FetchKeys(req.Context(), lookup) + if err != nil { + return util.ErrorResponse(err) + } + for req, res := range keys { + result[req] = res + } + response.Results = result + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + internalAPIMux.Handle(api.ServerKeyInputPublicKeyPath, + internal.MakeInternalAPI("inputPublicKeys", func(req *http.Request) util.JSONResponse { + request := api.InputPublicKeysRequest{} + response := api.InputPublicKeysResponse{} + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + store := make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult) + for req, res := range request.Keys { + store[req] = res + s.ImmutableCache.StoreServerKey(req, res) + } + if err := s.StoreKeys(req.Context(), store); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) +} diff --git a/serverkeyapi/serverkeyapi.go b/serverkeyapi/serverkeyapi.go new file mode 100644 index 000000000..61d807d64 --- /dev/null +++ b/serverkeyapi/serverkeyapi.go @@ -0,0 +1,83 @@ +package serverkeyapi + +import ( + "crypto/ed25519" + "encoding/base64" + + "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/serverkeyapi/api" + "github.com/matrix-org/dendrite/serverkeyapi/internal" + "github.com/matrix-org/dendrite/serverkeyapi/storage" + "github.com/matrix-org/dendrite/serverkeyapi/storage/cache" + "github.com/matrix-org/gomatrixserverlib" + "github.com/sirupsen/logrus" +) + +func SetupServerKeyAPIComponent( + base *basecomponent.BaseDendrite, + fedClient *gomatrixserverlib.FederationClient, +) api.ServerKeyInternalAPI { + innerDB, err := storage.NewDatabase( + string(base.Cfg.Database.ServerKey), + base.Cfg.DbProperties(), + base.Cfg.Matrix.ServerName, + base.Cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey), + base.Cfg.Matrix.KeyID, + ) + if err != nil { + logrus.WithError(err).Panicf("failed to connect to server key database") + } + + serverKeyDB, err := cache.NewKeyDatabase(innerDB, base.ImmutableCache) + if err != nil { + logrus.WithError(err).Panicf("failed to set up caching wrapper for server key database") + } + + internalAPI := internal.ServerKeyAPI{ + ImmutableCache: base.ImmutableCache, + FedClient: fedClient, + OurKeyRing: gomatrixserverlib.KeyRing{ + KeyFetchers: []gomatrixserverlib.KeyFetcher{ + &gomatrixserverlib.DirectKeyFetcher{ + Client: fedClient.Client, + }, + }, + KeyDatabase: serverKeyDB, + }, + } + + var b64e = base64.StdEncoding.WithPadding(base64.NoPadding) + for _, ps := range base.Cfg.Matrix.KeyPerspectives { + perspective := &gomatrixserverlib.PerspectiveKeyFetcher{ + PerspectiveServerName: ps.ServerName, + PerspectiveServerKeys: map[gomatrixserverlib.KeyID]ed25519.PublicKey{}, + Client: fedClient.Client, + } + + for _, key := range ps.Keys { + rawkey, err := b64e.DecodeString(key.PublicKey) + if err != nil { + logrus.WithError(err).WithFields(logrus.Fields{ + "server_name": ps.ServerName, + "public_key": key.PublicKey, + }).Warn("Couldn't parse perspective key") + continue + } + perspective.PerspectiveServerKeys[key.KeyID] = rawkey + } + + internalAPI.OurKeyRing.KeyFetchers = append( + internalAPI.OurKeyRing.KeyFetchers, + perspective, + ) + + logrus.WithFields(logrus.Fields{ + "server_name": ps.ServerName, + "num_public_keys": len(ps.Keys), + }).Info("Enabled perspective key fetcher") + } + + internalAPI.SetupHTTP(base.InternalAPIMux) + + return &internalAPI +} diff --git a/internal/keydb/cache/keydb.go b/serverkeyapi/storage/cache/keydb.go similarity index 91% rename from internal/keydb/cache/keydb.go rename to serverkeyapi/storage/cache/keydb.go index 87573ed2b..a0cdb9007 100644 --- a/internal/keydb/cache/keydb.go +++ b/serverkeyapi/storage/cache/keydb.go @@ -5,18 +5,17 @@ import ( "errors" "github.com/matrix-org/dendrite/internal/caching" - "github.com/matrix-org/dendrite/internal/keydb" "github.com/matrix-org/gomatrixserverlib" ) // A Database implements gomatrixserverlib.KeyDatabase and is used to store // the public keys for other matrix servers. type KeyDatabase struct { - inner keydb.Database + inner gomatrixserverlib.KeyDatabase cache caching.ImmutableCache } -func NewKeyDatabase(inner keydb.Database, cache caching.ImmutableCache) (*KeyDatabase, error) { +func NewKeyDatabase(inner gomatrixserverlib.KeyDatabase, cache caching.ImmutableCache) (*KeyDatabase, error) { if inner == nil { return nil, errors.New("inner database can't be nil") } diff --git a/internal/keydb/interface.go b/serverkeyapi/storage/interface.go similarity index 96% rename from internal/keydb/interface.go rename to serverkeyapi/storage/interface.go index c9a20fdd9..3a67ac55a 100644 --- a/internal/keydb/interface.go +++ b/serverkeyapi/storage/interface.go @@ -1,4 +1,4 @@ -package keydb +package storage import ( "context" diff --git a/internal/keydb/keydb.go b/serverkeyapi/storage/keydb.go similarity index 91% rename from internal/keydb/keydb.go rename to serverkeyapi/storage/keydb.go index ad6d56b86..b9389bd63 100644 --- a/internal/keydb/keydb.go +++ b/serverkeyapi/storage/keydb.go @@ -14,7 +14,7 @@ // +build !wasm -package keydb +package storage import ( "net/url" @@ -22,8 +22,8 @@ import ( "golang.org/x/crypto/ed25519" "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/dendrite/internal/keydb/postgres" - "github.com/matrix-org/dendrite/internal/keydb/sqlite3" + "github.com/matrix-org/dendrite/serverkeyapi/storage/postgres" + "github.com/matrix-org/dendrite/serverkeyapi/storage/sqlite3" "github.com/matrix-org/gomatrixserverlib" ) diff --git a/internal/keydb/keydb_wasm.go b/serverkeyapi/storage/keydb_wasm.go similarity index 88% rename from internal/keydb/keydb_wasm.go rename to serverkeyapi/storage/keydb_wasm.go index c80329027..3d2ca5505 100644 --- a/internal/keydb/keydb_wasm.go +++ b/serverkeyapi/storage/keydb_wasm.go @@ -12,7 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -package keydb +// +build wasm + +package storage import ( "fmt" @@ -21,7 +23,7 @@ import ( "golang.org/x/crypto/ed25519" "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/dendrite/internal/keydb/sqlite3" + "github.com/matrix-org/dendrite/serverkeyapi/storage/sqlite3" "github.com/matrix-org/gomatrixserverlib" ) @@ -41,7 +43,7 @@ func NewDatabase( case "postgres": return nil, fmt.Errorf("Cannot use postgres implementation") case "file": - return sqlite3.NewDatabase(uri.Path, serverName, serverKey, serverKeyID) + return sqlite3.NewDatabase(dataSourceName, serverName, serverKey, serverKeyID) default: return nil, fmt.Errorf("Cannot use postgres implementation") } diff --git a/internal/keydb/postgres/keydb.go b/serverkeyapi/storage/postgres/keydb.go similarity index 100% rename from internal/keydb/postgres/keydb.go rename to serverkeyapi/storage/postgres/keydb.go diff --git a/internal/keydb/postgres/server_key_table.go b/serverkeyapi/storage/postgres/server_key_table.go similarity index 100% rename from internal/keydb/postgres/server_key_table.go rename to serverkeyapi/storage/postgres/server_key_table.go diff --git a/internal/keydb/sqlite3/keydb.go b/serverkeyapi/storage/sqlite3/keydb.go similarity index 100% rename from internal/keydb/sqlite3/keydb.go rename to serverkeyapi/storage/sqlite3/keydb.go diff --git a/internal/keydb/sqlite3/server_key_table.go b/serverkeyapi/storage/sqlite3/server_key_table.go similarity index 100% rename from internal/keydb/sqlite3/server_key_table.go rename to serverkeyapi/storage/sqlite3/server_key_table.go From a7f12bce79436fab14ed97ba1cb049384e1e3522 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Wed, 27 May 2020 11:03:47 +0100 Subject: [PATCH 099/200] Convert remaining roomserver tables (#1065) * Convert invites table * Convert membership table * Factor out remaining functions except for *Updater structs * Remove 'implements' comments from long-forgotten interfaces. Move those comments to storage.Database --- roomserver/storage/interface.go | 51 +++++++++ .../storage/postgres/event_json_table.go | 5 +- .../postgres/event_state_keys_table.go | 5 +- .../storage/postgres/event_types_table.go | 5 +- roomserver/storage/postgres/events_table.go | 5 +- roomserver/storage/postgres/invite_table.go | 21 ++-- .../storage/postgres/membership_table.go | 41 +++---- roomserver/storage/postgres/prepare.go | 36 ------ .../storage/postgres/previous_events_table.go | 5 +- .../storage/postgres/room_aliases_table.go | 5 +- roomserver/storage/postgres/rooms_table.go | 5 +- roomserver/storage/postgres/sql.go | 50 -------- .../storage/postgres/state_block_table.go | 5 +- .../storage/postgres/state_snapshot_table.go | 5 +- roomserver/storage/postgres/storage.go | 97 +++++----------- .../storage/postgres/transactions_table.go | 5 +- .../storage/{sqlite3 => shared}/prepare.go | 16 +-- roomserver/storage/shared/storage.go | 69 +++++++---- .../storage/sqlite3/event_json_table.go | 5 +- .../storage/sqlite3/event_state_keys_table.go | 9 +- .../storage/sqlite3/event_types_table.go | 5 +- roomserver/storage/sqlite3/events_table.go | 5 +- roomserver/storage/sqlite3/invite_table.go | 19 +-- .../storage/sqlite3/membership_table.go | 58 +++++----- .../storage/sqlite3/previous_events_table.go | 5 +- .../storage/sqlite3/room_aliases_table.go | 5 +- roomserver/storage/sqlite3/rooms_table.go | 5 +- roomserver/storage/sqlite3/sql.go | 50 -------- .../storage/sqlite3/state_block_table.go | 5 +- .../storage/sqlite3/state_snapshot_table.go | 5 +- roomserver/storage/sqlite3/storage.go | 108 +++++------------- .../storage/sqlite3/transactions_table.go | 5 +- roomserver/storage/tables/interface.go | 24 ++++ 33 files changed, 317 insertions(+), 427 deletions(-) delete mode 100644 roomserver/storage/postgres/prepare.go delete mode 100644 roomserver/storage/postgres/sql.go rename roomserver/storage/{sqlite3 => shared}/prepare.go (66%) delete mode 100644 roomserver/storage/sqlite3/sql.go diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go index 1e0232d20..52e6a96b7 100644 --- a/roomserver/storage/interface.go +++ b/roomserver/storage/interface.go @@ -63,29 +63,80 @@ type Database interface { SnapshotNIDFromEventID(ctx context.Context, eventID string) (types.StateSnapshotNID, error) // Look up a room version from the room NID. GetRoomVersionForRoomNID(ctx context.Context, roomNID types.RoomNID) (gomatrixserverlib.RoomVersion, error) + // Stores a matrix room event in the database StoreEvent(ctx context.Context, event gomatrixserverlib.Event, txnAndSessionID *api.TransactionID, authEventNIDs []types.EventNID) (types.RoomNID, types.StateAtEvent, error) + // Look up the state entries for a list of string event IDs + // Returns an error if the there is an error talking to the database + // Returns a types.MissingEventError if the event IDs aren't in the database. StateEntriesForEventIDs(ctx context.Context, eventIDs []string) ([]types.StateEntry, error) + // Look up the string event state keys for a list of numeric event state keys + // Returns an error if there was a problem talking to the database. EventStateKeys(ctx context.Context, eventStateKeyNIDs []types.EventStateKeyNID) (map[types.EventStateKeyNID]string, error) + // Look up the numeric IDs for a list of events. + // Returns an error if there was a problem talking to the database. EventNIDs(ctx context.Context, eventIDs []string) (map[string]types.EventNID, error) + // Set the state at an event. FIXME TODO: "at" SetState(ctx context.Context, eventNID types.EventNID, stateNID types.StateSnapshotNID) error + // Lookup the event IDs for a batch of event numeric IDs. + // Returns an error if the retrieval went wrong. EventIDs(ctx context.Context, eventNIDs []types.EventNID) (map[types.EventNID]string, error) + // Look up the latest events in a room in preparation for an update. + // The RoomRecentEventsUpdater must have Commit or Rollback called on it if this doesn't return an error. + // Returns the latest events in the room and the last eventID sent to the log along with an updater. + // If this returns an error then no further action is required. GetLatestEventsForUpdate(ctx context.Context, roomNID types.RoomNID) (types.RoomRecentEventsUpdater, error) + // Look up event ID by transaction's info. + // This is used to determine if the room event is processed/processing already. + // Returns an empty string if no such event exists. GetTransactionEventID(ctx context.Context, transactionID string, sessionID int64, userID string) (string, error) + // Look up the numeric ID for the room. + // Returns 0 if the room doesn't exists. + // Returns an error if there was a problem talking to the database. RoomNID(ctx context.Context, roomID string) (types.RoomNID, error) // RoomNIDExcludingStubs is a special variation of RoomNID that will return 0 as if the room // does not exist if the room has no latest events. This can happen when we've received an // invite over federation for a room that we don't know anything else about yet. RoomNIDExcludingStubs(ctx context.Context, roomID string) (types.RoomNID, error) + // Look up event references for the latest events in the room and the current state snapshot. + // Returns the latest events, the current state and the maximum depth of the latest events plus 1. + // Returns an error if there was a problem talking to the database. LatestEventIDs(ctx context.Context, roomNID types.RoomNID) ([]gomatrixserverlib.EventReference, types.StateSnapshotNID, int64, error) + // Look up the active invites targeting a user in a room and return the + // numeric state key IDs for the user IDs who sent them. + // Returns an error if there was a problem talking to the database. GetInvitesForUser(ctx context.Context, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID) (senderUserIDs []types.EventStateKeyNID, err error) + // Save a given room alias with the room ID it refers to. + // Returns an error if there was a problem talking to the database. SetRoomAlias(ctx context.Context, alias string, roomID string, creatorUserID string) error + // Look up the room ID a given alias refers to. + // Returns an error if there was a problem talking to the database. GetRoomIDForAlias(ctx context.Context, alias string) (string, error) + // Look up all aliases referring to a given room ID. + // Returns an error if there was a problem talking to the database. GetAliasesForRoomID(ctx context.Context, roomID string) ([]string, error) + // Get the user ID of the creator of an alias. + // Returns an error if there was a problem talking to the database. GetCreatorIDForAlias(ctx context.Context, alias string) (string, error) + // Remove a given room alias. + // Returns an error if there was a problem talking to the database. RemoveRoomAlias(ctx context.Context, alias string) error + // Build a membership updater for the target user in a room. MembershipUpdater(ctx context.Context, roomID, targetUserID string, targetLocal bool, roomVersion gomatrixserverlib.RoomVersion) (types.MembershipUpdater, error) + // Lookup the membership of a given user in a given room. + // Returns the numeric ID of the latest membership event sent from this user + // in this room, along a boolean set to true if the user is still in this room, + // false if not. + // Returns an error if there was a problem talking to the database. GetMembership(ctx context.Context, roomNID types.RoomNID, requestSenderUserID string) (membershipEventNID types.EventNID, stillInRoom bool, err error) + // Lookup the membership event numeric IDs for all user that are or have + // been members of a given room. Only lookup events of "join" membership if + // joinOnly is set to true. + // Returns an error if there was a problem talking to the database. GetMembershipEventNIDsForRoom(ctx context.Context, roomNID types.RoomNID, joinOnly bool, localOnly bool) ([]types.EventNID, error) + // EventsFromIDs looks up the Events for a list of event IDs. Does not error if event was + // not found. + // Returns an error if the retrieval went wrong. EventsFromIDs(ctx context.Context, eventIDs []string) ([]types.Event, error) + // Look up the room version for a given room. GetRoomVersionForRoom(ctx context.Context, roomID string) (gomatrixserverlib.RoomVersion, error) } diff --git a/roomserver/storage/postgres/event_json_table.go b/roomserver/storage/postgres/event_json_table.go index a32629260..3bce9f086 100644 --- a/roomserver/storage/postgres/event_json_table.go +++ b/roomserver/storage/postgres/event_json_table.go @@ -21,6 +21,7 @@ import ( "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -65,10 +66,10 @@ func NewPostgresEventJSONTable(db *sql.DB) (tables.EventJSON, error) { if err != nil { return nil, err } - return s, statementList{ + return s, shared.StatementList{ {&s.insertEventJSONStmt, insertEventJSONSQL}, {&s.bulkSelectEventJSONStmt, bulkSelectEventJSONSQL}, - }.prepare(db) + }.Prepare(db) } func (s *eventJSONStatements) InsertEventJSON( diff --git a/roomserver/storage/postgres/event_state_keys_table.go b/roomserver/storage/postgres/event_state_keys_table.go index 81b9b06e8..b7e2cb6b7 100644 --- a/roomserver/storage/postgres/event_state_keys_table.go +++ b/roomserver/storage/postgres/event_state_keys_table.go @@ -21,6 +21,7 @@ import ( "github.com/lib/pq" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -81,12 +82,12 @@ func NewPostgresEventStateKeysTable(db *sql.DB) (tables.EventStateKeys, error) { if err != nil { return nil, err } - return s, statementList{ + return s, shared.StatementList{ {&s.insertEventStateKeyNIDStmt, insertEventStateKeyNIDSQL}, {&s.selectEventStateKeyNIDStmt, selectEventStateKeyNIDSQL}, {&s.bulkSelectEventStateKeyNIDStmt, bulkSelectEventStateKeyNIDSQL}, {&s.bulkSelectEventStateKeyStmt, bulkSelectEventStateKeySQL}, - }.prepare(db) + }.Prepare(db) } func (s *eventStateKeyStatements) InsertEventStateKeyNID( diff --git a/roomserver/storage/postgres/event_types_table.go b/roomserver/storage/postgres/event_types_table.go index aaba614a4..f8a7bff1b 100644 --- a/roomserver/storage/postgres/event_types_table.go +++ b/roomserver/storage/postgres/event_types_table.go @@ -22,6 +22,7 @@ import ( "github.com/matrix-org/dendrite/internal" "github.com/lib/pq" + "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -106,11 +107,11 @@ func NewPostgresEventTypesTable(db *sql.DB) (tables.EventTypes, error) { return nil, err } - return s, statementList{ + return s, shared.StatementList{ {&s.insertEventTypeNIDStmt, insertEventTypeNIDSQL}, {&s.selectEventTypeNIDStmt, selectEventTypeNIDSQL}, {&s.bulkSelectEventTypeNIDStmt, bulkSelectEventTypeNIDSQL}, - }.prepare(db) + }.Prepare(db) } func (s *eventTypeStatements) InsertEventTypeNID( diff --git a/roomserver/storage/postgres/events_table.go b/roomserver/storage/postgres/events_table.go index 5a567bf23..d39e9511a 100644 --- a/roomserver/storage/postgres/events_table.go +++ b/roomserver/storage/postgres/events_table.go @@ -22,6 +22,7 @@ import ( "github.com/lib/pq" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" @@ -144,7 +145,7 @@ func NewPostgresEventsTable(db *sql.DB) (tables.Events, error) { return nil, err } - return s, statementList{ + return s, shared.StatementList{ {&s.insertEventStmt, insertEventSQL}, {&s.selectEventStmt, selectEventSQL}, {&s.bulkSelectStateEventByIDStmt, bulkSelectStateEventByIDSQL}, @@ -159,7 +160,7 @@ func NewPostgresEventsTable(db *sql.DB) (tables.Events, error) { {&s.bulkSelectEventNIDStmt, bulkSelectEventNIDSQL}, {&s.selectMaxEventDepthStmt, selectMaxEventDepthSQL}, {&s.selectRoomNIDForEventNIDStmt, selectRoomNIDForEventNIDSQL}, - }.prepare(db) + }.Prepare(db) } func (s *eventStatements) InsertEvent( diff --git a/roomserver/storage/postgres/invite_table.go b/roomserver/storage/postgres/invite_table.go index f0fb919e6..55a5bc7d7 100644 --- a/roomserver/storage/postgres/invite_table.go +++ b/roomserver/storage/postgres/invite_table.go @@ -20,6 +20,8 @@ import ( "database/sql" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/roomserver/storage/shared" + "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -79,20 +81,21 @@ type inviteStatements struct { updateInviteRetiredStmt *sql.Stmt } -func (s *inviteStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(inviteSchema) +func NewPostgresInvitesTable(db *sql.DB) (tables.Invites, error) { + s := &inviteStatements{} + _, err := db.Exec(inviteSchema) if err != nil { - return + return nil, err } - return statementList{ + return s, shared.StatementList{ {&s.insertInviteEventStmt, insertInviteEventSQL}, {&s.selectInviteActiveForUserInRoomStmt, selectInviteActiveForUserInRoomSQL}, {&s.updateInviteRetiredStmt, updateInviteRetiredSQL}, - }.prepare(db) + }.Prepare(db) } -func (s *inviteStatements) insertInviteEvent( +func (s *inviteStatements) InsertInviteEvent( ctx context.Context, txn *sql.Tx, inviteEventID string, roomNID types.RoomNID, targetUserNID, senderUserNID types.EventStateKeyNID, @@ -111,7 +114,7 @@ func (s *inviteStatements) insertInviteEvent( return count != 0, nil } -func (s *inviteStatements) updateInviteRetired( +func (s *inviteStatements) UpdateInviteRetired( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, ) ([]string, error) { @@ -133,8 +136,8 @@ func (s *inviteStatements) updateInviteRetired( return eventIDs, rows.Err() } -// selectInviteActiveForUserInRoom returns a list of sender state key NIDs -func (s *inviteStatements) selectInviteActiveForUserInRoom( +// SelectInviteActiveForUserInRoom returns a list of sender state key NIDs +func (s *inviteStatements) SelectInviteActiveForUserInRoom( ctx context.Context, targetUserNID types.EventStateKeyNID, roomNID types.RoomNID, ) ([]types.EventStateKeyNID, error) { diff --git a/roomserver/storage/postgres/membership_table.go b/roomserver/storage/postgres/membership_table.go index f290a05f9..c635ce28a 100644 --- a/roomserver/storage/postgres/membership_table.go +++ b/roomserver/storage/postgres/membership_table.go @@ -20,17 +20,11 @@ import ( "database/sql" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/roomserver/storage/shared" + "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" ) -type membershipState int64 - -const ( - membershipStateLeaveOrBan membershipState = 1 - membershipStateInvite membershipState = 2 - membershipStateJoin membershipState = 3 -) - const membershipSchema = ` -- The membership table is used to coordinate updates between the invite table -- and the room state tables. @@ -115,13 +109,14 @@ type membershipStatements struct { updateMembershipStmt *sql.Stmt } -func (s *membershipStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(membershipSchema) +func NewPostgresMembershipTable(db *sql.DB) (tables.Membership, error) { + s := &membershipStatements{} + _, err := db.Exec(membershipSchema) if err != nil { - return + return nil, err } - return statementList{ + return s, shared.StatementList{ {&s.insertMembershipStmt, insertMembershipSQL}, {&s.selectMembershipForUpdateStmt, selectMembershipForUpdateSQL}, {&s.selectMembershipFromRoomAndTargetStmt, selectMembershipFromRoomAndTargetSQL}, @@ -130,10 +125,10 @@ func (s *membershipStatements) prepare(db *sql.DB) (err error) { {&s.selectMembershipsFromRoomStmt, selectMembershipsFromRoomSQL}, {&s.selectLocalMembershipsFromRoomStmt, selectLocalMembershipsFromRoomSQL}, {&s.updateMembershipStmt, updateMembershipSQL}, - }.prepare(db) + }.Prepare(db) } -func (s *membershipStatements) insertMembership( +func (s *membershipStatements) InsertMembership( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, localTarget bool, @@ -143,27 +138,27 @@ func (s *membershipStatements) insertMembership( return err } -func (s *membershipStatements) selectMembershipForUpdate( +func (s *membershipStatements) SelectMembershipForUpdate( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, -) (membership membershipState, err error) { +) (membership tables.MembershipState, err error) { err = internal.TxStmt(txn, s.selectMembershipForUpdateStmt).QueryRowContext( ctx, roomNID, targetUserNID, ).Scan(&membership) return } -func (s *membershipStatements) selectMembershipFromRoomAndTarget( +func (s *membershipStatements) SelectMembershipFromRoomAndTarget( ctx context.Context, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, -) (eventNID types.EventNID, membership membershipState, err error) { +) (eventNID types.EventNID, membership tables.MembershipState, err error) { err = s.selectMembershipFromRoomAndTargetStmt.QueryRowContext( ctx, roomNID, targetUserNID, ).Scan(&membership, &eventNID) return } -func (s *membershipStatements) selectMembershipsFromRoom( +func (s *membershipStatements) SelectMembershipsFromRoom( ctx context.Context, roomNID types.RoomNID, localOnly bool, ) (eventNIDs []types.EventNID, err error) { var stmt *sql.Stmt @@ -188,9 +183,9 @@ func (s *membershipStatements) selectMembershipsFromRoom( return eventNIDs, rows.Err() } -func (s *membershipStatements) selectMembershipsFromRoomAndMembership( +func (s *membershipStatements) SelectMembershipsFromRoomAndMembership( ctx context.Context, - roomNID types.RoomNID, membership membershipState, localOnly bool, + roomNID types.RoomNID, membership tables.MembershipState, localOnly bool, ) (eventNIDs []types.EventNID, err error) { var rows *sql.Rows var stmt *sql.Stmt @@ -215,10 +210,10 @@ func (s *membershipStatements) selectMembershipsFromRoomAndMembership( return eventNIDs, rows.Err() } -func (s *membershipStatements) updateMembership( +func (s *membershipStatements) UpdateMembership( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, - senderUserNID types.EventStateKeyNID, membership membershipState, + senderUserNID types.EventStateKeyNID, membership tables.MembershipState, eventNID types.EventNID, ) error { _, err := internal.TxStmt(txn, s.updateMembershipStmt).ExecContext( diff --git a/roomserver/storage/postgres/prepare.go b/roomserver/storage/postgres/prepare.go deleted file mode 100644 index 70b6e5161..000000000 --- a/roomserver/storage/postgres/prepare.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2017-2018 New Vector Ltd -// Copyright 2019-2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package postgres - -import ( - "database/sql" -) - -// a statementList is a list of SQL statements to prepare and a pointer to where to store the resulting prepared statement. -type statementList []struct { - statement **sql.Stmt - sql string -} - -// prepare the SQL for each statement in the list and assign the result to the prepared statement. -func (s statementList) prepare(db *sql.DB) (err error) { - for _, statement := range s { - if *statement.statement, err = db.Prepare(statement.sql); err != nil { - return - } - } - return -} diff --git a/roomserver/storage/postgres/previous_events_table.go b/roomserver/storage/postgres/previous_events_table.go index b3d32c953..ff0306967 100644 --- a/roomserver/storage/postgres/previous_events_table.go +++ b/roomserver/storage/postgres/previous_events_table.go @@ -20,6 +20,7 @@ import ( "database/sql" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -71,10 +72,10 @@ func NewPostgresPreviousEventsTable(db *sql.DB) (tables.PreviousEvents, error) { return nil, err } - return s, statementList{ + return s, shared.StatementList{ {&s.insertPreviousEventStmt, insertPreviousEventSQL}, {&s.selectPreviousEventExistsStmt, selectPreviousEventExistsSQL}, - }.prepare(db) + }.Prepare(db) } func (s *previousEventStatements) InsertPreviousEvent( diff --git a/roomserver/storage/postgres/room_aliases_table.go b/roomserver/storage/postgres/room_aliases_table.go index f869cf4fb..85042c54f 100644 --- a/roomserver/storage/postgres/room_aliases_table.go +++ b/roomserver/storage/postgres/room_aliases_table.go @@ -20,6 +20,7 @@ import ( "database/sql" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" ) @@ -66,13 +67,13 @@ func NewPostgresRoomAliasesTable(db *sql.DB) (tables.RoomAliases, error) { if err != nil { return nil, err } - return s, statementList{ + return s, shared.StatementList{ {&s.insertRoomAliasStmt, insertRoomAliasSQL}, {&s.selectRoomIDFromAliasStmt, selectRoomIDFromAliasSQL}, {&s.selectAliasesFromRoomIDStmt, selectAliasesFromRoomIDSQL}, {&s.selectCreatorIDFromAliasStmt, selectCreatorIDFromAliasSQL}, {&s.deleteRoomAliasStmt, deleteRoomAliasSQL}, - }.prepare(db) + }.Prepare(db) } func (s *roomAliasesStatements) InsertRoomAlias( diff --git a/roomserver/storage/postgres/rooms_table.go b/roomserver/storage/postgres/rooms_table.go index 98881390b..5a353ed1c 100644 --- a/roomserver/storage/postgres/rooms_table.go +++ b/roomserver/storage/postgres/rooms_table.go @@ -22,6 +22,7 @@ import ( "github.com/lib/pq" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" @@ -89,7 +90,7 @@ func NewPostgresRoomsTable(db *sql.DB) (tables.Rooms, error) { if err != nil { return nil, err } - return s, statementList{ + return s, shared.StatementList{ {&s.insertRoomNIDStmt, insertRoomNIDSQL}, {&s.selectRoomNIDStmt, selectRoomNIDSQL}, {&s.selectLatestEventNIDsStmt, selectLatestEventNIDsSQL}, @@ -97,7 +98,7 @@ func NewPostgresRoomsTable(db *sql.DB) (tables.Rooms, error) { {&s.updateLatestEventNIDsStmt, updateLatestEventNIDsSQL}, {&s.selectRoomVersionForRoomIDStmt, selectRoomVersionForRoomIDSQL}, {&s.selectRoomVersionForRoomNIDStmt, selectRoomVersionForRoomNIDSQL}, - }.prepare(db) + }.Prepare(db) } func (s *roomStatements) InsertRoomNID( diff --git a/roomserver/storage/postgres/sql.go b/roomserver/storage/postgres/sql.go deleted file mode 100644 index eb626dd88..000000000 --- a/roomserver/storage/postgres/sql.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2017-2018 New Vector Ltd -// Copyright 2019-2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package postgres - -import ( - "database/sql" -) - -type statements struct { - eventTypeStatements - eventStateKeyStatements - roomStatements - eventStatements - eventJSONStatements - stateSnapshotStatements - stateBlockStatements - previousEventStatements - roomAliasesStatements - inviteStatements - membershipStatements - transactionStatements -} - -func (s *statements) prepare(db *sql.DB) error { - var err error - - for _, prepare := range []func(db *sql.DB) error{ - s.inviteStatements.prepare, - s.membershipStatements.prepare, - } { - if err = prepare(db); err != nil { - return err - } - } - - return nil -} diff --git a/roomserver/storage/postgres/state_block_table.go b/roomserver/storage/postgres/state_block_table.go index d1aaaa003..dbe21d989 100644 --- a/roomserver/storage/postgres/state_block_table.go +++ b/roomserver/storage/postgres/state_block_table.go @@ -24,6 +24,7 @@ import ( "github.com/matrix-org/dendrite/internal" "github.com/lib/pq" + "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/util" @@ -95,12 +96,12 @@ func NewPostgresStateBlockTable(db *sql.DB) (tables.StateBlock, error) { return nil, err } - return s, statementList{ + return s, shared.StatementList{ {&s.insertStateDataStmt, insertStateDataSQL}, {&s.selectNextStateBlockNIDStmt, selectNextStateBlockNIDSQL}, {&s.bulkSelectStateBlockEntriesStmt, bulkSelectStateBlockEntriesSQL}, {&s.bulkSelectFilteredStateBlockEntriesStmt, bulkSelectFilteredStateBlockEntriesSQL}, - }.prepare(db) + }.Prepare(db) } func (s *stateBlockStatements) BulkInsertStateData( diff --git a/roomserver/storage/postgres/state_snapshot_table.go b/roomserver/storage/postgres/state_snapshot_table.go index 8971292fc..0f8f1c51e 100644 --- a/roomserver/storage/postgres/state_snapshot_table.go +++ b/roomserver/storage/postgres/state_snapshot_table.go @@ -21,6 +21,7 @@ import ( "fmt" "github.com/lib/pq" + "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -72,10 +73,10 @@ func NewPostgresStateSnapshotTable(db *sql.DB) (tables.StateSnapshot, error) { return nil, err } - return s, statementList{ + return s, shared.StatementList{ {&s.insertStateStmt, insertStateSQL}, {&s.bulkSelectStateBlockNIDsStmt, bulkSelectStateBlockNIDsSQL}, - }.prepare(db) + }.Prepare(db) } func (s *stateSnapshotStatements) InsertState( diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go index 03cfb7f0e..53a58076a 100644 --- a/roomserver/storage/postgres/storage.go +++ b/roomserver/storage/postgres/storage.go @@ -33,7 +33,6 @@ import ( // A Database is used to store room events and stream offsets. type Database struct { shared.Database - statements statements events tables.Events eventTypes tables.EventTypes eventStateKeys tables.EventStateKeys @@ -41,6 +40,8 @@ type Database struct { rooms tables.Rooms transactions tables.Transactions prevEvents tables.PreviousEvents + invites tables.Invites + membership tables.Membership db *sql.DB } @@ -52,9 +53,6 @@ func Open(dataSourceName string, dbProperties internal.DbProperties) (*Database, if d.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { return nil, err } - if err = d.statements.prepare(d.db); err != nil { - return nil, err - } d.eventStateKeys, err = NewPostgresEventStateKeysTable(d.db) if err != nil { return nil, err @@ -95,6 +93,14 @@ func Open(dataSourceName string, dbProperties internal.DbProperties) (*Database, if err != nil { return nil, err } + d.invites, err = NewPostgresInvitesTable(d.db) + if err != nil { + return nil, err + } + d.membership, err = NewPostgresMembershipTable(d.db) + if err != nil { + return nil, err + } d.Database = shared.Database{ DB: d.db, EventTypesTable: d.eventTypes, @@ -107,6 +113,8 @@ func Open(dataSourceName string, dbProperties internal.DbProperties) (*Database, StateSnapshotTable: stateSnapshot, PrevEventsTable: d.prevEvents, RoomAliasesTable: roomAliases, + InvitesTable: d.invites, + MembershipTable: d.membership, } return &d, nil } @@ -254,15 +262,6 @@ func (u *roomRecentEventsUpdater) MembershipUpdater(targetUserNID types.EventSta return u.d.membershipUpdaterTxn(u.ctx, u.txn, u.roomNID, targetUserNID, targetLocal) } -// GetInvitesForUser implements query.RoomserverQueryAPIDatabase -func (d *Database) GetInvitesForUser( - ctx context.Context, - roomNID types.RoomNID, - targetUserNID types.EventStateKeyNID, -) (senderUserIDs []types.EventStateKeyNID, err error) { - return d.statements.selectInviteActiveForUserInRoom(ctx, targetUserNID, roomNID) -} - // MembershipUpdater implements input.RoomEventDatabase func (d *Database) MembershipUpdater( ctx context.Context, roomID, targetUserID string, @@ -303,7 +302,7 @@ type membershipUpdater struct { d *Database roomNID types.RoomNID targetUserNID types.EventStateKeyNID - membership membershipState + membership tables.MembershipState } func (d *Database) membershipUpdaterTxn( @@ -314,11 +313,11 @@ func (d *Database) membershipUpdaterTxn( targetLocal bool, ) (types.MembershipUpdater, error) { - if err := d.statements.insertMembership(ctx, txn, roomNID, targetUserNID, targetLocal); err != nil { + if err := d.membership.InsertMembership(ctx, txn, roomNID, targetUserNID, targetLocal); err != nil { return nil, err } - membership, err := d.statements.selectMembershipForUpdate(ctx, txn, roomNID, targetUserNID) + membership, err := d.membership.SelectMembershipForUpdate(ctx, txn, roomNID, targetUserNID) if err != nil { return nil, err } @@ -330,17 +329,17 @@ func (d *Database) membershipUpdaterTxn( // IsInvite implements types.MembershipUpdater func (u *membershipUpdater) IsInvite() bool { - return u.membership == membershipStateInvite + return u.membership == tables.MembershipStateInvite } // IsJoin implements types.MembershipUpdater func (u *membershipUpdater) IsJoin() bool { - return u.membership == membershipStateJoin + return u.membership == tables.MembershipStateJoin } // IsLeave implements types.MembershipUpdater func (u *membershipUpdater) IsLeave() bool { - return u.membership == membershipStateLeaveOrBan + return u.membership == tables.MembershipStateLeaveOrBan } // SetToInvite implements types.MembershipUpdater @@ -349,15 +348,15 @@ func (u *membershipUpdater) SetToInvite(event gomatrixserverlib.Event) (bool, er if err != nil { return false, err } - inserted, err := u.d.statements.insertInviteEvent( + inserted, err := u.d.invites.InsertInviteEvent( u.ctx, u.txn, event.EventID(), u.roomNID, u.targetUserNID, senderUserNID, event.JSON(), ) if err != nil { return false, err } - if u.membership != membershipStateInvite { - if err = u.d.statements.updateMembership( - u.ctx, u.txn, u.roomNID, u.targetUserNID, senderUserNID, membershipStateInvite, 0, + if u.membership != tables.MembershipStateInvite { + if err = u.d.membership.UpdateMembership( + u.ctx, u.txn, u.roomNID, u.targetUserNID, senderUserNID, tables.MembershipStateInvite, 0, ); err != nil { return false, err } @@ -376,7 +375,7 @@ func (u *membershipUpdater) SetToJoin(senderUserID string, eventID string, isUpd // If this is a join event update, there is no invite to update if !isUpdate { - inviteEventIDs, err = u.d.statements.updateInviteRetired( + inviteEventIDs, err = u.d.invites.UpdateInviteRetired( u.ctx, u.txn, u.roomNID, u.targetUserNID, ) if err != nil { @@ -390,10 +389,10 @@ func (u *membershipUpdater) SetToJoin(senderUserID string, eventID string, isUpd return nil, err } - if u.membership != membershipStateJoin || isUpdate { - if err = u.d.statements.updateMembership( + if u.membership != tables.MembershipStateJoin || isUpdate { + if err = u.d.membership.UpdateMembership( u.ctx, u.txn, u.roomNID, u.targetUserNID, senderUserNID, - membershipStateJoin, nIDs[eventID], + tables.MembershipStateJoin, nIDs[eventID], ); err != nil { return nil, err } @@ -408,7 +407,7 @@ func (u *membershipUpdater) SetToLeave(senderUserID string, eventID string) ([]s if err != nil { return nil, err } - inviteEventIDs, err := u.d.statements.updateInviteRetired( + inviteEventIDs, err := u.d.invites.UpdateInviteRetired( u.ctx, u.txn, u.roomNID, u.targetUserNID, ) if err != nil { @@ -421,10 +420,10 @@ func (u *membershipUpdater) SetToLeave(senderUserID string, eventID string) ([]s return nil, err } - if u.membership != membershipStateLeaveOrBan { - if err = u.d.statements.updateMembership( + if u.membership != tables.MembershipStateLeaveOrBan { + if err = u.d.membership.UpdateMembership( u.ctx, u.txn, u.roomNID, u.targetUserNID, senderUserNID, - membershipStateLeaveOrBan, nIDs[eventID], + tables.MembershipStateLeaveOrBan, nIDs[eventID], ); err != nil { return nil, err } @@ -432,42 +431,6 @@ func (u *membershipUpdater) SetToLeave(senderUserID string, eventID string) ([]s return inviteEventIDs, nil } -// GetMembership implements query.RoomserverQueryAPIDB -func (d *Database) GetMembership( - ctx context.Context, roomNID types.RoomNID, requestSenderUserID string, -) (membershipEventNID types.EventNID, stillInRoom bool, err error) { - requestSenderUserNID, err := d.assignStateKeyNID(ctx, nil, requestSenderUserID) - if err != nil { - return - } - - senderMembershipEventNID, senderMembership, err := - d.statements.selectMembershipFromRoomAndTarget( - ctx, roomNID, requestSenderUserNID, - ) - if err == sql.ErrNoRows { - // The user has never been a member of that room - return 0, false, nil - } else if err != nil { - return - } - - return senderMembershipEventNID, senderMembership == membershipStateJoin, nil -} - -// GetMembershipEventNIDsForRoom implements query.RoomserverQueryAPIDB -func (d *Database) GetMembershipEventNIDsForRoom( - ctx context.Context, roomNID types.RoomNID, joinOnly bool, localOnly bool, -) ([]types.EventNID, error) { - if joinOnly { - return d.statements.selectMembershipsFromRoomAndMembership( - ctx, roomNID, membershipStateJoin, localOnly, - ) - } - - return d.statements.selectMembershipsFromRoom(ctx, roomNID, localOnly) -} - type transaction struct { ctx context.Context txn *sql.Tx diff --git a/roomserver/storage/postgres/transactions_table.go b/roomserver/storage/postgres/transactions_table.go index 7f7ef76ac..5e59ae16d 100644 --- a/roomserver/storage/postgres/transactions_table.go +++ b/roomserver/storage/postgres/transactions_table.go @@ -19,6 +19,7 @@ import ( "context" "database/sql" + "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" ) @@ -60,10 +61,10 @@ func NewPostgresTransactionsTable(db *sql.DB) (tables.Transactions, error) { return nil, err } - return s, statementList{ + return s, shared.StatementList{ {&s.insertTransactionStmt, insertTransactionSQL}, {&s.selectTransactionEventIDStmt, selectTransactionEventIDSQL}, - }.prepare(db) + }.Prepare(db) } func (s *transactionStatements) InsertTransaction( diff --git a/roomserver/storage/sqlite3/prepare.go b/roomserver/storage/shared/prepare.go similarity index 66% rename from roomserver/storage/sqlite3/prepare.go rename to roomserver/storage/shared/prepare.go index 482dfa2b9..1b3497fd8 100644 --- a/roomserver/storage/sqlite3/prepare.go +++ b/roomserver/storage/shared/prepare.go @@ -13,22 +13,22 @@ // See the License for the specific language governing permissions and // limitations under the License. -package sqlite3 +package shared import ( "database/sql" ) -// a statementList is a list of SQL statements to prepare and a pointer to where to store the resulting prepared statement. -type statementList []struct { - statement **sql.Stmt - sql string +// StatementList is a list of SQL statements to prepare and a pointer to where to store the resulting prepared statement. +type StatementList []struct { + Statement **sql.Stmt + SQL string } -// prepare the SQL for each statement in the list and assign the result to the prepared statement. -func (s statementList) prepare(db *sql.DB) (err error) { +// Prepare the SQL for each statement in the list and assign the result to the prepared statement. +func (s StatementList) Prepare(db *sql.DB) (err error) { for _, statement := range s { - if *statement.statement, err = db.Prepare(statement.sql); err != nil { + if *statement.Statement, err = db.Prepare(statement.SQL); err != nil { return } } diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index 29c6f73eb..17bcd6964 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -24,37 +24,34 @@ type Database struct { StateBlockTable tables.StateBlock RoomAliasesTable tables.RoomAliases PrevEventsTable tables.PreviousEvents + InvitesTable tables.Invites + MembershipTable tables.Membership } -// EventTypeNIDs implements state.RoomStateDatabase func (d *Database) EventTypeNIDs( ctx context.Context, eventTypes []string, ) (map[string]types.EventTypeNID, error) { return d.EventTypesTable.BulkSelectEventTypeNID(ctx, eventTypes) } -// EventStateKeys implements query.RoomserverQueryAPIDatabase func (d *Database) EventStateKeys( ctx context.Context, eventStateKeyNIDs []types.EventStateKeyNID, ) (map[types.EventStateKeyNID]string, error) { return d.EventStateKeysTable.BulkSelectEventStateKey(ctx, eventStateKeyNIDs) } -// EventStateKeyNIDs implements state.RoomStateDatabase func (d *Database) EventStateKeyNIDs( ctx context.Context, eventStateKeys []string, ) (map[string]types.EventStateKeyNID, error) { return d.EventStateKeysTable.BulkSelectEventStateKeyNID(ctx, eventStateKeys) } -// StateEntriesForEventIDs implements input.EventDatabase func (d *Database) StateEntriesForEventIDs( ctx context.Context, eventIDs []string, ) ([]types.StateEntry, error) { return d.EventsTable.BulkSelectStateEventByID(ctx, eventIDs) } -// StateEntriesForTuples implements state.RoomStateDatabase func (d *Database) StateEntriesForTuples( ctx context.Context, stateBlockNIDs []types.StateBlockNID, @@ -65,7 +62,6 @@ func (d *Database) StateEntriesForTuples( ) } -// AddState implements input.EventDatabase func (d *Database) AddState( ctx context.Context, roomNID types.RoomNID, @@ -90,28 +86,24 @@ func (d *Database) AddState( return } -// EventNIDs implements query.RoomserverQueryAPIDatabase func (d *Database) EventNIDs( ctx context.Context, eventIDs []string, ) (map[string]types.EventNID, error) { return d.EventsTable.BulkSelectEventNID(ctx, eventIDs) } -// SetState implements input.EventDatabase func (d *Database) SetState( ctx context.Context, eventNID types.EventNID, stateNID types.StateSnapshotNID, ) error { return d.EventsTable.UpdateEventState(ctx, eventNID, stateNID) } -// StateAtEventIDs implements input.EventDatabase func (d *Database) StateAtEventIDs( ctx context.Context, eventIDs []string, ) ([]types.StateAtEvent, error) { return d.EventsTable.BulkSelectStateAtEventByID(ctx, eventIDs) } -// SnapshotNIDFromEventID implements state.RoomStateDatabase func (d *Database) SnapshotNIDFromEventID( ctx context.Context, eventID string, ) (types.StateSnapshotNID, error) { @@ -119,14 +111,12 @@ func (d *Database) SnapshotNIDFromEventID( return stateNID, err } -// EventIDs implements input.RoomEventDatabase func (d *Database) EventIDs( ctx context.Context, eventNIDs []types.EventNID, ) (map[types.EventNID]string, error) { return d.EventsTable.BulkSelectEventID(ctx, eventNIDs) } -// EventsFromIDs implements query.RoomserverQueryAPIEventDB func (d *Database) EventsFromIDs(ctx context.Context, eventIDs []string) ([]types.Event, error) { nidMap, err := d.EventNIDs(ctx, eventIDs) if err != nil { @@ -141,7 +131,6 @@ func (d *Database) EventsFromIDs(ctx context.Context, eventIDs []string) ([]type return d.Events(ctx, nids) } -// RoomNID implements query.RoomserverQueryAPIDB func (d *Database) RoomNID(ctx context.Context, roomID string) (types.RoomNID, error) { roomNID, err := d.RoomsTable.SelectRoomNID(ctx, nil, roomID) if err == sql.ErrNoRows { @@ -150,7 +139,6 @@ func (d *Database) RoomNID(ctx context.Context, roomID string) (types.RoomNID, e return roomNID, err } -// RoomNIDExcludingStubs implements query.RoomserverQueryAPIDB func (d *Database) RoomNIDExcludingStubs(ctx context.Context, roomID string) (roomNID types.RoomNID, err error) { roomNID, err = d.RoomNID(ctx, roomID) if err != nil { @@ -167,7 +155,6 @@ func (d *Database) RoomNIDExcludingStubs(ctx context.Context, roomID string) (ro return } -// LatestEventIDs implements query.RoomserverQueryAPIDatabase func (d *Database) LatestEventIDs( ctx context.Context, roomNID types.RoomNID, ) (references []gomatrixserverlib.EventReference, currentStateSnapshotNID types.StateSnapshotNID, depth int64, err error) { @@ -190,14 +177,12 @@ func (d *Database) LatestEventIDs( return } -// StateBlockNIDs implements state.RoomStateDatabase func (d *Database) StateBlockNIDs( ctx context.Context, stateNIDs []types.StateSnapshotNID, ) ([]types.StateBlockNIDList, error) { return d.StateSnapshotTable.BulkSelectStateBlockNIDs(ctx, stateNIDs) } -// StateEntries implements state.RoomStateDatabase func (d *Database) StateEntries( ctx context.Context, stateBlockNIDs []types.StateBlockNID, ) ([]types.StateEntryList, error) { @@ -220,34 +205,70 @@ func (d *Database) GetRoomVersionForRoomNID( ) } -// SetRoomAlias implements alias.RoomserverAliasAPIDB func (d *Database) SetRoomAlias(ctx context.Context, alias string, roomID string, creatorUserID string) error { return d.RoomAliasesTable.InsertRoomAlias(ctx, alias, roomID, creatorUserID) } -// GetRoomIDForAlias implements alias.RoomserverAliasAPIDB func (d *Database) GetRoomIDForAlias(ctx context.Context, alias string) (string, error) { return d.RoomAliasesTable.SelectRoomIDFromAlias(ctx, alias) } -// GetAliasesForRoomID implements alias.RoomserverAliasAPIDB func (d *Database) GetAliasesForRoomID(ctx context.Context, roomID string) ([]string, error) { return d.RoomAliasesTable.SelectAliasesFromRoomID(ctx, roomID) } -// GetCreatorIDForAlias implements alias.RoomserverAliasAPIDB func (d *Database) GetCreatorIDForAlias( ctx context.Context, alias string, ) (string, error) { return d.RoomAliasesTable.SelectCreatorIDFromAlias(ctx, alias) } -// RemoveRoomAlias implements alias.RoomserverAliasAPIDB func (d *Database) RemoveRoomAlias(ctx context.Context, alias string) error { return d.RoomAliasesTable.DeleteRoomAlias(ctx, alias) } -// Events implements input.EventDatabase +func (d *Database) GetMembership( + ctx context.Context, roomNID types.RoomNID, requestSenderUserID string, +) (membershipEventNID types.EventNID, stillInRoom bool, err error) { + requestSenderUserNID, err := d.assignStateKeyNID(ctx, nil, requestSenderUserID) + if err != nil { + return + } + + senderMembershipEventNID, senderMembership, err := + d.MembershipTable.SelectMembershipFromRoomAndTarget( + ctx, roomNID, requestSenderUserNID, + ) + if err == sql.ErrNoRows { + // The user has never been a member of that room + return 0, false, nil + } else if err != nil { + return + } + + return senderMembershipEventNID, senderMembership == tables.MembershipStateJoin, nil +} + +func (d *Database) GetMembershipEventNIDsForRoom( + ctx context.Context, roomNID types.RoomNID, joinOnly bool, localOnly bool, +) ([]types.EventNID, error) { + if joinOnly { + return d.MembershipTable.SelectMembershipsFromRoomAndMembership( + ctx, roomNID, tables.MembershipStateJoin, localOnly, + ) + } + + return d.MembershipTable.SelectMembershipsFromRoom(ctx, roomNID, localOnly) +} + +func (d *Database) GetInvitesForUser( + ctx context.Context, + roomNID types.RoomNID, + targetUserNID types.EventStateKeyNID, +) (senderUserIDs []types.EventStateKeyNID, err error) { + return d.InvitesTable.SelectInviteActiveForUserInRoom(ctx, targetUserNID, roomNID) +} + func (d *Database) Events( ctx context.Context, eventNIDs []types.EventNID, ) ([]types.Event, error) { @@ -279,7 +300,6 @@ func (d *Database) Events( return results, nil } -// GetTransactionEventID implements input.EventDatabase func (d *Database) GetTransactionEventID( ctx context.Context, transactionID string, sessionID int64, userID string, @@ -291,7 +311,6 @@ func (d *Database) GetTransactionEventID( return eventID, err } -// StoreEvent implements input.EventDatabase func (d *Database) StoreEvent( ctx context.Context, event gomatrixserverlib.Event, txnAndSessionID *api.TransactionID, authEventNIDs []types.EventNID, diff --git a/roomserver/storage/sqlite3/event_json_table.go b/roomserver/storage/sqlite3/event_json_table.go index 34b067cb0..7ff4c1d4d 100644 --- a/roomserver/storage/sqlite3/event_json_table.go +++ b/roomserver/storage/sqlite3/event_json_table.go @@ -21,6 +21,7 @@ import ( "strings" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -59,10 +60,10 @@ func NewSqliteEventJSONTable(db *sql.DB) (tables.EventJSON, error) { if err != nil { return nil, err } - return s, statementList{ + return s, shared.StatementList{ {&s.insertEventJSONStmt, insertEventJSONSQL}, {&s.bulkSelectEventJSONStmt, bulkSelectEventJSONSQL}, - }.prepare(db) + }.Prepare(db) } func (s *eventJSONStatements) InsertEventJSON( diff --git a/roomserver/storage/sqlite3/event_state_keys_table.go b/roomserver/storage/sqlite3/event_state_keys_table.go index 0d3d323fb..2d2c04d26 100644 --- a/roomserver/storage/sqlite3/event_state_keys_table.go +++ b/roomserver/storage/sqlite3/event_state_keys_table.go @@ -21,6 +21,7 @@ import ( "strings" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -75,12 +76,12 @@ func NewSqliteEventStateKeysTable(db *sql.DB) (tables.EventStateKeys, error) { if err != nil { return nil, err } - return s, statementList{ + return s, shared.StatementList{ {&s.insertEventStateKeyNIDStmt, insertEventStateKeyNIDSQL}, {&s.selectEventStateKeyNIDStmt, selectEventStateKeyNIDSQL}, {&s.bulkSelectEventStateKeyNIDStmt, bulkSelectEventStateKeyNIDSQL}, {&s.bulkSelectEventStateKeyStmt, bulkSelectEventStateKeySQL}, - }.prepare(db) + }.Prepare(db) } func (s *eventStateKeyStatements) InsertEventStateKeyNID( @@ -89,7 +90,7 @@ func (s *eventStateKeyStatements) InsertEventStateKeyNID( var eventStateKeyNID int64 var err error var res sql.Result - insertStmt := txn.Stmt(s.insertEventStateKeyNIDStmt) + insertStmt := internal.TxStmt(txn, s.insertEventStateKeyNIDStmt) if res, err = insertStmt.ExecContext(ctx, eventStateKey); err == nil { eventStateKeyNID, err = res.LastInsertId() } @@ -100,7 +101,7 @@ func (s *eventStateKeyStatements) SelectEventStateKeyNID( ctx context.Context, txn *sql.Tx, eventStateKey string, ) (types.EventStateKeyNID, error) { var eventStateKeyNID int64 - stmt := txn.Stmt(s.selectEventStateKeyNIDStmt) + stmt := internal.TxStmt(txn, s.selectEventStateKeyNIDStmt) err := stmt.QueryRowContext(ctx, eventStateKey).Scan(&eventStateKeyNID) return types.EventStateKeyNID(eventStateKeyNID), err } diff --git a/roomserver/storage/sqlite3/event_types_table.go b/roomserver/storage/sqlite3/event_types_table.go index d47be5453..6d229bbd7 100644 --- a/roomserver/storage/sqlite3/event_types_table.go +++ b/roomserver/storage/sqlite3/event_types_table.go @@ -21,6 +21,7 @@ import ( "strings" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -90,12 +91,12 @@ func NewSqliteEventTypesTable(db *sql.DB) (tables.EventTypes, error) { return nil, err } - return s, statementList{ + return s, shared.StatementList{ {&s.insertEventTypeNIDStmt, insertEventTypeNIDSQL}, {&s.insertEventTypeNIDResultStmt, insertEventTypeNIDResultSQL}, {&s.selectEventTypeNIDStmt, selectEventTypeNIDSQL}, {&s.bulkSelectEventTypeNIDStmt, bulkSelectEventTypeNIDSQL}, - }.prepare(db) + }.Prepare(db) } func (s *eventTypeStatements) InsertEventTypeNID( diff --git a/roomserver/storage/sqlite3/events_table.go b/roomserver/storage/sqlite3/events_table.go index 247faa68f..491399ce7 100644 --- a/roomserver/storage/sqlite3/events_table.go +++ b/roomserver/storage/sqlite3/events_table.go @@ -23,6 +23,7 @@ import ( "strings" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" @@ -120,7 +121,7 @@ func NewSqliteEventsTable(db *sql.DB) (tables.Events, error) { return nil, err } - return s, statementList{ + return s, shared.StatementList{ {&s.insertEventStmt, insertEventSQL}, {&s.selectEventStmt, selectEventSQL}, {&s.bulkSelectStateEventByIDStmt, bulkSelectStateEventByIDSQL}, @@ -134,7 +135,7 @@ func NewSqliteEventsTable(db *sql.DB) (tables.Events, error) { {&s.bulkSelectEventIDStmt, bulkSelectEventIDSQL}, {&s.bulkSelectEventNIDStmt, bulkSelectEventNIDSQL}, {&s.selectRoomNIDForEventNIDStmt, selectRoomNIDForEventNIDSQL}, - }.prepare(db) + }.Prepare(db) } func (s *eventStatements) InsertEvent( diff --git a/roomserver/storage/sqlite3/invite_table.go b/roomserver/storage/sqlite3/invite_table.go index a42d18a73..fbe2b0729 100644 --- a/roomserver/storage/sqlite3/invite_table.go +++ b/roomserver/storage/sqlite3/invite_table.go @@ -20,6 +20,8 @@ import ( "database/sql" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/roomserver/storage/shared" + "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -66,21 +68,22 @@ type inviteStatements struct { selectInvitesAboutToRetireStmt *sql.Stmt } -func (s *inviteStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(inviteSchema) +func NewSqliteInvitesTable(db *sql.DB) (tables.Invites, error) { + s := &inviteStatements{} + _, err := db.Exec(inviteSchema) if err != nil { - return + return nil, err } - return statementList{ + return s, shared.StatementList{ {&s.insertInviteEventStmt, insertInviteEventSQL}, {&s.selectInviteActiveForUserInRoomStmt, selectInviteActiveForUserInRoomSQL}, {&s.updateInviteRetiredStmt, updateInviteRetiredSQL}, {&s.selectInvitesAboutToRetireStmt, selectInvitesAboutToRetireSQL}, - }.prepare(db) + }.Prepare(db) } -func (s *inviteStatements) insertInviteEvent( +func (s *inviteStatements) InsertInviteEvent( ctx context.Context, txn *sql.Tx, inviteEventID string, roomNID types.RoomNID, targetUserNID, senderUserNID types.EventStateKeyNID, @@ -101,7 +104,7 @@ func (s *inviteStatements) insertInviteEvent( return count != 0, nil } -func (s *inviteStatements) updateInviteRetired( +func (s *inviteStatements) UpdateInviteRetired( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, ) (eventIDs []string, err error) { @@ -127,7 +130,7 @@ func (s *inviteStatements) updateInviteRetired( } // selectInviteActiveForUserInRoom returns a list of sender state key NIDs -func (s *inviteStatements) selectInviteActiveForUserInRoom( +func (s *inviteStatements) SelectInviteActiveForUserInRoom( ctx context.Context, targetUserNID types.EventStateKeyNID, roomNID types.RoomNID, ) ([]types.EventStateKeyNID, error) { diff --git a/roomserver/storage/sqlite3/membership_table.go b/roomserver/storage/sqlite3/membership_table.go index 34108af43..5f3076c5a 100644 --- a/roomserver/storage/sqlite3/membership_table.go +++ b/roomserver/storage/sqlite3/membership_table.go @@ -20,17 +20,11 @@ import ( "database/sql" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/roomserver/storage/shared" + "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" ) -type membershipState int64 - -const ( - membershipStateLeaveOrBan membershipState = 1 - membershipStateInvite membershipState = 2 - membershipStateJoin membershipState = 3 -) - const membershipSchema = ` CREATE TABLE IF NOT EXISTS roomserver_membership ( room_nid INTEGER NOT NULL, @@ -91,13 +85,14 @@ type membershipStatements struct { updateMembershipStmt *sql.Stmt } -func (s *membershipStatements) prepare(db *sql.DB) (err error) { - _, err = db.Exec(membershipSchema) +func NewSqliteMembershipTable(db *sql.DB) (tables.Membership, error) { + s := &membershipStatements{} + _, err := db.Exec(membershipSchema) if err != nil { - return + return nil, err } - return statementList{ + return s, shared.StatementList{ {&s.insertMembershipStmt, insertMembershipSQL}, {&s.selectMembershipForUpdateStmt, selectMembershipForUpdateSQL}, {&s.selectMembershipFromRoomAndTargetStmt, selectMembershipFromRoomAndTargetSQL}, @@ -106,10 +101,10 @@ func (s *membershipStatements) prepare(db *sql.DB) (err error) { {&s.selectMembershipsFromRoomStmt, selectMembershipsFromRoomSQL}, {&s.selectLocalMembershipsFromRoomStmt, selectLocalMembershipsFromRoomSQL}, {&s.updateMembershipStmt, updateMembershipSQL}, - }.prepare(db) + }.Prepare(db) } -func (s *membershipStatements) insertMembership( +func (s *membershipStatements) InsertMembership( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, localTarget bool, @@ -119,10 +114,10 @@ func (s *membershipStatements) insertMembership( return err } -func (s *membershipStatements) selectMembershipForUpdate( +func (s *membershipStatements) SelectMembershipForUpdate( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, -) (membership membershipState, err error) { +) (membership tables.MembershipState, err error) { stmt := internal.TxStmt(txn, s.selectMembershipForUpdateStmt) err = stmt.QueryRowContext( ctx, roomNID, targetUserNID, @@ -130,26 +125,25 @@ func (s *membershipStatements) selectMembershipForUpdate( return } -func (s *membershipStatements) selectMembershipFromRoomAndTarget( - ctx context.Context, txn *sql.Tx, +func (s *membershipStatements) SelectMembershipFromRoomAndTarget( + ctx context.Context, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, -) (eventNID types.EventNID, membership membershipState, err error) { - selectStmt := internal.TxStmt(txn, s.selectMembershipFromRoomAndTargetStmt) - err = selectStmt.QueryRowContext( +) (eventNID types.EventNID, membership tables.MembershipState, err error) { + err = s.selectMembershipFromRoomAndTargetStmt.QueryRowContext( ctx, roomNID, targetUserNID, ).Scan(&membership, &eventNID) return } -func (s *membershipStatements) selectMembershipsFromRoom( - ctx context.Context, txn *sql.Tx, +func (s *membershipStatements) SelectMembershipsFromRoom( + ctx context.Context, roomNID types.RoomNID, localOnly bool, ) (eventNIDs []types.EventNID, err error) { var selectStmt *sql.Stmt if localOnly { - selectStmt = internal.TxStmt(txn, s.selectLocalMembershipsFromRoomStmt) + selectStmt = s.selectLocalMembershipsFromRoomStmt } else { - selectStmt = internal.TxStmt(txn, s.selectMembershipsFromRoomStmt) + selectStmt = s.selectMembershipsFromRoomStmt } rows, err := selectStmt.QueryContext(ctx, roomNID) if err != nil { @@ -167,15 +161,15 @@ func (s *membershipStatements) selectMembershipsFromRoom( return } -func (s *membershipStatements) selectMembershipsFromRoomAndMembership( - ctx context.Context, txn *sql.Tx, - roomNID types.RoomNID, membership membershipState, localOnly bool, +func (s *membershipStatements) SelectMembershipsFromRoomAndMembership( + ctx context.Context, + roomNID types.RoomNID, membership tables.MembershipState, localOnly bool, ) (eventNIDs []types.EventNID, err error) { var stmt *sql.Stmt if localOnly { - stmt = internal.TxStmt(txn, s.selectLocalMembershipsFromRoomAndMembershipStmt) + stmt = s.selectLocalMembershipsFromRoomAndMembershipStmt } else { - stmt = internal.TxStmt(txn, s.selectMembershipsFromRoomAndMembershipStmt) + stmt = s.selectMembershipsFromRoomAndMembershipStmt } rows, err := stmt.QueryContext(ctx, roomNID, membership) if err != nil { @@ -193,10 +187,10 @@ func (s *membershipStatements) selectMembershipsFromRoomAndMembership( return } -func (s *membershipStatements) updateMembership( +func (s *membershipStatements) UpdateMembership( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, - senderUserNID types.EventStateKeyNID, membership membershipState, + senderUserNID types.EventStateKeyNID, membership tables.MembershipState, eventNID types.EventNID, ) error { stmt := internal.TxStmt(txn, s.updateMembershipStmt) diff --git a/roomserver/storage/sqlite3/previous_events_table.go b/roomserver/storage/sqlite3/previous_events_table.go index 6b758ccc6..2b187bba3 100644 --- a/roomserver/storage/sqlite3/previous_events_table.go +++ b/roomserver/storage/sqlite3/previous_events_table.go @@ -20,6 +20,7 @@ import ( "database/sql" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -63,10 +64,10 @@ func NewSqlitePrevEventsTable(db *sql.DB) (tables.PreviousEvents, error) { return nil, err } - return s, statementList{ + return s, shared.StatementList{ {&s.insertPreviousEventStmt, insertPreviousEventSQL}, {&s.selectPreviousEventExistsStmt, selectPreviousEventExistsSQL}, - }.prepare(db) + }.Prepare(db) } func (s *previousEventStatements) InsertPreviousEvent( diff --git a/roomserver/storage/sqlite3/room_aliases_table.go b/roomserver/storage/sqlite3/room_aliases_table.go index 686391f68..da5f9161a 100644 --- a/roomserver/storage/sqlite3/room_aliases_table.go +++ b/roomserver/storage/sqlite3/room_aliases_table.go @@ -20,6 +20,7 @@ import ( "database/sql" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" ) @@ -67,13 +68,13 @@ func NewSqliteRoomAliasesTable(db *sql.DB) (tables.RoomAliases, error) { if err != nil { return nil, err } - return s, statementList{ + return s, shared.StatementList{ {&s.insertRoomAliasStmt, insertRoomAliasSQL}, {&s.selectRoomIDFromAliasStmt, selectRoomIDFromAliasSQL}, {&s.selectAliasesFromRoomIDStmt, selectAliasesFromRoomIDSQL}, {&s.selectCreatorIDFromAliasStmt, selectCreatorIDFromAliasSQL}, {&s.deleteRoomAliasStmt, deleteRoomAliasSQL}, - }.prepare(db) + }.Prepare(db) } func (s *roomAliasesStatements) InsertRoomAlias( diff --git a/roomserver/storage/sqlite3/rooms_table.go b/roomserver/storage/sqlite3/rooms_table.go index 75b8fec90..d22fceb3b 100644 --- a/roomserver/storage/sqlite3/rooms_table.go +++ b/roomserver/storage/sqlite3/rooms_table.go @@ -22,6 +22,7 @@ import ( "errors" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" @@ -78,7 +79,7 @@ func NewSqliteRoomsTable(db *sql.DB) (tables.Rooms, error) { if err != nil { return nil, err } - return s, statementList{ + return s, shared.StatementList{ {&s.insertRoomNIDStmt, insertRoomNIDSQL}, {&s.selectRoomNIDStmt, selectRoomNIDSQL}, {&s.selectLatestEventNIDsStmt, selectLatestEventNIDsSQL}, @@ -86,7 +87,7 @@ func NewSqliteRoomsTable(db *sql.DB) (tables.Rooms, error) { {&s.updateLatestEventNIDsStmt, updateLatestEventNIDsSQL}, {&s.selectRoomVersionForRoomIDStmt, selectRoomVersionForRoomIDSQL}, {&s.selectRoomVersionForRoomNIDStmt, selectRoomVersionForRoomNIDSQL}, - }.prepare(db) + }.Prepare(db) } func (s *roomStatements) InsertRoomNID( diff --git a/roomserver/storage/sqlite3/sql.go b/roomserver/storage/sqlite3/sql.go deleted file mode 100644 index df994d508..000000000 --- a/roomserver/storage/sqlite3/sql.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2017-2018 New Vector Ltd -// Copyright 2019-2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sqlite3 - -import ( - "database/sql" -) - -type statements struct { - eventTypeStatements - eventStateKeyStatements - roomStatements - eventStatements - eventJSONStatements - stateSnapshotStatements - stateBlockStatements - previousEventStatements - roomAliasesStatements - inviteStatements - membershipStatements - transactionStatements -} - -func (s *statements) prepare(db *sql.DB) error { - var err error - - for _, prepare := range []func(db *sql.DB) error{ - s.inviteStatements.prepare, - s.membershipStatements.prepare, - } { - if err = prepare(db); err != nil { - return err - } - } - - return nil -} diff --git a/roomserver/storage/sqlite3/state_block_table.go b/roomserver/storage/sqlite3/state_block_table.go index e3ca0b3c8..399fdbf63 100644 --- a/roomserver/storage/sqlite3/state_block_table.go +++ b/roomserver/storage/sqlite3/state_block_table.go @@ -23,6 +23,7 @@ import ( "strings" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/util" @@ -86,12 +87,12 @@ func NewSqliteStateBlockTable(db *sql.DB) (tables.StateBlock, error) { return nil, err } - return s, statementList{ + return s, shared.StatementList{ {&s.insertStateDataStmt, insertStateDataSQL}, {&s.selectNextStateBlockNIDStmt, selectNextStateBlockNIDSQL}, {&s.bulkSelectStateBlockEntriesStmt, bulkSelectStateBlockEntriesSQL}, {&s.bulkSelectFilteredStateBlockEntriesStmt, bulkSelectFilteredStateBlockEntriesSQL}, - }.prepare(db) + }.Prepare(db) } func (s *stateBlockStatements) BulkInsertStateData( diff --git a/roomserver/storage/sqlite3/state_snapshot_table.go b/roomserver/storage/sqlite3/state_snapshot_table.go index 42eae1a7f..5de3f639a 100644 --- a/roomserver/storage/sqlite3/state_snapshot_table.go +++ b/roomserver/storage/sqlite3/state_snapshot_table.go @@ -23,6 +23,7 @@ import ( "strings" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" ) @@ -60,10 +61,10 @@ func NewSqliteStateSnapshotTable(db *sql.DB) (tables.StateSnapshot, error) { return nil, err } - return s, statementList{ + return s, shared.StatementList{ {&s.insertStateStmt, insertStateSQL}, {&s.bulkSelectStateBlockNIDsStmt, bulkSelectStateBlockNIDsSQL}, - }.prepare(db) + }.Prepare(db) } func (s *stateSnapshotStatements) InsertState( diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index 16697f1b9..16d893044 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -34,7 +34,6 @@ import ( // A Database is used to store room events and stream offsets. type Database struct { shared.Database - statements statements events tables.Events eventJSON tables.EventJSON eventTypes tables.EventTypes @@ -42,6 +41,8 @@ type Database struct { rooms tables.Rooms transactions tables.Transactions prevEvents tables.PreviousEvents + invites tables.Invites + membership tables.Membership db *sql.DB } @@ -72,9 +73,7 @@ func Open(dataSourceName string) (*Database, error) { // acquire the global mutex and never unlock it because it is waiting for a connection // which it will never obtain. d.db.SetMaxOpenConns(20) - if err = d.statements.prepare(d.db); err != nil { - return nil, err - } + d.eventStateKeys, err = NewSqliteEventStateKeysTable(d.db) if err != nil { return nil, err @@ -115,6 +114,14 @@ func Open(dataSourceName string) (*Database, error) { if err != nil { return nil, err } + d.invites, err = NewSqliteInvitesTable(d.db) + if err != nil { + return nil, err + } + d.membership, err = NewSqliteMembershipTable(d.db) + if err != nil { + return nil, err + } d.Database = shared.Database{ DB: d.db, EventsTable: d.events, @@ -127,6 +134,8 @@ func Open(dataSourceName string) (*Database, error) { StateSnapshotTable: stateSnapshot, PrevEventsTable: d.prevEvents, RoomAliasesTable: roomAliases, + InvitesTable: d.invites, + MembershipTable: d.membership, } return &d, nil } @@ -305,15 +314,6 @@ func (u *roomRecentEventsUpdater) MembershipUpdater(targetUserNID types.EventSta return } -// GetInvitesForUser implements query.RoomserverQueryAPIDatabase -func (d *Database) GetInvitesForUser( - ctx context.Context, - roomNID types.RoomNID, - targetUserNID types.EventStateKeyNID, -) (senderUserIDs []types.EventStateKeyNID, err error) { - return d.statements.selectInviteActiveForUserInRoom(ctx, targetUserNID, roomNID) -} - // MembershipUpdater implements input.RoomEventDatabase func (d *Database) MembershipUpdater( ctx context.Context, roomID, targetUserID string, @@ -367,7 +367,7 @@ type membershipUpdater struct { d *Database roomNID types.RoomNID targetUserNID types.EventStateKeyNID - membership membershipState + membership tables.MembershipState } func (d *Database) membershipUpdaterTxn( @@ -378,11 +378,11 @@ func (d *Database) membershipUpdaterTxn( targetLocal bool, ) (types.MembershipUpdater, error) { - if err := d.statements.insertMembership(ctx, txn, roomNID, targetUserNID, targetLocal); err != nil { + if err := d.membership.InsertMembership(ctx, txn, roomNID, targetUserNID, targetLocal); err != nil { return nil, err } - membership, err := d.statements.selectMembershipForUpdate(ctx, txn, roomNID, targetUserNID) + membership, err := d.membership.SelectMembershipForUpdate(ctx, txn, roomNID, targetUserNID) if err != nil { return nil, err } @@ -395,17 +395,17 @@ func (d *Database) membershipUpdaterTxn( // IsInvite implements types.MembershipUpdater func (u *membershipUpdater) IsInvite() bool { - return u.membership == membershipStateInvite + return u.membership == tables.MembershipStateInvite } // IsJoin implements types.MembershipUpdater func (u *membershipUpdater) IsJoin() bool { - return u.membership == membershipStateJoin + return u.membership == tables.MembershipStateJoin } // IsLeave implements types.MembershipUpdater func (u *membershipUpdater) IsLeave() bool { - return u.membership == membershipStateLeaveOrBan + return u.membership == tables.MembershipStateLeaveOrBan } // SetToInvite implements types.MembershipUpdater @@ -415,15 +415,15 @@ func (u *membershipUpdater) SetToInvite(event gomatrixserverlib.Event) (inserted if err != nil { return err } - inserted, err = u.d.statements.insertInviteEvent( + inserted, err = u.d.invites.InsertInviteEvent( u.ctx, txn, event.EventID(), u.roomNID, u.targetUserNID, senderUserNID, event.JSON(), ) if err != nil { return err } - if u.membership != membershipStateInvite { - if err = u.d.statements.updateMembership( - u.ctx, txn, u.roomNID, u.targetUserNID, senderUserNID, membershipStateInvite, 0, + if u.membership != tables.MembershipStateInvite { + if err = u.d.membership.UpdateMembership( + u.ctx, txn, u.roomNID, u.targetUserNID, senderUserNID, tables.MembershipStateInvite, 0, ); err != nil { return err } @@ -443,7 +443,7 @@ func (u *membershipUpdater) SetToJoin(senderUserID string, eventID string, isUpd // If this is a join event update, there is no invite to update if !isUpdate { - inviteEventIDs, err = u.d.statements.updateInviteRetired( + inviteEventIDs, err = u.d.invites.UpdateInviteRetired( u.ctx, txn, u.roomNID, u.targetUserNID, ) if err != nil { @@ -457,10 +457,10 @@ func (u *membershipUpdater) SetToJoin(senderUserID string, eventID string, isUpd return err } - if u.membership != membershipStateJoin || isUpdate { - if err = u.d.statements.updateMembership( + if u.membership != tables.MembershipStateJoin || isUpdate { + if err = u.d.membership.UpdateMembership( u.ctx, txn, u.roomNID, u.targetUserNID, senderUserNID, - membershipStateJoin, nIDs[eventID], + tables.MembershipStateJoin, nIDs[eventID], ); err != nil { return err } @@ -478,7 +478,7 @@ func (u *membershipUpdater) SetToLeave(senderUserID string, eventID string) (inv if err != nil { return err } - inviteEventIDs, err = u.d.statements.updateInviteRetired( + inviteEventIDs, err = u.d.invites.UpdateInviteRetired( u.ctx, txn, u.roomNID, u.targetUserNID, ) if err != nil { @@ -491,10 +491,10 @@ func (u *membershipUpdater) SetToLeave(senderUserID string, eventID string) (inv return err } - if u.membership != membershipStateLeaveOrBan { - if err = u.d.statements.updateMembership( + if u.membership != tables.MembershipStateLeaveOrBan { + if err = u.d.membership.UpdateMembership( u.ctx, txn, u.roomNID, u.targetUserNID, senderUserNID, - membershipStateLeaveOrBan, nIDs[eventID], + tables.MembershipStateLeaveOrBan, nIDs[eventID], ); err != nil { return err } @@ -504,52 +504,6 @@ func (u *membershipUpdater) SetToLeave(senderUserID string, eventID string) (inv return } -// GetMembership implements query.RoomserverQueryAPIDB -func (d *Database) GetMembership( - ctx context.Context, roomNID types.RoomNID, requestSenderUserID string, -) (membershipEventNID types.EventNID, stillInRoom bool, err error) { - err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { - requestSenderUserNID, err := d.assignStateKeyNID(ctx, txn, requestSenderUserID) - if err != nil { - return err - } - - membershipEventNID, _, err = - d.statements.selectMembershipFromRoomAndTarget( - ctx, txn, roomNID, requestSenderUserNID, - ) - if err == sql.ErrNoRows { - // The user has never been a member of that room - return nil - } - if err != nil { - return err - } - stillInRoom = true - return nil - }) - - return -} - -// GetMembershipEventNIDsForRoom implements query.RoomserverQueryAPIDB -func (d *Database) GetMembershipEventNIDsForRoom( - ctx context.Context, roomNID types.RoomNID, joinOnly bool, localOnly bool, -) (eventNIDs []types.EventNID, err error) { - err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { - if joinOnly { - eventNIDs, err = d.statements.selectMembershipsFromRoomAndMembership( - ctx, txn, roomNID, membershipStateJoin, localOnly, - ) - return nil - } - - eventNIDs, err = d.statements.selectMembershipsFromRoom(ctx, txn, roomNID, localOnly) - return nil - }) - return -} - type transaction struct { ctx context.Context txn *sql.Tx diff --git a/roomserver/storage/sqlite3/transactions_table.go b/roomserver/storage/sqlite3/transactions_table.go index 37ea15c0f..7b489ac6e 100644 --- a/roomserver/storage/sqlite3/transactions_table.go +++ b/roomserver/storage/sqlite3/transactions_table.go @@ -20,6 +20,7 @@ import ( "database/sql" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" ) @@ -54,10 +55,10 @@ func NewSqliteTransactionsTable(db *sql.DB) (tables.Transactions, error) { return nil, err } - return s, statementList{ + return s, shared.StatementList{ {&s.insertTransactionStmt, insertTransactionSQL}, {&s.selectTransactionEventIDStmt, selectTransactionEventIDSQL}, - }.prepare(db) + }.Prepare(db) } func (s *transactionStatements) InsertTransaction( diff --git a/roomserver/storage/tables/interface.go b/roomserver/storage/tables/interface.go index 56ef5dd39..11cff8a8b 100644 --- a/roomserver/storage/tables/interface.go +++ b/roomserver/storage/tables/interface.go @@ -96,3 +96,27 @@ type PreviousEvents interface { // Returns sql.ErrNoRows if the event reference doesn't exist. SelectPreviousEventExists(ctx context.Context, txn *sql.Tx, eventID string, eventReferenceSHA256 []byte) error } + +type Invites interface { + InsertInviteEvent(ctx context.Context, txn *sql.Tx, inviteEventID string, roomNID types.RoomNID, targetUserNID, senderUserNID types.EventStateKeyNID, inviteEventJSON []byte) (bool, error) + UpdateInviteRetired(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID) ([]string, error) + // SelectInviteActiveForUserInRoom returns a list of sender state key NIDs + SelectInviteActiveForUserInRoom(ctx context.Context, targetUserNID types.EventStateKeyNID, roomNID types.RoomNID) ([]types.EventStateKeyNID, error) +} + +type MembershipState int64 + +const ( + MembershipStateLeaveOrBan MembershipState = 1 + MembershipStateInvite MembershipState = 2 + MembershipStateJoin MembershipState = 3 +) + +type Membership interface { + InsertMembership(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, localTarget bool) error + SelectMembershipForUpdate(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID) (MembershipState, error) + SelectMembershipFromRoomAndTarget(ctx context.Context, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID) (types.EventNID, MembershipState, error) + SelectMembershipsFromRoom(ctx context.Context, roomNID types.RoomNID, localOnly bool) (eventNIDs []types.EventNID, err error) + SelectMembershipsFromRoomAndMembership(ctx context.Context, roomNID types.RoomNID, membership MembershipState, localOnly bool) (eventNIDs []types.EventNID, err error) + UpdateMembership(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, senderUserNID types.EventStateKeyNID, membership MembershipState, eventNID types.EventNID) error +} From 406b47267eca9e1be2f63b6c18f598e98618e7a0 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 27 May 2020 11:16:27 +0100 Subject: [PATCH 100/200] Return 500 when processing a transaction fails fatally (#1066) --- federationapi/routing/send.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index 6cf44018e..b514af0ab 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -76,6 +76,7 @@ func Send( resp, err := t.processTransaction() if err != nil { util.GetLogger(httpReq.Context()).WithError(err).Error("t.processTransaction failed") + return util.ErrorResponse(err) } // https://matrix.org/docs/spec/server_server/r0.1.3#put-matrix-federation-v1-send-txnid @@ -117,7 +118,7 @@ type txnFederationClient interface { func (t *txnReq) processTransaction() (*gomatrixserverlib.RespSend, error) { results := make(map[string]gomatrixserverlib.PDUResult) - var pdus []gomatrixserverlib.HeaderedEvent + pdus := []gomatrixserverlib.HeaderedEvent{} for _, pdu := range t.PDUs { var header struct { RoomID string `json:"room_id"` From 57841fc35ebb6f591903e20694dad3548bf71ce8 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 27 May 2020 12:16:53 +0100 Subject: [PATCH 101/200] Read batches from incoming channels (#1067) --- federationsender/queue/destinationqueue.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/federationsender/queue/destinationqueue.go b/federationsender/queue/destinationqueue.go index 45faa287c..09dac464f 100644 --- a/federationsender/queue/destinationqueue.go +++ b/federationsender/queue/destinationqueue.go @@ -110,12 +110,26 @@ func (oq *destinationQueue) backgroundSend() { // of the queue and they will all be added to transactions // in order. oq.pendingPDUs = append(oq.pendingPDUs, pdu) + // If there are any more things waiting in the channel queue + // then read them. This is safe because we guarantee only + // having one goroutine per destination queue, so the channel + // isn't being consumed anywhere else. + for len(oq.incomingPDUs) > 0 { + oq.pendingPDUs = append(oq.pendingPDUs, <-oq.incomingPDUs) + } case edu := <-oq.incomingEDUs: // Likewise for EDUs, although we should probably not try // too hard with some EDUs (like typing notifications) after // a certain amount of time has passed. // TODO: think about EDU expiry some more oq.pendingEDUs = append(oq.pendingEDUs, edu) + // If there are any more things waiting in the channel queue + // then read them. This is safe because we guarantee only + // having one goroutine per destination queue, so the channel + // isn't being consumed anywhere else. + for len(oq.incomingEDUs) > 0 { + oq.pendingEDUs = append(oq.pendingEDUs, <-oq.incomingEDUs) + } case invite := <-oq.incomingInvites: // There's no strict ordering requirement for invites like // there is for transactions, so we put the invite onto the @@ -126,6 +140,13 @@ func (oq *destinationQueue) backgroundSend() { []*gomatrixserverlib.InviteV2Request{invite}, oq.pendingInvites..., ) + // If there are any more things waiting in the channel queue + // then read them. This is safe because we guarantee only + // having one goroutine per destination queue, so the channel + // isn't being consumed anywhere else. + for len(oq.incomingInvites) > 0 { + oq.pendingInvites = append(oq.pendingInvites, <-oq.incomingInvites) + } case <-time.After(time.Second * 30): // The worker is idle so stop the goroutine. It'll // get restarted automatically the next time we From 02fe38e1f74a61f900ff5d8a4e2d5870ba737386 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 28 May 2020 10:05:04 +0100 Subject: [PATCH 102/200] Per-user-per-device sync streams (#1068) * Per-user-per-device sync streams * Tweaks * Tweaks * Pass full device into CompleteSync * Set user IDs and device IDs properly in tests * Add new test, fix TestNewEventAndWasPreviouslyJoinedToRoom * nolint a function that is not used yet * Add test for waking up single device * Hopefully unstick test * Try to ensure that TestCorrectStreamWakeup doesn't block forever * Update tests --- build/scripts/build-test-lint.sh | 2 +- syncapi/storage/interface.go | 2 +- syncapi/storage/shared/syncserver.go | 4 +- syncapi/storage/storage_test.go | 4 +- syncapi/sync/notifier.go | 83 +++++++++++++++++++++------ syncapi/sync/notifier_test.go | 86 +++++++++++++++++++++------- syncapi/sync/requestpool.go | 12 ++-- syncapi/sync/userstream.go | 38 ++++++------ 8 files changed, 161 insertions(+), 70 deletions(-) diff --git a/build/scripts/build-test-lint.sh b/build/scripts/build-test-lint.sh index d2b2b4b16..4b18ca2f8 100755 --- a/build/scripts/build-test-lint.sh +++ b/build/scripts/build-test-lint.sh @@ -13,4 +13,4 @@ go build ./cmd/... ./scripts/find-lint.sh echo "Testing..." -go test ./... +go test -v ./... diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index 3e9ff0533..7e1a40fd6 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -58,7 +58,7 @@ type Database interface { // ID. IncrementalSync(ctx context.Context, device authtypes.Device, fromPos, toPos types.StreamingToken, numRecentEventsPerRoom int, wantFullState bool) (*types.Response, error) // CompleteSync returns a complete /sync API response for the given user. - CompleteSync(ctx context.Context, userID string, numRecentEventsPerRoom int) (*types.Response, error) + CompleteSync(ctx context.Context, device authtypes.Device, numRecentEventsPerRoom int) (*types.Response, error) // GetAccountDataInRange returns all account data for a given user inserted or // updated between two given positions // Returns a map following the format data[roomID] = []dataTypes diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go index 145a3dbf3..888f85e0b 100644 --- a/syncapi/storage/shared/syncserver.go +++ b/syncapi/storage/shared/syncserver.go @@ -666,10 +666,10 @@ func (d *Database) getResponseWithPDUsForCompleteSync( } func (d *Database) CompleteSync( - ctx context.Context, userID string, numRecentEventsPerRoom int, + ctx context.Context, device authtypes.Device, numRecentEventsPerRoom int, ) (*types.Response, error) { res, toPos, joinedRoomIDs, err := d.getResponseWithPDUsForCompleteSync( - ctx, userID, numRecentEventsPerRoom, + ctx, device.UserID, numRecentEventsPerRoom, ) if err != nil { return nil, err diff --git a/syncapi/storage/storage_test.go b/syncapi/storage/storage_test.go index 6b0458527..bb8554f43 100644 --- a/syncapi/storage/storage_test.go +++ b/syncapi/storage/storage_test.go @@ -181,7 +181,7 @@ func TestSyncResponse(t *testing.T) { Name: "CompleteSync limited", DoSync: func() (*types.Response, error) { // limit set to 5 - return db.CompleteSync(ctx, testUserIDA, 5) + return db.CompleteSync(ctx, testUserDeviceA, 5) }, // want the last 5 events WantTimeline: events[len(events)-5:], @@ -193,7 +193,7 @@ func TestSyncResponse(t *testing.T) { { Name: "CompleteSync", DoSync: func() (*types.Response, error) { - return db.CompleteSync(ctx, testUserIDA, len(events)+1) + return db.CompleteSync(ctx, testUserDeviceA, len(events)+1) }, WantTimeline: events, // We want no state at all as that field in /sync is the delta between the token (beginning of time) diff --git a/syncapi/sync/notifier.go b/syncapi/sync/notifier.go index b3ed5cd03..9b410a0c4 100644 --- a/syncapi/sync/notifier.go +++ b/syncapi/sync/notifier.go @@ -37,8 +37,8 @@ type Notifier struct { streamLock *sync.Mutex // The latest sync position currPos types.StreamingToken - // A map of user_id => UserStream which can be used to wake a given user's /sync request. - userStreams map[string]*UserStream + // A map of user_id => device_id => UserStream which can be used to wake a given user's /sync request. + userDeviceStreams map[string]map[string]*UserDeviceStream // The last time we cleaned out stale entries from the userStreams map lastCleanUpTime time.Time } @@ -50,7 +50,7 @@ func NewNotifier(pos types.StreamingToken) *Notifier { return &Notifier{ currPos: pos, roomIDToJoinedUsers: make(map[string]userIDSet), - userStreams: make(map[string]*UserStream), + userDeviceStreams: make(map[string]map[string]*UserDeviceStream), streamLock: &sync.Mutex{}, lastCleanUpTime: time.Now(), } @@ -123,7 +123,7 @@ func (n *Notifier) OnNewEvent( // GetListener returns a UserStreamListener that can be used to wait for // updates for a user. Must be closed. // notify for anything before sincePos -func (n *Notifier) GetListener(req syncRequest) UserStreamListener { +func (n *Notifier) GetListener(req syncRequest) UserDeviceStreamListener { // Do what synapse does: https://github.com/matrix-org/synapse/blob/v0.20.0/synapse/notifier.py#L298 // - Bucket request into a lookup map keyed off a list of joined room IDs and separately a user ID // - Incoming events wake requests for a matching room ID @@ -137,7 +137,7 @@ func (n *Notifier) GetListener(req syncRequest) UserStreamListener { n.removeEmptyUserStreams() - return n.fetchUserStream(req.device.UserID, true).GetListener(req.ctx) + return n.fetchUserDeviceStream(req.device.UserID, req.device.ID, true).GetListener(req.ctx) } // Load the membership states required to notify users correctly. @@ -173,27 +173,69 @@ func (n *Notifier) setUsersJoinedToRooms(roomIDToUserIDs map[string][]string) { } } +// wakeupUsers will wake up the sync strems for all of the devices for all of the +// specified user IDs. func (n *Notifier) wakeupUsers(userIDs []string, newPos types.StreamingToken) { for _, userID := range userIDs { - stream := n.fetchUserStream(userID, false) - if stream != nil { + for _, stream := range n.fetchUserStreams(userID) { + if stream == nil { + continue + } stream.Broadcast(newPos) // wake up all goroutines Wait()ing on this stream } } } -// fetchUserStream retrieves a stream unique to the given user. If makeIfNotExists is true, +// wakeupUserDevice will wake up the sync stream for a specific user device. Other +// device streams will be left alone. +// nolint:unused +func (n *Notifier) wakeupUserDevice(userDevices map[string]string, newPos types.StreamingToken) { + for userID, deviceID := range userDevices { + if stream := n.fetchUserDeviceStream(userID, deviceID, false); stream != nil { + stream.Broadcast(newPos) // wake up all goroutines Wait()ing on this stream + } + } +} + +// fetchUserDeviceStream retrieves a stream unique to the given device. If makeIfNotExists is true, +// a stream will be made for this device if one doesn't exist and it will be returned. This +// function does not wait for data to be available on the stream. +// NB: Callers should have locked the mutex before calling this function. +func (n *Notifier) fetchUserDeviceStream(userID, deviceID string, makeIfNotExists bool) *UserDeviceStream { + _, ok := n.userDeviceStreams[userID] + if !ok { + if !makeIfNotExists { + return nil + } + n.userDeviceStreams[userID] = map[string]*UserDeviceStream{} + } + stream, ok := n.userDeviceStreams[userID][deviceID] + if !ok { + if !makeIfNotExists { + return nil + } + // TODO: Unbounded growth of streams (1 per user) + if stream = NewUserDeviceStream(userID, deviceID, n.currPos); stream != nil { + n.userDeviceStreams[userID][deviceID] = stream + } + } + return stream +} + +// fetchUserStreams retrieves all streams for the given user. If makeIfNotExists is true, // a stream will be made for this user if one doesn't exist and it will be returned. This // function does not wait for data to be available on the stream. // NB: Callers should have locked the mutex before calling this function. -func (n *Notifier) fetchUserStream(userID string, makeIfNotExists bool) *UserStream { - stream, ok := n.userStreams[userID] - if !ok && makeIfNotExists { - // TODO: Unbounded growth of streams (1 per user) - stream = NewUserStream(userID, n.currPos) - n.userStreams[userID] = stream +func (n *Notifier) fetchUserStreams(userID string) []*UserDeviceStream { + user, ok := n.userDeviceStreams[userID] + if !ok { + return []*UserDeviceStream{} } - return stream + streams := []*UserDeviceStream{} + for _, stream := range user { + streams = append(streams, stream) + } + return streams } // Not thread-safe: must be called on the OnNewEvent goroutine only @@ -236,9 +278,14 @@ func (n *Notifier) removeEmptyUserStreams() { n.lastCleanUpTime = now deleteBefore := now.Add(-5 * time.Minute) - for key, value := range n.userStreams { - if value.TimeOfLastNonEmpty().Before(deleteBefore) { - delete(n.userStreams, key) + for user, byUser := range n.userDeviceStreams { + for device, stream := range byUser { + if stream.TimeOfLastNonEmpty().Before(deleteBefore) { + delete(n.userDeviceStreams[user], device) + } + if len(n.userDeviceStreams[user]) == 0 { + delete(n.userDeviceStreams, user) + } } } } diff --git a/syncapi/sync/notifier_test.go b/syncapi/sync/notifier_test.go index 7d979fcc9..14ddef20a 100644 --- a/syncapi/sync/notifier_test.go +++ b/syncapi/sync/notifier_test.go @@ -41,9 +41,11 @@ var ( ) var ( - roomID = "!test:localhost" - alice = "@alice:localhost" - bob = "@bob:localhost" + roomID = "!test:localhost" + alice = "@alice:localhost" + aliceDev = "alicedevice" + bob = "@bob:localhost" + bobDev = "bobdev" ) func init() { @@ -107,7 +109,7 @@ func mustEqualPositions(t *testing.T, got, want types.StreamingToken) { // Test that the current position is returned if a request is already behind. func TestImmediateNotification(t *testing.T) { n := NewNotifier(syncPositionBefore) - pos, err := waitForEvents(n, newTestSyncRequest(alice, syncPositionVeryOld)) + pos, err := waitForEvents(n, newTestSyncRequest(alice, aliceDev, syncPositionVeryOld)) if err != nil { t.Fatalf("TestImmediateNotification error: %s", err) } @@ -124,7 +126,7 @@ func TestNewEventAndJoinedToRoom(t *testing.T) { var wg sync.WaitGroup wg.Add(1) go func() { - pos, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionBefore)) + pos, err := waitForEvents(n, newTestSyncRequest(bob, bobDev, syncPositionBefore)) if err != nil { t.Errorf("TestNewEventAndJoinedToRoom error: %w", err) } @@ -132,7 +134,7 @@ func TestNewEventAndJoinedToRoom(t *testing.T) { wg.Done() }() - stream := lockedFetchUserStream(n, bob) + stream := lockedFetchUserStream(n, bob, bobDev) waitForBlocking(stream, 1) n.OnNewEvent(&randomMessageEvent, "", nil, syncPositionAfter) @@ -140,6 +142,43 @@ func TestNewEventAndJoinedToRoom(t *testing.T) { wg.Wait() } +func TestCorrectStream(t *testing.T) { + n := NewNotifier(syncPositionBefore) + stream := lockedFetchUserStream(n, bob, bobDev) + if stream.UserID != bob { + t.Fatalf("expected user %q, got %q", bob, stream.UserID) + } + if stream.DeviceID != bobDev { + t.Fatalf("expected device %q, got %q", bobDev, stream.DeviceID) + } +} + +func TestCorrectStreamWakeup(t *testing.T) { + n := NewNotifier(syncPositionBefore) + awoken := make(chan string) + + streamone := lockedFetchUserStream(n, alice, "one") + streamtwo := lockedFetchUserStream(n, alice, "two") + + go func() { + select { + case <-streamone.signalChannel: + awoken <- "one" + case <-streamtwo.signalChannel: + awoken <- "two" + } + }() + + time.Sleep(1 * time.Second) + + wake := "two" + n.wakeupUserDevice(map[string]string{alice: wake}, syncPositionAfter) + + if result := <-awoken; result != wake { + t.Fatalf("expected to wake %q, got %q", wake, result) + } +} + // Test that an invite unblocks the request func TestNewInviteEventForUser(t *testing.T) { n := NewNotifier(syncPositionBefore) @@ -150,7 +189,7 @@ func TestNewInviteEventForUser(t *testing.T) { var wg sync.WaitGroup wg.Add(1) go func() { - pos, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionBefore)) + pos, err := waitForEvents(n, newTestSyncRequest(bob, bobDev, syncPositionBefore)) if err != nil { t.Errorf("TestNewInviteEventForUser error: %w", err) } @@ -158,7 +197,7 @@ func TestNewInviteEventForUser(t *testing.T) { wg.Done() }() - stream := lockedFetchUserStream(n, bob) + stream := lockedFetchUserStream(n, bob, bobDev) waitForBlocking(stream, 1) n.OnNewEvent(&aliceInviteBobEvent, "", nil, syncPositionAfter) @@ -176,7 +215,7 @@ func TestEDUWakeup(t *testing.T) { var wg sync.WaitGroup wg.Add(1) go func() { - pos, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionAfter)) + pos, err := waitForEvents(n, newTestSyncRequest(bob, bobDev, syncPositionAfter)) if err != nil { t.Errorf("TestNewInviteEventForUser error: %w", err) } @@ -184,7 +223,7 @@ func TestEDUWakeup(t *testing.T) { wg.Done() }() - stream := lockedFetchUserStream(n, bob) + stream := lockedFetchUserStream(n, bob, bobDev) waitForBlocking(stream, 1) n.OnNewEvent(&aliceInviteBobEvent, "", nil, syncPositionNewEDU) @@ -202,7 +241,7 @@ func TestMultipleRequestWakeup(t *testing.T) { var wg sync.WaitGroup wg.Add(3) poll := func() { - pos, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionBefore)) + pos, err := waitForEvents(n, newTestSyncRequest(bob, bobDev, syncPositionBefore)) if err != nil { t.Errorf("TestMultipleRequestWakeup error: %w", err) } @@ -213,7 +252,7 @@ func TestMultipleRequestWakeup(t *testing.T) { go poll() go poll() - stream := lockedFetchUserStream(n, bob) + stream := lockedFetchUserStream(n, bob, bobDev) waitForBlocking(stream, 3) n.OnNewEvent(&randomMessageEvent, "", nil, syncPositionAfter) @@ -240,24 +279,24 @@ func TestNewEventAndWasPreviouslyJoinedToRoom(t *testing.T) { // Make bob leave the room leaveWG.Add(1) go func() { - pos, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionBefore)) + pos, err := waitForEvents(n, newTestSyncRequest(bob, bobDev, syncPositionBefore)) if err != nil { t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom error: %w", err) } mustEqualPositions(t, pos, syncPositionAfter) leaveWG.Done() }() - bobStream := lockedFetchUserStream(n, bob) + bobStream := lockedFetchUserStream(n, bob, bobDev) waitForBlocking(bobStream, 1) n.OnNewEvent(&bobLeaveEvent, "", nil, syncPositionAfter) leaveWG.Wait() // send an event into the room. Make sure alice gets it. Bob should not. var aliceWG sync.WaitGroup - aliceStream := lockedFetchUserStream(n, alice) + aliceStream := lockedFetchUserStream(n, alice, aliceDev) aliceWG.Add(1) go func() { - pos, err := waitForEvents(n, newTestSyncRequest(alice, syncPositionAfter)) + pos, err := waitForEvents(n, newTestSyncRequest(alice, aliceDev, syncPositionAfter)) if err != nil { t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom error: %w", err) } @@ -267,7 +306,7 @@ func TestNewEventAndWasPreviouslyJoinedToRoom(t *testing.T) { go func() { // this should timeout with an error (but the main goroutine won't wait for the timeout explicitly) - _, err := waitForEvents(n, newTestSyncRequest(bob, syncPositionAfter)) + _, err := waitForEvents(n, newTestSyncRequest(bob, bobDev, syncPositionAfter)) if err == nil { t.Errorf("TestNewEventAndWasPreviouslyJoinedToRoom expect error but got nil") } @@ -300,7 +339,7 @@ func waitForEvents(n *Notifier, req syncRequest) (types.StreamingToken, error) { } // Wait until something is Wait()ing on the user stream. -func waitForBlocking(s *UserStream, numBlocking uint) { +func waitForBlocking(s *UserDeviceStream, numBlocking uint) { for numBlocking != s.NumWaiting() { // This is horrible but I don't want to add a signalling mechanism JUST for testing. time.Sleep(1 * time.Microsecond) @@ -309,16 +348,19 @@ func waitForBlocking(s *UserStream, numBlocking uint) { // lockedFetchUserStream invokes Notifier.fetchUserStream, respecting Notifier.streamLock. // A new stream is made if it doesn't exist already. -func lockedFetchUserStream(n *Notifier, userID string) *UserStream { +func lockedFetchUserStream(n *Notifier, userID, deviceID string) *UserDeviceStream { n.streamLock.Lock() defer n.streamLock.Unlock() - return n.fetchUserStream(userID, true) + return n.fetchUserDeviceStream(userID, deviceID, true) } -func newTestSyncRequest(userID string, since types.StreamingToken) syncRequest { +func newTestSyncRequest(userID, deviceID string, since types.StreamingToken) syncRequest { return syncRequest{ - device: authtypes.Device{UserID: userID}, + device: authtypes.Device{ + UserID: userID, + ID: deviceID, + }, timeout: 1 * time.Minute, since: &since, wantFullState: false, diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index 82ce283b6..bd29b3338 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -47,7 +47,6 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *authtype var syncData *types.Response // Extract values from request - userID := device.UserID syncReq, err := newSyncRequest(req, *device) if err != nil { return util.JSONResponse{ @@ -56,10 +55,11 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *authtype } } logger := util.GetLogger(req.Context()).WithFields(log.Fields{ - "userID": userID, - "since": syncReq.since, - "timeout": syncReq.timeout, - "limit": syncReq.limit, + "userID": device.UserID, + "deviceID": device.ID, + "since": syncReq.since, + "timeout": syncReq.timeout, + "limit": syncReq.limit, }) currPos := rp.notifier.CurrentPosition() @@ -136,7 +136,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *authtype func (rp *RequestPool) currentSyncForUser(req syncRequest, latestPos types.StreamingToken) (res *types.Response, err error) { // TODO: handle ignored users if req.since == nil { - res, err = rp.db.CompleteSync(req.ctx, req.device.UserID, req.limit) + res, err = rp.db.CompleteSync(req.ctx, req.device, req.limit) } else { res, err = rp.db.IncrementalSync(req.ctx, req.device, *req.since, latestPos, req.limit, req.wantFullState) } diff --git a/syncapi/sync/userstream.go b/syncapi/sync/userstream.go index b2eafa3dc..ff9a4d003 100644 --- a/syncapi/sync/userstream.go +++ b/syncapi/sync/userstream.go @@ -23,12 +23,13 @@ import ( "github.com/matrix-org/dendrite/syncapi/types" ) -// UserStream represents a communication mechanism between the /sync request goroutine +// UserDeviceStream represents a communication mechanism between the /sync request goroutine // and the underlying sync server goroutines. // Goroutines can get a UserStreamListener to wait for updates, and can Broadcast() // updates. -type UserStream struct { - UserID string +type UserDeviceStream struct { + UserID string + DeviceID string // The lock that protects changes to this struct lock sync.Mutex // Closed when there is an update. @@ -41,18 +42,19 @@ type UserStream struct { numWaiting uint } -// UserStreamListener allows a sync request to wait for updates for a user. -type UserStreamListener struct { - userStream *UserStream +// UserDeviceStreamListener allows a sync request to wait for updates for a user. +type UserDeviceStreamListener struct { + userStream *UserDeviceStream // Whether the stream has been closed hasClosed bool } -// NewUserStream creates a new user stream -func NewUserStream(userID string, currPos types.StreamingToken) *UserStream { - return &UserStream{ +// NewUserDeviceStream creates a new user stream +func NewUserDeviceStream(userID, deviceID string, currPos types.StreamingToken) *UserDeviceStream { + return &UserDeviceStream{ UserID: userID, + DeviceID: deviceID, timeOfLastChannel: time.Now(), pos: currPos, signalChannel: make(chan struct{}), @@ -62,18 +64,18 @@ func NewUserStream(userID string, currPos types.StreamingToken) *UserStream { // GetListener returns UserStreamListener that a sync request can use to wait // for new updates with. // UserStreamListener must be closed -func (s *UserStream) GetListener(ctx context.Context) UserStreamListener { +func (s *UserDeviceStream) GetListener(ctx context.Context) UserDeviceStreamListener { s.lock.Lock() defer s.lock.Unlock() s.numWaiting++ // We decrement when UserStreamListener is closed - listener := UserStreamListener{ + listener := UserDeviceStreamListener{ userStream: s, } // Lets be a bit paranoid here and check that Close() is being called - runtime.SetFinalizer(&listener, func(l *UserStreamListener) { + runtime.SetFinalizer(&listener, func(l *UserDeviceStreamListener) { if !l.hasClosed { l.Close() } @@ -83,7 +85,7 @@ func (s *UserStream) GetListener(ctx context.Context) UserStreamListener { } // Broadcast a new sync position for this user. -func (s *UserStream) Broadcast(pos types.StreamingToken) { +func (s *UserDeviceStream) Broadcast(pos types.StreamingToken) { s.lock.Lock() defer s.lock.Unlock() @@ -96,7 +98,7 @@ func (s *UserStream) Broadcast(pos types.StreamingToken) { // NumWaiting returns the number of goroutines waiting for waiting for updates. // Used for metrics and testing. -func (s *UserStream) NumWaiting() uint { +func (s *UserDeviceStream) NumWaiting() uint { s.lock.Lock() defer s.lock.Unlock() return s.numWaiting @@ -105,7 +107,7 @@ func (s *UserStream) NumWaiting() uint { // TimeOfLastNonEmpty returns the last time that the number of waiting listeners // was non-empty, may be time.Now() if number of waiting listeners is currently // non-empty. -func (s *UserStream) TimeOfLastNonEmpty() time.Time { +func (s *UserDeviceStream) TimeOfLastNonEmpty() time.Time { s.lock.Lock() defer s.lock.Unlock() @@ -118,7 +120,7 @@ func (s *UserStream) TimeOfLastNonEmpty() time.Time { // GetSyncPosition returns last sync position which the UserStream was // notified about -func (s *UserStreamListener) GetSyncPosition() types.StreamingToken { +func (s *UserDeviceStreamListener) GetSyncPosition() types.StreamingToken { s.userStream.lock.Lock() defer s.userStream.lock.Unlock() @@ -130,7 +132,7 @@ func (s *UserStreamListener) GetSyncPosition() types.StreamingToken { // sincePos specifies from which point we want to be notified about. If there // has already been an update after sincePos we'll return a closed channel // immediately. -func (s *UserStreamListener) GetNotifyChannel(sincePos types.StreamingToken) <-chan struct{} { +func (s *UserDeviceStreamListener) GetNotifyChannel(sincePos types.StreamingToken) <-chan struct{} { s.userStream.lock.Lock() defer s.userStream.lock.Unlock() @@ -147,7 +149,7 @@ func (s *UserStreamListener) GetNotifyChannel(sincePos types.StreamingToken) <-c } // Close cleans up resources used -func (s *UserStreamListener) Close() { +func (s *UserDeviceStreamListener) Close() { s.userStream.lock.Lock() defer s.userStream.lock.Unlock() From a6f995eb45f0d012f79aa66f727cb6260455b334 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Thu, 28 May 2020 11:15:21 +0100 Subject: [PATCH 103/200] Merge Updater structs (#1069) * Move Updater structs to shared and use it for postgres * Add constructors for NewXXXUpdater and a useTxns flag In sqlite, we set useTxns=false and comment why. * Handle nil txn * Handle nil in transaction * Missed one * Close the txn at the right time * Don't close the transaction as we reuse it between calls --- go.sum | 2 - roomserver/storage/postgres/storage.go | 390 ++---------------- .../storage/shared/membership_updater.go | 183 ++++++++ roomserver/storage/shared/prepare.go | 24 ++ .../shared/room_recent_events_updater.go | 120 ++++++ roomserver/storage/shared/storage.go | 13 + roomserver/storage/sqlite3/invite_table.go | 5 +- roomserver/storage/sqlite3/storage.go | 388 +---------------- 8 files changed, 381 insertions(+), 744 deletions(-) create mode 100644 roomserver/storage/shared/membership_updater.go create mode 100644 roomserver/storage/shared/room_recent_events_updater.go diff --git a/go.sum b/go.sum index aa0c8dd57..3398195b6 100644 --- a/go.sum +++ b/go.sum @@ -352,8 +352,6 @@ github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5/go.mod h1:NgPCr+ github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg= github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 h1:eqE5OnGx9ZMWmrRbD3KF/3KtTunw0iQulI7YxOIdxo4= github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4/go.mod h1:3WluEZ9QXSwU30tWYqktnpC1x9mwZKx1r8uAv8Iq+a4= -github.com/matrix-org/go-sqlite3-js v0.0.0-20200326102434-98eda28055bd h1:C1FV4dRKF1uuGK8UH01+IoW6zZpfsTV1MvQimZvt418= -github.com/matrix-org/go-sqlite3-js v0.0.0-20200326102434-98eda28055bd/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 h1:Yb+Wlf/iHhWlLWd+kCgG+Fsg4Dc+xBl7hptfK7lD0zY= github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bhrnp3Ky1qgx/fzCtCALOoGYylh2tpS9K4= diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go index 53a58076a..971f2b9e6 100644 --- a/roomserver/storage/postgres/storage.go +++ b/roomserver/storage/postgres/storage.go @@ -9,14 +9,13 @@ // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implie // See the License for the specific language governing permissions and // limitations under the License. package postgres import ( - "context" "database/sql" "github.com/matrix-org/dendrite/internal" @@ -25,423 +24,84 @@ import ( // Import the postgres database driver. _ "github.com/lib/pq" "github.com/matrix-org/dendrite/roomserver/storage/shared" - "github.com/matrix-org/dendrite/roomserver/storage/tables" - "github.com/matrix-org/dendrite/roomserver/types" - "github.com/matrix-org/gomatrixserverlib" ) // A Database is used to store room events and stream offsets. type Database struct { shared.Database - events tables.Events - eventTypes tables.EventTypes - eventStateKeys tables.EventStateKeys - eventJSON tables.EventJSON - rooms tables.Rooms - transactions tables.Transactions - prevEvents tables.PreviousEvents - invites tables.Invites - membership tables.Membership - db *sql.DB } // Open a postgres database. // nolint: gocyclo func Open(dataSourceName string, dbProperties internal.DbProperties) (*Database, error) { var d Database + var db *sql.DB var err error - if d.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { + if db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { return nil, err } - d.eventStateKeys, err = NewPostgresEventStateKeysTable(d.db) + eventStateKeys, err := NewPostgresEventStateKeysTable(db) if err != nil { return nil, err } - d.eventTypes, err = NewPostgresEventTypesTable(d.db) + eventTypes, err := NewPostgresEventTypesTable(db) if err != nil { return nil, err } - d.eventJSON, err = NewPostgresEventJSONTable(d.db) + eventJSON, err := NewPostgresEventJSONTable(db) if err != nil { return nil, err } - d.events, err = NewPostgresEventsTable(d.db) + events, err := NewPostgresEventsTable(db) if err != nil { return nil, err } - d.rooms, err = NewPostgresRoomsTable(d.db) + rooms, err := NewPostgresRoomsTable(db) if err != nil { return nil, err } - d.transactions, err = NewPostgresTransactionsTable(d.db) + transactions, err := NewPostgresTransactionsTable(db) if err != nil { return nil, err } - stateBlock, err := NewPostgresStateBlockTable(d.db) + stateBlock, err := NewPostgresStateBlockTable(db) if err != nil { return nil, err } - stateSnapshot, err := NewPostgresStateSnapshotTable(d.db) + stateSnapshot, err := NewPostgresStateSnapshotTable(db) if err != nil { return nil, err } - roomAliases, err := NewPostgresRoomAliasesTable(d.db) + roomAliases, err := NewPostgresRoomAliasesTable(db) if err != nil { return nil, err } - d.prevEvents, err = NewPostgresPreviousEventsTable(d.db) + prevEvents, err := NewPostgresPreviousEventsTable(db) if err != nil { return nil, err } - d.invites, err = NewPostgresInvitesTable(d.db) + invites, err := NewPostgresInvitesTable(db) if err != nil { return nil, err } - d.membership, err = NewPostgresMembershipTable(d.db) + membership, err := NewPostgresMembershipTable(db) if err != nil { return nil, err } d.Database = shared.Database{ - DB: d.db, - EventTypesTable: d.eventTypes, - EventStateKeysTable: d.eventStateKeys, - EventJSONTable: d.eventJSON, - EventsTable: d.events, - RoomsTable: d.rooms, - TransactionsTable: d.transactions, + DB: db, + EventTypesTable: eventTypes, + EventStateKeysTable: eventStateKeys, + EventJSONTable: eventJSON, + EventsTable: events, + RoomsTable: rooms, + TransactionsTable: transactions, StateBlockTable: stateBlock, StateSnapshotTable: stateSnapshot, - PrevEventsTable: d.prevEvents, + PrevEventsTable: prevEvents, RoomAliasesTable: roomAliases, - InvitesTable: d.invites, - MembershipTable: d.membership, + InvitesTable: invites, + MembershipTable: membership, } return &d, nil } - -func (d *Database) assignRoomNID( - ctx context.Context, txn *sql.Tx, - roomID string, roomVersion gomatrixserverlib.RoomVersion, -) (types.RoomNID, error) { - // Check if we already have a numeric ID in the database. - roomNID, err := d.rooms.SelectRoomNID(ctx, txn, roomID) - if err == sql.ErrNoRows { - // We don't have a numeric ID so insert one into the database. - roomNID, err = d.rooms.InsertRoomNID(ctx, txn, roomID, roomVersion) - if err == sql.ErrNoRows { - // We raced with another insert so run the select again. - roomNID, err = d.rooms.SelectRoomNID(ctx, txn, roomID) - } - } - return roomNID, err -} - -func (d *Database) assignStateKeyNID( - ctx context.Context, txn *sql.Tx, eventStateKey string, -) (types.EventStateKeyNID, error) { - // Check if we already have a numeric ID in the database. - eventStateKeyNID, err := d.eventStateKeys.SelectEventStateKeyNID(ctx, txn, eventStateKey) - if err == sql.ErrNoRows { - // We don't have a numeric ID so insert one into the database. - eventStateKeyNID, err = d.eventStateKeys.InsertEventStateKeyNID(ctx, txn, eventStateKey) - if err == sql.ErrNoRows { - // We raced with another insert so run the select again. - eventStateKeyNID, err = d.eventStateKeys.SelectEventStateKeyNID(ctx, txn, eventStateKey) - } - } - return eventStateKeyNID, err -} - -// GetLatestEventsForUpdate implements input.EventDatabase -func (d *Database) GetLatestEventsForUpdate( - ctx context.Context, roomNID types.RoomNID, -) (types.RoomRecentEventsUpdater, error) { - txn, err := d.db.Begin() - if err != nil { - return nil, err - } - eventNIDs, lastEventNIDSent, currentStateSnapshotNID, err := - d.rooms.SelectLatestEventsNIDsForUpdate(ctx, txn, roomNID) - if err != nil { - txn.Rollback() // nolint: errcheck - return nil, err - } - stateAndRefs, err := d.events.BulkSelectStateAtEventAndReference(ctx, txn, eventNIDs) - if err != nil { - txn.Rollback() // nolint: errcheck - return nil, err - } - var lastEventIDSent string - if lastEventNIDSent != 0 { - lastEventIDSent, err = d.events.SelectEventID(ctx, txn, lastEventNIDSent) - if err != nil { - txn.Rollback() // nolint: errcheck - return nil, err - } - } - return &roomRecentEventsUpdater{ - transaction{ctx, txn}, d, roomNID, stateAndRefs, lastEventIDSent, currentStateSnapshotNID, - }, nil -} - -type roomRecentEventsUpdater struct { - transaction - d *Database - roomNID types.RoomNID - latestEvents []types.StateAtEventAndReference - lastEventIDSent string - currentStateSnapshotNID types.StateSnapshotNID -} - -// RoomVersion implements types.RoomRecentEventsUpdater -func (u *roomRecentEventsUpdater) RoomVersion() (version gomatrixserverlib.RoomVersion) { - version, _ = u.d.GetRoomVersionForRoomNID(u.ctx, u.roomNID) - return -} - -// LatestEvents implements types.RoomRecentEventsUpdater -func (u *roomRecentEventsUpdater) LatestEvents() []types.StateAtEventAndReference { - return u.latestEvents -} - -// LastEventIDSent implements types.RoomRecentEventsUpdater -func (u *roomRecentEventsUpdater) LastEventIDSent() string { - return u.lastEventIDSent -} - -// CurrentStateSnapshotNID implements types.RoomRecentEventsUpdater -func (u *roomRecentEventsUpdater) CurrentStateSnapshotNID() types.StateSnapshotNID { - return u.currentStateSnapshotNID -} - -// StorePreviousEvents implements types.RoomRecentEventsUpdater -func (u *roomRecentEventsUpdater) StorePreviousEvents(eventNID types.EventNID, previousEventReferences []gomatrixserverlib.EventReference) error { - for _, ref := range previousEventReferences { - if err := u.d.prevEvents.InsertPreviousEvent(u.ctx, u.txn, ref.EventID, ref.EventSHA256, eventNID); err != nil { - return err - } - } - return nil -} - -// IsReferenced implements types.RoomRecentEventsUpdater -func (u *roomRecentEventsUpdater) IsReferenced(eventReference gomatrixserverlib.EventReference) (bool, error) { - err := u.d.prevEvents.SelectPreviousEventExists(u.ctx, u.txn, eventReference.EventID, eventReference.EventSHA256) - if err == nil { - return true, nil - } - if err == sql.ErrNoRows { - return false, nil - } - return false, err -} - -// SetLatestEvents implements types.RoomRecentEventsUpdater -func (u *roomRecentEventsUpdater) SetLatestEvents( - roomNID types.RoomNID, latest []types.StateAtEventAndReference, lastEventNIDSent types.EventNID, - currentStateSnapshotNID types.StateSnapshotNID, -) error { - eventNIDs := make([]types.EventNID, len(latest)) - for i := range latest { - eventNIDs[i] = latest[i].EventNID - } - return u.d.rooms.UpdateLatestEventNIDs(u.ctx, u.txn, roomNID, eventNIDs, lastEventNIDSent, currentStateSnapshotNID) -} - -// HasEventBeenSent implements types.RoomRecentEventsUpdater -func (u *roomRecentEventsUpdater) HasEventBeenSent(eventNID types.EventNID) (bool, error) { - return u.d.events.SelectEventSentToOutput(u.ctx, u.txn, eventNID) -} - -// MarkEventAsSent implements types.RoomRecentEventsUpdater -func (u *roomRecentEventsUpdater) MarkEventAsSent(eventNID types.EventNID) error { - return u.d.events.UpdateEventSentToOutput(u.ctx, u.txn, eventNID) -} - -func (u *roomRecentEventsUpdater) MembershipUpdater(targetUserNID types.EventStateKeyNID, targetLocal bool) (types.MembershipUpdater, error) { - return u.d.membershipUpdaterTxn(u.ctx, u.txn, u.roomNID, targetUserNID, targetLocal) -} - -// MembershipUpdater implements input.RoomEventDatabase -func (d *Database) MembershipUpdater( - ctx context.Context, roomID, targetUserID string, - targetLocal bool, roomVersion gomatrixserverlib.RoomVersion, -) (types.MembershipUpdater, error) { - txn, err := d.db.Begin() - if err != nil { - return nil, err - } - succeeded := false - defer func() { - if !succeeded { - txn.Rollback() // nolint: errcheck - } - }() - - roomNID, err := d.assignRoomNID(ctx, txn, roomID, roomVersion) - if err != nil { - return nil, err - } - - targetUserNID, err := d.assignStateKeyNID(ctx, txn, targetUserID) - if err != nil { - return nil, err - } - - updater, err := d.membershipUpdaterTxn(ctx, txn, roomNID, targetUserNID, targetLocal) - if err != nil { - return nil, err - } - - succeeded = true - return updater, nil -} - -type membershipUpdater struct { - transaction - d *Database - roomNID types.RoomNID - targetUserNID types.EventStateKeyNID - membership tables.MembershipState -} - -func (d *Database) membershipUpdaterTxn( - ctx context.Context, - txn *sql.Tx, - roomNID types.RoomNID, - targetUserNID types.EventStateKeyNID, - targetLocal bool, -) (types.MembershipUpdater, error) { - - if err := d.membership.InsertMembership(ctx, txn, roomNID, targetUserNID, targetLocal); err != nil { - return nil, err - } - - membership, err := d.membership.SelectMembershipForUpdate(ctx, txn, roomNID, targetUserNID) - if err != nil { - return nil, err - } - - return &membershipUpdater{ - transaction{ctx, txn}, d, roomNID, targetUserNID, membership, - }, nil -} - -// IsInvite implements types.MembershipUpdater -func (u *membershipUpdater) IsInvite() bool { - return u.membership == tables.MembershipStateInvite -} - -// IsJoin implements types.MembershipUpdater -func (u *membershipUpdater) IsJoin() bool { - return u.membership == tables.MembershipStateJoin -} - -// IsLeave implements types.MembershipUpdater -func (u *membershipUpdater) IsLeave() bool { - return u.membership == tables.MembershipStateLeaveOrBan -} - -// SetToInvite implements types.MembershipUpdater -func (u *membershipUpdater) SetToInvite(event gomatrixserverlib.Event) (bool, error) { - senderUserNID, err := u.d.assignStateKeyNID(u.ctx, u.txn, event.Sender()) - if err != nil { - return false, err - } - inserted, err := u.d.invites.InsertInviteEvent( - u.ctx, u.txn, event.EventID(), u.roomNID, u.targetUserNID, senderUserNID, event.JSON(), - ) - if err != nil { - return false, err - } - if u.membership != tables.MembershipStateInvite { - if err = u.d.membership.UpdateMembership( - u.ctx, u.txn, u.roomNID, u.targetUserNID, senderUserNID, tables.MembershipStateInvite, 0, - ); err != nil { - return false, err - } - } - return inserted, nil -} - -// SetToJoin implements types.MembershipUpdater -func (u *membershipUpdater) SetToJoin(senderUserID string, eventID string, isUpdate bool) ([]string, error) { - var inviteEventIDs []string - - senderUserNID, err := u.d.assignStateKeyNID(u.ctx, u.txn, senderUserID) - if err != nil { - return nil, err - } - - // If this is a join event update, there is no invite to update - if !isUpdate { - inviteEventIDs, err = u.d.invites.UpdateInviteRetired( - u.ctx, u.txn, u.roomNID, u.targetUserNID, - ) - if err != nil { - return nil, err - } - } - - // Look up the NID of the new join event - nIDs, err := u.d.EventNIDs(u.ctx, []string{eventID}) - if err != nil { - return nil, err - } - - if u.membership != tables.MembershipStateJoin || isUpdate { - if err = u.d.membership.UpdateMembership( - u.ctx, u.txn, u.roomNID, u.targetUserNID, senderUserNID, - tables.MembershipStateJoin, nIDs[eventID], - ); err != nil { - return nil, err - } - } - - return inviteEventIDs, nil -} - -// SetToLeave implements types.MembershipUpdater -func (u *membershipUpdater) SetToLeave(senderUserID string, eventID string) ([]string, error) { - senderUserNID, err := u.d.assignStateKeyNID(u.ctx, u.txn, senderUserID) - if err != nil { - return nil, err - } - inviteEventIDs, err := u.d.invites.UpdateInviteRetired( - u.ctx, u.txn, u.roomNID, u.targetUserNID, - ) - if err != nil { - return nil, err - } - - // Look up the NID of the new leave event - nIDs, err := u.d.EventNIDs(u.ctx, []string{eventID}) - if err != nil { - return nil, err - } - - if u.membership != tables.MembershipStateLeaveOrBan { - if err = u.d.membership.UpdateMembership( - u.ctx, u.txn, u.roomNID, u.targetUserNID, senderUserNID, - tables.MembershipStateLeaveOrBan, nIDs[eventID], - ); err != nil { - return nil, err - } - } - return inviteEventIDs, nil -} - -type transaction struct { - ctx context.Context - txn *sql.Tx -} - -// Commit implements types.Transaction -func (t *transaction) Commit() error { - return t.txn.Commit() -} - -// Rollback implements types.Transaction -func (t *transaction) Rollback() error { - return t.txn.Rollback() -} diff --git a/roomserver/storage/shared/membership_updater.go b/roomserver/storage/shared/membership_updater.go new file mode 100644 index 000000000..5ddf6d84d --- /dev/null +++ b/roomserver/storage/shared/membership_updater.go @@ -0,0 +1,183 @@ +package shared + +import ( + "context" + "database/sql" + + "github.com/matrix-org/dendrite/roomserver/storage/tables" + "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/gomatrixserverlib" +) + +type membershipUpdater struct { + transaction + d *Database + roomNID types.RoomNID + targetUserNID types.EventStateKeyNID + membership tables.MembershipState +} + +func NewMembershipUpdater( + ctx context.Context, d *Database, roomID, targetUserID string, + targetLocal bool, roomVersion gomatrixserverlib.RoomVersion, + useTxns bool, +) (types.MembershipUpdater, error) { + txn, err := d.DB.Begin() + if err != nil { + return nil, err + } + succeeded := false + defer func() { + if !succeeded { + txn.Rollback() // nolint: errcheck + } + }() + + roomNID, err := d.assignRoomNID(ctx, txn, roomID, roomVersion) + if err != nil { + return nil, err + } + + targetUserNID, err := d.assignStateKeyNID(ctx, txn, targetUserID) + if err != nil { + return nil, err + } + + updater, err := d.membershipUpdaterTxn(ctx, txn, roomNID, targetUserNID, targetLocal) + if err != nil { + return nil, err + } + + succeeded = true + if !useTxns { + txn.Commit() // nolint: errcheck + updater.transaction.txn = nil + } + return updater, nil +} + +func (d *Database) membershipUpdaterTxn( + ctx context.Context, + txn *sql.Tx, + roomNID types.RoomNID, + targetUserNID types.EventStateKeyNID, + targetLocal bool, +) (*membershipUpdater, error) { + + if err := d.MembershipTable.InsertMembership(ctx, txn, roomNID, targetUserNID, targetLocal); err != nil { + return nil, err + } + + membership, err := d.MembershipTable.SelectMembershipForUpdate(ctx, txn, roomNID, targetUserNID) + if err != nil { + return nil, err + } + + return &membershipUpdater{ + transaction{ctx, txn}, d, roomNID, targetUserNID, membership, + }, nil +} + +// IsInvite implements types.MembershipUpdater +func (u *membershipUpdater) IsInvite() bool { + return u.membership == tables.MembershipStateInvite +} + +// IsJoin implements types.MembershipUpdater +func (u *membershipUpdater) IsJoin() bool { + return u.membership == tables.MembershipStateJoin +} + +// IsLeave implements types.MembershipUpdater +func (u *membershipUpdater) IsLeave() bool { + return u.membership == tables.MembershipStateLeaveOrBan +} + +// SetToInvite implements types.MembershipUpdater +func (u *membershipUpdater) SetToInvite(event gomatrixserverlib.Event) (bool, error) { + senderUserNID, err := u.d.assignStateKeyNID(u.ctx, u.txn, event.Sender()) + if err != nil { + return false, err + } + inserted, err := u.d.InvitesTable.InsertInviteEvent( + u.ctx, u.txn, event.EventID(), u.roomNID, u.targetUserNID, senderUserNID, event.JSON(), + ) + if err != nil { + return false, err + } + if u.membership != tables.MembershipStateInvite { + if err = u.d.MembershipTable.UpdateMembership( + u.ctx, u.txn, u.roomNID, u.targetUserNID, senderUserNID, tables.MembershipStateInvite, 0, + ); err != nil { + return false, err + } + } + return inserted, nil +} + +// SetToJoin implements types.MembershipUpdater +func (u *membershipUpdater) SetToJoin(senderUserID string, eventID string, isUpdate bool) ([]string, error) { + var inviteEventIDs []string + + senderUserNID, err := u.d.assignStateKeyNID(u.ctx, u.txn, senderUserID) + if err != nil { + return nil, err + } + + // If this is a join event update, there is no invite to update + if !isUpdate { + inviteEventIDs, err = u.d.InvitesTable.UpdateInviteRetired( + u.ctx, u.txn, u.roomNID, u.targetUserNID, + ) + if err != nil { + return nil, err + } + } + + // Look up the NID of the new join event + nIDs, err := u.d.EventNIDs(u.ctx, []string{eventID}) + if err != nil { + return nil, err + } + + if u.membership != tables.MembershipStateJoin || isUpdate { + if err = u.d.MembershipTable.UpdateMembership( + u.ctx, u.txn, u.roomNID, u.targetUserNID, senderUserNID, + tables.MembershipStateJoin, nIDs[eventID], + ); err != nil { + return nil, err + } + } + + return inviteEventIDs, nil +} + +// SetToLeave implements types.MembershipUpdater +func (u *membershipUpdater) SetToLeave(senderUserID string, eventID string) ([]string, error) { + senderUserNID, err := u.d.assignStateKeyNID(u.ctx, u.txn, senderUserID) + if err != nil { + return nil, err + } + inviteEventIDs, err := u.d.InvitesTable.UpdateInviteRetired( + u.ctx, u.txn, u.roomNID, u.targetUserNID, + ) + if err != nil { + return nil, err + } + + // Look up the NID of the new leave event + nIDs, err := u.d.EventNIDs(u.ctx, []string{eventID}) + if err != nil { + return nil, err + } + + if u.membership != tables.MembershipStateLeaveOrBan { + if err = u.d.MembershipTable.UpdateMembership( + u.ctx, u.txn, u.roomNID, u.targetUserNID, senderUserNID, + tables.MembershipStateLeaveOrBan, nIDs[eventID], + ); err != nil { + return nil, err + } + } + return inviteEventIDs, nil +} diff --git a/roomserver/storage/shared/prepare.go b/roomserver/storage/shared/prepare.go index 1b3497fd8..65ceec1cc 100644 --- a/roomserver/storage/shared/prepare.go +++ b/roomserver/storage/shared/prepare.go @@ -16,6 +16,7 @@ package shared import ( + "context" "database/sql" ) @@ -34,3 +35,26 @@ func (s StatementList) Prepare(db *sql.DB) (err error) { } return } + +type transaction struct { + ctx context.Context + txn *sql.Tx +} + +// Commit implements types.Transaction +func (t *transaction) Commit() error { + if t.txn == nil { + // The Updater structs can operate in useTxns=false mode. The code will still call this though. + return nil + } + return t.txn.Commit() +} + +// Rollback implements types.Transaction +func (t *transaction) Rollback() error { + if t.txn == nil { + // The Updater structs can operate in useTxns=false mode. The code will still call this though. + return nil + } + return t.txn.Rollback() +} diff --git a/roomserver/storage/shared/room_recent_events_updater.go b/roomserver/storage/shared/room_recent_events_updater.go new file mode 100644 index 000000000..8131f712d --- /dev/null +++ b/roomserver/storage/shared/room_recent_events_updater.go @@ -0,0 +1,120 @@ +package shared + +import ( + "context" + "database/sql" + + "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/gomatrixserverlib" +) + +type roomRecentEventsUpdater struct { + transaction + d *Database + roomNID types.RoomNID + latestEvents []types.StateAtEventAndReference + lastEventIDSent string + currentStateSnapshotNID types.StateSnapshotNID +} + +func NewRoomRecentEventsUpdater(d *Database, ctx context.Context, roomNID types.RoomNID, useTxns bool) (types.RoomRecentEventsUpdater, error) { + txn, err := d.DB.Begin() + if err != nil { + return nil, err + } + eventNIDs, lastEventNIDSent, currentStateSnapshotNID, err := + d.RoomsTable.SelectLatestEventsNIDsForUpdate(ctx, txn, roomNID) + if err != nil { + txn.Rollback() // nolint: errcheck + return nil, err + } + stateAndRefs, err := d.EventsTable.BulkSelectStateAtEventAndReference(ctx, txn, eventNIDs) + if err != nil { + txn.Rollback() // nolint: errcheck + return nil, err + } + var lastEventIDSent string + if lastEventNIDSent != 0 { + lastEventIDSent, err = d.EventsTable.SelectEventID(ctx, txn, lastEventNIDSent) + if err != nil { + txn.Rollback() // nolint: errcheck + return nil, err + } + } + if !useTxns { + txn.Commit() // nolint: errcheck + txn = nil + } + return &roomRecentEventsUpdater{ + transaction{ctx, txn}, d, roomNID, stateAndRefs, lastEventIDSent, currentStateSnapshotNID, + }, nil +} + +// RoomVersion implements types.RoomRecentEventsUpdater +func (u *roomRecentEventsUpdater) RoomVersion() (version gomatrixserverlib.RoomVersion) { + version, _ = u.d.GetRoomVersionForRoomNID(u.ctx, u.roomNID) + return +} + +// LatestEvents implements types.RoomRecentEventsUpdater +func (u *roomRecentEventsUpdater) LatestEvents() []types.StateAtEventAndReference { + return u.latestEvents +} + +// LastEventIDSent implements types.RoomRecentEventsUpdater +func (u *roomRecentEventsUpdater) LastEventIDSent() string { + return u.lastEventIDSent +} + +// CurrentStateSnapshotNID implements types.RoomRecentEventsUpdater +func (u *roomRecentEventsUpdater) CurrentStateSnapshotNID() types.StateSnapshotNID { + return u.currentStateSnapshotNID +} + +// StorePreviousEvents implements types.RoomRecentEventsUpdater +func (u *roomRecentEventsUpdater) StorePreviousEvents(eventNID types.EventNID, previousEventReferences []gomatrixserverlib.EventReference) error { + for _, ref := range previousEventReferences { + if err := u.d.PrevEventsTable.InsertPreviousEvent(u.ctx, u.txn, ref.EventID, ref.EventSHA256, eventNID); err != nil { + return err + } + } + return nil +} + +// IsReferenced implements types.RoomRecentEventsUpdater +func (u *roomRecentEventsUpdater) IsReferenced(eventReference gomatrixserverlib.EventReference) (bool, error) { + err := u.d.PrevEventsTable.SelectPreviousEventExists(u.ctx, u.txn, eventReference.EventID, eventReference.EventSHA256) + if err == nil { + return true, nil + } + if err == sql.ErrNoRows { + return false, nil + } + return false, err +} + +// SetLatestEvents implements types.RoomRecentEventsUpdater +func (u *roomRecentEventsUpdater) SetLatestEvents( + roomNID types.RoomNID, latest []types.StateAtEventAndReference, lastEventNIDSent types.EventNID, + currentStateSnapshotNID types.StateSnapshotNID, +) error { + eventNIDs := make([]types.EventNID, len(latest)) + for i := range latest { + eventNIDs[i] = latest[i].EventNID + } + return u.d.RoomsTable.UpdateLatestEventNIDs(u.ctx, u.txn, roomNID, eventNIDs, lastEventNIDSent, currentStateSnapshotNID) +} + +// HasEventBeenSent implements types.RoomRecentEventsUpdater +func (u *roomRecentEventsUpdater) HasEventBeenSent(eventNID types.EventNID) (bool, error) { + return u.d.EventsTable.SelectEventSentToOutput(u.ctx, u.txn, eventNID) +} + +// MarkEventAsSent implements types.RoomRecentEventsUpdater +func (u *roomRecentEventsUpdater) MarkEventAsSent(eventNID types.EventNID) error { + return u.d.EventsTable.UpdateEventSentToOutput(u.ctx, u.txn, eventNID) +} + +func (u *roomRecentEventsUpdater) MembershipUpdater(targetUserNID types.EventStateKeyNID, targetLocal bool) (types.MembershipUpdater, error) { + return u.d.membershipUpdaterTxn(u.ctx, u.txn, u.roomNID, targetUserNID, targetLocal) +} diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index 17bcd6964..bb5b51562 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -311,6 +311,19 @@ func (d *Database) GetTransactionEventID( return eventID, err } +func (d *Database) MembershipUpdater( + ctx context.Context, roomID, targetUserID string, + targetLocal bool, roomVersion gomatrixserverlib.RoomVersion, +) (types.MembershipUpdater, error) { + return NewMembershipUpdater(ctx, d, roomID, targetUserID, targetLocal, roomVersion, true) +} + +func (d *Database) GetLatestEventsForUpdate( + ctx context.Context, roomNID types.RoomNID, +) (types.RoomRecentEventsUpdater, error) { + return NewRoomRecentEventsUpdater(d, ctx, roomNID, true) +} + func (d *Database) StoreEvent( ctx context.Context, event gomatrixserverlib.Event, txnAndSessionID *api.TransactionID, authEventNIDs []types.EventNID, diff --git a/roomserver/storage/sqlite3/invite_table.go b/roomserver/storage/sqlite3/invite_table.go index fbe2b0729..7dcc2dc0b 100644 --- a/roomserver/storage/sqlite3/invite_table.go +++ b/roomserver/storage/sqlite3/invite_table.go @@ -90,7 +90,6 @@ func (s *inviteStatements) InsertInviteEvent( inviteEventJSON []byte, ) (bool, error) { stmt := internal.TxStmt(txn, s.insertInviteEventStmt) - defer stmt.Close() // nolint: errcheck result, err := stmt.ExecContext( ctx, inviteEventID, roomNID, targetUserNID, senderUserNID, inviteEventJSON, ) @@ -109,7 +108,7 @@ func (s *inviteStatements) UpdateInviteRetired( txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, ) (eventIDs []string, err error) { // gather all the event IDs we will retire - stmt := txn.Stmt(s.selectInvitesAboutToRetireStmt) + stmt := internal.TxStmt(txn, s.selectInvitesAboutToRetireStmt) rows, err := stmt.QueryContext(ctx, roomNID, targetUserNID) if err != nil { return nil, err @@ -124,7 +123,7 @@ func (s *inviteStatements) UpdateInviteRetired( } // now retire the invites - stmt = txn.Stmt(s.updateInviteRetiredStmt) + stmt = internal.TxStmt(txn, s.updateInviteRetiredStmt) _, err = stmt.ExecContext(ctx, roomNID, targetUserNID) return } diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index 16d893044..54a2f2658 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -140,387 +140,27 @@ func Open(dataSourceName string) (*Database, error) { return &d, nil } -func (d *Database) assignRoomNID( - ctx context.Context, txn *sql.Tx, - roomID string, roomVersion gomatrixserverlib.RoomVersion, -) (roomNID types.RoomNID, err error) { - // Check if we already have a numeric ID in the database. - roomNID, err = d.rooms.SelectRoomNID(ctx, txn, roomID) - if err == sql.ErrNoRows { - // We don't have a numeric ID so insert one into the database. - roomNID, err = d.rooms.InsertRoomNID(ctx, txn, roomID, roomVersion) - if err == nil { - // Now get the numeric ID back out of the database - roomNID, err = d.rooms.SelectRoomNID(ctx, txn, roomID) - } - } - return -} - -func (d *Database) assignStateKeyNID( - ctx context.Context, txn *sql.Tx, eventStateKey string, -) (eventStateKeyNID types.EventStateKeyNID, err error) { - // Check if we already have a numeric ID in the database. - eventStateKeyNID, err = d.eventStateKeys.SelectEventStateKeyNID(ctx, txn, eventStateKey) - if err == sql.ErrNoRows { - // We don't have a numeric ID so insert one into the database. - eventStateKeyNID, err = d.eventStateKeys.InsertEventStateKeyNID(ctx, txn, eventStateKey) - if err == sql.ErrNoRows { - // We raced with another insert so run the select again. - eventStateKeyNID, err = d.eventStateKeys.SelectEventStateKeyNID(ctx, txn, eventStateKey) - } - } - return -} - -// GetLatestEventsForUpdate implements input.EventDatabase func (d *Database) GetLatestEventsForUpdate( ctx context.Context, roomNID types.RoomNID, ) (types.RoomRecentEventsUpdater, error) { - txn, err := d.db.Begin() - if err != nil { - return nil, err - } - eventNIDs, lastEventNIDSent, currentStateSnapshotNID, err := - d.rooms.SelectLatestEventsNIDsForUpdate(ctx, txn, roomNID) - if err != nil { - txn.Rollback() // nolint: errcheck - return nil, err - } - stateAndRefs, err := d.events.BulkSelectStateAtEventAndReference(ctx, txn, eventNIDs) - if err != nil { - txn.Rollback() // nolint: errcheck - return nil, err - } - var lastEventIDSent string - if lastEventNIDSent != 0 { - lastEventIDSent, err = d.events.SelectEventID(ctx, txn, lastEventNIDSent) - if err != nil { - txn.Rollback() // nolint: errcheck - return nil, err - } - } - - // FIXME: we probably want to support long-lived txns in sqlite somehow, but we don't because we get - // 'database is locked' errors caused by multiple write txns (one being the long-lived txn created here) - // so for now let's not use a long-lived txn at all, and just commit it here and set the txn to nil so - // we fail fast if someone tries to use the underlying txn object. - err = txn.Commit() - if err != nil { - return nil, err - } - return &roomRecentEventsUpdater{ - transaction{ctx, nil}, d, roomNID, stateAndRefs, lastEventIDSent, currentStateSnapshotNID, - }, nil + // TODO: Do not use transactions. We should be holding open this transaction but we cannot have + // multiple write transactions on sqlite. The code will perform additional + // write transactions independent of this one which will consistently cause + // 'database is locked' errors. As sqlite doesn't support multi-process on the + // same DB anyway, and we only execute updates sequentially, the only worries + // are for rolling back when things go wrong. (atomicity) + return shared.NewRoomRecentEventsUpdater(&d.Database, ctx, roomNID, false) } -type roomRecentEventsUpdater struct { - transaction - d *Database - roomNID types.RoomNID - latestEvents []types.StateAtEventAndReference - lastEventIDSent string - currentStateSnapshotNID types.StateSnapshotNID -} - -// RoomVersion implements types.RoomRecentEventsUpdater -func (u *roomRecentEventsUpdater) RoomVersion() (version gomatrixserverlib.RoomVersion) { - version, _ = u.d.GetRoomVersionForRoomNID(u.ctx, u.roomNID) - return -} - -// LatestEvents implements types.RoomRecentEventsUpdater -func (u *roomRecentEventsUpdater) LatestEvents() []types.StateAtEventAndReference { - return u.latestEvents -} - -// LastEventIDSent implements types.RoomRecentEventsUpdater -func (u *roomRecentEventsUpdater) LastEventIDSent() string { - return u.lastEventIDSent -} - -// CurrentStateSnapshotNID implements types.RoomRecentEventsUpdater -func (u *roomRecentEventsUpdater) CurrentStateSnapshotNID() types.StateSnapshotNID { - return u.currentStateSnapshotNID -} - -// StorePreviousEvents implements types.RoomRecentEventsUpdater -func (u *roomRecentEventsUpdater) StorePreviousEvents(eventNID types.EventNID, previousEventReferences []gomatrixserverlib.EventReference) error { - err := internal.WithTransaction(u.d.db, func(txn *sql.Tx) error { - for _, ref := range previousEventReferences { - if err := u.d.prevEvents.InsertPreviousEvent(u.ctx, txn, ref.EventID, ref.EventSHA256, eventNID); err != nil { - return err - } - } - return nil - }) - return err -} - -// IsReferenced implements types.RoomRecentEventsUpdater -func (u *roomRecentEventsUpdater) IsReferenced(eventReference gomatrixserverlib.EventReference) (res bool, err error) { - err = internal.WithTransaction(u.d.db, func(txn *sql.Tx) error { - err := u.d.prevEvents.SelectPreviousEventExists(u.ctx, txn, eventReference.EventID, eventReference.EventSHA256) - if err == nil { - res = true - err = nil - } - if err == sql.ErrNoRows { - res = false - err = nil - } - return err - }) - return -} - -// SetLatestEvents implements types.RoomRecentEventsUpdater -func (u *roomRecentEventsUpdater) SetLatestEvents( - roomNID types.RoomNID, latest []types.StateAtEventAndReference, lastEventNIDSent types.EventNID, - currentStateSnapshotNID types.StateSnapshotNID, -) error { - err := internal.WithTransaction(u.d.db, func(txn *sql.Tx) error { - eventNIDs := make([]types.EventNID, len(latest)) - for i := range latest { - eventNIDs[i] = latest[i].EventNID - } - return u.d.rooms.UpdateLatestEventNIDs(u.ctx, txn, roomNID, eventNIDs, lastEventNIDSent, currentStateSnapshotNID) - }) - return err -} - -// HasEventBeenSent implements types.RoomRecentEventsUpdater -func (u *roomRecentEventsUpdater) HasEventBeenSent(eventNID types.EventNID) (res bool, err error) { - err = internal.WithTransaction(u.d.db, func(txn *sql.Tx) error { - res, err = u.d.events.SelectEventSentToOutput(u.ctx, txn, eventNID) - return err - }) - return -} - -// MarkEventAsSent implements types.RoomRecentEventsUpdater -func (u *roomRecentEventsUpdater) MarkEventAsSent(eventNID types.EventNID) error { - err := internal.WithTransaction(u.d.db, func(txn *sql.Tx) error { - return u.d.events.UpdateEventSentToOutput(u.ctx, txn, eventNID) - }) - return err -} - -func (u *roomRecentEventsUpdater) MembershipUpdater(targetUserNID types.EventStateKeyNID, targetLocal bool) (mu types.MembershipUpdater, err error) { - err = internal.WithTransaction(u.d.db, func(txn *sql.Tx) error { - mu, err = u.d.membershipUpdaterTxn(u.ctx, txn, u.roomNID, targetUserNID, targetLocal) - return err - }) - return -} - -// MembershipUpdater implements input.RoomEventDatabase func (d *Database) MembershipUpdater( ctx context.Context, roomID, targetUserID string, targetLocal bool, roomVersion gomatrixserverlib.RoomVersion, ) (updater types.MembershipUpdater, err error) { - var txn *sql.Tx - txn, err = d.db.Begin() - if err != nil { - return nil, err - } - succeeded := false - defer func() { - if !succeeded { - txn.Rollback() // nolint: errcheck - } else { - // TODO: We should be holding open this transaction but we cannot have - // multiple write transactions on sqlite. The code will perform additional - // write transactions independent of this one which will consistently cause - // 'database is locked' errors. For now, we'll break up the transaction and - // hope we don't race too catastrophically. Long term, we should be able to - // thread in txn objects where appropriate (either at the interface level or - // bring matrix business logic into the storage layer). - txerr := txn.Commit() - if err == nil && txerr != nil { - err = txerr - } - } - }() - - roomNID, err := d.assignRoomNID(ctx, txn, roomID, roomVersion) - if err != nil { - return nil, err - } - - targetUserNID, err := d.assignStateKeyNID(ctx, txn, targetUserID) - if err != nil { - return nil, err - } - - updater, err = d.membershipUpdaterTxn(ctx, txn, roomNID, targetUserNID, targetLocal) - if err != nil { - return nil, err - } - - succeeded = true - return updater, nil -} - -type membershipUpdater struct { - transaction - d *Database - roomNID types.RoomNID - targetUserNID types.EventStateKeyNID - membership tables.MembershipState -} - -func (d *Database) membershipUpdaterTxn( - ctx context.Context, - txn *sql.Tx, - roomNID types.RoomNID, - targetUserNID types.EventStateKeyNID, - targetLocal bool, -) (types.MembershipUpdater, error) { - - if err := d.membership.InsertMembership(ctx, txn, roomNID, targetUserNID, targetLocal); err != nil { - return nil, err - } - - membership, err := d.membership.SelectMembershipForUpdate(ctx, txn, roomNID, targetUserNID) - if err != nil { - return nil, err - } - - return &membershipUpdater{ - // purposefully set the txn to nil so if we try to use it we panic and fail fast - transaction{ctx, nil}, d, roomNID, targetUserNID, membership, - }, nil -} - -// IsInvite implements types.MembershipUpdater -func (u *membershipUpdater) IsInvite() bool { - return u.membership == tables.MembershipStateInvite -} - -// IsJoin implements types.MembershipUpdater -func (u *membershipUpdater) IsJoin() bool { - return u.membership == tables.MembershipStateJoin -} - -// IsLeave implements types.MembershipUpdater -func (u *membershipUpdater) IsLeave() bool { - return u.membership == tables.MembershipStateLeaveOrBan -} - -// SetToInvite implements types.MembershipUpdater -func (u *membershipUpdater) SetToInvite(event gomatrixserverlib.Event) (inserted bool, err error) { - err = internal.WithTransaction(u.d.db, func(txn *sql.Tx) error { - senderUserNID, err := u.d.assignStateKeyNID(u.ctx, txn, event.Sender()) - if err != nil { - return err - } - inserted, err = u.d.invites.InsertInviteEvent( - u.ctx, txn, event.EventID(), u.roomNID, u.targetUserNID, senderUserNID, event.JSON(), - ) - if err != nil { - return err - } - if u.membership != tables.MembershipStateInvite { - if err = u.d.membership.UpdateMembership( - u.ctx, txn, u.roomNID, u.targetUserNID, senderUserNID, tables.MembershipStateInvite, 0, - ); err != nil { - return err - } - } - return nil - }) - return -} - -// SetToJoin implements types.MembershipUpdater -func (u *membershipUpdater) SetToJoin(senderUserID string, eventID string, isUpdate bool) (inviteEventIDs []string, err error) { - err = internal.WithTransaction(u.d.db, func(txn *sql.Tx) error { - senderUserNID, err := u.d.assignStateKeyNID(u.ctx, txn, senderUserID) - if err != nil { - return err - } - - // If this is a join event update, there is no invite to update - if !isUpdate { - inviteEventIDs, err = u.d.invites.UpdateInviteRetired( - u.ctx, txn, u.roomNID, u.targetUserNID, - ) - if err != nil { - return err - } - } - - // Look up the NID of the new join event - nIDs, err := u.d.EventNIDs(u.ctx, []string{eventID}) - if err != nil { - return err - } - - if u.membership != tables.MembershipStateJoin || isUpdate { - if err = u.d.membership.UpdateMembership( - u.ctx, txn, u.roomNID, u.targetUserNID, senderUserNID, - tables.MembershipStateJoin, nIDs[eventID], - ); err != nil { - return err - } - } - return nil - }) - - return -} - -// SetToLeave implements types.MembershipUpdater -func (u *membershipUpdater) SetToLeave(senderUserID string, eventID string) (inviteEventIDs []string, err error) { - err = internal.WithTransaction(u.d.db, func(txn *sql.Tx) error { - senderUserNID, err := u.d.assignStateKeyNID(u.ctx, txn, senderUserID) - if err != nil { - return err - } - inviteEventIDs, err = u.d.invites.UpdateInviteRetired( - u.ctx, txn, u.roomNID, u.targetUserNID, - ) - if err != nil { - return err - } - - // Look up the NID of the new leave event - nIDs, err := u.d.EventNIDs(u.ctx, []string{eventID}) - if err != nil { - return err - } - - if u.membership != tables.MembershipStateLeaveOrBan { - if err = u.d.membership.UpdateMembership( - u.ctx, txn, u.roomNID, u.targetUserNID, senderUserNID, - tables.MembershipStateLeaveOrBan, nIDs[eventID], - ); err != nil { - return err - } - } - return nil - }) - return -} - -type transaction struct { - ctx context.Context - txn *sql.Tx -} - -// Commit implements types.Transaction -func (t *transaction) Commit() error { - if t.txn == nil { - return nil - } - return t.txn.Commit() -} - -// Rollback implements types.Transaction -func (t *transaction) Rollback() error { - if t.txn == nil { - return nil - } - return t.txn.Rollback() + // TODO: Do not use transactions. We should be holding open this transaction but we cannot have + // multiple write transactions on sqlite. The code will perform additional + // write transactions independent of this one which will consistently cause + // 'database is locked' errors. As sqlite doesn't support multi-process on the + // same DB anyway, and we only execute updates sequentially, the only worries + // are for rolling back when things go wrong. (atomicity) + return shared.NewMembershipUpdater(ctx, &d.Database, roomID, targetUserID, targetLocal, roomVersion, false) } From 2411007c4bab8b5c851f3e1e9552cd7a69bbcd9b Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 28 May 2020 12:28:21 +0100 Subject: [PATCH 104/200] Parse URIs correctly --- serverkeyapi/storage/sqlite3/keydb.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/serverkeyapi/storage/sqlite3/keydb.go b/serverkeyapi/storage/sqlite3/keydb.go index d1dc61c98..1ad6e7e88 100644 --- a/serverkeyapi/storage/sqlite3/keydb.go +++ b/serverkeyapi/storage/sqlite3/keydb.go @@ -17,6 +17,8 @@ package sqlite3 import ( "context" + "errors" + "net/url" "time" "golang.org/x/crypto/ed25519" @@ -44,7 +46,19 @@ func NewDatabase( serverKey ed25519.PublicKey, serverKeyID gomatrixserverlib.KeyID, ) (*Database, error) { - db, err := sqlutil.Open(internal.SQLiteDriverName(), dataSourceName, nil) + uri, err := url.Parse(dataSourceName) + if err != nil { + return nil, err + } + var cs string + if uri.Opaque != "" { // file:filename.db + cs = uri.Opaque + } else if uri.Path != "" { // file:///path/to/filename.db + cs = uri.Path + } else { + return nil, errors.New("no filename or path in connect string") + } + db, err := sqlutil.Open(internal.SQLiteDriverName(), cs, nil) if err != nil { return nil, err } From f123d7d65b43b5ded3abcbaa032832261f2b2686 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 28 May 2020 12:44:34 +0100 Subject: [PATCH 105/200] Hopefully fix get_missing_events (#1070) --- federationapi/routing/missingevents.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/federationapi/routing/missingevents.go b/federationapi/routing/missingevents.go index ae91c5891..f93e0eb41 100644 --- a/federationapi/routing/missingevents.go +++ b/federationapi/routing/missingevents.go @@ -60,9 +60,14 @@ func GetMissingEvents( } eventsResponse.Events = filterEvents(eventsResponse.Events, gme.MinDepth, roomID) + + resp := gomatrixserverlib.RespMissingEvents{ + Events: gomatrixserverlib.UnwrapEventHeaders(eventsResponse.Events), + } + return util.JSONResponse{ Code: http.StatusOK, - JSON: eventsResponse, + JSON: resp, } } From 5307c499fe23090c409b1254b57423288ee3e609 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 28 May 2020 13:44:16 +0100 Subject: [PATCH 106/200] Update gomatrixserverlib (#1071) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index fe2cff768..4365ea503 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200521102632-2a81324a04ae + github.com/matrix-org/gomatrixserverlib v0.0.0-20200528122156-fbb320a2ee61 github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible diff --git a/go.sum b/go.sum index 3398195b6..c08cfa5d5 100644 --- a/go.sum +++ b/go.sum @@ -356,8 +356,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 h1:Yb+Wlf github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bhrnp3Ky1qgx/fzCtCALOoGYylh2tpS9K4= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200521102632-2a81324a04ae h1:kFMh2aU3pMCkVCUeH57rtgm05XImbxKOHFYeUp80RCk= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200521102632-2a81324a04ae/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200528122156-fbb320a2ee61 h1:3rgoGvj/skUWg+u9E6ycEFs2ZGenEjr28ZtAhAhmZeM= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200528122156-fbb320a2ee61/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f/go.mod h1:y0oDTjZDv5SM9a2rp3bl+CU+bvTRINQsdb7YlDql5Go= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= From fe5cf6f880cdb906006a6ca215872949c3388272 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Fri, 29 May 2020 13:50:06 +0100 Subject: [PATCH 107/200] fedsender: de-duplicate without sorting server names (#1073) --- federationsender/api/perform.go | 5 +++-- federationsender/internal/perform.go | 15 +++++++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/federationsender/api/perform.go b/federationsender/api/perform.go index a73ba0475..2a1834b61 100644 --- a/federationsender/api/perform.go +++ b/federationsender/api/perform.go @@ -44,8 +44,9 @@ func (h *httpFederationSenderInternalAPI) PerformDirectoryLookup( } type PerformJoinRequest struct { - RoomID string `json:"room_id"` - UserID string `json:"user_id"` + RoomID string `json:"room_id"` + UserID string `json:"user_id"` + // The sorted list of servers to try. Servers will be tried sequentially, after de-duplication. ServerNames types.ServerNames `json:"server_names"` Content map[string]interface{} `json:"content"` } diff --git a/federationsender/internal/perform.go b/federationsender/internal/perform.go index 9791afefa..383ce4888 100644 --- a/federationsender/internal/perform.go +++ b/federationsender/internal/perform.go @@ -46,8 +46,19 @@ func (r *FederationSenderInternalAPI) PerformJoin( supportedVersions = append(supportedVersions, version) } - // Deduplicate the server names we were provided. - util.SortAndUnique(request.ServerNames) + // Deduplicate the server names we were provided but keep the ordering + // as this encodes useful information about which servers are most likely + // to respond. + seenSet := make(map[gomatrixserverlib.ServerName]bool) + var uniqueList []gomatrixserverlib.ServerName + for _, srv := range request.ServerNames { + if seenSet[srv] { + continue + } + seenSet[srv] = true + uniqueList = append(uniqueList, srv) + } + request.ServerNames = uniqueList // Try each server that we were provided until we land on one that // successfully completes the make-join send-join dance. From 97c64bdb6d1ce6ee4684f209dfa96db3545e240b Mon Sep 17 00:00:00 2001 From: Kegsay Date: Fri, 29 May 2020 15:34:15 +0100 Subject: [PATCH 108/200] p2p: de-dupe and shuffle public rooms; implement keydb (#1074) - We don't want a serverKeyAPI as fetching keys doesn't need a DB. - De-dupe rooms so we don't see them multiple times, but shuffle the alias we join via so we don't all flood a single server. --- cmd/dendritejs/keyfetcher.go | 5 +++++ cmd/dendritejs/main.go | 9 +++------ publicroomsapi/directory/public_rooms.go | 18 ++++++++++++++++++ 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/cmd/dendritejs/keyfetcher.go b/cmd/dendritejs/keyfetcher.go index ee4905d4f..47a81c41e 100644 --- a/cmd/dendritejs/keyfetcher.go +++ b/cmd/dendritejs/keyfetcher.go @@ -82,3 +82,8 @@ func (f *libp2pKeyFetcher) FetchKeys( func (f *libp2pKeyFetcher) FetcherName() string { return "libp2pKeyFetcher" } + +// no-op function for storing keys - we don't do any work to fetch them so don't bother storing. +func (f *libp2pKeyFetcher) StoreKeys(ctx context.Context, results map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult) error { + return nil +} diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index b8517bf9d..45f23d9ad 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -37,7 +37,6 @@ import ( "github.com/matrix-org/dendrite/publicroomsapi" "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/roomserver" - "github.com/matrix-org/dendrite/serverkeyapi" "github.com/matrix-org/dendrite/syncapi" go_http_js_libp2p "github.com/matrix-org/go-http-js-libp2p" "github.com/matrix-org/gomatrixserverlib" @@ -197,14 +196,12 @@ func main() { deviceDB := base.CreateDeviceDB() federation := createFederationClient(cfg, node) - serverKeyAPI := serverkeyapi.SetupServerKeyAPIComponent( - base, federation, - ) + fetcher := &libp2pKeyFetcher{} keyRing := gomatrixserverlib.KeyRing{ KeyFetchers: []gomatrixserverlib.KeyFetcher{ - &libp2pKeyFetcher{}, + fetcher, }, - KeyDatabase: serverKeyAPI, + KeyDatabase: fetcher, } p2pPublicRoomProvider := NewLibP2PPublicRoomsProvider(node) diff --git a/publicroomsapi/directory/public_rooms.go b/publicroomsapi/directory/public_rooms.go index 7bd6740eb..bc40da516 100644 --- a/publicroomsapi/directory/public_rooms.go +++ b/publicroomsapi/directory/public_rooms.go @@ -16,6 +16,7 @@ package directory import ( "context" + "math/rand" "net/http" "strconv" "sync" @@ -96,6 +97,23 @@ func GetPostPublicRoomsWithExternal( // downcasting `limit` is safe as we know it isn't bigger than request.Limit which is int16 fedRooms := bulkFetchPublicRoomsFromServers(req.Context(), fedClient, extRoomsProvider.Homeservers(), int16(limit)) response.Chunk = append(response.Chunk, fedRooms...) + + // de-duplicate rooms with the same room ID. We can join the room via any of these aliases as we know these servers + // are alive and well, so we arbitrarily pick one (purposefully shuffling them to spread the load a bit) + var publicRooms []gomatrixserverlib.PublicRoom + haveRoomIDs := make(map[string]bool) + rand.Shuffle(len(response.Chunk), func(i, j int) { + response.Chunk[i], response.Chunk[j] = response.Chunk[j], response.Chunk[i] + }) + for _, r := range response.Chunk { + if haveRoomIDs[r.RoomID] { + continue + } + haveRoomIDs[r.RoomID] = true + publicRooms = append(publicRooms, r) + } + response.Chunk = publicRooms + return util.JSONResponse{ Code: http.StatusOK, JSON: response, From 1f43c24f8602dfbc95620e9d34fac78a7b449c11 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 29 May 2020 21:00:28 +0100 Subject: [PATCH 109/200] Add Dockerfile for building p2p riot $ docker build -t dendritejs -f DendriteJS.Dockerfile . $ docker run --rm -p 8090:80 dendritejs --- build/docker/DendriteJS.Dockerfile | 112 +++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 build/docker/DendriteJS.Dockerfile diff --git a/build/docker/DendriteJS.Dockerfile b/build/docker/DendriteJS.Dockerfile new file mode 100644 index 000000000..2dc6ef01c --- /dev/null +++ b/build/docker/DendriteJS.Dockerfile @@ -0,0 +1,112 @@ +# This dockerfile will build dendritejs and hook it up to riot-web, build that then dump the +# resulting HTML/JS onto an nginx container for hosting. It requires no specific build context +# as it pulls archives straight from github branches. +FROM golang:1.13.7-alpine3.11 AS gobuild + +# TODO: This does nothing currently ;) +# or /dns4/rendezvous.matrix.org/tcp/8443/wss/p2p-websocket-star/ +# or whatever you want! +ENV RENDEZVOUS_SERVER=/ip4/127.0.0.1/tcp/9090/ws/p2p-websocket-star/ + +# Download and build dendrite +WORKDIR /build +ADD https://github.com/matrix-org/dendrite/archive/master.tar.gz /build/master.tar.gz +RUN tar xvfz master.tar.gz +WORKDIR /build/dendrite-master +RUN GOOS=js GOARCH=wasm go build -o main.wasm ./cmd/dendritejs + + +FROM node:14-stretch AS jsbuild +# apparently some deps require python +RUN apt-get update && apt-get -y install python + +# Download riot-web and libp2p repos +WORKDIR /build +ADD https://github.com/matrix-org/go-http-js-libp2p/archive/master.tar.gz /build/libp2p.tar.gz +RUN tar xvfz libp2p.tar.gz +ADD https://github.com/vector-im/riot-web/archive/matthew/p2p.tar.gz /build/p2p.tar.gz +RUN tar xvfz p2p.tar.gz + +# Install deps for riot-web, symlink in libp2p repo and build that too +WORKDIR /build/riot-web-matthew-p2p +RUN yarn install +RUN ln -s /build/go-http-js-libp2p-master /build/riot-web-matthew-p2p/node_modules/go-http-js-libp2p +RUN (cd node_modules/go-http-js-libp2p && yarn install) +COPY --from=gobuild /build/dendrite-master/main.wasm ./src/vector/dendrite.wasm +# build it all +RUN yarn build:p2p + +SHELL ["/bin/bash", "-c"] +RUN echo $'\ +{ \n\ + "default_server_config": { \n\ + "m.homeserver": { \n\ + "base_url": "https://p2p.riot.im", \n\ + "server_name": "p2p.riot.im" \n\ + }, \n\ + "m.identity_server": { \n\ + "base_url": "https://vector.im" \n\ + } \n\ + }, \n\ + "disable_custom_urls": false, \n\ + "disable_guests": true, \n\ + "disable_login_language_selector": false, \n\ + "disable_3pid_login": true, \n\ + "brand": "Riot", \n\ + "integrations_ui_url": "https://scalar.vector.im/", \n\ + "integrations_rest_url": "https://scalar.vector.im/api", \n\ + "integrations_widgets_urls": [ \n\ + "https://scalar.vector.im/_matrix/integrations/v1", \n\ + "https://scalar.vector.im/api", \n\ + "https://scalar-staging.vector.im/_matrix/integrations/v1", \n\ + "https://scalar-staging.vector.im/api", \n\ + "https://scalar-staging.riot.im/scalar/api" \n\ + ], \n\ + "integrations_jitsi_widget_url": "https://scalar.vector.im/api/widgets/jitsi.html", \n\ + "bug_report_endpoint_url": "https://riot.im/bugreports/submit", \n\ + "defaultCountryCode": "GB", \n\ + "showLabsSettings": false, \n\ + "features": { \n\ + "feature_pinning": "labs", \n\ + "feature_custom_status": "labs", \n\ + "feature_custom_tags": "labs", \n\ + "feature_state_counters": "labs" \n\ + }, \n\ + "default_federate": true, \n\ + "default_theme": "light", \n\ + "roomDirectory": { \n\ + "servers": [ \n\ + "matrix.org" \n\ + ] \n\ + }, \n\ + "welcomeUserId": "", \n\ + "piwik": { \n\ + "url": "https://piwik.riot.im/", \n\ + "whitelistedHSUrls": ["https://matrix.org"], \n\ + "whitelistedISUrls": ["https://vector.im", "https://matrix.org"], \n\ + "siteId": 1 \n\ + }, \n\ + "enable_presence_by_hs_url": { \n\ + "https://matrix.org": false, \n\ + "https://matrix-client.matrix.org": false \n\ + }, \n\ + "settingDefaults": { \n\ + "breadcrumbs": true \n\ + } \n\ +}' > webapp/config.json + +FROM nginx +# Add "Service-Worker-Allowed: /" header so the worker can sniff traffic on this domain rather +# than just the path this gets hosted under. NB this newline echo syntax only works on bash. +SHELL ["/bin/bash", "-c"] +RUN echo $'\ +server { \n\ + listen 80; \n\ + add_header \'Service-Worker-Allowed\' \'/\'; \n\ + location / { \n\ + root /usr/share/nginx/html; \n\ + index index.html index.htm; \n\ + } \n\ +}' > /etc/nginx/conf.d/default.conf +RUN sed -i 's/}/ application\/wasm wasm;\n}/g' /etc/nginx/mime.types +COPY --from=jsbuild /build/riot-web-matthew-p2p/webapp /usr/share/nginx/html \ No newline at end of file From a5d822004dd93d6f6a7ed73371aeb4bfb163b5ba Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 1 Jun 2020 17:50:19 +0100 Subject: [PATCH 110/200] Send-to-device support (#1072) * Groundwork for send-to-device messaging * Update sample config * Add unstable routing for now * Send to device consumer in sync API * Start the send-to-device consumer * fix indentation in dendrite-config.yaml * Create send-to-device database tables, other tweaks * Add some logic for send-to-device messages, add them into sync stream * Handle incoming send-to-device messages, count them with EDU stream pos * Undo changes to test * pq.Array * Fix sync * Logging * Fix a couple of transaction things, fix client API * Add send-to-device test, hopefully fix bugs * Comments * Refactor a bit * Fix schema * Fix queries * Debug logging * Fix storing and retrieving of send-to-device messages * Try to avoid database locks * Update sync position * Use latest sync position * Jiggle about sync a bit * Fix tests * Break out the retrieval from the update/delete behaviour * Comments * nolint on getResponseWithPDUsForCompleteSync * Try to line up sync tokens again * Implement wildcard * Add all send-to-device tests to whitelist, what could possibly go wrong? * Only care about wildcard when targeted locally * Deduplicate transactions * Handle tokens properly, return immediately if waiting send-to-device messages * Fix sync * Update sytest-whitelist * Fix copyright notice (need to do more of this) * Comments, copyrights * Return errors from Do, fix dendritejs * Review comments * Comments * Constructor for TransactionWriter * defletions * Update gomatrixserverlib, sytest-blacklist --- clientapi/producers/eduserver.go | 26 +++ clientapi/routing/routing.go | 25 +++ clientapi/routing/sendtodevice.go | 70 +++++++ cmd/dendrite-client-api-server/main.go | 2 +- cmd/dendrite-demo-libp2p/main.go | 2 +- cmd/dendrite-edu-server/main.go | 3 +- cmd/dendrite-federation-api-server/main.go | 2 +- cmd/dendrite-monolith-server/main.go | 2 +- cmd/dendritejs/main.go | 3 +- dendrite-config.yaml | 7 +- eduserver/api/input.go | 42 ++++- eduserver/api/output.go | 19 +- eduserver/cache/cache.go | 17 ++ eduserver/cache/cache_test.go | 4 + eduserver/eduserver.go | 15 +- eduserver/input/input.go | 98 +++++++++- federationapi/routing/send.go | 19 ++ federationapi/routing/send_test.go | 8 + go.mod | 2 +- go.sum | 4 +- internal/config/config.go | 2 + internal/sql.go | 62 +++++++ syncapi/consumers/eduserver_sendtodevice.go | 113 ++++++++++++ .../{eduserver.go => eduserver_typing.go} | 0 syncapi/storage/interface.go | 32 +++- .../storage/postgres/send_to_device_table.go | 171 +++++++++++++++++ syncapi/storage/postgres/syncserver.go | 6 + syncapi/storage/shared/syncserver.go | 148 ++++++++++++++- .../storage/sqlite3/send_to_device_table.go | 172 ++++++++++++++++++ syncapi/storage/sqlite3/syncserver.go | 6 + syncapi/storage/storage_test.go | 99 +++++++++- syncapi/storage/tables/interface.go | 39 ++++ syncapi/sync/notifier.go | 16 +- syncapi/sync/notifier_test.go | 2 +- syncapi/sync/requestpool.go | 75 ++++++-- syncapi/syncapi.go | 9 +- syncapi/types/types.go | 23 ++- sytest-blacklist | 5 + sytest-whitelist | 12 ++ 39 files changed, 1302 insertions(+), 60 deletions(-) create mode 100644 clientapi/routing/sendtodevice.go create mode 100644 syncapi/consumers/eduserver_sendtodevice.go rename syncapi/consumers/{eduserver.go => eduserver_typing.go} (100%) create mode 100644 syncapi/storage/postgres/send_to_device_table.go create mode 100644 syncapi/storage/sqlite3/send_to_device_table.go diff --git a/clientapi/producers/eduserver.go b/clientapi/producers/eduserver.go index 30c40fb7f..102c1fad8 100644 --- a/clientapi/producers/eduserver.go +++ b/clientapi/producers/eduserver.go @@ -14,6 +14,7 @@ package producers import ( "context" + "encoding/json" "time" "github.com/matrix-org/dendrite/eduserver/api" @@ -52,3 +53,28 @@ func (p *EDUServerProducer) SendTyping( return err } + +// SendToDevice sends a typing event to EDU server +func (p *EDUServerProducer) SendToDevice( + ctx context.Context, sender, userID, deviceID, eventType string, + message interface{}, +) error { + js, err := json.Marshal(message) + if err != nil { + return err + } + requestData := api.InputSendToDeviceEvent{ + UserID: userID, + DeviceID: deviceID, + SendToDeviceEvent: gomatrixserverlib.SendToDeviceEvent{ + Sender: sender, + Type: eventType, + Content: js, + }, + } + request := api.InputSendToDeviceEventRequest{ + InputSendToDeviceEvent: requestData, + } + response := api.InputSendToDeviceEventResponse{} + return p.InputAPI.InputSendToDeviceEvent(ctx, &request, &response) +} diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 934d9f065..83e399ac3 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -274,6 +274,31 @@ func Setup( }), ).Methods(http.MethodPut, http.MethodOptions) + r0mux.Handle("/sendToDevice/{eventType}/{txnID}", + internal.MakeAuthAPI("send_to_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + txnID := vars["txnID"] + return SendToDevice(req, device, eduProducer, transactionsCache, vars["eventType"], &txnID) + }), + ).Methods(http.MethodPut, http.MethodOptions) + + // This is only here because sytest refers to /unstable for this endpoint + // rather than r0. It's an exact duplicate of the above handler. + // TODO: Remove this if/when sytest is fixed! + unstableMux.Handle("/sendToDevice/{eventType}/{txnID}", + internal.MakeAuthAPI("send_to_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + txnID := vars["txnID"] + return SendToDevice(req, device, eduProducer, transactionsCache, vars["eventType"], &txnID) + }), + ).Methods(http.MethodPut, http.MethodOptions) + r0mux.Handle("/account/whoami", internal.MakeAuthAPI("whoami", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return Whoami(req, device) diff --git a/clientapi/routing/sendtodevice.go b/clientapi/routing/sendtodevice.go new file mode 100644 index 000000000..5d3060d74 --- /dev/null +++ b/clientapi/routing/sendtodevice.go @@ -0,0 +1,70 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routing + +import ( + "encoding/json" + "net/http" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/clientapi/httputil" + "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/clientapi/producers" + "github.com/matrix-org/dendrite/internal/transactions" + "github.com/matrix-org/util" +) + +// SendToDevice handles PUT /_matrix/client/r0/sendToDevice/{eventType}/{txnId} +// sends the device events to the EDU Server +func SendToDevice( + req *http.Request, device *authtypes.Device, + eduProducer *producers.EDUServerProducer, + txnCache *transactions.Cache, + eventType string, txnID *string, +) util.JSONResponse { + if txnID != nil { + if res, ok := txnCache.FetchTransaction(device.AccessToken, *txnID); ok { + return *res + } + } + + var httpReq struct { + Messages map[string]map[string]json.RawMessage `json:"messages"` + } + resErr := httputil.UnmarshalJSONRequest(req, &httpReq) + if resErr != nil { + return *resErr + } + + for userID, byUser := range httpReq.Messages { + for deviceID, message := range byUser { + if err := eduProducer.SendToDevice( + req.Context(), device.UserID, userID, deviceID, eventType, message, + ); err != nil { + util.GetLogger(req.Context()).WithError(err).Error("eduProducer.SendToDevice failed") + return jsonerror.InternalServerError() + } + } + } + + res := util.JSONResponse{ + Code: http.StatusOK, + JSON: struct{}{}, + } + + if txnID != nil { + txnCache.AddTransaction(device.AccessToken, *txnID, &res) + } + + return res +} diff --git a/cmd/dendrite-client-api-server/main.go b/cmd/dendrite-client-api-server/main.go index e06adf8ff..f919243de 100644 --- a/cmd/dendrite-client-api-server/main.go +++ b/cmd/dendrite-client-api-server/main.go @@ -39,7 +39,7 @@ func main() { rsAPI := base.CreateHTTPRoomserverAPIs() fsAPI := base.CreateHTTPFederationSenderAPIs() rsAPI.SetFederationSenderAPI(fsAPI) - eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New()) + eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New(), deviceDB) clientapi.SetupClientAPIComponent( base, deviceDB, accountDB, federation, keyRing, diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index fc56b9bbf..e9d01fd95 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -148,7 +148,7 @@ func main() { &base.Base, keyRing, federation, ) eduInputAPI := eduserver.SetupEDUServerComponent( - &base.Base, cache.New(), + &base.Base, cache.New(), deviceDB, ) asAPI := appservice.SetupAppServiceAPIComponent( &base.Base, accountDB, deviceDB, federation, rsAPI, transactions.New(), diff --git a/cmd/dendrite-edu-server/main.go b/cmd/dendrite-edu-server/main.go index 66e17e577..ca0460f8d 100644 --- a/cmd/dendrite-edu-server/main.go +++ b/cmd/dendrite-edu-server/main.go @@ -29,8 +29,9 @@ func main() { logrus.WithError(err).Warn("BaseDendrite close failed") } }() + deviceDB := base.CreateDeviceDB() - eduserver.SetupEDUServerComponent(base, cache.New()) + eduserver.SetupEDUServerComponent(base, cache.New(), deviceDB) base.SetupAndServeHTTP(string(base.Cfg.Bind.EDUServer), string(base.Cfg.Listen.EDUServer)) diff --git a/cmd/dendrite-federation-api-server/main.go b/cmd/dendrite-federation-api-server/main.go index 5425d117f..af63b549a 100644 --- a/cmd/dendrite-federation-api-server/main.go +++ b/cmd/dendrite-federation-api-server/main.go @@ -39,7 +39,7 @@ func main() { rsAPI := base.CreateHTTPRoomserverAPIs() asAPI := base.CreateHTTPAppServiceAPIs() rsAPI.SetFederationSenderAPI(fsAPI) - eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New()) + eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New(), deviceDB) eduProducer := producers.NewEDUServerProducer(eduInputAPI) federationapi.SetupFederationAPIComponent( diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 8367cd9da..ef114ccd1 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -87,7 +87,7 @@ func main() { } eduInputAPI := eduserver.SetupEDUServerComponent( - base, cache.New(), + base, cache.New(), deviceDB, ) if base.EnableHTTPAPIs { eduInputAPI = base.CreateHTTPEDUServerAPIs() diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 45f23d9ad..9a02e71e9 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -175,6 +175,7 @@ func main() { cfg.Database.SyncAPI = "file:/idb/dendritejs_syncapi.db" cfg.Kafka.Topics.UserUpdates = "user_updates" cfg.Kafka.Topics.OutputTypingEvent = "output_typing_event" + cfg.Kafka.Topics.OutputSendToDeviceEvent = "output_send_to_device_event" cfg.Kafka.Topics.OutputClientData = "output_client_data" cfg.Kafka.Topics.OutputRoomEvent = "output_room_event" cfg.Matrix.TrustedIDServers = []string{ @@ -206,7 +207,7 @@ func main() { p2pPublicRoomProvider := NewLibP2PPublicRoomsProvider(node) rsAPI := roomserver.SetupRoomServerComponent(base, keyRing, federation) - eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New()) + eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New(), deviceDB) asQuery := appservice.SetupAppServiceAPIComponent( base, accountDB, deviceDB, federation, rsAPI, transactions.New(), ) diff --git a/dendrite-config.yaml b/dendrite-config.yaml index 1802b8b73..a5b295977 100644 --- a/dendrite-config.yaml +++ b/dendrite-config.yaml @@ -104,7 +104,8 @@ kafka: topics: output_room_event: roomserverOutput output_client_data: clientapiOutput - output_typing_event: eduServerOutput + output_typing_event: eduServerTypingOutput + output_send_to_device_event: eduServerSendToDeviceOutput user_updates: userUpdates # The postgres connection configs for connecting to the databases e.g a postgres:// URI @@ -137,8 +138,8 @@ listen: federation_sender: "localhost:7776" appservice_api: "localhost:7777" edu_server: "localhost:7778" - key_server: "localhost:7779" - server_key_api: "localhost:7780" + key_server: "localhost:7779" + server_key_api: "localhost:7780" # The configuration for tracing the dendrite components. tracing: diff --git a/eduserver/api/input.go b/eduserver/api/input.go index 8b5b6d76a..fa7f30cb6 100644 --- a/eduserver/api/input.go +++ b/eduserver/api/input.go @@ -1,3 +1,7 @@ +// Copyright 2017 Vector Creations Ltd +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -37,6 +41,12 @@ type InputTypingEvent struct { OriginServerTS gomatrixserverlib.Timestamp `json:"origin_server_ts"` } +type InputSendToDeviceEvent struct { + UserID string `json:"user_id"` + DeviceID string `json:"device_id"` + gomatrixserverlib.SendToDeviceEvent +} + // InputTypingEventRequest is a request to EDUServerInputAPI type InputTypingEventRequest struct { InputTypingEvent InputTypingEvent `json:"input_typing_event"` @@ -45,6 +55,14 @@ type InputTypingEventRequest struct { // InputTypingEventResponse is a response to InputTypingEvents type InputTypingEventResponse struct{} +// InputSendToDeviceEventRequest is a request to EDUServerInputAPI +type InputSendToDeviceEventRequest struct { + InputSendToDeviceEvent InputSendToDeviceEvent `json:"input_send_to_device_event"` +} + +// InputSendToDeviceEventResponse is a response to InputSendToDeviceEventRequest +type InputSendToDeviceEventResponse struct{} + // EDUServerInputAPI is used to write events to the typing server. type EDUServerInputAPI interface { InputTypingEvent( @@ -52,11 +70,20 @@ type EDUServerInputAPI interface { request *InputTypingEventRequest, response *InputTypingEventResponse, ) error + + InputSendToDeviceEvent( + ctx context.Context, + request *InputSendToDeviceEventRequest, + response *InputSendToDeviceEventResponse, + ) error } // EDUServerInputTypingEventPath is the HTTP path for the InputTypingEvent API. const EDUServerInputTypingEventPath = "/eduserver/input" +// EDUServerInputSendToDeviceEventPath is the HTTP path for the InputSendToDeviceEvent API. +const EDUServerInputSendToDeviceEventPath = "/eduserver/sendToDevice" + // NewEDUServerInputAPIHTTP creates a EDUServerInputAPI implemented by talking to a HTTP POST API. func NewEDUServerInputAPIHTTP(eduServerURL string, httpClient *http.Client) (EDUServerInputAPI, error) { if httpClient == nil { @@ -70,7 +97,7 @@ type httpEDUServerInputAPI struct { httpClient *http.Client } -// InputRoomEvents implements EDUServerInputAPI +// InputTypingEvent implements EDUServerInputAPI func (h *httpEDUServerInputAPI) InputTypingEvent( ctx context.Context, request *InputTypingEventRequest, @@ -82,3 +109,16 @@ func (h *httpEDUServerInputAPI) InputTypingEvent( apiURL := h.eduServerURL + EDUServerInputTypingEventPath return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } + +// InputSendToDeviceEvent implements EDUServerInputAPI +func (h *httpEDUServerInputAPI) InputSendToDeviceEvent( + ctx context.Context, + request *InputSendToDeviceEventRequest, + response *InputSendToDeviceEventResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "InputSendToDeviceEvent") + defer span.Finish() + + apiURL := h.eduServerURL + EDUServerInputSendToDeviceEventPath + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} diff --git a/eduserver/api/output.go b/eduserver/api/output.go index 8696acf49..e6ded8413 100644 --- a/eduserver/api/output.go +++ b/eduserver/api/output.go @@ -1,3 +1,7 @@ +// Copyright 2017 Vector Creations Ltd +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -12,7 +16,11 @@ package api -import "time" +import ( + "time" + + "github.com/matrix-org/gomatrixserverlib" +) // OutputTypingEvent is an entry in typing server output kafka log. // This contains the event with extra fields used to create 'm.typing' event @@ -32,3 +40,12 @@ type TypingEvent struct { UserID string `json:"user_id"` Typing bool `json:"typing"` } + +// OutputSendToDeviceEvent is an entry in the send-to-device output kafka log. +// This contains the full event content, along with the user ID and device ID +// to which it is destined. +type OutputSendToDeviceEvent struct { + UserID string `json:"user_id"` + DeviceID string `json:"device_id"` + gomatrixserverlib.SendToDeviceEvent +} diff --git a/eduserver/cache/cache.go b/eduserver/cache/cache.go index 46f7a2b13..dd535a6d2 100644 --- a/eduserver/cache/cache.go +++ b/eduserver/cache/cache.go @@ -1,3 +1,7 @@ +// Copyright 2017 Vector Creations Ltd +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -109,6 +113,19 @@ func (t *EDUCache) AddTypingUser( return t.GetLatestSyncPosition() } +// AddSendToDeviceMessage increases the sync position for +// send-to-device updates. +// Returns the sync position before update, as the caller +// will use this to record the current stream position +// at the time that the send-to-device message was sent. +func (t *EDUCache) AddSendToDeviceMessage() int64 { + t.Lock() + defer t.Unlock() + latestSyncPosition := t.latestSyncPosition + t.latestSyncPosition++ + return latestSyncPosition +} + // addUser with mutex lock & replace the previous timer. // Returns the latest typing sync position after update. func (t *EDUCache) addUser( diff --git a/eduserver/cache/cache_test.go b/eduserver/cache/cache_test.go index d1b2f8bd5..c7d01879f 100644 --- a/eduserver/cache/cache_test.go +++ b/eduserver/cache/cache_test.go @@ -1,3 +1,7 @@ +// Copyright 2017 Vector Creations Ltd +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at diff --git a/eduserver/eduserver.go b/eduserver/eduserver.go index 14fbd3328..6f664eb67 100644 --- a/eduserver/eduserver.go +++ b/eduserver/eduserver.go @@ -1,3 +1,7 @@ +// Copyright 2017 Vector Creations Ltd +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -13,6 +17,7 @@ package eduserver import ( + "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/eduserver/input" @@ -26,11 +31,15 @@ import ( func SetupEDUServerComponent( base *basecomponent.BaseDendrite, eduCache *cache.EDUCache, + deviceDB devices.Database, ) api.EDUServerInputAPI { inputAPI := &input.EDUServerInputAPI{ - Cache: eduCache, - Producer: base.KafkaProducer, - OutputTypingEventTopic: string(base.Cfg.Kafka.Topics.OutputTypingEvent), + Cache: eduCache, + DeviceDB: deviceDB, + Producer: base.KafkaProducer, + OutputTypingEventTopic: string(base.Cfg.Kafka.Topics.OutputTypingEvent), + OutputSendToDeviceEventTopic: string(base.Cfg.Kafka.Topics.OutputSendToDeviceEvent), + ServerName: base.Cfg.Matrix.ServerName, } inputAPI.SetupHTTP(base.InternalAPIMux) diff --git a/eduserver/input/input.go b/eduserver/input/input.go index 73777e323..4e3051950 100644 --- a/eduserver/input/input.go +++ b/eduserver/input/input.go @@ -1,3 +1,7 @@ +// Copyright 2017 Vector Creations Ltd +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -20,11 +24,13 @@ import ( "github.com/Shopify/sarama" "github.com/gorilla/mux" + "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" + "github.com/sirupsen/logrus" ) // EDUServerInputAPI implements api.EDUServerInputAPI @@ -33,8 +39,14 @@ type EDUServerInputAPI struct { Cache *cache.EDUCache // The kafka topic to output new typing events to. OutputTypingEventTopic string + // The kafka topic to output new send to device events to. + OutputSendToDeviceEventTopic string // kafka producer Producer sarama.SyncProducer + // device database + DeviceDB devices.Database + // our server name + ServerName gomatrixserverlib.ServerName } // InputTypingEvent implements api.EDUServerInputAPI @@ -54,10 +66,20 @@ func (t *EDUServerInputAPI) InputTypingEvent( t.Cache.RemoveUser(ite.UserID, ite.RoomID) } - return t.sendEvent(ite) + return t.sendTypingEvent(ite) } -func (t *EDUServerInputAPI) sendEvent(ite *api.InputTypingEvent) error { +// InputTypingEvent implements api.EDUServerInputAPI +func (t *EDUServerInputAPI) InputSendToDeviceEvent( + ctx context.Context, + request *api.InputSendToDeviceEventRequest, + response *api.InputSendToDeviceEventResponse, +) error { + ise := &request.InputSendToDeviceEvent + return t.sendToDeviceEvent(ise) +} + +func (t *EDUServerInputAPI) sendTypingEvent(ite *api.InputTypingEvent) error { ev := &api.TypingEvent{ Type: gomatrixserverlib.MTyping, RoomID: ite.RoomID, @@ -90,6 +112,65 @@ func (t *EDUServerInputAPI) sendEvent(ite *api.InputTypingEvent) error { return err } +func (t *EDUServerInputAPI) sendToDeviceEvent(ise *api.InputSendToDeviceEvent) error { + devices := []string{} + localpart, domain, err := gomatrixserverlib.SplitID('@', ise.UserID) + if err != nil { + return err + } + + // If the event is targeted locally then we want to expand the wildcard + // out into individual device IDs so that we can send them to each respective + // device. If the event isn't targeted locally then we can't expand the + // wildcard as we don't know about the remote devices, so instead we leave it + // as-is, so that the federation sender can send it on with the wildcard intact. + if domain == t.ServerName && ise.DeviceID == "*" { + devs, err := t.DeviceDB.GetDevicesByLocalpart(context.TODO(), localpart) + if err != nil { + return err + } + for _, dev := range devs { + devices = append(devices, dev.ID) + } + } else { + devices = append(devices, ise.DeviceID) + } + + for _, device := range devices { + ote := &api.OutputSendToDeviceEvent{ + UserID: ise.UserID, + DeviceID: device, + SendToDeviceEvent: ise.SendToDeviceEvent, + } + + logrus.WithFields(logrus.Fields{ + "user_id": ise.UserID, + "device_id": ise.DeviceID, + "event_type": ise.Type, + }).Info("handling send-to-device message") + + eventJSON, err := json.Marshal(ote) + if err != nil { + logrus.WithError(err).Error("sendToDevice failed json.Marshal") + return err + } + + m := &sarama.ProducerMessage{ + Topic: string(t.OutputSendToDeviceEventTopic), + Key: sarama.StringEncoder(ote.UserID), + Value: sarama.ByteEncoder(eventJSON), + } + + _, _, err = t.Producer.SendMessage(m) + if err != nil { + logrus.WithError(err).Error("sendToDevice failed t.Producer.SendMessage") + return err + } + } + + return nil +} + // SetupHTTP adds the EDUServerInputAPI handlers to the http.ServeMux. func (t *EDUServerInputAPI) SetupHTTP(internalAPIMux *mux.Router) { internalAPIMux.Handle(api.EDUServerInputTypingEventPath, @@ -105,4 +186,17 @@ func (t *EDUServerInputAPI) SetupHTTP(internalAPIMux *mux.Router) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) + internalAPIMux.Handle(api.EDUServerInputSendToDeviceEventPath, + internal.MakeInternalAPI("inputSendToDeviceEvents", func(req *http.Request) util.JSONResponse { + var request api.InputSendToDeviceEventRequest + var response api.InputSendToDeviceEventResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := t.InputSendToDeviceEvent(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) } diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index b514af0ab..74b4c0144 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -265,6 +265,25 @@ func (t *txnReq) processEDUs(edus []gomatrixserverlib.EDU) { if err := t.eduProducer.SendTyping(t.context, typingPayload.UserID, typingPayload.RoomID, typingPayload.Typing, 30*1000); err != nil { util.GetLogger(t.context).WithError(err).Error("Failed to send typing event to edu server") } + case gomatrixserverlib.MDirectToDevice: + // https://matrix.org/docs/spec/server_server/r0.1.3#m-direct-to-device-schema + var directPayload gomatrixserverlib.ToDeviceMessage + if err := json.Unmarshal(e.Content, &directPayload); err != nil { + util.GetLogger(t.context).WithError(err).Error("Failed to unmarshal send-to-device events") + continue + } + for userID, byUser := range directPayload.Messages { + for deviceID, message := range byUser { + // TODO: check that the user and the device actually exist here + if err := t.eduProducer.SendToDevice(t.context, directPayload.Sender, userID, deviceID, directPayload.Type, message); err != nil { + util.GetLogger(t.context).WithError(err).WithFields(logrus.Fields{ + "sender": directPayload.Sender, + "user_id": userID, + "device_id": deviceID, + }).Error("Failed to send send-to-device event to edu server") + } + } + } default: util.GetLogger(t.context).WithField("type", e.Type).Warn("unhandled edu") } diff --git a/federationapi/routing/send_test.go b/federationapi/routing/send_test.go index cb8aec6f5..3e28a3476 100644 --- a/federationapi/routing/send_test.go +++ b/federationapi/routing/send_test.go @@ -77,6 +77,14 @@ func (p *testEDUProducer) InputTypingEvent( return nil } +func (p *testEDUProducer) InputSendToDeviceEvent( + ctx context.Context, + request *eduAPI.InputSendToDeviceEventRequest, + response *eduAPI.InputSendToDeviceEventResponse, +) error { + return nil +} + type testRoomserverAPI struct { inputRoomEvents []api.InputRoomEvent queryStateAfterEvents func(*api.QueryStateAfterEventsRequest) api.QueryStateAfterEventsResponse diff --git a/go.mod b/go.mod index 4365ea503..cc60e1a22 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200528122156-fbb320a2ee61 + github.com/matrix-org/gomatrixserverlib v0.0.0-20200601162724-79e93fe989cf github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible diff --git a/go.sum b/go.sum index c08cfa5d5..6d9c2725d 100644 --- a/go.sum +++ b/go.sum @@ -356,8 +356,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 h1:Yb+Wlf github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bhrnp3Ky1qgx/fzCtCALOoGYylh2tpS9K4= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200528122156-fbb320a2ee61 h1:3rgoGvj/skUWg+u9E6ycEFs2ZGenEjr28ZtAhAhmZeM= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200528122156-fbb320a2ee61/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200601162724-79e93fe989cf h1:iT2dfJ6JmYNRZBQTXeCNwsZIvfkBbFggzclM8iKnbR0= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200601162724-79e93fe989cf/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f/go.mod h1:y0oDTjZDv5SM9a2rp3bl+CU+bvTRINQsdb7YlDql5Go= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= diff --git a/internal/config/config.go b/internal/config/config.go index 2a95069a4..a20cc0ead 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -152,6 +152,8 @@ type Dendrite struct { OutputClientData Topic `yaml:"output_client_data"` // Topic for eduserver/api.OutputTypingEvent events. OutputTypingEvent Topic `yaml:"output_typing_event"` + // Topic for eduserver/api.OutputSendToDeviceEvent events. + OutputSendToDeviceEvent Topic `yaml:"output_send_to_device_event"` // Topic for user updates (profile, presence) UserUpdates Topic `yaml:"user_updates"` } diff --git a/internal/sql.go b/internal/sql.go index d6a5a3086..546954bd5 100644 --- a/internal/sql.go +++ b/internal/sql.go @@ -1,4 +1,6 @@ // Copyright 2017 Vector Creations Ltd +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -16,9 +18,12 @@ package internal import ( "database/sql" + "errors" "fmt" "runtime" "time" + + "go.uber.org/atomic" ) // A Transaction is something that can be committed or rolledback. @@ -107,3 +112,60 @@ type DbProperties interface { MaxOpenConns() int ConnMaxLifetime() time.Duration } + +// TransactionWriter allows queuing database writes so that you don't +// contend on database locks in, e.g. SQLite. Only one task will run +// at a time on a given TransactionWriter. +type TransactionWriter struct { + running atomic.Bool + todo chan transactionWriterTask +} + +func NewTransactionWriter() *TransactionWriter { + return &TransactionWriter{ + todo: make(chan transactionWriterTask), + } +} + +// transactionWriterTask represents a specific task. +type transactionWriterTask struct { + db *sql.DB + f func(txn *sql.Tx) error + wait chan error +} + +// Do queues a task to be run by a TransactionWriter. The function +// provided will be ran within a transaction as supplied by the +// database parameter. This will block until the task is finished. +func (w *TransactionWriter) Do(db *sql.DB, f func(txn *sql.Tx) error) error { + if w.todo == nil { + return errors.New("not initialised") + } + if !w.running.Load() { + go w.run() + } + task := transactionWriterTask{ + db: db, + f: f, + wait: make(chan error, 1), + } + w.todo <- task + return <-task.wait +} + +// run processes the tasks for a given transaction writer. Only one +// of these goroutines will run at a time. A transaction will be +// opened using the database object from the task and then this will +// be passed as a parameter to the task function. +func (w *TransactionWriter) run() { + if !w.running.CAS(false, true) { + return + } + defer w.running.Store(false) + for task := range w.todo { + task.wait <- WithTransaction(task.db, func(txn *sql.Tx) error { + return task.f(txn) + }) + close(task.wait) + } +} diff --git a/syncapi/consumers/eduserver_sendtodevice.go b/syncapi/consumers/eduserver_sendtodevice.go new file mode 100644 index 000000000..487018031 --- /dev/null +++ b/syncapi/consumers/eduserver_sendtodevice.go @@ -0,0 +1,113 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package consumers + +import ( + "context" + "encoding/json" + + "github.com/Shopify/sarama" + "github.com/matrix-org/dendrite/eduserver/api" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/dendrite/syncapi/sync" + "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" + log "github.com/sirupsen/logrus" +) + +// OutputSendToDeviceEventConsumer consumes events that originated in the EDU server. +type OutputSendToDeviceEventConsumer struct { + sendToDeviceConsumer *internal.ContinualConsumer + db storage.Database + serverName gomatrixserverlib.ServerName // our server name + notifier *sync.Notifier +} + +// NewOutputSendToDeviceEventConsumer creates a new OutputSendToDeviceEventConsumer. +// Call Start() to begin consuming from the EDU server. +func NewOutputSendToDeviceEventConsumer( + cfg *config.Dendrite, + kafkaConsumer sarama.Consumer, + n *sync.Notifier, + store storage.Database, +) *OutputSendToDeviceEventConsumer { + + consumer := internal.ContinualConsumer{ + Topic: string(cfg.Kafka.Topics.OutputSendToDeviceEvent), + Consumer: kafkaConsumer, + PartitionStore: store, + } + + s := &OutputSendToDeviceEventConsumer{ + sendToDeviceConsumer: &consumer, + db: store, + serverName: cfg.Matrix.ServerName, + notifier: n, + } + + consumer.ProcessMessage = s.onMessage + + return s +} + +// Start consuming from EDU api +func (s *OutputSendToDeviceEventConsumer) Start() error { + return s.sendToDeviceConsumer.Start() +} + +func (s *OutputSendToDeviceEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { + var output api.OutputSendToDeviceEvent + if err := json.Unmarshal(msg.Value, &output); err != nil { + // If the message was invalid, log it and move on to the next message in the stream + log.WithError(err).Errorf("EDU server output log: message parse failure") + return err + } + + _, domain, err := gomatrixserverlib.SplitID('@', output.UserID) + if err != nil { + return err + } + if domain != s.serverName { + return nil + } + + util.GetLogger(context.TODO()).WithFields(log.Fields{ + "sender": output.Sender, + "user_id": output.UserID, + "device_id": output.DeviceID, + "event_type": output.Type, + }).Info("sync API received send-to-device event from EDU server") + + streamPos := s.db.AddSendToDevice() + + _, err = s.db.StoreNewSendForDeviceMessage( + context.TODO(), streamPos, output.UserID, output.DeviceID, output.SendToDeviceEvent, + ) + if err != nil { + log.WithError(err).Errorf("failed to store send-to-device message") + return err + } + + s.notifier.OnNewSendToDevice( + output.UserID, + []string{output.DeviceID}, + types.NewStreamToken(0, streamPos), + ) + + return nil +} diff --git a/syncapi/consumers/eduserver.go b/syncapi/consumers/eduserver_typing.go similarity index 100% rename from syncapi/consumers/eduserver.go rename to syncapi/consumers/eduserver_typing.go diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index 7e1a40fd6..566e5d589 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -55,10 +55,12 @@ type Database interface { // sync response for the given user. Events returned will include any client // transaction IDs associated with the given device. These transaction IDs come // from when the device sent the event via an API that included a transaction - // ID. - IncrementalSync(ctx context.Context, device authtypes.Device, fromPos, toPos types.StreamingToken, numRecentEventsPerRoom int, wantFullState bool) (*types.Response, error) - // CompleteSync returns a complete /sync API response for the given user. - CompleteSync(ctx context.Context, device authtypes.Device, numRecentEventsPerRoom int) (*types.Response, error) + // ID. A response object must be provided for IncrementaSync to populate - it + // will not create one. + IncrementalSync(ctx context.Context, res *types.Response, device authtypes.Device, fromPos, toPos types.StreamingToken, numRecentEventsPerRoom int, wantFullState bool) (*types.Response, error) + // CompleteSync returns a complete /sync API response for the given user. A response object + // must be provided for CompleteSync to populate - it will not create one. + CompleteSync(ctx context.Context, res *types.Response, device authtypes.Device, numRecentEventsPerRoom int) (*types.Response, error) // GetAccountDataInRange returns all account data for a given user inserted or // updated between two given positions // Returns a map following the format data[roomID] = []dataTypes @@ -104,4 +106,26 @@ type Database interface { StreamEventsToEvents(device *authtypes.Device, in []types.StreamEvent) []gomatrixserverlib.HeaderedEvent // SyncStreamPosition returns the latest position in the sync stream. Returns 0 if there are no events yet. SyncStreamPosition(ctx context.Context) (types.StreamPosition, error) + // AddSendToDevice increases the EDU position in the cache and returns the stream position. + AddSendToDevice() types.StreamPosition + // SendToDeviceUpdatesForSync returns a list of send-to-device updates. It returns three lists: + // - "events": a list of send-to-device events that should be included in the sync + // - "changes": a list of send-to-device events that should be updated in the database by + // CleanSendToDeviceUpdates + // - "deletions": a list of send-to-device events which have been confirmed as sent and + // can be deleted altogether by CleanSendToDeviceUpdates + // The token supplied should be the current requested sync token, e.g. from the "since" + // parameter. + SendToDeviceUpdatesForSync(ctx context.Context, userID, deviceID string, token types.StreamingToken) (events []types.SendToDeviceEvent, changes []types.SendToDeviceNID, deletions []types.SendToDeviceNID, err error) + // StoreNewSendForDeviceMessage stores a new send-to-device event for a user's device. + StoreNewSendForDeviceMessage(ctx context.Context, streamPos types.StreamPosition, userID, deviceID string, event gomatrixserverlib.SendToDeviceEvent) (types.StreamPosition, error) + // CleanSendToDeviceUpdates will update or remove any send-to-device updates based on the + // result to a previous call to SendDeviceUpdatesForSync. This is separate as it allows + // SendToDeviceUpdatesForSync to be called multiple times if needed (e.g. before and after + // starting to wait for an incremental sync with timeout). + // The token supplied should be the current requested sync token, e.g. from the "since" + // parameter. + CleanSendToDeviceUpdates(ctx context.Context, toUpdate, toDelete []types.SendToDeviceNID, token types.StreamingToken) (err error) + // SendToDeviceUpdatesWaiting returns true if there are send-to-device updates waiting to be sent. + SendToDeviceUpdatesWaiting(ctx context.Context, userID, deviceID string) (bool, error) } diff --git a/syncapi/storage/postgres/send_to_device_table.go b/syncapi/storage/postgres/send_to_device_table.go new file mode 100644 index 000000000..335a05ef1 --- /dev/null +++ b/syncapi/storage/postgres/send_to_device_table.go @@ -0,0 +1,171 @@ +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package postgres + +import ( + "context" + "database/sql" + "encoding/json" + + "github.com/lib/pq" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/syncapi/storage/tables" + "github.com/matrix-org/dendrite/syncapi/types" +) + +const sendToDeviceSchema = ` +CREATE SEQUENCE IF NOT EXISTS syncapi_send_to_device_id; + +-- Stores send-to-device messages. +CREATE TABLE IF NOT EXISTS syncapi_send_to_device ( + -- The ID that uniquely identifies this message. + id BIGINT PRIMARY KEY DEFAULT nextval('syncapi_send_to_device_id'), + -- The user ID to send the message to. + user_id TEXT NOT NULL, + -- The device ID to send the message to. + device_id TEXT NOT NULL, + -- The event content JSON. + content TEXT NOT NULL, + -- The token that was supplied to the /sync at the time that this + -- message was included in a sync response, or NULL if we haven't + -- included it in a /sync response yet. + sent_by_token TEXT +); +` + +const insertSendToDeviceMessageSQL = ` + INSERT INTO syncapi_send_to_device (user_id, device_id, content) + VALUES ($1, $2, $3) +` + +const countSendToDeviceMessagesSQL = ` + SELECT COUNT(*) + FROM syncapi_send_to_device + WHERE user_id = $1 AND device_id = $2 +` + +const selectSendToDeviceMessagesSQL = ` + SELECT id, user_id, device_id, content, sent_by_token + FROM syncapi_send_to_device + WHERE user_id = $1 AND device_id = $2 + ORDER BY id DESC +` + +const updateSentSendToDeviceMessagesSQL = ` + UPDATE syncapi_send_to_device SET sent_by_token = $1 + WHERE id = ANY($2) +` + +const deleteSendToDeviceMessagesSQL = ` + DELETE FROM syncapi_send_to_device WHERE id = ANY($1) +` + +type sendToDeviceStatements struct { + insertSendToDeviceMessageStmt *sql.Stmt + countSendToDeviceMessagesStmt *sql.Stmt + selectSendToDeviceMessagesStmt *sql.Stmt + updateSentSendToDeviceMessagesStmt *sql.Stmt + deleteSendToDeviceMessagesStmt *sql.Stmt +} + +func NewPostgresSendToDeviceTable(db *sql.DB) (tables.SendToDevice, error) { + s := &sendToDeviceStatements{} + _, err := db.Exec(sendToDeviceSchema) + if err != nil { + return nil, err + } + if s.insertSendToDeviceMessageStmt, err = db.Prepare(insertSendToDeviceMessageSQL); err != nil { + return nil, err + } + if s.countSendToDeviceMessagesStmt, err = db.Prepare(countSendToDeviceMessagesSQL); err != nil { + return nil, err + } + if s.selectSendToDeviceMessagesStmt, err = db.Prepare(selectSendToDeviceMessagesSQL); err != nil { + return nil, err + } + if s.updateSentSendToDeviceMessagesStmt, err = db.Prepare(updateSentSendToDeviceMessagesSQL); err != nil { + return nil, err + } + if s.deleteSendToDeviceMessagesStmt, err = db.Prepare(deleteSendToDeviceMessagesSQL); err != nil { + return nil, err + } + return s, nil +} + +func (s *sendToDeviceStatements) InsertSendToDeviceMessage( + ctx context.Context, txn *sql.Tx, userID, deviceID, content string, +) (err error) { + _, err = internal.TxStmt(txn, s.insertSendToDeviceMessageStmt).ExecContext(ctx, userID, deviceID, content) + return +} + +func (s *sendToDeviceStatements) CountSendToDeviceMessages( + ctx context.Context, txn *sql.Tx, userID, deviceID string, +) (count int, err error) { + row := internal.TxStmt(txn, s.countSendToDeviceMessagesStmt).QueryRowContext(ctx, userID, deviceID) + if err = row.Scan(&count); err != nil { + return + } + return count, nil +} + +func (s *sendToDeviceStatements) SelectSendToDeviceMessages( + ctx context.Context, txn *sql.Tx, userID, deviceID string, +) (events []types.SendToDeviceEvent, err error) { + rows, err := internal.TxStmt(txn, s.selectSendToDeviceMessagesStmt).QueryContext(ctx, userID, deviceID) + if err != nil { + return + } + defer internal.CloseAndLogIfError(ctx, rows, "SelectSendToDeviceMessages: rows.close() failed") + + for rows.Next() { + var id types.SendToDeviceNID + var userID, deviceID, content string + var sentByToken *string + if err = rows.Scan(&id, &userID, &deviceID, &content, &sentByToken); err != nil { + return + } + event := types.SendToDeviceEvent{ + ID: id, + UserID: userID, + DeviceID: deviceID, + } + if err = json.Unmarshal([]byte(content), &event.SendToDeviceEvent); err != nil { + return + } + if sentByToken != nil { + if token, err := types.NewStreamTokenFromString(*sentByToken); err == nil { + event.SentByToken = &token + } + } + events = append(events, event) + } + + return events, rows.Err() +} + +func (s *sendToDeviceStatements) UpdateSentSendToDeviceMessages( + ctx context.Context, txn *sql.Tx, token string, nids []types.SendToDeviceNID, +) (err error) { + _, err = txn.Stmt(s.updateSentSendToDeviceMessagesStmt).ExecContext(ctx, token, pq.Array(nids)) + return +} + +func (s *sendToDeviceStatements) DeleteSendToDeviceMessages( + ctx context.Context, txn *sql.Tx, nids []types.SendToDeviceNID, +) (err error) { + _, err = txn.Stmt(s.deleteSendToDeviceMessagesStmt).ExecContext(ctx, pq.Array(nids)) + return +} diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go index dc73350a2..8a8f964a5 100644 --- a/syncapi/storage/postgres/syncserver.go +++ b/syncapi/storage/postgres/syncserver.go @@ -69,6 +69,10 @@ func NewDatabase(dbDataSourceName string, dbProperties internal.DbProperties) (* if err != nil { return nil, err } + sendToDevice, err := NewPostgresSendToDeviceTable(d.db) + if err != nil { + return nil, err + } d.Database = shared.Database{ DB: d.db, Invites: invites, @@ -77,6 +81,8 @@ func NewDatabase(dbDataSourceName string, dbProperties internal.DbProperties) (* Topology: topology, CurrentRoomState: currState, BackwardExtremities: backwardExtremities, + SendToDevice: sendToDevice, + SendToDeviceWriter: internal.NewTransactionWriter(), EDUCache: cache.New(), } return &d, nil diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go index 888f85e0b..497c043a6 100644 --- a/syncapi/storage/shared/syncserver.go +++ b/syncapi/storage/shared/syncserver.go @@ -1,3 +1,17 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package shared import ( @@ -27,6 +41,8 @@ type Database struct { Topology tables.Topology CurrentRoomState tables.CurrentRoomState BackwardExtremities tables.BackwardsExtremities + SendToDevice tables.SendToDevice + SendToDeviceWriter *internal.TransactionWriter EDUCache *cache.EDUCache } @@ -89,6 +105,10 @@ func (d *Database) RemoveTypingUser( return types.StreamPosition(d.EDUCache.RemoveUser(userID, roomID)) } +func (d *Database) AddSendToDevice() types.StreamPosition { + return types.StreamPosition(d.EDUCache.AddSendToDeviceMessage()) +} + func (d *Database) SetTypingTimeoutCallback(fn cache.TimeoutCallbackFn) { d.EDUCache.SetTimeoutCallback(fn) } @@ -528,14 +548,14 @@ func (d *Database) addEDUDeltaToResponse( } func (d *Database) IncrementalSync( - ctx context.Context, + ctx context.Context, res *types.Response, device authtypes.Device, fromPos, toPos types.StreamingToken, numRecentEventsPerRoom int, wantFullState bool, ) (*types.Response, error) { nextBatchPos := fromPos.WithUpdates(toPos) - res := types.NewResponse(nextBatchPos) + res.NextBatch = nextBatchPos.String() var joinedRoomIDs []string var err error @@ -568,12 +588,12 @@ func (d *Database) IncrementalSync( // getResponseWithPDUsForCompleteSync creates a response and adds all PDUs needed // to it. It returns toPos and joinedRoomIDs for use of adding EDUs. +// nolint:nakedret func (d *Database) getResponseWithPDUsForCompleteSync( - ctx context.Context, + ctx context.Context, res *types.Response, userID string, numRecentEventsPerRoom int, ) ( - res *types.Response, toPos types.StreamingToken, joinedRoomIDs []string, err error, @@ -604,7 +624,7 @@ func (d *Database) getResponseWithPDUsForCompleteSync( To: toPos.PDUPosition(), } - res = types.NewResponse(toPos) + res.NextBatch = toPos.String() // Extract room state and recent events for all rooms the user is joined to. joinedRoomIDs, err = d.CurrentRoomState.SelectRoomIDsWithMembership(ctx, txn, userID, gomatrixserverlib.Join) @@ -662,14 +682,15 @@ func (d *Database) getResponseWithPDUsForCompleteSync( } succeeded = true - return res, toPos, joinedRoomIDs, err + return //res, toPos, joinedRoomIDs, err } func (d *Database) CompleteSync( - ctx context.Context, device authtypes.Device, numRecentEventsPerRoom int, + ctx context.Context, res *types.Response, + device authtypes.Device, numRecentEventsPerRoom int, ) (*types.Response, error) { - res, toPos, joinedRoomIDs, err := d.getResponseWithPDUsForCompleteSync( - ctx, device.UserID, numRecentEventsPerRoom, + toPos, joinedRoomIDs, err := d.getResponseWithPDUsForCompleteSync( + ctx, res, device.UserID, numRecentEventsPerRoom, ) if err != nil { return nil, err @@ -1028,6 +1049,115 @@ func (d *Database) currentStateStreamEventsForRoom( return s, nil } +func (d *Database) SendToDeviceUpdatesWaiting( + ctx context.Context, userID, deviceID string, +) (bool, error) { + count, err := d.SendToDevice.CountSendToDeviceMessages(ctx, nil, userID, deviceID) + if err != nil { + return false, err + } + return count > 0, nil +} + +func (d *Database) AddSendToDeviceEvent( + ctx context.Context, txn *sql.Tx, + userID, deviceID, content string, +) error { + return d.SendToDevice.InsertSendToDeviceMessage( + ctx, txn, userID, deviceID, content, + ) +} + +func (d *Database) StoreNewSendForDeviceMessage( + ctx context.Context, streamPos types.StreamPosition, userID, deviceID string, event gomatrixserverlib.SendToDeviceEvent, +) (types.StreamPosition, error) { + j, err := json.Marshal(event) + if err != nil { + return streamPos, err + } + // Delegate the database write task to the SendToDeviceWriter. It'll guarantee + // that we don't lock the table for writes in more than one place. + err = d.SendToDeviceWriter.Do(d.DB, func(txn *sql.Tx) error { + return d.AddSendToDeviceEvent( + ctx, txn, userID, deviceID, string(j), + ) + }) + if err != nil { + return streamPos, err + } + return streamPos, nil +} + +func (d *Database) SendToDeviceUpdatesForSync( + ctx context.Context, + userID, deviceID string, + token types.StreamingToken, +) ([]types.SendToDeviceEvent, []types.SendToDeviceNID, []types.SendToDeviceNID, error) { + // First of all, get our send-to-device updates for this user. + events, err := d.SendToDevice.SelectSendToDeviceMessages(ctx, nil, userID, deviceID) + if err != nil { + return nil, nil, nil, fmt.Errorf("d.SendToDevice.SelectSendToDeviceMessages: %w", err) + } + + // If there's nothing to do then stop here. + if len(events) == 0 { + return nil, nil, nil, nil + } + + // Work out whether we need to update any of the database entries. + toReturn := []types.SendToDeviceEvent{} + toUpdate := []types.SendToDeviceNID{} + toDelete := []types.SendToDeviceNID{} + for _, event := range events { + if event.SentByToken == nil { + // If the event has no sent-by token yet then we haven't attempted to send + // it. Record the current requested sync token in the database. + toUpdate = append(toUpdate, event.ID) + toReturn = append(toReturn, event) + event.SentByToken = &token + } else if token.IsAfter(*event.SentByToken) { + // The event had a sync token, therefore we've sent it before. The current + // sync token is now after the stored one so we can assume that the client + // successfully completed the previous sync (it would re-request it otherwise) + // so we can remove the entry from the database. + toDelete = append(toDelete, event.ID) + } else { + // It looks like the sync is being re-requested, maybe it timed out or + // failed. Re-send any that should have been acknowledged by now. + toReturn = append(toReturn, event) + } + } + + return toReturn, toUpdate, toDelete, nil +} + +func (d *Database) CleanSendToDeviceUpdates( + ctx context.Context, + toUpdate, toDelete []types.SendToDeviceNID, + token types.StreamingToken, +) (err error) { + if len(toUpdate) == 0 && len(toDelete) == 0 { + return nil + } + // If we need to write to the database then we'll ask the SendToDeviceWriter to + // do that for us. It'll guarantee that we don't lock the table for writes in + // more than one place. + err = d.SendToDeviceWriter.Do(d.DB, func(txn *sql.Tx) error { + // Delete any send-to-device messages marked for deletion. + if e := d.SendToDevice.DeleteSendToDeviceMessages(ctx, txn, toDelete); e != nil { + return fmt.Errorf("d.SendToDevice.DeleteSendToDeviceMessages: %w", e) + } + + // Now update any outstanding send-to-device messages with the new sync token. + if e := d.SendToDevice.UpdateSentSendToDeviceMessages(ctx, txn, token.String(), toUpdate); e != nil { + return fmt.Errorf("d.SendToDevice.UpdateSentSendToDeviceMessages: %w", err) + } + + return nil + }) + return +} + // There may be some overlap where events in stateEvents are already in recentEvents, so filter // them out so we don't include them twice in the /sync response. They should be in recentEvents // only, so clients get to the correct state once they have rolled forward. diff --git a/syncapi/storage/sqlite3/send_to_device_table.go b/syncapi/storage/sqlite3/send_to_device_table.go new file mode 100644 index 000000000..0d03f23ef --- /dev/null +++ b/syncapi/storage/sqlite3/send_to_device_table.go @@ -0,0 +1,172 @@ +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + "encoding/json" + "strings" + + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/syncapi/storage/tables" + "github.com/matrix-org/dendrite/syncapi/types" +) + +const sendToDeviceSchema = ` +-- Stores send-to-device messages. +CREATE TABLE IF NOT EXISTS syncapi_send_to_device ( + -- The ID that uniquely identifies this message. + id INTEGER PRIMARY KEY AUTOINCREMENT, + -- The user ID to send the message to. + user_id TEXT NOT NULL, + -- The device ID to send the message to. + device_id TEXT NOT NULL, + -- The event content JSON. + content TEXT NOT NULL, + -- The token that was supplied to the /sync at the time that this + -- message was included in a sync response, or NULL if we haven't + -- included it in a /sync response yet. + sent_by_token TEXT +); +` + +const insertSendToDeviceMessageSQL = ` + INSERT INTO syncapi_send_to_device (user_id, device_id, content) + VALUES ($1, $2, $3) +` + +const countSendToDeviceMessagesSQL = ` + SELECT COUNT(*) + FROM syncapi_send_to_device + WHERE user_id = $1 AND device_id = $2 +` + +const selectSendToDeviceMessagesSQL = ` + SELECT id, user_id, device_id, content, sent_by_token + FROM syncapi_send_to_device + WHERE user_id = $1 AND device_id = $2 + ORDER BY id DESC +` + +const updateSentSendToDeviceMessagesSQL = ` + UPDATE syncapi_send_to_device SET sent_by_token = $1 + WHERE id IN ($2) +` + +const deleteSendToDeviceMessagesSQL = ` + DELETE FROM syncapi_send_to_device WHERE id IN ($1) +` + +type sendToDeviceStatements struct { + insertSendToDeviceMessageStmt *sql.Stmt + selectSendToDeviceMessagesStmt *sql.Stmt + countSendToDeviceMessagesStmt *sql.Stmt +} + +func NewSqliteSendToDeviceTable(db *sql.DB) (tables.SendToDevice, error) { + s := &sendToDeviceStatements{} + _, err := db.Exec(sendToDeviceSchema) + if err != nil { + return nil, err + } + if s.countSendToDeviceMessagesStmt, err = db.Prepare(countSendToDeviceMessagesSQL); err != nil { + return nil, err + } + if s.insertSendToDeviceMessageStmt, err = db.Prepare(insertSendToDeviceMessageSQL); err != nil { + return nil, err + } + if s.selectSendToDeviceMessagesStmt, err = db.Prepare(selectSendToDeviceMessagesSQL); err != nil { + return nil, err + } + return s, nil +} + +func (s *sendToDeviceStatements) InsertSendToDeviceMessage( + ctx context.Context, txn *sql.Tx, userID, deviceID, content string, +) (err error) { + _, err = internal.TxStmt(txn, s.insertSendToDeviceMessageStmt).ExecContext(ctx, userID, deviceID, content) + return +} + +func (s *sendToDeviceStatements) CountSendToDeviceMessages( + ctx context.Context, txn *sql.Tx, userID, deviceID string, +) (count int, err error) { + row := internal.TxStmt(txn, s.countSendToDeviceMessagesStmt).QueryRowContext(ctx, userID, deviceID) + if err = row.Scan(&count); err != nil { + return + } + return count, nil +} + +func (s *sendToDeviceStatements) SelectSendToDeviceMessages( + ctx context.Context, txn *sql.Tx, userID, deviceID string, +) (events []types.SendToDeviceEvent, err error) { + rows, err := internal.TxStmt(txn, s.selectSendToDeviceMessagesStmt).QueryContext(ctx, userID, deviceID) + if err != nil { + return + } + defer internal.CloseAndLogIfError(ctx, rows, "SelectSendToDeviceMessages: rows.close() failed") + + for rows.Next() { + var id types.SendToDeviceNID + var userID, deviceID, content string + var sentByToken *string + if err = rows.Scan(&id, &userID, &deviceID, &content, &sentByToken); err != nil { + return + } + event := types.SendToDeviceEvent{ + ID: id, + UserID: userID, + DeviceID: deviceID, + } + if err = json.Unmarshal([]byte(content), &event.SendToDeviceEvent); err != nil { + return + } + if sentByToken != nil { + if token, err := types.NewStreamTokenFromString(*sentByToken); err == nil { + event.SentByToken = &token + } + } + events = append(events, event) + } + + return events, rows.Err() +} + +func (s *sendToDeviceStatements) UpdateSentSendToDeviceMessages( + ctx context.Context, txn *sql.Tx, token string, nids []types.SendToDeviceNID, +) (err error) { + query := strings.Replace(updateSentSendToDeviceMessagesSQL, "($2)", internal.QueryVariadic(1+len(nids)), 1) + params := make([]interface{}, 1+len(nids)) + params[0] = token + for k, v := range nids { + params[k+1] = v + } + _, err = txn.ExecContext(ctx, query, params...) + return +} + +func (s *sendToDeviceStatements) DeleteSendToDeviceMessages( + ctx context.Context, txn *sql.Tx, nids []types.SendToDeviceNID, +) (err error) { + query := strings.Replace(deleteSendToDeviceMessagesSQL, "($1)", internal.QueryVariadic(len(nids)), 1) + params := make([]interface{}, 1+len(nids)) + for k, v := range nids { + params[k] = v + } + _, err = txn.ExecContext(ctx, query, params...) + return +} diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index 8ab1d4040..5ba07617e 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -95,6 +95,10 @@ func (d *SyncServerDatasource) prepare() (err error) { if err != nil { return err } + sendToDevice, err := NewSqliteSendToDeviceTable(d.db) + if err != nil { + return err + } d.Database = shared.Database{ DB: d.db, Invites: invites, @@ -103,6 +107,8 @@ func (d *SyncServerDatasource) prepare() (err error) { BackwardExtremities: bwExtrem, CurrentRoomState: roomState, Topology: topology, + SendToDevice: sendToDevice, + SendToDeviceWriter: internal.NewTransactionWriter(), EDUCache: cache.New(), } return nil diff --git a/syncapi/storage/storage_test.go b/syncapi/storage/storage_test.go index bb8554f43..4661ede4d 100644 --- a/syncapi/storage/storage_test.go +++ b/syncapi/storage/storage_test.go @@ -3,6 +3,7 @@ package storage_test import ( "context" "crypto/ed25519" + "encoding/json" "fmt" "testing" "time" @@ -157,7 +158,8 @@ func TestSyncResponse(t *testing.T) { from := types.NewStreamToken( // pretend we are at the penultimate event positions[len(positions)-2], types.StreamPosition(0), ) - return db.IncrementalSync(ctx, testUserDeviceA, from, latest, 5, false) + res := types.NewResponse() + return db.IncrementalSync(ctx, res, testUserDeviceA, from, latest, 5, false) }, WantTimeline: events[len(events)-1:], }, @@ -169,8 +171,9 @@ func TestSyncResponse(t *testing.T) { from := types.NewStreamToken( // pretend we are 10 events behind positions[len(positions)-11], types.StreamPosition(0), ) + res := types.NewResponse() // limit is set to 5 - return db.IncrementalSync(ctx, testUserDeviceA, from, latest, 5, false) + return db.IncrementalSync(ctx, res, testUserDeviceA, from, latest, 5, false) }, // want the last 5 events, NOT the last 10. WantTimeline: events[len(events)-5:], @@ -180,8 +183,9 @@ func TestSyncResponse(t *testing.T) { { Name: "CompleteSync limited", DoSync: func() (*types.Response, error) { + res := types.NewResponse() // limit set to 5 - return db.CompleteSync(ctx, testUserDeviceA, 5) + return db.CompleteSync(ctx, res, testUserDeviceA, 5) }, // want the last 5 events WantTimeline: events[len(events)-5:], @@ -193,7 +197,8 @@ func TestSyncResponse(t *testing.T) { { Name: "CompleteSync", DoSync: func() (*types.Response, error) { - return db.CompleteSync(ctx, testUserDeviceA, len(events)+1) + res := types.NewResponse() + return db.CompleteSync(ctx, res, testUserDeviceA, len(events)+1) }, WantTimeline: events, // We want no state at all as that field in /sync is the delta between the token (beginning of time) @@ -234,7 +239,8 @@ func TestGetEventsInRangeWithPrevBatch(t *testing.T) { positions[len(positions)-2], types.StreamPosition(0), ) - res, err := db.IncrementalSync(ctx, testUserDeviceA, from, latest, 5, false) + res := types.NewResponse() + res, err = db.IncrementalSync(ctx, res, testUserDeviceA, from, latest, 5, false) if err != nil { t.Fatalf("failed to IncrementalSync with latest token") } @@ -512,6 +518,89 @@ func TestGetEventsInRangeWithEventsInsertedLikeBackfill(t *testing.T) { } } +func TestSendToDeviceBehaviour(t *testing.T) { + //t.Parallel() + db := MustCreateDatabase(t) + + // At this point there should be no messages. We haven't sent anything + // yet. + events, updates, deletions, err := db.SendToDeviceUpdatesForSync(ctx, "alice", "one", types.NewStreamToken(0, 0)) + if err != nil { + t.Fatal(err) + } + if len(events) != 0 || len(updates) != 0 || len(deletions) != 0 { + t.Fatal("first call should have no updates") + } + err = db.CleanSendToDeviceUpdates(context.Background(), updates, deletions, types.NewStreamToken(0, 0)) + if err != nil { + return + } + + // Try sending a message. + streamPos, err := db.StoreNewSendForDeviceMessage(ctx, types.StreamPosition(0), "alice", "one", gomatrixserverlib.SendToDeviceEvent{ + Sender: "bob", + Type: "m.type", + Content: json.RawMessage("{}"), + }) + if err != nil { + t.Fatal(err) + } + + // At this point we should get exactly one message. We're sending the sync position + // that we were given from the update and the send-to-device update will be updated + // in the database to reflect that this was the sync position we sent the message at. + events, updates, deletions, err = db.SendToDeviceUpdatesForSync(ctx, "alice", "one", types.NewStreamToken(0, streamPos)) + if err != nil { + t.Fatal(err) + } + if len(events) != 1 || len(updates) != 1 || len(deletions) != 0 { + t.Fatal("second call should have one update") + } + err = db.CleanSendToDeviceUpdates(context.Background(), updates, deletions, types.NewStreamToken(0, streamPos)) + if err != nil { + return + } + + // At this point we should still have one message because we haven't progressed the + // sync position yet. This is equivalent to the client failing to /sync and retrying + // with the same position. + events, updates, deletions, err = db.SendToDeviceUpdatesForSync(ctx, "alice", "one", types.NewStreamToken(0, streamPos)) + if err != nil { + t.Fatal(err) + } + if len(events) != 1 || len(updates) != 0 || len(deletions) != 0 { + t.Fatal("third call should have one update still") + } + err = db.CleanSendToDeviceUpdates(context.Background(), updates, deletions, types.NewStreamToken(0, streamPos)) + if err != nil { + return + } + + // At this point we should now have no updates, because we've progressed the sync + // position. Therefore the update from before will not be sent again. + events, updates, deletions, err = db.SendToDeviceUpdatesForSync(ctx, "alice", "one", types.NewStreamToken(0, streamPos+1)) + if err != nil { + t.Fatal(err) + } + if len(events) != 0 || len(updates) != 0 || len(deletions) != 1 { + t.Fatal("fourth call should have no updates") + } + err = db.CleanSendToDeviceUpdates(context.Background(), updates, deletions, types.NewStreamToken(0, streamPos+1)) + if err != nil { + return + } + + // At this point we should still have no updates, because no new updates have been + // sent. + events, updates, deletions, err = db.SendToDeviceUpdatesForSync(ctx, "alice", "one", types.NewStreamToken(0, streamPos+2)) + if err != nil { + t.Fatal(err) + } + if len(events) != 0 || len(updates) != 0 || len(deletions) != 0 { + t.Fatal("fifth call should have no updates") + } +} + func assertEventsEqual(t *testing.T, msg string, checkRoomID bool, gots []gomatrixserverlib.ClientEvent, wants []gomatrixserverlib.HeaderedEvent) { if len(gots) != len(wants) { t.Fatalf("%s response returned %d events, want %d", msg, len(gots), len(wants)) diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go index bc3b6941a..0b7d15951 100644 --- a/syncapi/storage/tables/interface.go +++ b/syncapi/storage/tables/interface.go @@ -1,3 +1,17 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package tables import ( @@ -94,3 +108,28 @@ type BackwardsExtremities interface { // DeleteBackwardExtremity removes a backwards extremity for a room, if one existed. DeleteBackwardExtremity(ctx context.Context, txn *sql.Tx, roomID, knownEventID string) (err error) } + +// SendToDevice tracks send-to-device messages which are sent to individual +// clients. Each message gets inserted into this table at the point that we +// receive it from the EDU server. +// +// We're supposed to try and do our best to deliver send-to-device messages +// once, but the only way that we can really guarantee that they have been +// delivered is if the client successfully requests the next sync as given +// in the next_batch. Each time the device syncs, we will request all of the +// updates that either haven't been sent yet, along with all updates that we +// *have* sent but we haven't confirmed to have been received yet. If it's the +// first time we're sending a given update then we update the table to say +// what the "since" parameter was when we tried to send it. +// +// When the client syncs again, if their "since" parameter is *later* than +// the recorded one, we drop the entry from the DB as it's "sent". If the +// sync parameter isn't later then we will keep including the updates in the +// sync response, as the client is seemingly trying to repeat the same /sync. +type SendToDevice interface { + InsertSendToDeviceMessage(ctx context.Context, txn *sql.Tx, userID, deviceID, content string) (err error) + SelectSendToDeviceMessages(ctx context.Context, txn *sql.Tx, userID, deviceID string) (events []types.SendToDeviceEvent, err error) + UpdateSentSendToDeviceMessages(ctx context.Context, txn *sql.Tx, token string, nids []types.SendToDeviceNID) (err error) + DeleteSendToDeviceMessages(ctx context.Context, txn *sql.Tx, nids []types.SendToDeviceNID) (err error) + CountSendToDeviceMessages(ctx context.Context, txn *sql.Tx, userID, deviceID string) (count int, err error) +} diff --git a/syncapi/sync/notifier.go b/syncapi/sync/notifier.go index 9b410a0c4..325e75351 100644 --- a/syncapi/sync/notifier.go +++ b/syncapi/sync/notifier.go @@ -120,6 +120,18 @@ func (n *Notifier) OnNewEvent( } } +func (n *Notifier) OnNewSendToDevice( + userID string, deviceIDs []string, + posUpdate types.StreamingToken, +) { + n.streamLock.Lock() + defer n.streamLock.Unlock() + latestPos := n.currPos.WithUpdates(posUpdate) + n.currPos = latestPos + + n.wakeupUserDevice(userID, deviceIDs, latestPos) +} + // GetListener returns a UserStreamListener that can be used to wait for // updates for a user. Must be closed. // notify for anything before sincePos @@ -189,8 +201,8 @@ func (n *Notifier) wakeupUsers(userIDs []string, newPos types.StreamingToken) { // wakeupUserDevice will wake up the sync stream for a specific user device. Other // device streams will be left alone. // nolint:unused -func (n *Notifier) wakeupUserDevice(userDevices map[string]string, newPos types.StreamingToken) { - for userID, deviceID := range userDevices { +func (n *Notifier) wakeupUserDevice(userID string, deviceIDs []string, newPos types.StreamingToken) { + for _, deviceID := range deviceIDs { if stream := n.fetchUserDeviceStream(userID, deviceID, false); stream != nil { stream.Broadcast(newPos) // wake up all goroutines Wait()ing on this stream } diff --git a/syncapi/sync/notifier_test.go b/syncapi/sync/notifier_test.go index 14ddef20a..132315573 100644 --- a/syncapi/sync/notifier_test.go +++ b/syncapi/sync/notifier_test.go @@ -172,7 +172,7 @@ func TestCorrectStreamWakeup(t *testing.T) { time.Sleep(1 * time.Second) wake := "two" - n.wakeupUserDevice(map[string]string{alice: wake}, syncPositionAfter) + n.wakeupUserDevice(alice, []string{wake}, syncPositionAfter) if result := <-awoken; result != wake { t.Fatalf("expected to wake %q, got %q", wake, result) diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index bd29b3338..8b93cad45 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -1,4 +1,6 @@ // Copyright 2017 Vector Creations Ltd +// Copyright 2017-2018 New Vector Ltd +// Copyright 2019-2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,6 +17,7 @@ package sync import ( + "context" "net/http" "time" @@ -54,17 +57,18 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *authtype JSON: jsonerror.Unknown(err.Error()), } } + logger := util.GetLogger(req.Context()).WithFields(log.Fields{ - "userID": device.UserID, - "deviceID": device.ID, - "since": syncReq.since, - "timeout": syncReq.timeout, - "limit": syncReq.limit, + "user_id": device.UserID, + "device_id": device.ID, + "since": syncReq.since, + "timeout": syncReq.timeout, + "limit": syncReq.limit, }) currPos := rp.notifier.CurrentPosition() - if shouldReturnImmediately(syncReq) { + if rp.shouldReturnImmediately(syncReq) { syncData, err = rp.currentSyncForUser(*syncReq, currPos) if err != nil { logger.WithError(err).Error("rp.currentSyncForUser failed") @@ -116,7 +120,6 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *authtype // response. This ensures that we don't waste the hard work // of calculating the sync only to get timed out before we // can respond - syncData, err = rp.currentSyncForUser(*syncReq, currPos) if err != nil { logger.WithError(err).Error("rp.currentSyncForUser failed") @@ -134,19 +137,59 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *authtype } func (rp *RequestPool) currentSyncForUser(req syncRequest, latestPos types.StreamingToken) (res *types.Response, err error) { - // TODO: handle ignored users - if req.since == nil { - res, err = rp.db.CompleteSync(req.ctx, req.device, req.limit) - } else { - res, err = rp.db.IncrementalSync(req.ctx, req.device, *req.since, latestPos, req.limit, req.wantFullState) + res = types.NewResponse() + + since := types.NewStreamToken(0, 0) + if req.since != nil { + since = *req.since } + // See if we have any new tasks to do for the send-to-device messaging. + events, updates, deletions, err := rp.db.SendToDeviceUpdatesForSync(req.ctx, req.device.UserID, req.device.ID, since) + if err != nil { + return nil, err + } + + // TODO: handle ignored users + if req.since == nil { + res, err = rp.db.CompleteSync(req.ctx, res, req.device, req.limit) + } else { + res, err = rp.db.IncrementalSync(req.ctx, res, req.device, *req.since, latestPos, req.limit, req.wantFullState) + } if err != nil { return } accountDataFilter := gomatrixserverlib.DefaultEventFilter() // TODO: use filter provided in req instead res, err = rp.appendAccountData(res, req.device.UserID, req, latestPos.PDUPosition(), &accountDataFilter) + if err != nil { + return + } + + // Before we return the sync response, make sure that we take action on + // any send-to-device database updates or deletions that we need to do. + // Then add the updates into the sync response. + if len(updates) > 0 || len(deletions) > 0 { + // Handle the updates and deletions in the database. + err = rp.db.CleanSendToDeviceUpdates(context.Background(), updates, deletions, since) + if err != nil { + return + } + } + if len(events) > 0 { + // Add the updates into the sync response. + for _, event := range events { + res.ToDevice.Events = append(res.ToDevice.Events, event.SendToDeviceEvent) + } + + // Get the next_batch from the sync response and increase the + // EDU counter. + if pos, perr := types.NewStreamTokenFromString(res.NextBatch); perr == nil { + pos.Positions[1]++ + res.NextBatch = pos.String() + } + } + return } @@ -238,6 +281,10 @@ func (rp *RequestPool) appendAccountData( // shouldReturnImmediately returns whether the /sync request is an initial sync, // or timeout=0, or full_state=true, in any of the cases the request should // return immediately. -func shouldReturnImmediately(syncReq *syncRequest) bool { - return syncReq.since == nil || syncReq.timeout == 0 || syncReq.wantFullState +func (rp *RequestPool) shouldReturnImmediately(syncReq *syncRequest) bool { + if syncReq.since == nil || syncReq.timeout == 0 || syncReq.wantFullState { + return true + } + waiting, werr := rp.db.SendToDeviceUpdatesWaiting(context.TODO(), syncReq.device.UserID, syncReq.device.ID) + return werr == nil && waiting } diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go index 9251f6182..762f4e9d3 100644 --- a/syncapi/syncapi.go +++ b/syncapi/syncapi.go @@ -78,7 +78,14 @@ func SetupSyncAPIComponent( base.Cfg, base.KafkaConsumer, notifier, syncDB, ) if err = typingConsumer.Start(); err != nil { - logrus.WithError(err).Panicf("failed to start typing server consumer") + logrus.WithError(err).Panicf("failed to start typing consumer") + } + + sendToDeviceConsumer := consumers.NewOutputSendToDeviceEventConsumer( + base.Cfg, base.KafkaConsumer, notifier, syncDB, + ) + if err = sendToDeviceConsumer.Start(); err != nil { + logrus.WithError(err).Panicf("failed to start send-to-device consumer") } routing.Setup(base.PublicAPIMux, requestPool, syncDB, deviceDB, federation, rsAPI, cfg) diff --git a/syncapi/types/types.go b/syncapi/types/types.go index caa1b3ade..c1f09fba5 100644 --- a/syncapi/types/types.go +++ b/syncapi/types/types.go @@ -296,13 +296,14 @@ type Response struct { Invite map[string]InviteResponse `json:"invite"` Leave map[string]LeaveResponse `json:"leave"` } `json:"rooms"` + ToDevice struct { + Events []gomatrixserverlib.SendToDeviceEvent `json:"events"` + } `json:"to_device"` } // NewResponse creates an empty response with initialised maps. -func NewResponse(token StreamingToken) *Response { - res := Response{ - NextBatch: token.String(), - } +func NewResponse() *Response { + res := Response{} // Pre-initialise the maps. Synapse will return {} even if there are no rooms under a specific section, // so let's do the same thing. Bonus: this means we can't get dreaded 'assignment to entry in nil map' errors. res.Rooms.Join = make(map[string]JoinResponse) @@ -315,6 +316,7 @@ func NewResponse(token StreamingToken) *Response { // This also applies to NewJoinResponse, NewInviteResponse and NewLeaveResponse. res.AccountData.Events = make([]gomatrixserverlib.ClientEvent, 0) res.Presence.Events = make([]gomatrixserverlib.ClientEvent, 0) + res.ToDevice.Events = make([]gomatrixserverlib.SendToDeviceEvent, 0) return &res } @@ -326,7 +328,8 @@ func (r *Response) IsEmpty() bool { len(r.Rooms.Invite) == 0 && len(r.Rooms.Leave) == 0 && len(r.AccountData.Events) == 0 && - len(r.Presence.Events) == 0 + len(r.Presence.Events) == 0 && + len(r.ToDevice.Events) == 0 } // JoinResponse represents a /sync response for a room which is under the 'join' key. @@ -393,3 +396,13 @@ func NewLeaveResponse() *LeaveResponse { res.Timeline.Events = make([]gomatrixserverlib.ClientEvent, 0) return &res } + +type SendToDeviceNID int + +type SendToDeviceEvent struct { + gomatrixserverlib.SendToDeviceEvent + ID SendToDeviceNID + UserID string + DeviceID string + SentByToken *StreamingToken +} diff --git a/sytest-blacklist b/sytest-blacklist index caad25455..1efc207f7 100644 --- a/sytest-blacklist +++ b/sytest-blacklist @@ -39,3 +39,8 @@ Ignore invite in incremental sync # Blacklisted because this test calls /r0/events which we don't implement New room members see their own join event Existing members see new members' join events + +# Blacklisted because the federation work for these hasn't been finished yet. +Can recv device messages over federation +Device messages over federation wake up /sync +Wildcard device messages over federation wake up /sync diff --git a/sytest-whitelist b/sytest-whitelist index d4e6be9a4..6236b28ed 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -289,3 +289,15 @@ Existing members see new members' join events Inbound federation can receive events Inbound federation can receive redacted events Can logout current device +Can send a message directly to a device using PUT /sendToDevice +Can recv a device message using /sync +Can recv device messages until they are acknowledged +Device messages with the same txn_id are deduplicated +Device messages wake up /sync +# TODO: separate PR for: Can recv device messages over federation +# TODO: separate PR for: Device messages over federation wake up /sync +Can send messages with a wildcard device id +Can send messages with a wildcard device id to two devices +Wildcard device messages wake up /sync +# TODO: separate PR for: Wildcard device messages over federation wake up /sync +Can send a to-device message to two users which both receive it using /sync From cfc137652ed0d783d946836cf4e9e18267a438dc Mon Sep 17 00:00:00 2001 From: Kegsay Date: Mon, 1 Jun 2020 18:34:08 +0100 Subject: [PATCH 111/200] Add a way to force federationsender to retry sending transactions (#1077) * Add a way to force federationsender to retry sending transactions And use it in P2P mode when we pick up new nodes. * Linting * Use atomic bool to stop us blocking on the channel --- cmd/dendritejs/main.go | 3 +- cmd/dendritejs/publicrooms.go | 29 +++++++++++++++++-- federationsender/api/api.go | 6 ++++ federationsender/api/perform.go | 22 +++++++++++++++ federationsender/federationsender.go | 2 +- federationsender/internal/api.go | 17 +++++++++++ federationsender/internal/perform.go | 13 +++++++++ federationsender/queue/destinationqueue.go | 33 ++++++++++++++++++++-- federationsender/queue/queue.go | 16 +++++++++++ 9 files changed, 135 insertions(+), 6 deletions(-) diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 9a02e71e9..72008aa9a 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -39,6 +39,7 @@ import ( "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/syncapi" go_http_js_libp2p "github.com/matrix-org/go-http-js-libp2p" + "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" @@ -204,7 +205,6 @@ func main() { }, KeyDatabase: fetcher, } - p2pPublicRoomProvider := NewLibP2PPublicRoomsProvider(node) rsAPI := roomserver.SetupRoomServerComponent(base, keyRing, federation) eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New(), deviceDB) @@ -213,6 +213,7 @@ func main() { ) fedSenderAPI := federationsender.SetupFederationSenderComponent(base, federation, rsAPI, &keyRing) rsAPI.SetFederationSenderAPI(fedSenderAPI) + p2pPublicRoomProvider := NewLibP2PPublicRoomsProvider(node, fedSenderAPI) clientapi.SetupClientAPIComponent( base, deviceDB, accountDB, diff --git a/cmd/dendritejs/publicrooms.go b/cmd/dendritejs/publicrooms.go index 5246cf535..5032bc15f 100644 --- a/cmd/dendritejs/publicrooms.go +++ b/cmd/dendritejs/publicrooms.go @@ -17,23 +17,48 @@ package main import ( + "context" + + "github.com/matrix-org/dendrite/federationsender/api" go_http_js_libp2p "github.com/matrix-org/go-http-js-libp2p" + "github.com/matrix-org/gomatrixserverlib" ) type libp2pPublicRoomsProvider struct { node *go_http_js_libp2p.P2pLocalNode providers []go_http_js_libp2p.PeerInfo + fedSender api.FederationSenderInternalAPI } -func NewLibP2PPublicRoomsProvider(node *go_http_js_libp2p.P2pLocalNode) *libp2pPublicRoomsProvider { +func NewLibP2PPublicRoomsProvider(node *go_http_js_libp2p.P2pLocalNode, fedSender api.FederationSenderInternalAPI) *libp2pPublicRoomsProvider { p := &libp2pPublicRoomsProvider{ - node: node, + node: node, + fedSender: fedSender, } node.RegisterFoundProviders(p.foundProviders) return p } func (p *libp2pPublicRoomsProvider) foundProviders(peerInfos []go_http_js_libp2p.PeerInfo) { + // work out the diff then poke for new ones + seen := make(map[string]bool, len(p.providers)) + for _, pr := range p.providers { + seen[pr.Id] = true + } + var newPeers []gomatrixserverlib.ServerName + for _, pi := range peerInfos { + if !seen[pi.Id] { + newPeers = append(newPeers, gomatrixserverlib.ServerName(pi.Id)) + } + } + if len(newPeers) > 0 { + var res api.PerformServersAliveResponse + // ignore errors, we don't care. + p.fedSender.PerformServersAlive(context.Background(), &api.PerformServersAliveRequest{ + Servers: newPeers, + }, &res) + } + p.providers = peerInfos } diff --git a/federationsender/api/api.go b/federationsender/api/api.go index 678f02e68..4eb20cb67 100644 --- a/federationsender/api/api.go +++ b/federationsender/api/api.go @@ -42,6 +42,12 @@ type FederationSenderInternalAPI interface { request *PerformLeaveRequest, response *PerformLeaveResponse, ) error + // Notifies the federation sender that these servers may be online and to retry sending messages. + PerformServersAlive( + ctx context.Context, + request *PerformServersAliveRequest, + response *PerformServersAliveResponse, + ) error } // NewFederationSenderInternalAPIHTTP creates a FederationSenderInternalAPI implemented by talking to a HTTP POST API. diff --git a/federationsender/api/perform.go b/federationsender/api/perform.go index 2a1834b61..5e4d7fe97 100644 --- a/federationsender/api/perform.go +++ b/federationsender/api/perform.go @@ -18,6 +18,9 @@ const ( // FederationSenderPerformLeaveRequestPath is the HTTP path for the PerformLeaveRequest API. FederationSenderPerformLeaveRequestPath = "/federationsender/performLeaveRequest" + + // FederationSenderPerformServersAlivePath is the HTTP path for the PerformServersAlive API. + FederationSenderPerformServersAlivePath = "/federationsender/performServersAlive" ) type PerformDirectoryLookupRequest struct { @@ -88,3 +91,22 @@ func (h *httpFederationSenderInternalAPI) PerformLeave( apiURL := h.federationSenderURL + FederationSenderPerformLeaveRequestPath return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } + +type PerformServersAliveRequest struct { + Servers []gomatrixserverlib.ServerName +} + +type PerformServersAliveResponse struct { +} + +func (h *httpFederationSenderInternalAPI) PerformServersAlive( + ctx context.Context, + request *PerformServersAliveRequest, + response *PerformServersAliveResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "PerformServersAlive") + defer span.Finish() + + apiURL := h.federationSenderURL + FederationSenderPerformServersAlivePath + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} diff --git a/federationsender/federationsender.go b/federationsender/federationsender.go index cca847a51..9e5cc8dd1 100644 --- a/federationsender/federationsender.go +++ b/federationsender/federationsender.go @@ -67,7 +67,7 @@ func SetupFederationSenderComponent( queryAPI := internal.NewFederationSenderInternalAPI( federationSenderDB, base.Cfg, roomserverProducer, federation, keyRing, - statistics, + statistics, queues, ) queryAPI.SetupHTTP(base.InternalAPIMux) diff --git a/federationsender/internal/api.go b/federationsender/internal/api.go index dd3942583..edf8fb4e9 100644 --- a/federationsender/internal/api.go +++ b/federationsender/internal/api.go @@ -7,6 +7,7 @@ import ( "github.com/gorilla/mux" "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/federationsender/producers" + "github.com/matrix-org/dendrite/federationsender/queue" "github.com/matrix-org/dendrite/federationsender/storage" "github.com/matrix-org/dendrite/federationsender/types" "github.com/matrix-org/dendrite/internal" @@ -24,6 +25,7 @@ type FederationSenderInternalAPI struct { producer *producers.RoomserverProducer federation *gomatrixserverlib.FederationClient keyRing *gomatrixserverlib.KeyRing + queues *queue.OutgoingQueues } func NewFederationSenderInternalAPI( @@ -32,6 +34,7 @@ func NewFederationSenderInternalAPI( federation *gomatrixserverlib.FederationClient, keyRing *gomatrixserverlib.KeyRing, statistics *types.Statistics, + queues *queue.OutgoingQueues, ) *FederationSenderInternalAPI { return &FederationSenderInternalAPI{ db: db, @@ -40,6 +43,7 @@ func NewFederationSenderInternalAPI( federation: federation, keyRing: keyRing, statistics: statistics, + queues: queues, } } @@ -112,4 +116,17 @@ func (f *FederationSenderInternalAPI) SetupHTTP(internalAPIMux *mux.Router) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) + internalAPIMux.Handle(api.FederationSenderPerformServersAlivePath, + internal.MakeInternalAPI("PerformServersAliveRequest", func(req *http.Request) util.JSONResponse { + var request api.PerformServersAliveRequest + var response api.PerformServersAliveResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := f.PerformServersAlive(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) } diff --git a/federationsender/internal/perform.go b/federationsender/internal/perform.go index 383ce4888..c601e9604 100644 --- a/federationsender/internal/perform.go +++ b/federationsender/internal/perform.go @@ -275,3 +275,16 @@ func (r *FederationSenderInternalAPI) PerformLeave( request.RoomID, len(request.ServerNames), ) } + +// PerformServersAlive implements api.FederationSenderInternalAPI +func (r *FederationSenderInternalAPI) PerformServersAlive( + ctx context.Context, + request *api.PerformServersAliveRequest, + response *api.PerformServersAliveResponse, +) (err error) { + for _, srv := range request.Servers { + r.queues.RetryServer(srv) + } + + return nil +} diff --git a/federationsender/queue/destinationqueue.go b/federationsender/queue/destinationqueue.go index 09dac464f..4ab610de7 100644 --- a/federationsender/queue/destinationqueue.go +++ b/federationsender/queue/destinationqueue.go @@ -39,6 +39,7 @@ type destinationQueue struct { origin gomatrixserverlib.ServerName // origin of requests destination gomatrixserverlib.ServerName // destination of requests running atomic.Bool // is the queue worker running? + backingOff atomic.Bool // true if we're backing off statistics *types.ServerStatistics // statistics about this remote server incomingPDUs chan *gomatrixserverlib.HeaderedEvent // PDUs to send incomingEDUs chan *gomatrixserverlib.EDU // EDUs to send @@ -47,6 +48,28 @@ type destinationQueue struct { pendingPDUs []*gomatrixserverlib.HeaderedEvent // owned by backgroundSend pendingEDUs []*gomatrixserverlib.EDU // owned by backgroundSend pendingInvites []*gomatrixserverlib.InviteV2Request // owned by backgroundSend + retryServerCh chan bool // interrupts backoff +} + +// retry will clear the blacklist state and attempt to send built up events to the server, +// resetting and interrupting any backoff timers. +func (oq *destinationQueue) retry() { + // TODO: We don't send all events in the case where the server has been blacklisted as we + // drop events instead then. This means we will send the oldest N events (chan size, currently 128) + // and then skip ahead a lot which feels non-ideal but equally we can't persist thousands of events + // in-memory to maybe-send it one day. Ideally we would just shove these pending events in a database + // so we can send a lot of events. + oq.statistics.Success() + // if we were backing off, swap to not backing off and interrupt the select. + // We need to use an atomic bool here to prevent multiple calls to retry() blocking on the channel + // as it is unbuffered. + if oq.backingOff.CAS(true, false) { + oq.retryServerCh <- true + } + if !oq.running.Load() { + log.Infof("Restarting queue for %s", oq.destination) + go oq.backgroundSend() + } } // Send event adds the event to the pending queue for the destination. @@ -155,9 +178,15 @@ func (oq *destinationQueue) backgroundSend() { } // If we are backing off this server then wait for the - // backoff duration to complete first. + // backoff duration to complete first, or until explicitly + // told to retry. if backoff, duration := oq.statistics.BackoffDuration(); backoff { - <-time.After(duration) + oq.backingOff.Store(true) + select { + case <-time.After(duration): + case <-oq.retryServerCh: + } + oq.backingOff.Store(false) } // How many things do we have waiting? diff --git a/federationsender/queue/queue.go b/federationsender/queue/queue.go index aae6c53a0..386a3397f 100644 --- a/federationsender/queue/queue.go +++ b/federationsender/queue/queue.go @@ -52,6 +52,12 @@ func NewOutgoingQueues( } } +func (oqs *OutgoingQueues) getQueueIfExists(destination gomatrixserverlib.ServerName) *destinationQueue { + oqs.queuesMutex.Lock() + defer oqs.queuesMutex.Unlock() + return oqs.queues[destination] +} + func (oqs *OutgoingQueues) getQueue(destination gomatrixserverlib.ServerName) *destinationQueue { oqs.queuesMutex.Lock() defer oqs.queuesMutex.Unlock() @@ -66,6 +72,7 @@ func (oqs *OutgoingQueues) getQueue(destination gomatrixserverlib.ServerName) *d incomingPDUs: make(chan *gomatrixserverlib.HeaderedEvent, 128), incomingEDUs: make(chan *gomatrixserverlib.EDU, 128), incomingInvites: make(chan *gomatrixserverlib.InviteV2Request, 128), + retryServerCh: make(chan bool), } oqs.queues[destination] = oq } @@ -160,6 +167,15 @@ func (oqs *OutgoingQueues) SendEDU( return nil } +// RetryServer attempts to resend events to the given server if we had given up. +func (oqs *OutgoingQueues) RetryServer(srv gomatrixserverlib.ServerName) { + q := oqs.getQueueIfExists(srv) + if q == nil { + return + } + q.retry() +} + // filterAndDedupeDests removes our own server from the list of destinations // and deduplicates any servers in the list that may appear more than once. func filterAndDedupeDests(origin gomatrixserverlib.ServerName, destinations []gomatrixserverlib.ServerName) ( From 895c8f03c031a613f8f603eba753056ae4eb83b9 Mon Sep 17 00:00:00 2001 From: S7evinK Date: Mon, 1 Jun 2020 19:34:29 +0200 Subject: [PATCH 112/200] Fix pg user already exists (#1076) * Return newly created error if user already exists (#1002) Signed-off-by: Till Faelligen * Rename variable * Remove check for account and use returned error * Return ErrUserExists Signed-off-by: Till Faelligen * State that CreateAccount will return err ErrUserExists if the user exists Signed-off-by: Till Faelligen * Also check sqlite for constraint error * Revert "Also check sqlite for constraint error" This reverts commit 7d310514 * Check for sqlite3 constraint error * Add documentation to CreateAccount * Move ErrUserExists to accounts package * Revert "Move ErrUserExists to accounts package" Import Cycle.. This reverts commit be3d4cda Co-authored-by: Kegsay --- appservice/appservice.go | 10 ++++++---- clientapi/auth/storage/accounts/interface.go | 3 +++ clientapi/auth/storage/accounts/postgres/storage.go | 4 ++-- clientapi/auth/storage/accounts/sqlite3/storage.go | 10 +++++----- clientapi/routing/register.go | 11 ++++++----- cmd/create-account/main.go | 5 +---- internal/sql.go | 3 +++ 7 files changed, 26 insertions(+), 20 deletions(-) diff --git a/appservice/appservice.go b/appservice/appservice.go index be5b30e2b..68cf52e79 100644 --- a/appservice/appservice.go +++ b/appservice/appservice.go @@ -16,6 +16,7 @@ package appservice import ( "context" + "errors" "net/http" "sync" "time" @@ -29,6 +30,7 @@ import ( "github.com/matrix-org/dendrite/appservice/workers" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/basecomponent" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/transactions" @@ -117,12 +119,12 @@ func generateAppServiceAccount( ctx := context.Background() // Create an account for the application service - acc, err := accountsDB.CreateAccount(ctx, as.SenderLocalpart, "", as.ID) + _, err := accountsDB.CreateAccount(ctx, as.SenderLocalpart, "", as.ID) if err != nil { + if errors.Is(err, internal.ErrUserExists) { // This account already exists + return nil + } return err - } else if acc == nil { - // This account already exists - return nil } // Create a dummy device with a dummy token for the application service diff --git a/clientapi/auth/storage/accounts/interface.go b/clientapi/auth/storage/accounts/interface.go index a860f809d..4d1941a23 100644 --- a/clientapi/auth/storage/accounts/interface.go +++ b/clientapi/auth/storage/accounts/interface.go @@ -29,6 +29,9 @@ type Database interface { GetProfileByLocalpart(ctx context.Context, localpart string) (*authtypes.Profile, error) SetAvatarURL(ctx context.Context, localpart string, avatarURL string) error SetDisplayName(ctx context.Context, localpart string, displayName string) error + // CreateAccount makes a new account with the given login name and password, and creates an empty profile + // for this account. If no password is supplied, the account will be a passwordless account. If the + // account already exists, it will return nil, ErrUserExists. CreateAccount(ctx context.Context, localpart, plaintextPassword, appserviceID string) (*authtypes.Account, error) CreateGuestAccount(ctx context.Context) (*authtypes.Account, error) UpdateMemberships(ctx context.Context, eventsToAdd []gomatrixserverlib.Event, idsToRemove []string) error diff --git a/clientapi/auth/storage/accounts/postgres/storage.go b/clientapi/auth/storage/accounts/postgres/storage.go index 4a1832674..4be5dca94 100644 --- a/clientapi/auth/storage/accounts/postgres/storage.go +++ b/clientapi/auth/storage/accounts/postgres/storage.go @@ -138,7 +138,7 @@ func (d *Database) CreateGuestAccount(ctx context.Context) (acc *authtypes.Accou // CreateAccount makes a new account with the given login name and password, and creates an empty profile // for this account. If no password is supplied, the account will be a passwordless account. If the -// account already exists, it will return nil, nil. +// account already exists, it will return nil, ErrUserExists. func (d *Database) CreateAccount( ctx context.Context, localpart, plaintextPassword, appserviceID string, ) (acc *authtypes.Account, err error) { @@ -164,7 +164,7 @@ func (d *Database) createAccount( } if err := d.profiles.insertProfile(ctx, txn, localpart); err != nil { if internal.IsUniqueConstraintViolationErr(err) { - return nil, nil + return nil, internal.ErrUserExists } return nil, err } diff --git a/clientapi/auth/storage/accounts/sqlite3/storage.go b/clientapi/auth/storage/accounts/sqlite3/storage.go index 7dec87294..4b4367082 100644 --- a/clientapi/auth/storage/accounts/sqlite3/storage.go +++ b/clientapi/auth/storage/accounts/sqlite3/storage.go @@ -27,8 +27,8 @@ import ( "github.com/matrix-org/gomatrixserverlib" "golang.org/x/crypto/bcrypt" - // Import the postgres database driver. - _ "github.com/mattn/go-sqlite3" + // Import the sqlite3 database driver. + "github.com/mattn/go-sqlite3" ) // Database represents an account database @@ -148,7 +148,7 @@ func (d *Database) CreateGuestAccount(ctx context.Context) (acc *authtypes.Accou // CreateAccount makes a new account with the given login name and password, and creates an empty profile // for this account. If no password is supplied, the account will be a passwordless account. If the -// account already exists, it will return nil, nil. +// account already exists, it will return nil, ErrUserExists. func (d *Database) CreateAccount( ctx context.Context, localpart, plaintextPassword, appserviceID string, ) (acc *authtypes.Account, err error) { @@ -172,8 +172,8 @@ func (d *Database) createAccount( } } if err := d.profiles.insertProfile(ctx, txn, localpart); err != nil { - if internal.IsUniqueConstraintViolationErr(err) { - return nil, nil + if errors.Is(err, sqlite3.ErrConstraint) { + return nil, internal.ErrUserExists } return nil, err } diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index 40a2862f8..d356db2cb 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -830,15 +830,16 @@ func completeRegistration( acc, err := accountDB.CreateAccount(ctx, username, password, appserviceID) if err != nil { + if errors.Is(err, internal.ErrUserExists) { // user already exists + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.UserInUse("Desired user ID is already taken."), + } + } return util.JSONResponse{ Code: http.StatusInternalServerError, JSON: jsonerror.Unknown("failed to create account: " + err.Error()), } - } else if acc == nil { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.UserInUse("Desired user ID is already taken."), - } } // Increment prometheus counter for created users diff --git a/cmd/create-account/main.go b/cmd/create-account/main.go index eca9b2fe6..9c1e45932 100644 --- a/cmd/create-account/main.go +++ b/cmd/create-account/main.go @@ -69,13 +69,10 @@ func main() { os.Exit(1) } - account, err := accountDB.CreateAccount(context.Background(), *username, *password, "") + _, err = accountDB.CreateAccount(context.Background(), *username, *password, "") if err != nil { fmt.Println(err.Error()) os.Exit(1) - } else if account == nil { - fmt.Println("Username already exists") - os.Exit(1) } deviceDB, err := devices.NewDatabase(*database, nil, serverName) diff --git a/internal/sql.go b/internal/sql.go index 546954bd5..e3c10afca 100644 --- a/internal/sql.go +++ b/internal/sql.go @@ -26,6 +26,9 @@ import ( "go.uber.org/atomic" ) +// ErrUserExists is returned if a username already exists in the database. +var ErrUserExists = errors.New("Username already exists") + // A Transaction is something that can be committed or rolledback. type Transaction interface { // Commit the transaction From 42e797de5e60f1e5e84d55655de74cc983a932ff Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 1 Jun 2020 18:41:58 +0100 Subject: [PATCH 113/200] Unbreak dendritejs --- .../storage/accounts/sqlite3/constraint.go | 27 +++++++++++++++++++ .../accounts/sqlite3/constraint_wasm.go | 21 +++++++++++++++ .../auth/storage/accounts/sqlite3/storage.go | 4 +-- 3 files changed, 49 insertions(+), 3 deletions(-) create mode 100644 clientapi/auth/storage/accounts/sqlite3/constraint.go create mode 100644 clientapi/auth/storage/accounts/sqlite3/constraint_wasm.go diff --git a/clientapi/auth/storage/accounts/sqlite3/constraint.go b/clientapi/auth/storage/accounts/sqlite3/constraint.go new file mode 100644 index 000000000..32f96c8e4 --- /dev/null +++ b/clientapi/auth/storage/accounts/sqlite3/constraint.go @@ -0,0 +1,27 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !wasm + +package sqlite3 + +import ( + "errors" + + "github.com/mattn/go-sqlite3" +) + +func isConstraintError(err error) bool { + return errors.Is(err, sqlite3.ErrConstraint) +} diff --git a/clientapi/auth/storage/accounts/sqlite3/constraint_wasm.go b/clientapi/auth/storage/accounts/sqlite3/constraint_wasm.go new file mode 100644 index 000000000..b5077c070 --- /dev/null +++ b/clientapi/auth/storage/accounts/sqlite3/constraint_wasm.go @@ -0,0 +1,21 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build wasm + +package sqlite3 + +func isConstraintError(err error) bool { + return false +} \ No newline at end of file diff --git a/clientapi/auth/storage/accounts/sqlite3/storage.go b/clientapi/auth/storage/accounts/sqlite3/storage.go index 4b4367082..5233001f2 100644 --- a/clientapi/auth/storage/accounts/sqlite3/storage.go +++ b/clientapi/auth/storage/accounts/sqlite3/storage.go @@ -26,9 +26,7 @@ import ( "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" "golang.org/x/crypto/bcrypt" - // Import the sqlite3 database driver. - "github.com/mattn/go-sqlite3" ) // Database represents an account database @@ -172,7 +170,7 @@ func (d *Database) createAccount( } } if err := d.profiles.insertProfile(ctx, txn, localpart); err != nil { - if errors.Is(err, sqlite3.ErrConstraint) { + if isConstraintError(err) { return nil, internal.ErrUserExists } return nil, err From a07fc0ef352ecc4ef21247cbbc6abaf536b31df3 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 2 Jun 2020 09:34:36 +0100 Subject: [PATCH 114/200] Return correct error to LookupRoomAlias --- clientapi/routing/directory.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clientapi/routing/directory.go b/clientapi/routing/directory.go index b1dfb56fb..3d4b5f5bc 100644 --- a/clientapi/routing/directory.go +++ b/clientapi/routing/directory.go @@ -77,7 +77,7 @@ func DirectoryRoom( if fedErr != nil { // TODO: Return 502 if the remote server errored. // TODO: Return 504 if the remote server timed out. - util.GetLogger(req.Context()).WithError(err).Error("federation.LookupRoomAlias failed") + util.GetLogger(req.Context()).WithError(fedErr).Error("federation.LookupRoomAlias failed") return jsonerror.InternalServerError() } res.RoomID = fedRes.RoomID From 353a5d6fc25cb0e31ccb4cd433fed6112089c0af Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 2 Jun 2020 11:03:01 +0100 Subject: [PATCH 115/200] Remove p2p log lines --- cmd/dendritejs/jsServer.go | 4 ---- cmd/dendritejs/keyfetcher.go | 3 --- 2 files changed, 7 deletions(-) diff --git a/cmd/dendritejs/jsServer.go b/cmd/dendritejs/jsServer.go index a5ac574d8..31f122645 100644 --- a/cmd/dendritejs/jsServer.go +++ b/cmd/dendritejs/jsServer.go @@ -49,15 +49,11 @@ func (h *JSServer) OnRequestFromJS(this js.Value, args []js.Value) interface{} { // we need to put this in an immediately invoked goroutine. go func() { resolve := pargs[0] - fmt.Println("Received request:") - fmt.Printf("%s\n", httpStr) resStr, err := h.handle(httpStr) errStr := "" if err != nil { errStr = err.Error() } - fmt.Println("Sending response:") - fmt.Printf("%s\n", resStr) resolve.Invoke(map[string]interface{}{ "result": resStr, "error": errStr, diff --git a/cmd/dendritejs/keyfetcher.go b/cmd/dendritejs/keyfetcher.go index 47a81c41e..0073b7e6b 100644 --- a/cmd/dendritejs/keyfetcher.go +++ b/cmd/dendritejs/keyfetcher.go @@ -23,7 +23,6 @@ import ( "github.com/libp2p/go-libp2p-core/peer" "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" ) const libp2pMatrixKeyID = "ed25519:libp2p-dendrite" @@ -63,8 +62,6 @@ func (f *libp2pKeyFetcher) FetchKeys( if err != nil { return nil, fmt.Errorf("Failed to extract raw bytes from public key: %w", err) } - util.GetLogger(ctx).Info("libp2pKeyFetcher.FetchKeys: Using public key %v for server name %s", pubKeyBytes, req.ServerName) - b64Key := gomatrixserverlib.Base64String(pubKeyBytes) res[req] = gomatrixserverlib.PublicKeyLookupResult{ VerifyKey: gomatrixserverlib.VerifyKey{ From 484b6f694c74104c641cfcb240219f287754e090 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 2 Jun 2020 11:29:47 +0100 Subject: [PATCH 116/200] Use gomatrixserverlib device structs (#1079) --- federationapi/routing/devices.go | 32 ++++++++++++++++++-------------- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/federationapi/routing/devices.go b/federationapi/routing/devices.go index 49686c761..caf5fe592 100644 --- a/federationapi/routing/devices.go +++ b/federationapi/routing/devices.go @@ -15,19 +15,13 @@ package routing import ( "net/http" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/userutil" + "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) -type userDevicesResponse struct { - UserID string `json:"user_id"` - StreamID int `json:"stream_id"` - Devices []authtypes.Device `json:"devices"` -} - // GetUserDevices for the given user id func GetUserDevices( req *http.Request, @@ -42,20 +36,30 @@ func GetUserDevices( } } + response := gomatrixserverlib.RespUserDevices{ + UserID: userID, + // TODO: we should return an incrementing stream ID each time the device + // list changes for delta changes to be recognised + StreamID: 0, + } + devs, err := deviceDB.GetDevicesByLocalpart(req.Context(), localpart) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("deviceDB.GetDevicesByLocalPart failed") return jsonerror.InternalServerError() } + for _, dev := range devs { + device := gomatrixserverlib.RespUserDevice{ + DeviceID: dev.ID, + DisplayName: dev.DisplayName, + Keys: []gomatrixserverlib.RespUserDeviceKeys{}, + } + response.Devices = append(response.Devices, device) + } + return util.JSONResponse{ Code: 200, - // TODO: we should return an incrementing stream ID each time the device - // list changes for delta changes to be recognised - JSON: userDevicesResponse{ - UserID: userID, - StreamID: 0, - Devices: devs, - }, + JSON: response, } } diff --git a/go.mod b/go.mod index cc60e1a22..014ddd7d9 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200601162724-79e93fe989cf + github.com/matrix-org/gomatrixserverlib v0.0.0-20200602095934-0117edafc57a github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible diff --git a/go.sum b/go.sum index 6d9c2725d..70e980b9a 100644 --- a/go.sum +++ b/go.sum @@ -356,8 +356,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 h1:Yb+Wlf github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bhrnp3Ky1qgx/fzCtCALOoGYylh2tpS9K4= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200601162724-79e93fe989cf h1:iT2dfJ6JmYNRZBQTXeCNwsZIvfkBbFggzclM8iKnbR0= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200601162724-79e93fe989cf/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200602095934-0117edafc57a h1:TX8oRfmzq84CFTcEuk1uvDPzV+jA9SlRSEj8MU/W3o0= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200602095934-0117edafc57a/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f/go.mod h1:y0oDTjZDv5SM9a2rp3bl+CU+bvTRINQsdb7YlDql5Go= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= From 794c63e757ce59fb8f934099b6ca2b1ccc0fa31e Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 2 Jun 2020 12:42:36 +0100 Subject: [PATCH 117/200] Reset backoff on incoming federation (#1080) * Reset backoffs in response to incoming federation requests * Federation wakeups no more than once per minute per origin --- federationapi/routing/routing.go | 42 +++++++++++++++++--------------- internal/httpapi.go | 31 +++++++++++++++++++++++ 2 files changed, 54 insertions(+), 19 deletions(-) diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index 86d3192a2..b75a7941d 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -48,7 +48,7 @@ func Setup( asAPI appserviceAPI.AppServiceQueryAPI, producer *producers.RoomserverProducer, eduProducer *producers.EDUServerProducer, - federationSenderAPI federationSenderAPI.FederationSenderInternalAPI, + fsAPI federationSenderAPI.FederationSenderInternalAPI, keys gomatrixserverlib.KeyRing, federation *gomatrixserverlib.FederationClient, accountDB accounts.Database, @@ -58,6 +58,10 @@ func Setup( v1fedmux := publicAPIMux.PathPrefix(pathPrefixV1Federation).Subrouter() v2fedmux := publicAPIMux.PathPrefix(pathPrefixV2Federation).Subrouter() + wakeup := &internal.FederationWakeups{ + FsAPI: fsAPI, + } + localKeys := internal.MakeExternalAPI("localkeys", func(req *http.Request) util.JSONResponse { return LocalKeys(cfg) }) @@ -71,7 +75,7 @@ func Setup( v2keysmux.Handle("/server", localKeys).Methods(http.MethodGet) v1fedmux.Handle("/send/{txnID}", internal.MakeFedAPI( - "federation_send", cfg.Matrix.ServerName, keys, + "federation_send", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq)) if err != nil { @@ -85,7 +89,7 @@ func Setup( )).Methods(http.MethodPut, http.MethodOptions) v2fedmux.Handle("/invite/{roomID}/{eventID}", internal.MakeFedAPI( - "federation_invite", cfg.Matrix.ServerName, keys, + "federation_invite", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq)) if err != nil { @@ -105,7 +109,7 @@ func Setup( )).Methods(http.MethodPost, http.MethodOptions) v1fedmux.Handle("/exchange_third_party_invite/{roomID}", internal.MakeFedAPI( - "exchange_third_party_invite", cfg.Matrix.ServerName, keys, + "exchange_third_party_invite", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq)) if err != nil { @@ -118,7 +122,7 @@ func Setup( )).Methods(http.MethodPut, http.MethodOptions) v1fedmux.Handle("/event/{eventID}", internal.MakeFedAPI( - "federation_get_event", cfg.Matrix.ServerName, keys, + "federation_get_event", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq)) if err != nil { @@ -131,7 +135,7 @@ func Setup( )).Methods(http.MethodGet) v1fedmux.Handle("/state/{roomID}", internal.MakeFedAPI( - "federation_get_state", cfg.Matrix.ServerName, keys, + "federation_get_state", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq)) if err != nil { @@ -144,7 +148,7 @@ func Setup( )).Methods(http.MethodGet) v1fedmux.Handle("/state_ids/{roomID}", internal.MakeFedAPI( - "federation_get_state_ids", cfg.Matrix.ServerName, keys, + "federation_get_state_ids", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq)) if err != nil { @@ -157,7 +161,7 @@ func Setup( )).Methods(http.MethodGet) v1fedmux.Handle("/event_auth/{roomID}/{eventID}", internal.MakeFedAPI( - "federation_get_event_auth", cfg.Matrix.ServerName, keys, + "federation_get_event_auth", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { vars := mux.Vars(httpReq) return GetEventAuth( @@ -167,16 +171,16 @@ func Setup( )).Methods(http.MethodGet) v1fedmux.Handle("/query/directory", internal.MakeFedAPI( - "federation_query_room_alias", cfg.Matrix.ServerName, keys, + "federation_query_room_alias", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { return RoomAliasToID( - httpReq, federation, cfg, rsAPI, federationSenderAPI, + httpReq, federation, cfg, rsAPI, fsAPI, ) }, )).Methods(http.MethodGet) v1fedmux.Handle("/query/profile", internal.MakeFedAPI( - "federation_query_profile", cfg.Matrix.ServerName, keys, + "federation_query_profile", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { return GetProfile( httpReq, accountDB, cfg, asAPI, @@ -185,7 +189,7 @@ func Setup( )).Methods(http.MethodGet) v1fedmux.Handle("/user/devices/{userID}", internal.MakeFedAPI( - "federation_user_devices", cfg.Matrix.ServerName, keys, + "federation_user_devices", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq)) if err != nil { @@ -198,7 +202,7 @@ func Setup( )).Methods(http.MethodGet) v1fedmux.Handle("/make_join/{roomID}/{eventID}", internal.MakeFedAPI( - "federation_make_join", cfg.Matrix.ServerName, keys, + "federation_make_join", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq)) if err != nil { @@ -227,7 +231,7 @@ func Setup( )).Methods(http.MethodGet) v1fedmux.Handle("/send_join/{roomID}/{eventID}", internal.MakeFedAPI( - "federation_send_join", cfg.Matrix.ServerName, keys, + "federation_send_join", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq)) if err != nil { @@ -249,7 +253,7 @@ func Setup( )).Methods(http.MethodPut) v2fedmux.Handle("/send_join/{roomID}/{eventID}", internal.MakeFedAPI( - "federation_send_join", cfg.Matrix.ServerName, keys, + "federation_send_join", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq)) if err != nil { @@ -264,7 +268,7 @@ func Setup( )).Methods(http.MethodPut) v1fedmux.Handle("/make_leave/{roomID}/{eventID}", internal.MakeFedAPI( - "federation_make_leave", cfg.Matrix.ServerName, keys, + "federation_make_leave", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq)) if err != nil { @@ -279,7 +283,7 @@ func Setup( )).Methods(http.MethodGet) v2fedmux.Handle("/send_leave/{roomID}/{eventID}", internal.MakeFedAPI( - "federation_send_leave", cfg.Matrix.ServerName, keys, + "federation_send_leave", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq)) if err != nil { @@ -301,7 +305,7 @@ func Setup( )).Methods(http.MethodGet) v1fedmux.Handle("/get_missing_events/{roomID}", internal.MakeFedAPI( - "federation_get_missing_events", cfg.Matrix.ServerName, keys, + "federation_get_missing_events", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq)) if err != nil { @@ -312,7 +316,7 @@ func Setup( )).Methods(http.MethodPost) v1fedmux.Handle("/backfill/{roomID}", internal.MakeFedAPI( - "federation_backfill", cfg.Matrix.ServerName, keys, + "federation_backfill", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq)) if err != nil { diff --git a/internal/httpapi.go b/internal/httpapi.go index 07bbacdd6..bacd1768a 100644 --- a/internal/httpapi.go +++ b/internal/httpapi.go @@ -1,17 +1,20 @@ package internal import ( + "context" "io" "net/http" "net/http/httptest" "net/http/httputil" "os" "strings" + "sync" "time" "github.com/gorilla/mux" "github.com/matrix-org/dendrite/clientapi/auth" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + federationsenderAPI "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/httpapis" "github.com/matrix-org/gomatrixserverlib" @@ -170,6 +173,7 @@ func MakeFedAPI( metricsName string, serverName gomatrixserverlib.ServerName, keyRing gomatrixserverlib.KeyRing, + wakeup *FederationWakeups, f func(*http.Request, *gomatrixserverlib.FederationRequest) util.JSONResponse, ) http.Handler { h := func(req *http.Request) util.JSONResponse { @@ -179,11 +183,38 @@ func MakeFedAPI( if fedReq == nil { return errResp } + go wakeup.Wakeup(req.Context(), fedReq.Origin()) return f(req, fedReq) } return MakeExternalAPI(metricsName, h) } +type FederationWakeups struct { + FsAPI federationsenderAPI.FederationSenderInternalAPI + origins sync.Map +} + +func (f *FederationWakeups) Wakeup(ctx context.Context, origin gomatrixserverlib.ServerName) { + key, keyok := f.origins.Load(origin) + if keyok { + lastTime, ok := key.(time.Time) + if ok && time.Since(lastTime) < time.Minute { + return + } + } + aliveReq := federationsenderAPI.PerformServersAliveRequest{ + Servers: []gomatrixserverlib.ServerName{origin}, + } + aliveRes := federationsenderAPI.PerformServersAliveResponse{} + if err := f.FsAPI.PerformServersAlive(ctx, &aliveReq, &aliveRes); err != nil { + util.GetLogger(ctx).WithError(err).WithFields(logrus.Fields{ + "origin": origin, + }).Warn("incoming federation request failed to notify server alive") + } else { + f.origins.Store(origin, time.Now()) + } +} + // SetupHTTPAPI registers an HTTP API mux under /api and sets up a metrics // listener. func SetupHTTPAPI(servMux *http.ServeMux, publicApiMux *mux.Router, internalApiMux *mux.Router, cfg *config.Dendrite, enableHTTPAPIs bool) { From 02b150fd13f64bae551d5a7fd967caf4479981b8 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 2 Jun 2020 15:01:13 +0100 Subject: [PATCH 118/200] Only store our own aliases in publicroomsapi (#1081) Otherwise we just store the latest aliases submitted from a server, which is not what we want. --- cmd/dendrite-demo-libp2p/main.go | 2 +- .../storage/postgreswithdht/storage.go | 4 ++-- .../storage/postgreswithpubsub/storage.go | 4 ++-- cmd/dendrite-demo-libp2p/storage/storage.go | 21 ++++++++++--------- cmd/dendrite-monolith-server/main.go | 2 +- cmd/dendrite-public-rooms-api-server/main.go | 2 +- cmd/dendritejs/main.go | 2 +- publicroomsapi/storage/postgres/storage.go | 11 +++++++--- publicroomsapi/storage/sqlite3/storage.go | 11 +++++++--- publicroomsapi/storage/storage.go | 11 +++++----- publicroomsapi/storage/storage_wasm.go | 5 +++-- 11 files changed, 44 insertions(+), 31 deletions(-) diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index e9d01fd95..c00316ec7 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -166,7 +166,7 @@ func main() { eduProducer := producers.NewEDUServerProducer(eduInputAPI) federationapi.SetupFederationAPIComponent(&base.Base, accountDB, deviceDB, federation, keyRing, rsAPI, asAPI, fsAPI, eduProducer) mediaapi.SetupMediaAPIComponent(&base.Base, deviceDB) - publicRoomsDB, err := storage.NewPublicRoomsServerDatabaseWithPubSub(string(base.Base.Cfg.Database.PublicRoomsAPI), base.LibP2PPubsub) + publicRoomsDB, err := storage.NewPublicRoomsServerDatabaseWithPubSub(string(base.Base.Cfg.Database.PublicRoomsAPI), base.LibP2PPubsub, cfg.Matrix.ServerName) if err != nil { logrus.WithError(err).Panicf("failed to connect to public rooms db") } diff --git a/cmd/dendrite-demo-libp2p/storage/postgreswithdht/storage.go b/cmd/dendrite-demo-libp2p/storage/postgreswithdht/storage.go index cd2804c97..d2cb36a8b 100644 --- a/cmd/dendrite-demo-libp2p/storage/postgreswithdht/storage.go +++ b/cmd/dendrite-demo-libp2p/storage/postgreswithdht/storage.go @@ -44,8 +44,8 @@ type PublicRoomsServerDatabase struct { } // NewPublicRoomsServerDatabase creates a new public rooms server database. -func NewPublicRoomsServerDatabase(dataSourceName string, dht *dht.IpfsDHT) (*PublicRoomsServerDatabase, error) { - pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName, nil) +func NewPublicRoomsServerDatabase(dataSourceName string, dht *dht.IpfsDHT, localServerName gomatrixserverlib.ServerName) (*PublicRoomsServerDatabase, error) { + pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName, nil, localServerName) if err != nil { return nil, err } diff --git a/cmd/dendrite-demo-libp2p/storage/postgreswithpubsub/storage.go b/cmd/dendrite-demo-libp2p/storage/postgreswithpubsub/storage.go index e4372c64f..cf642eb38 100644 --- a/cmd/dendrite-demo-libp2p/storage/postgreswithpubsub/storage.go +++ b/cmd/dendrite-demo-libp2p/storage/postgreswithpubsub/storage.go @@ -47,8 +47,8 @@ type PublicRoomsServerDatabase struct { } // NewPublicRoomsServerDatabase creates a new public rooms server database. -func NewPublicRoomsServerDatabase(dataSourceName string, pubsub *pubsub.PubSub) (*PublicRoomsServerDatabase, error) { - pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName, nil) +func NewPublicRoomsServerDatabase(dataSourceName string, pubsub *pubsub.PubSub, localServerName gomatrixserverlib.ServerName) (*PublicRoomsServerDatabase, error) { + pg, err := postgres.NewPublicRoomsServerDatabase(dataSourceName, nil, localServerName) if err != nil { return nil, err } diff --git a/cmd/dendrite-demo-libp2p/storage/storage.go b/cmd/dendrite-demo-libp2p/storage/storage.go index 668edbaa3..2d8dc1817 100644 --- a/cmd/dendrite-demo-libp2p/storage/storage.go +++ b/cmd/dendrite-demo-libp2p/storage/storage.go @@ -23,39 +23,40 @@ import ( "github.com/matrix-org/dendrite/cmd/dendrite-demo-libp2p/storage/postgreswithpubsub" "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/publicroomsapi/storage/sqlite3" + "github.com/matrix-org/gomatrixserverlib" ) const schemePostgres = "postgres" const schemeFile = "file" // NewPublicRoomsServerDatabase opens a database connection. -func NewPublicRoomsServerDatabaseWithDHT(dataSourceName string, dht *dht.IpfsDHT) (storage.Database, error) { +func NewPublicRoomsServerDatabaseWithDHT(dataSourceName string, dht *dht.IpfsDHT, localServerName gomatrixserverlib.ServerName) (storage.Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { - return postgreswithdht.NewPublicRoomsServerDatabase(dataSourceName, dht) + return postgreswithdht.NewPublicRoomsServerDatabase(dataSourceName, dht, localServerName) } switch uri.Scheme { case schemePostgres: - return postgreswithdht.NewPublicRoomsServerDatabase(dataSourceName, dht) + return postgreswithdht.NewPublicRoomsServerDatabase(dataSourceName, dht, localServerName) case schemeFile: - return sqlite3.NewPublicRoomsServerDatabase(dataSourceName) + return sqlite3.NewPublicRoomsServerDatabase(dataSourceName, localServerName) default: - return postgreswithdht.NewPublicRoomsServerDatabase(dataSourceName, dht) + return postgreswithdht.NewPublicRoomsServerDatabase(dataSourceName, dht, localServerName) } } // NewPublicRoomsServerDatabase opens a database connection. -func NewPublicRoomsServerDatabaseWithPubSub(dataSourceName string, pubsub *pubsub.PubSub) (storage.Database, error) { +func NewPublicRoomsServerDatabaseWithPubSub(dataSourceName string, pubsub *pubsub.PubSub, localServerName gomatrixserverlib.ServerName) (storage.Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { - return postgreswithpubsub.NewPublicRoomsServerDatabase(dataSourceName, pubsub) + return postgreswithpubsub.NewPublicRoomsServerDatabase(dataSourceName, pubsub, localServerName) } switch uri.Scheme { case schemePostgres: - return postgreswithpubsub.NewPublicRoomsServerDatabase(dataSourceName, pubsub) + return postgreswithpubsub.NewPublicRoomsServerDatabase(dataSourceName, pubsub, localServerName) case schemeFile: - return sqlite3.NewPublicRoomsServerDatabase(dataSourceName) + return sqlite3.NewPublicRoomsServerDatabase(dataSourceName, localServerName) default: - return postgreswithpubsub.NewPublicRoomsServerDatabase(dataSourceName, pubsub) + return postgreswithpubsub.NewPublicRoomsServerDatabase(dataSourceName, pubsub, localServerName) } } diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index ef114ccd1..41907cc0c 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -120,7 +120,7 @@ func main() { eduProducer := producers.NewEDUServerProducer(eduInputAPI) federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, keyRing, rsAPI, asAPI, fsAPI, eduProducer) mediaapi.SetupMediaAPIComponent(base, deviceDB) - publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties()) + publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties(), cfg.Matrix.ServerName) if err != nil { logrus.WithError(err).Panicf("failed to connect to public rooms db") } diff --git a/cmd/dendrite-public-rooms-api-server/main.go b/cmd/dendrite-public-rooms-api-server/main.go index cf9d09c11..d506e32d1 100644 --- a/cmd/dendrite-public-rooms-api-server/main.go +++ b/cmd/dendrite-public-rooms-api-server/main.go @@ -32,7 +32,7 @@ func main() { rsAPI := base.CreateHTTPRoomserverAPIs() rsAPI.SetFederationSenderAPI(fsAPI) - publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties()) + publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties(), cfg.Matrix.ServerName) if err != nil { logrus.WithError(err).Panicf("failed to connect to public rooms db") } diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 72008aa9a..c1aef44d9 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -223,7 +223,7 @@ func main() { eduProducer := producers.NewEDUServerProducer(eduInputAPI) federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, rsAPI, asQuery, fedSenderAPI, eduProducer) mediaapi.SetupMediaAPIComponent(base, deviceDB) - publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI)) + publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), cfg.Matrix.ServerName) if err != nil { logrus.WithError(err).Panicf("failed to connect to public rooms db") } diff --git a/publicroomsapi/storage/postgres/storage.go b/publicroomsapi/storage/postgres/storage.go index 691199a0b..5f0f551a7 100644 --- a/publicroomsapi/storage/postgres/storage.go +++ b/publicroomsapi/storage/postgres/storage.go @@ -30,20 +30,22 @@ import ( type PublicRoomsServerDatabase struct { db *sql.DB internal.PartitionOffsetStatements - statements publicRoomsStatements + statements publicRoomsStatements + localServerName gomatrixserverlib.ServerName } type attributeValue interface{} // NewPublicRoomsServerDatabase creates a new public rooms server database. -func NewPublicRoomsServerDatabase(dataSourceName string, dbProperties internal.DbProperties) (*PublicRoomsServerDatabase, error) { +func NewPublicRoomsServerDatabase(dataSourceName string, dbProperties internal.DbProperties, localServerName gomatrixserverlib.ServerName) (*PublicRoomsServerDatabase, error) { var db *sql.DB var err error if db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { return nil, err } storage := PublicRoomsServerDatabase{ - db: db, + db: db, + localServerName: localServerName, } if err = storage.PartitionOffsetStatements.Prepare(db, "publicroomsapi"); err != nil { return nil, err @@ -243,6 +245,9 @@ func (d *PublicRoomsServerDatabase) updateBooleanAttribute( func (d *PublicRoomsServerDatabase) updateRoomAliases( ctx context.Context, aliasesEvent gomatrixserverlib.Event, ) error { + if aliasesEvent.StateKey() == nil || *aliasesEvent.StateKey() != string(d.localServerName) { + return nil // only store our own aliases + } var content internal.AliasesContent if err := json.Unmarshal(aliasesEvent.Content(), &content); err != nil { return err diff --git a/publicroomsapi/storage/sqlite3/storage.go b/publicroomsapi/storage/sqlite3/storage.go index b81222dd3..898101127 100644 --- a/publicroomsapi/storage/sqlite3/storage.go +++ b/publicroomsapi/storage/sqlite3/storage.go @@ -32,20 +32,22 @@ import ( type PublicRoomsServerDatabase struct { db *sql.DB internal.PartitionOffsetStatements - statements publicRoomsStatements + statements publicRoomsStatements + localServerName gomatrixserverlib.ServerName } type attributeValue interface{} // NewPublicRoomsServerDatabase creates a new public rooms server database. -func NewPublicRoomsServerDatabase(dataSourceName string) (*PublicRoomsServerDatabase, error) { +func NewPublicRoomsServerDatabase(dataSourceName string, localServerName gomatrixserverlib.ServerName) (*PublicRoomsServerDatabase, error) { var db *sql.DB var err error if db, err = sqlutil.Open(internal.SQLiteDriverName(), dataSourceName, nil); err != nil { return nil, err } storage := PublicRoomsServerDatabase{ - db: db, + db: db, + localServerName: localServerName, } if err = storage.PartitionOffsetStatements.Prepare(db, "publicroomsapi"); err != nil { return nil, err @@ -245,6 +247,9 @@ func (d *PublicRoomsServerDatabase) updateBooleanAttribute( func (d *PublicRoomsServerDatabase) updateRoomAliases( ctx context.Context, aliasesEvent gomatrixserverlib.Event, ) error { + if aliasesEvent.StateKey() == nil || *aliasesEvent.StateKey() != string(d.localServerName) { + return nil // only store our own aliases + } var content internal.AliasesContent if err := json.Unmarshal(aliasesEvent.Content(), &content); err != nil { return err diff --git a/publicroomsapi/storage/storage.go b/publicroomsapi/storage/storage.go index 978d9a3c9..507b9fa6c 100644 --- a/publicroomsapi/storage/storage.go +++ b/publicroomsapi/storage/storage.go @@ -22,23 +22,24 @@ import ( "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/publicroomsapi/storage/postgres" "github.com/matrix-org/dendrite/publicroomsapi/storage/sqlite3" + "github.com/matrix-org/gomatrixserverlib" ) const schemePostgres = "postgres" const schemeFile = "file" // NewPublicRoomsServerDatabase opens a database connection. -func NewPublicRoomsServerDatabase(dataSourceName string, dbProperties internal.DbProperties) (Database, error) { +func NewPublicRoomsServerDatabase(dataSourceName string, dbProperties internal.DbProperties, localServerName gomatrixserverlib.ServerName) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { - return postgres.NewPublicRoomsServerDatabase(dataSourceName, dbProperties) + return postgres.NewPublicRoomsServerDatabase(dataSourceName, dbProperties, localServerName) } switch uri.Scheme { case schemePostgres: - return postgres.NewPublicRoomsServerDatabase(dataSourceName, dbProperties) + return postgres.NewPublicRoomsServerDatabase(dataSourceName, dbProperties, localServerName) case schemeFile: - return sqlite3.NewPublicRoomsServerDatabase(dataSourceName) + return sqlite3.NewPublicRoomsServerDatabase(dataSourceName, localServerName) default: - return postgres.NewPublicRoomsServerDatabase(dataSourceName, dbProperties) + return postgres.NewPublicRoomsServerDatabase(dataSourceName, dbProperties, localServerName) } } diff --git a/publicroomsapi/storage/storage_wasm.go b/publicroomsapi/storage/storage_wasm.go index 2157ce110..fa3356c03 100644 --- a/publicroomsapi/storage/storage_wasm.go +++ b/publicroomsapi/storage/storage_wasm.go @@ -19,10 +19,11 @@ import ( "net/url" "github.com/matrix-org/dendrite/publicroomsapi/storage/sqlite3" + "github.com/matrix-org/gomatrixserverlib" ) // NewPublicRoomsServerDatabase opens a database connection. -func NewPublicRoomsServerDatabase(dataSourceName string) (Database, error) { +func NewPublicRoomsServerDatabase(dataSourceName string, localServerName gomatrixserverlib.ServerName) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return nil, err @@ -31,7 +32,7 @@ func NewPublicRoomsServerDatabase(dataSourceName string) (Database, error) { case "postgres": return nil, fmt.Errorf("Cannot use postgres implementation") case "file": - return sqlite3.NewPublicRoomsServerDatabase(uri.Path) + return sqlite3.NewPublicRoomsServerDatabase(uri.Path, localServerName) default: return nil, fmt.Errorf("Cannot use postgres implementation") } From e37720be441aa57aa8b30ce80b73cc151d1b2b0c Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 2 Jun 2020 16:18:01 +0100 Subject: [PATCH 119/200] Sort public rooms again by member count (#1083) --- publicroomsapi/directory/public_rooms.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/publicroomsapi/directory/public_rooms.go b/publicroomsapi/directory/public_rooms.go index bc40da516..df9df8ff9 100644 --- a/publicroomsapi/directory/public_rooms.go +++ b/publicroomsapi/directory/public_rooms.go @@ -18,6 +18,7 @@ import ( "context" "math/rand" "net/http" + "sort" "strconv" "sync" "time" @@ -112,6 +113,11 @@ func GetPostPublicRoomsWithExternal( haveRoomIDs[r.RoomID] = true publicRooms = append(publicRooms, r) } + // sort by member count + sort.SliceStable(publicRooms, func(i, j int) bool { + return publicRooms[i].JoinedMembersCount > publicRooms[j].JoinedMembersCount + }) + response.Chunk = publicRooms return util.JSONResponse{ From e598e80d7659f3da323a92c07e5092172e16ca2c Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 2 Jun 2020 16:20:50 +0100 Subject: [PATCH 120/200] Delegate responsibility for marking room versions as supported/stable to gomatrixserverlib (#1082) --- go.mod | 2 +- go.sum | 4 +-- roomserver/version/version.go | 62 ++++++----------------------------- 3 files changed, 13 insertions(+), 55 deletions(-) diff --git a/go.mod b/go.mod index 014ddd7d9..021cc02a4 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200602095934-0117edafc57a + github.com/matrix-org/gomatrixserverlib v0.0.0-20200602125825-24ff01093eca github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible diff --git a/go.sum b/go.sum index 70e980b9a..3f235d51e 100644 --- a/go.sum +++ b/go.sum @@ -356,8 +356,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 h1:Yb+Wlf github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bhrnp3Ky1qgx/fzCtCALOoGYylh2tpS9K4= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200602095934-0117edafc57a h1:TX8oRfmzq84CFTcEuk1uvDPzV+jA9SlRSEj8MU/W3o0= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200602095934-0117edafc57a/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200602125825-24ff01093eca h1:s/dJePRDKjD1fGeoTnEYFqPmp1v7fC6GTd6iFwCKxw8= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200602125825-24ff01093eca/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f/go.mod h1:y0oDTjZDv5SM9a2rp3bl+CU+bvTRINQsdb7YlDql5Go= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= diff --git a/roomserver/version/version.go b/roomserver/version/version.go index ddd0f23a6..f2b15ec39 100644 --- a/roomserver/version/version.go +++ b/roomserver/version/version.go @@ -20,42 +20,6 @@ import ( "github.com/matrix-org/gomatrixserverlib" ) -// RoomVersionDescription contains information about a room version, -// namely whether it is marked as supported or stable in this server -// version. -// A version is supported if the server has some support for rooms -// that are this version. A version is marked as stable or unstable -// in order to hint whether the version should be used to clients -// calling the /capabilities endpoint. -// https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-capabilities -type RoomVersionDescription struct { - Supported bool - Stable bool -} - -var roomVersions = map[gomatrixserverlib.RoomVersion]RoomVersionDescription{ - gomatrixserverlib.RoomVersionV1: RoomVersionDescription{ - Supported: true, - Stable: true, - }, - gomatrixserverlib.RoomVersionV2: RoomVersionDescription{ - Supported: true, - Stable: true, - }, - gomatrixserverlib.RoomVersionV3: RoomVersionDescription{ - Supported: true, - Stable: true, - }, - gomatrixserverlib.RoomVersionV4: RoomVersionDescription{ - Supported: true, - Stable: true, - }, - gomatrixserverlib.RoomVersionV5: RoomVersionDescription{ - Supported: true, - Stable: true, - }, -} - // DefaultRoomVersion contains the room version that will, by // default, be used to create new rooms on this server. func DefaultRoomVersion() gomatrixserverlib.RoomVersion { @@ -64,43 +28,37 @@ func DefaultRoomVersion() gomatrixserverlib.RoomVersion { // RoomVersions returns a map of all known room versions to this // server. -func RoomVersions() map[gomatrixserverlib.RoomVersion]RoomVersionDescription { - return roomVersions +func RoomVersions() map[gomatrixserverlib.RoomVersion]gomatrixserverlib.RoomVersionDescription { + return gomatrixserverlib.RoomVersions() } // SupportedRoomVersions returns a map of descriptions for room // versions that are supported by this homeserver. -func SupportedRoomVersions() map[gomatrixserverlib.RoomVersion]RoomVersionDescription { - versions := make(map[gomatrixserverlib.RoomVersion]RoomVersionDescription) - for id, version := range RoomVersions() { - if version.Supported { - versions[id] = version - } - } - return versions +func SupportedRoomVersions() map[gomatrixserverlib.RoomVersion]gomatrixserverlib.RoomVersionDescription { + return gomatrixserverlib.SupportedRoomVersions() } // RoomVersion returns information about a specific room version. // An UnknownVersionError is returned if the version is not known // to the server. -func RoomVersion(version gomatrixserverlib.RoomVersion) (RoomVersionDescription, error) { - if version, ok := roomVersions[version]; ok { +func RoomVersion(version gomatrixserverlib.RoomVersion) (gomatrixserverlib.RoomVersionDescription, error) { + if version, ok := gomatrixserverlib.RoomVersions()[version]; ok { return version, nil } - return RoomVersionDescription{}, UnknownVersionError{version} + return gomatrixserverlib.RoomVersionDescription{}, UnknownVersionError{version} } // SupportedRoomVersion returns information about a specific room // version. An UnknownVersionError is returned if the version is not // known to the server, or an UnsupportedVersionError is returned if // the version is known but specifically marked as unsupported. -func SupportedRoomVersion(version gomatrixserverlib.RoomVersion) (RoomVersionDescription, error) { +func SupportedRoomVersion(version gomatrixserverlib.RoomVersion) (gomatrixserverlib.RoomVersionDescription, error) { result, err := RoomVersion(version) if err != nil { - return RoomVersionDescription{}, err + return gomatrixserverlib.RoomVersionDescription{}, err } if !result.Supported { - return RoomVersionDescription{}, UnsupportedVersionError{version} + return gomatrixserverlib.RoomVersionDescription{}, UnsupportedVersionError{version} } return result, nil } From dc3338d1f299555148cbf406c5b2bd823f3ca038 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 2 Jun 2020 16:35:07 +0100 Subject: [PATCH 121/200] Remove ENV to avoid confusion --- build/docker/DendriteJS.Dockerfile | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/build/docker/DendriteJS.Dockerfile b/build/docker/DendriteJS.Dockerfile index 2dc6ef01c..9a84a370a 100644 --- a/build/docker/DendriteJS.Dockerfile +++ b/build/docker/DendriteJS.Dockerfile @@ -3,11 +3,6 @@ # as it pulls archives straight from github branches. FROM golang:1.13.7-alpine3.11 AS gobuild -# TODO: This does nothing currently ;) -# or /dns4/rendezvous.matrix.org/tcp/8443/wss/p2p-websocket-star/ -# or whatever you want! -ENV RENDEZVOUS_SERVER=/ip4/127.0.0.1/tcp/9090/ws/p2p-websocket-star/ - # Download and build dendrite WORKDIR /build ADD https://github.com/matrix-org/dendrite/archive/master.tar.gz /build/master.tar.gz @@ -109,4 +104,4 @@ server { \n\ } \n\ }' > /etc/nginx/conf.d/default.conf RUN sed -i 's/}/ application\/wasm wasm;\n}/g' /etc/nginx/mime.types -COPY --from=jsbuild /build/riot-web-matthew-p2p/webapp /usr/share/nginx/html \ No newline at end of file +COPY --from=jsbuild /build/riot-web-matthew-p2p/webapp /usr/share/nginx/html From 17c92ad10e7c1c6e17eed62d91fbc130866fdf91 Mon Sep 17 00:00:00 2001 From: Aiden McClelland <3732071+dr-bonez@users.noreply.github.com> Date: Tue, 2 Jun 2020 14:02:24 -0600 Subject: [PATCH 122/200] Adds support for adding a proxy to the HTTP Client from the config (#1055) * adds support for defining an proxy for the http client within the config * alphabetize imports * goimports * comments --- internal/basecomponent/base.go | 12 +++++++++++- internal/config/config.go | 10 ++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/internal/basecomponent/base.go b/internal/basecomponent/base.go index a48323726..0fc95e824 100644 --- a/internal/basecomponent/base.go +++ b/internal/basecomponent/base.go @@ -16,6 +16,7 @@ package basecomponent import ( "database/sql" + "fmt" "io" "net/http" "net/url" @@ -94,7 +95,16 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, enableHTTPAPIs logrus.WithError(err).Warnf("Failed to create cache") } + client := http.Client{Timeout: HTTPClientTimeout} + if cfg.Proxy != nil { + client.Transport = &http.Transport{Proxy: http.ProxyURL(&url.URL{ + Scheme: cfg.Proxy.Protocol, + Host: fmt.Sprintf("%s:%d", cfg.Proxy.Host, cfg.Proxy.Port), + })} + } + httpmux := mux.NewRouter() + return &BaseDendrite{ componentName: componentName, EnableHTTPAPIs: enableHTTPAPIs, @@ -103,7 +113,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, enableHTTPAPIs ImmutableCache: cache, PublicAPIMux: httpmux.PathPrefix(httpapis.PublicPathPrefix).Subrouter().UseEncodedPath(), InternalAPIMux: httpmux.PathPrefix(httpapis.InternalPathPrefix).Subrouter().UseEncodedPath(), - httpClient: &http.Client{Timeout: HTTPClientTimeout}, + httpClient: &client, KafkaConsumer: kafkaConsumer, KafkaProducer: kafkaProducer, } diff --git a/internal/config/config.go b/internal/config/config.go index a20cc0ead..597995855 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -268,6 +268,16 @@ type Dendrite struct { // The config for logging informations. Each hook will be added to logrus. Logging []LogrusHook `yaml:"logging"` + // The config for setting a proxy to use for server->server requests + Proxy *struct { + // The protocol for the proxy (http / https / socks5) + Protocol string `yaml:"protocol"` + // The host where the proxy is listening + Host string `yaml:"host"` + // The port on which the proxy is listening + Port uint16 `yaml:"port"` + } `yaml:"proxy"` + // Any information derived from the configuration options for later use. Derived struct { Registration struct { From e21d7d4bafb8c785e75369bebd5270cc67cb36d4 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Wed, 3 Jun 2020 16:03:54 +0100 Subject: [PATCH 123/200] Update DendriteJS.Dockerfile --- build/docker/DendriteJS.Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build/docker/DendriteJS.Dockerfile b/build/docker/DendriteJS.Dockerfile index 9a84a370a..4467c9c70 100644 --- a/build/docker/DendriteJS.Dockerfile +++ b/build/docker/DendriteJS.Dockerfile @@ -1,6 +1,10 @@ # This dockerfile will build dendritejs and hook it up to riot-web, build that then dump the # resulting HTML/JS onto an nginx container for hosting. It requires no specific build context # as it pulls archives straight from github branches. +# +# $ docker build -t dendritejs -f DendriteJS.Dockerfile . +# $ docker run --rm -p 8888:80 dendritejs +# Then visit http://localhost:8888 FROM golang:1.13.7-alpine3.11 AS gobuild # Download and build dendrite From 8a6152ca70aa86df651db54efb3b1ec68ec9ec78 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 4 Jun 2020 10:53:39 +0100 Subject: [PATCH 124/200] Enable room version 6 (#1087) * Return bad request on CS API /send if bad JSON * Return some more M_BAD_JSON in the right places * nolint because damnit gocyclo all I added was a type check for an error * Update gomatrixserverlib * Update gomatrixserverlib * Update sytest-whitelist * Update gomatrixserverlib * Update sytest-whitelist * NotJSON -> BadJSON --- clientapi/routing/membership.go | 8 +++++++- clientapi/routing/profile.go | 20 ++++++++++++++++++-- clientapi/routing/sendevent.go | 5 +++++ federationapi/routing/join.go | 7 ++++++- federationapi/routing/leave.go | 5 +++++ go.mod | 2 +- go.sum | 4 ++-- sytest-whitelist | 12 ++++++++++++ 8 files changed, 56 insertions(+), 7 deletions(-) diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index cd0ce7328..74d92a059 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -276,7 +276,13 @@ func checkAndProcessThreepid( Code: http.StatusNotFound, JSON: jsonerror.NotFound(err.Error()), } - } else if err != nil { + } else if e, ok := err.(gomatrixserverlib.BadJSONError); ok { + return inviteStored, &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.BadJSON(e.Error()), + } + } + if err != nil { util.GetLogger(req.Context()).WithError(err).Error("threepid.CheckAndProcessInvite failed") er := jsonerror.InternalServerError() return inviteStored, &er diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index 1c1ee8036..dbf6ef1d9 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -91,6 +91,7 @@ func GetAvatarURL( } // SetAvatarURL implements PUT /profile/{userID}/avatar_url +// nolint:gocyclo func SetAvatarURL( req *http.Request, accountDB accounts.Database, device *authtypes.Device, userID string, producer *producers.UserUpdateProducer, cfg *config.Dendrite, @@ -156,7 +157,14 @@ func SetAvatarURL( events, err := buildMembershipEvents( req.Context(), memberships, newProfile, userID, cfg, evTime, rsAPI, ) - if err != nil { + switch e := err.(type) { + case nil: + case gomatrixserverlib.BadJSONError: + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.BadJSON(e.Error()), + } + default: util.GetLogger(req.Context()).WithError(err).Error("buildMembershipEvents failed") return jsonerror.InternalServerError() } @@ -205,6 +213,7 @@ func GetDisplayName( } // SetDisplayName implements PUT /profile/{userID}/displayname +// nolint:gocyclo func SetDisplayName( req *http.Request, accountDB accounts.Database, device *authtypes.Device, userID string, producer *producers.UserUpdateProducer, cfg *config.Dendrite, @@ -270,7 +279,14 @@ func SetDisplayName( events, err := buildMembershipEvents( req.Context(), memberships, newProfile, userID, cfg, evTime, rsAPI, ) - if err != nil { + switch e := err.(type) { + case nil: + case gomatrixserverlib.BadJSONError: + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.BadJSON(e.Error()), + } + default: util.GetLogger(req.Context()).WithError(err).Error("buildMembershipEvents failed") return jsonerror.InternalServerError() } diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go index aa7317094..a8f8893ba 100644 --- a/clientapi/routing/sendevent.go +++ b/clientapi/routing/sendevent.go @@ -154,6 +154,11 @@ func generateSendEvent( Code: http.StatusNotFound, JSON: jsonerror.NotFound("Room does not exist"), } + } else if e, ok := err.(gomatrixserverlib.BadJSONError); ok { + return nil, &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.BadJSON(e.Error()), + } } else if err != nil { util.GetLogger(req.Context()).WithError(err).Error("internal.BuildEvent failed") resErr := jsonerror.InternalServerError() diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go index 617a6cab0..5b4e8db3e 100644 --- a/federationapi/routing/join.go +++ b/federationapi/routing/join.go @@ -102,6 +102,11 @@ func MakeJoin( Code: http.StatusNotFound, JSON: jsonerror.NotFound("Room does not exist"), } + } else if e, ok := err.(gomatrixserverlib.BadJSONError); ok { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.BadJSON(e.Error()), + } } else if err != nil { util.GetLogger(httpReq.Context()).WithError(err).Error("internal.BuildEvent failed") return jsonerror.InternalServerError() @@ -157,7 +162,7 @@ func SendJoin( if err != nil { return util.JSONResponse{ Code: http.StatusBadRequest, - JSON: jsonerror.NotJSON("The request body could not be decoded into valid JSON. " + err.Error()), + JSON: jsonerror.BadJSON("The request body could not be decoded into valid JSON: " + err.Error()), } } diff --git a/federationapi/routing/leave.go b/federationapi/routing/leave.go index bd226d7ee..62ca11457 100644 --- a/federationapi/routing/leave.go +++ b/federationapi/routing/leave.go @@ -76,6 +76,11 @@ func MakeLeave( Code: http.StatusNotFound, JSON: jsonerror.NotFound("Room does not exist"), } + } else if e, ok := err.(gomatrixserverlib.BadJSONError); ok { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.BadJSON(e.Error()), + } } else if err != nil { util.GetLogger(httpReq.Context()).WithError(err).Error("internal.BuildEvent failed") return jsonerror.InternalServerError() diff --git a/go.mod b/go.mod index 021cc02a4..5510e8e0e 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200602125825-24ff01093eca + github.com/matrix-org/gomatrixserverlib v0.0.0-20200604085359-baf0c20ac96f github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible diff --git a/go.sum b/go.sum index 3f235d51e..5df03cbaa 100644 --- a/go.sum +++ b/go.sum @@ -356,8 +356,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 h1:Yb+Wlf github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bhrnp3Ky1qgx/fzCtCALOoGYylh2tpS9K4= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200602125825-24ff01093eca h1:s/dJePRDKjD1fGeoTnEYFqPmp1v7fC6GTd6iFwCKxw8= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200602125825-24ff01093eca/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200604085359-baf0c20ac96f h1:G0B8Yu/5RVg4PiNhr9Adw3g7RLmQ95wnOL7bz6tUKrU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200604085359-baf0c20ac96f/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f/go.mod h1:y0oDTjZDv5SM9a2rp3bl+CU+bvTRINQsdb7YlDql5Go= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= diff --git a/sytest-whitelist b/sytest-whitelist index 6236b28ed..c1c0c13ac 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -301,3 +301,15 @@ Can send messages with a wildcard device id to two devices Wildcard device messages wake up /sync # TODO: separate PR for: Wildcard device messages over federation wake up /sync Can send a to-device message to two users which both receive it using /sync +User can create and send/receive messages in a room with version 6 +local user can join room with version 6 +User can invite local user to room with version 6 +remote user can join room with version 6 +User can invite remote user to room with version 6 +Remote user can backfill in a room with version 6 +Inbound: send_join rejects invalid JSON for room version 6 +Outbound federation rejects backfill containing invalid JSON for events in room version 6 +Invalid JSON integers +Invalid JSON special values +Invalid JSON floats +Outbound federation will ignore a missing event with bad JSON for room version 6 From 225b72bd42fb104bccd4a6b98e770b7473eb00f7 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 4 Jun 2020 10:54:10 +0100 Subject: [PATCH 125/200] Don't reset counters before successful outgoing federation request (#1089) * Don't reset counters before successful outgoing federation request on incoming federation request * Comments --- federationsender/queue/destinationqueue.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/federationsender/queue/destinationqueue.go b/federationsender/queue/destinationqueue.go index 4ab610de7..bf9042f45 100644 --- a/federationsender/queue/destinationqueue.go +++ b/federationsender/queue/destinationqueue.go @@ -59,8 +59,11 @@ func (oq *destinationQueue) retry() { // and then skip ahead a lot which feels non-ideal but equally we can't persist thousands of events // in-memory to maybe-send it one day. Ideally we would just shove these pending events in a database // so we can send a lot of events. - oq.statistics.Success() - // if we were backing off, swap to not backing off and interrupt the select. + // + // Interrupt the backoff. If the federation request that happens as a result of this is successful + // then the counters will be reset there and the backoff will cancel. If the federation request + // fails then we will retry at the current backoff interval, so as to prevent us from spamming + // homeservers which are behaving badly. // We need to use an atomic bool here to prevent multiple calls to retry() blocking on the channel // as it is unbuffered. if oq.backingOff.CAS(true, false) { From 8c3f51d624aea751e61575e3d2c401f976d1f8ef Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 4 Jun 2020 11:13:40 +0100 Subject: [PATCH 126/200] Update are-we-synapse-yet (#1092) --- are-we-synapse-yet.list | 26 +++++++++++++++++++++++--- are-we-synapse-yet.py | 2 ++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/are-we-synapse-yet.list b/are-we-synapse-yet.list index cb90d6282..c088c8b5e 100644 --- a/are-we-synapse-yet.list +++ b/are-we-synapse-yet.list @@ -455,6 +455,19 @@ rmv User can invite remote user to room with version 5 rmv Remote user can backfill in a room with version 5 rmv Can reject invites over federation for rooms with version 5 rmv Can receive redactions from regular users over federation in room version 5 +rmv User can create and send/receive messages in a room with version 6 +rmv User can create and send/receive messages in a room with version 6 (2 subtests) +rmv local user can join room with version 6 +rmv User can invite local user to room with version 6 +rmv remote user can join room with version 6 +rmv User can invite remote user to room with version 6 +rmv Remote user can backfill in a room with version 6 +rmv Can reject invites over federation for rooms with version 6 +rmv Can receive redactions from regular users over federation in room version 6 +rmv Inbound federation rejects invites which include invalid JSON for room version 6 +rmv Outbound federation rejects invite response which include invalid JSON for room version 6 +rmv Inbound federation rejects invite rejections which include invalid JSON for room version 6 +rmv Server rejects invalid JSON in a version 6 room pre Presence changes are reported to local room members f,pre Presence changes are also reported to remote room members pre Presence changes to UNAVAILABLE are reported to local room members @@ -536,11 +549,11 @@ std Can recv device messages until they are acknowledged std Device messages with the same txn_id are deduplicated std Device messages wake up /sync std Can recv device messages over federation -std Device messages over federation wake up /sync +fsd Device messages over federation wake up /sync std Can send messages with a wildcard device id std Can send messages with a wildcard device id to two devices std Wildcard device messages wake up /sync -std Wildcard device messages over federation wake up /sync +fsd Wildcard device messages over federation wake up /sync adm /whois nsp /purge_history nsp /purge_history by ts @@ -578,6 +591,7 @@ frv A pair of servers can establish a join in a v2 room fsj Outbound federation rejects send_join responses with no m.room.create event frv Outbound federation rejects m.room.create events with an unknown room version fsj Event with an invalid signature in the send_join response should not cause room join to fail +fsj Inbound: send_join rejects invalid JSON for room version 6 fed Outbound federation can send events fed Inbound federation can receive events fed Inbound federation can receive redacted events @@ -636,6 +650,7 @@ fst Name/topic keys are correct fau Remote servers cannot set power levels in rooms without existing powerlevels fau Remote servers should reject attempts by non-creators to set the power levels fau Inbound federation rejects typing notifications from wrong remote +fau Users cannot set notifications powerlevel higher than their own fed Forward extremities remain so even after the next events are populated as outliers fau Banned servers cannot send events fau Banned servers cannot /make_join @@ -833,4 +848,9 @@ gst Guest user can call /events on another world_readable room (SYN-606) gst Real user can call /events on another world_readable room (SYN-606) gst Events come down the correct room pub Asking for a remote rooms list, but supplying the local server's name, returns the local rooms list -std Can send a to-device message to two users which both receive it using /sync \ No newline at end of file +std Can send a to-device message to two users which both receive it using /sync +fme Outbound federation will ignore a missing event with bad JSON for room version 6 +fbk Outbound federation rejects backfill containing invalid JSON for events in room version 6 +jso Invalid JSON integers +jso Invalid JSON floats +jso Invalid JSON special values \ No newline at end of file diff --git a/are-we-synapse-yet.py b/are-we-synapse-yet.py index ffed8d384..5d5128479 100755 --- a/are-we-synapse-yet.py +++ b/are-we-synapse-yet.py @@ -50,6 +50,7 @@ test_mappings = { "fpb": "Public Room API", "fdk": "Device Key APIs", "fed": "Federation API", + "fsd": "Send-to-Device APIs", }, "client_apis": { @@ -99,6 +100,7 @@ test_mappings = { "ign": "Ignore Users", "udr": "User Directory APIs", "app": "Application Services API", + "jso": "Enforced canonical JSON", }, } From feb32ba365a460e4dcd3e77d0d4aed61d7579610 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Thu, 4 Jun 2020 11:14:08 +0100 Subject: [PATCH 127/200] Encode v3 event IDs correctly (#1090) --- federationapi/routing/routing.go | 94 +++++++------------------------- internal/basecomponent/base.go | 13 ++++- internal/httpapi.go | 9 ++- 3 files changed, 39 insertions(+), 77 deletions(-) diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index b75a7941d..e6c6df658 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -37,6 +37,9 @@ const ( ) // Setup registers HTTP handlers with the given ServeMux. +// The provided publicAPIMux MUST have `UseEncodedPath()` enabled or else routes will incorrectly +// path unescape twice (once from the router, once from MakeFedAPI). We need to have this enabled +// so we can decode paths like foo/bar%2Fbaz as [foo, bar/baz] - by default it will decode to [foo, bar, baz] // // Due to Setup being used to call many other functions, a gocyclo nolint is // applied: @@ -76,11 +79,7 @@ func Setup( v1fedmux.Handle("/send/{txnID}", internal.MakeFedAPI( "federation_send", cfg.Matrix.ServerName, keys, wakeup, - func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq)) - if err != nil { - return util.ErrorResponse(err) - } + func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return Send( httpReq, request, gomatrixserverlib.TransactionID(vars["txnID"]), cfg, rsAPI, producer, eduProducer, keys, federation, @@ -90,11 +89,7 @@ func Setup( v2fedmux.Handle("/invite/{roomID}/{eventID}", internal.MakeFedAPI( "federation_invite", cfg.Matrix.ServerName, keys, wakeup, - func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq)) - if err != nil { - return util.ErrorResponse(err) - } + func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return Invite( httpReq, request, vars["roomID"], vars["eventID"], cfg, producer, keys, @@ -110,11 +105,7 @@ func Setup( v1fedmux.Handle("/exchange_third_party_invite/{roomID}", internal.MakeFedAPI( "exchange_third_party_invite", cfg.Matrix.ServerName, keys, wakeup, - func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq)) - if err != nil { - return util.ErrorResponse(err) - } + func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return ExchangeThirdPartyInvite( httpReq, request, vars["roomID"], rsAPI, cfg, federation, producer, ) @@ -123,11 +114,7 @@ func Setup( v1fedmux.Handle("/event/{eventID}", internal.MakeFedAPI( "federation_get_event", cfg.Matrix.ServerName, keys, wakeup, - func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq)) - if err != nil { - return util.ErrorResponse(err) - } + func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return GetEvent( httpReq.Context(), request, rsAPI, vars["eventID"], cfg.Matrix.ServerName, ) @@ -136,11 +123,7 @@ func Setup( v1fedmux.Handle("/state/{roomID}", internal.MakeFedAPI( "federation_get_state", cfg.Matrix.ServerName, keys, wakeup, - func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq)) - if err != nil { - return util.ErrorResponse(err) - } + func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return GetState( httpReq.Context(), request, rsAPI, vars["roomID"], ) @@ -149,11 +132,7 @@ func Setup( v1fedmux.Handle("/state_ids/{roomID}", internal.MakeFedAPI( "federation_get_state_ids", cfg.Matrix.ServerName, keys, wakeup, - func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq)) - if err != nil { - return util.ErrorResponse(err) - } + func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return GetStateIDs( httpReq.Context(), request, rsAPI, vars["roomID"], ) @@ -162,8 +141,7 @@ func Setup( v1fedmux.Handle("/event_auth/{roomID}/{eventID}", internal.MakeFedAPI( "federation_get_event_auth", cfg.Matrix.ServerName, keys, wakeup, - func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars := mux.Vars(httpReq) + func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return GetEventAuth( httpReq.Context(), request, rsAPI, vars["roomID"], vars["eventID"], ) @@ -172,7 +150,7 @@ func Setup( v1fedmux.Handle("/query/directory", internal.MakeFedAPI( "federation_query_room_alias", cfg.Matrix.ServerName, keys, wakeup, - func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { + func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return RoomAliasToID( httpReq, federation, cfg, rsAPI, fsAPI, ) @@ -181,7 +159,7 @@ func Setup( v1fedmux.Handle("/query/profile", internal.MakeFedAPI( "federation_query_profile", cfg.Matrix.ServerName, keys, wakeup, - func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { + func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return GetProfile( httpReq, accountDB, cfg, asAPI, ) @@ -190,11 +168,7 @@ func Setup( v1fedmux.Handle("/user/devices/{userID}", internal.MakeFedAPI( "federation_user_devices", cfg.Matrix.ServerName, keys, wakeup, - func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq)) - if err != nil { - return util.ErrorResponse(err) - } + func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return GetUserDevices( httpReq, deviceDB, vars["userID"], ) @@ -203,11 +177,7 @@ func Setup( v1fedmux.Handle("/make_join/{roomID}/{eventID}", internal.MakeFedAPI( "federation_make_join", cfg.Matrix.ServerName, keys, wakeup, - func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq)) - if err != nil { - return util.ErrorResponse(err) - } + func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { roomID := vars["roomID"] eventID := vars["eventID"] queryVars := httpReq.URL.Query() @@ -232,11 +202,7 @@ func Setup( v1fedmux.Handle("/send_join/{roomID}/{eventID}", internal.MakeFedAPI( "federation_send_join", cfg.Matrix.ServerName, keys, wakeup, - func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq)) - if err != nil { - return util.ErrorResponse(err) - } + func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { roomID := vars["roomID"] eventID := vars["eventID"] res := SendJoin( @@ -254,11 +220,7 @@ func Setup( v2fedmux.Handle("/send_join/{roomID}/{eventID}", internal.MakeFedAPI( "federation_send_join", cfg.Matrix.ServerName, keys, wakeup, - func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq)) - if err != nil { - return util.ErrorResponse(err) - } + func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { roomID := vars["roomID"] eventID := vars["eventID"] return SendJoin( @@ -269,11 +231,7 @@ func Setup( v1fedmux.Handle("/make_leave/{roomID}/{eventID}", internal.MakeFedAPI( "federation_make_leave", cfg.Matrix.ServerName, keys, wakeup, - func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq)) - if err != nil { - return util.ErrorResponse(err) - } + func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { roomID := vars["roomID"] eventID := vars["eventID"] return MakeLeave( @@ -284,11 +242,7 @@ func Setup( v2fedmux.Handle("/send_leave/{roomID}/{eventID}", internal.MakeFedAPI( "federation_send_leave", cfg.Matrix.ServerName, keys, wakeup, - func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq)) - if err != nil { - return util.ErrorResponse(err) - } + func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { roomID := vars["roomID"] eventID := vars["eventID"] return SendLeave( @@ -306,22 +260,14 @@ func Setup( v1fedmux.Handle("/get_missing_events/{roomID}", internal.MakeFedAPI( "federation_get_missing_events", cfg.Matrix.ServerName, keys, wakeup, - func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq)) - if err != nil { - return util.ErrorResponse(err) - } + func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return GetMissingEvents(httpReq, request, rsAPI, vars["roomID"]) }, )).Methods(http.MethodPost) v1fedmux.Handle("/backfill/{roomID}", internal.MakeFedAPI( "federation_backfill", cfg.Matrix.ServerName, keys, wakeup, - func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(httpReq)) - if err != nil { - return util.ErrorResponse(err) - } + func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return Backfill(httpReq, request, rsAPI, vars["roomID"], cfg) }, )).Methods(http.MethodGet) diff --git a/internal/basecomponent/base.go b/internal/basecomponent/base.go index 0fc95e824..beaf0e86d 100644 --- a/internal/basecomponent/base.go +++ b/internal/basecomponent/base.go @@ -103,7 +103,18 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, enableHTTPAPIs })} } - httpmux := mux.NewRouter() + // Ideally we would only use SkipClean on routes which we know can allow '/' but due to + // https://github.com/gorilla/mux/issues/460 we have to attach this at the top router. + // When used in conjunction with UseEncodedPath() we get the behaviour we want when parsing + // path parameters: + // /foo/bar%2Fbaz == [foo, bar%2Fbaz] (from UseEncodedPath) + // /foo/bar%2F%2Fbaz == [foo, bar%2F%2Fbaz] (from SkipClean) + // In particular, rooms v3 event IDs are not urlsafe and can include '/' and because they + // are randomly generated it results in flakey tests. + // We need to be careful with media APIs if they read from a filesystem to make sure they + // are not inadvertently reading paths without cleaning, else this could introduce a + // directory traversal attack e.g /../../../etc/passwd + httpmux := mux.NewRouter().SkipClean(true) return &BaseDendrite{ componentName: componentName, diff --git a/internal/httpapi.go b/internal/httpapi.go index bacd1768a..991a98619 100644 --- a/internal/httpapi.go +++ b/internal/httpapi.go @@ -174,7 +174,7 @@ func MakeFedAPI( serverName gomatrixserverlib.ServerName, keyRing gomatrixserverlib.KeyRing, wakeup *FederationWakeups, - f func(*http.Request, *gomatrixserverlib.FederationRequest) util.JSONResponse, + f func(*http.Request, *gomatrixserverlib.FederationRequest, map[string]string) util.JSONResponse, ) http.Handler { h := func(req *http.Request) util.JSONResponse { fedReq, errResp := gomatrixserverlib.VerifyHTTPRequest( @@ -184,7 +184,12 @@ func MakeFedAPI( return errResp } go wakeup.Wakeup(req.Context(), fedReq.Origin()) - return f(req, fedReq) + vars, err := URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + + return f(req, fedReq, vars) } return MakeExternalAPI(metricsName, h) } From d4f9a4bb97c0c66963d7fd9c1395614a2198112e Mon Sep 17 00:00:00 2001 From: Kegsay Date: Thu, 4 Jun 2020 11:17:37 +0100 Subject: [PATCH 128/200] Fix #632 and send spec-compliant transactions to the AS (#1091) --- appservice/workers/transaction_scheduler.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/appservice/workers/transaction_scheduler.go b/appservice/workers/transaction_scheduler.go index 208056192..63ec58aa1 100644 --- a/appservice/workers/transaction_scheduler.go +++ b/appservice/workers/transaction_scheduler.go @@ -21,6 +21,7 @@ import ( "fmt" "math" "net/http" + "net/url" "time" "github.com/matrix-org/dendrite/appservice/storage" @@ -207,9 +208,15 @@ func send( txnID int, transaction []byte, ) error { - // POST a transaction to our AS - address := fmt.Sprintf("%s/transactions/%d", appservice.URL, txnID) - resp, err := client.Post(address, "application/json", bytes.NewBuffer(transaction)) + // PUT a transaction to our AS + // https://matrix.org/docs/spec/application_service/r0.1.2#put-matrix-app-v1-transactions-txnid + address := fmt.Sprintf("%s/transactions/%d?access_token=%s", appservice.URL, txnID, url.QueryEscape(appservice.HSToken)) + req, err := http.NewRequest("PUT", address, bytes.NewBuffer(transaction)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + resp, err := client.Do(req) if err != nil { return err } From e7d1ac84c32a10f8deb7bcfc94531386312179c7 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Thu, 4 Jun 2020 11:18:08 +0100 Subject: [PATCH 129/200] Add ParseFileURI and use it when dealing with file URIs (#1088) * Add ParseFileURI and use it when dealing with file URIs Fixes #1059 * Missing file * Linting --- appservice/storage/sqlite3/storage.go | 6 ++++- appservice/storage/storage_wasm.go | 2 +- .../auth/storage/accounts/sqlite3/storage.go | 6 ++++- .../auth/storage/accounts/storage_wasm.go | 2 +- .../auth/storage/devices/sqlite3/storage.go | 6 ++++- .../auth/storage/devices/storage_wasm.go | 2 +- federationsender/storage/sqlite3/storage.go | 6 ++++- federationsender/storage/storage_wasm.go | 2 +- internal/basecomponent/base.go | 9 +++---- internal/sqlutil/uri.go | 24 +++++++++++++++++++ mediaapi/storage/sqlite3/storage.go | 6 ++++- mediaapi/storage/storage_wasm.go | 2 +- publicroomsapi/storage/sqlite3/storage.go | 6 ++++- publicroomsapi/storage/storage_wasm.go | 2 +- roomserver/storage/sqlite3/storage.go | 12 +--------- serverkeyapi/storage/sqlite3/keydb.go | 12 +--------- syncapi/storage/sqlite3/syncserver.go | 12 +--------- 17 files changed, 66 insertions(+), 51 deletions(-) create mode 100644 internal/sqlutil/uri.go diff --git a/appservice/storage/sqlite3/storage.go b/appservice/storage/sqlite3/storage.go index 275c66526..2238b3ff9 100644 --- a/appservice/storage/sqlite3/storage.go +++ b/appservice/storage/sqlite3/storage.go @@ -37,7 +37,11 @@ type Database struct { func NewDatabase(dataSourceName string) (*Database, error) { var result Database var err error - if result.db, err = sqlutil.Open(internal.SQLiteDriverName(), dataSourceName, nil); err != nil { + cs, err := sqlutil.ParseFileURI(dataSourceName) + if err != nil { + return nil, err + } + if result.db, err = sqlutil.Open(internal.SQLiteDriverName(), cs, nil); err != nil { return nil, err } if err = result.prepare(); err != nil { diff --git a/appservice/storage/storage_wasm.go b/appservice/storage/storage_wasm.go index de1acf92f..a6144b435 100644 --- a/appservice/storage/storage_wasm.go +++ b/appservice/storage/storage_wasm.go @@ -34,7 +34,7 @@ func NewDatabase( case "postgres": return nil, fmt.Errorf("Cannot use postgres implementation") case "file": - return sqlite3.NewDatabase(uri.Path) + return sqlite3.NewDatabase(dataSourceName) default: return nil, fmt.Errorf("Cannot use postgres implementation") } diff --git a/clientapi/auth/storage/accounts/sqlite3/storage.go b/clientapi/auth/storage/accounts/sqlite3/storage.go index 5233001f2..31426e471 100644 --- a/clientapi/auth/storage/accounts/sqlite3/storage.go +++ b/clientapi/auth/storage/accounts/sqlite3/storage.go @@ -48,7 +48,11 @@ type Database struct { func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) { var db *sql.DB var err error - if db, err = sqlutil.Open(internal.SQLiteDriverName(), dataSourceName, nil); err != nil { + cs, err := sqlutil.ParseFileURI(dataSourceName) + if err != nil { + return nil, err + } + if db, err = sqlutil.Open(internal.SQLiteDriverName(), cs, nil); err != nil { return nil, err } partitions := internal.PartitionOffsetStatements{} diff --git a/clientapi/auth/storage/accounts/storage_wasm.go b/clientapi/auth/storage/accounts/storage_wasm.go index 7cf50de79..81e77cf79 100644 --- a/clientapi/auth/storage/accounts/storage_wasm.go +++ b/clientapi/auth/storage/accounts/storage_wasm.go @@ -36,7 +36,7 @@ func NewDatabase( case "postgres": return nil, fmt.Errorf("Cannot use postgres implementation") case "file": - return sqlite3.NewDatabase(uri.Path, serverName) + return sqlite3.NewDatabase(dataSourceName, serverName) default: return nil, fmt.Errorf("Cannot use postgres implementation") } diff --git a/clientapi/auth/storage/devices/sqlite3/storage.go b/clientapi/auth/storage/devices/sqlite3/storage.go index 5cd894517..e05a53b46 100644 --- a/clientapi/auth/storage/devices/sqlite3/storage.go +++ b/clientapi/auth/storage/devices/sqlite3/storage.go @@ -41,7 +41,11 @@ type Database struct { func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) (*Database, error) { var db *sql.DB var err error - if db, err = sqlutil.Open(internal.SQLiteDriverName(), dataSourceName, nil); err != nil { + cs, err := sqlutil.ParseFileURI(dataSourceName) + if err != nil { + return nil, err + } + if db, err = sqlutil.Open(internal.SQLiteDriverName(), cs, nil); err != nil { return nil, err } d := devicesStatements{} diff --git a/clientapi/auth/storage/devices/storage_wasm.go b/clientapi/auth/storage/devices/storage_wasm.go index c89ad887b..14c19e74b 100644 --- a/clientapi/auth/storage/devices/storage_wasm.go +++ b/clientapi/auth/storage/devices/storage_wasm.go @@ -36,7 +36,7 @@ func NewDatabase( case "postgres": return nil, fmt.Errorf("Cannot use postgres implementation") case "file": - return sqlite3.NewDatabase(uri.Path, serverName) + return sqlite3.NewDatabase(dataSourceName, serverName) default: return nil, fmt.Errorf("Cannot use postgres implementation") } diff --git a/federationsender/storage/sqlite3/storage.go b/federationsender/storage/sqlite3/storage.go index dc60ebb01..8699fc19a 100644 --- a/federationsender/storage/sqlite3/storage.go +++ b/federationsender/storage/sqlite3/storage.go @@ -38,7 +38,11 @@ type Database struct { func NewDatabase(dataSourceName string) (*Database, error) { var result Database var err error - if result.db, err = sqlutil.Open(internal.SQLiteDriverName(), dataSourceName, nil); err != nil { + cs, err := sqlutil.ParseFileURI(dataSourceName) + if err != nil { + return nil, err + } + if result.db, err = sqlutil.Open(internal.SQLiteDriverName(), cs, nil); err != nil { return nil, err } if err = result.prepare(); err != nil { diff --git a/federationsender/storage/storage_wasm.go b/federationsender/storage/storage_wasm.go index 3d071bfef..f593fd448 100644 --- a/federationsender/storage/storage_wasm.go +++ b/federationsender/storage/storage_wasm.go @@ -33,7 +33,7 @@ func NewDatabase( } switch uri.Scheme { case "file": - return sqlite3.NewDatabase(uri.Path) + return sqlite3.NewDatabase(dataSourceName) case "postgres": return nil, fmt.Errorf("Cannot use postgres implementation") default: diff --git a/internal/basecomponent/base.go b/internal/basecomponent/base.go index beaf0e86d..42d236f85 100644 --- a/internal/basecomponent/base.go +++ b/internal/basecomponent/base.go @@ -277,12 +277,9 @@ func setupNaffka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) { uri, err := url.Parse(string(cfg.Database.Naffka)) if err != nil || uri.Scheme == "file" { var cs string - if uri.Opaque != "" { // file:filename.db - cs = uri.Opaque - } else if uri.Path != "" { // file:///path/to/filename.db - cs = uri.Path - } else { - logrus.Panic("file uri has no filename") + cs, err = sqlutil.ParseFileURI(string(cfg.Database.Naffka)) + if err != nil { + logrus.WithError(err).Panic("Failed to parse naffka database file URI") } db, err = sqlutil.Open(internal.SQLiteDriverName(), cs, nil) if err != nil { diff --git a/internal/sqlutil/uri.go b/internal/sqlutil/uri.go new file mode 100644 index 000000000..f72e02426 --- /dev/null +++ b/internal/sqlutil/uri.go @@ -0,0 +1,24 @@ +package sqlutil + +import ( + "fmt" + "net/url" +) + +// ParseFileURI returns the filepath in the given file: URI. Specifically, this will handle +// both relative (file:foo.db) and absolute (file:///path/to/foo) paths. +func ParseFileURI(dataSourceName string) (string, error) { + uri, err := url.Parse(dataSourceName) + if err != nil { + return "", err + } + var cs string + if uri.Opaque != "" { // file:filename.db + cs = uri.Opaque + } else if uri.Path != "" { // file:///path/to/filename.db + cs = uri.Path + } else { + return "", fmt.Errorf("invalid file uri: %s", dataSourceName) + } + return cs, nil +} diff --git a/mediaapi/storage/sqlite3/storage.go b/mediaapi/storage/sqlite3/storage.go index 6da7132d8..ea81f9120 100644 --- a/mediaapi/storage/sqlite3/storage.go +++ b/mediaapi/storage/sqlite3/storage.go @@ -37,7 +37,11 @@ type Database struct { func Open(dataSourceName string) (*Database, error) { var d Database var err error - if d.db, err = sqlutil.Open(internal.SQLiteDriverName(), dataSourceName, nil); err != nil { + cs, err := sqlutil.ParseFileURI(dataSourceName) + if err != nil { + return nil, err + } + if d.db, err = sqlutil.Open(internal.SQLiteDriverName(), cs, nil); err != nil { return nil, err } if err = d.statements.prepare(d.db); err != nil { diff --git a/mediaapi/storage/storage_wasm.go b/mediaapi/storage/storage_wasm.go index aa188997f..78de2cb82 100644 --- a/mediaapi/storage/storage_wasm.go +++ b/mediaapi/storage/storage_wasm.go @@ -35,7 +35,7 @@ func Open( case "postgres": return nil, fmt.Errorf("Cannot use postgres implementation") case "file": - return sqlite3.Open(uri.Path) + return sqlite3.Open(dataSourceName) default: return nil, fmt.Errorf("Cannot use postgres implementation") } diff --git a/publicroomsapi/storage/sqlite3/storage.go b/publicroomsapi/storage/sqlite3/storage.go index 898101127..f51051823 100644 --- a/publicroomsapi/storage/sqlite3/storage.go +++ b/publicroomsapi/storage/sqlite3/storage.go @@ -42,7 +42,11 @@ type attributeValue interface{} func NewPublicRoomsServerDatabase(dataSourceName string, localServerName gomatrixserverlib.ServerName) (*PublicRoomsServerDatabase, error) { var db *sql.DB var err error - if db, err = sqlutil.Open(internal.SQLiteDriverName(), dataSourceName, nil); err != nil { + cs, err := sqlutil.ParseFileURI(dataSourceName) + if err != nil { + return nil, err + } + if db, err = sqlutil.Open(internal.SQLiteDriverName(), cs, nil); err != nil { return nil, err } storage := PublicRoomsServerDatabase{ diff --git a/publicroomsapi/storage/storage_wasm.go b/publicroomsapi/storage/storage_wasm.go index fa3356c03..70ceeaf85 100644 --- a/publicroomsapi/storage/storage_wasm.go +++ b/publicroomsapi/storage/storage_wasm.go @@ -32,7 +32,7 @@ func NewPublicRoomsServerDatabase(dataSourceName string, localServerName gomatri case "postgres": return nil, fmt.Errorf("Cannot use postgres implementation") case "file": - return sqlite3.NewPublicRoomsServerDatabase(uri.Path, localServerName) + return sqlite3.NewPublicRoomsServerDatabase(dataSourceName, localServerName) default: return nil, fmt.Errorf("Cannot use postgres implementation") } diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index 54a2f2658..5596b5e3c 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -18,8 +18,6 @@ package sqlite3 import ( "context" "database/sql" - "errors" - "net/url" "github.com/matrix-org/dendrite/internal/sqlutil" @@ -50,18 +48,10 @@ type Database struct { // nolint: gocyclo func Open(dataSourceName string) (*Database, error) { var d Database - uri, err := url.Parse(dataSourceName) + cs, err := sqlutil.ParseFileURI(dataSourceName) if err != nil { return nil, err } - var cs string - if uri.Opaque != "" { // file:filename.db - cs = uri.Opaque - } else if uri.Path != "" { // file:///path/to/filename.db - cs = uri.Path - } else { - return nil, errors.New("no filename or path in connect string") - } if d.db, err = sqlutil.Open(internal.SQLiteDriverName(), cs, nil); err != nil { return nil, err } diff --git a/serverkeyapi/storage/sqlite3/keydb.go b/serverkeyapi/storage/sqlite3/keydb.go index 1ad6e7e88..e8cb78bd6 100644 --- a/serverkeyapi/storage/sqlite3/keydb.go +++ b/serverkeyapi/storage/sqlite3/keydb.go @@ -17,8 +17,6 @@ package sqlite3 import ( "context" - "errors" - "net/url" "time" "golang.org/x/crypto/ed25519" @@ -46,18 +44,10 @@ func NewDatabase( serverKey ed25519.PublicKey, serverKeyID gomatrixserverlib.KeyID, ) (*Database, error) { - uri, err := url.Parse(dataSourceName) + cs, err := sqlutil.ParseFileURI(dataSourceName) if err != nil { return nil, err } - var cs string - if uri.Opaque != "" { // file:filename.db - cs = uri.Opaque - } else if uri.Path != "" { // file:///path/to/filename.db - cs = uri.Path - } else { - return nil, errors.New("no filename or path in connect string") - } db, err := sqlutil.Open(internal.SQLiteDriverName(), cs, nil) if err != nil { return nil, err diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index 5ba07617e..38ce5bcf0 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -17,8 +17,6 @@ package sqlite3 import ( "database/sql" - "errors" - "net/url" "github.com/matrix-org/dendrite/internal/sqlutil" @@ -43,18 +41,10 @@ type SyncServerDatasource struct { // nolint: gocyclo func NewDatabase(dataSourceName string) (*SyncServerDatasource, error) { var d SyncServerDatasource - uri, err := url.Parse(dataSourceName) + cs, err := sqlutil.ParseFileURI(dataSourceName) if err != nil { return nil, err } - var cs string - if uri.Opaque != "" { // file:filename.db - cs = uri.Opaque - } else if uri.Path != "" { // file:///path/to/filename.db - cs = uri.Path - } else { - return nil, errors.New("no filename or path in connect string") - } if d.db, err = sqlutil.Open(internal.SQLiteDriverName(), cs, nil); err != nil { return nil, err } From 097cdf879cb83324e238ee489354e87d77ee4dc9 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Thu, 4 Jun 2020 11:50:57 +0100 Subject: [PATCH 130/200] s/Base64String/Base64Bytes/g (#1093) * s/Base64String/Base64Bytes/g Requires https://github.com/matrix-org/gomatrixserverlib/pull/203 to land first * update gmsl --- clientapi/auth/storage/accounts/sqlite3/constraint_wasm.go | 2 +- clientapi/threepid/invites.go | 2 +- cmd/dendrite-demo-libp2p/mdnslistener.go | 2 +- cmd/dendritejs/keyfetcher.go | 2 +- federationapi/routing/keys.go | 2 +- go.mod | 2 +- go.sum | 4 ++-- serverkeyapi/storage/postgres/keydb.go | 2 +- serverkeyapi/storage/sqlite3/keydb.go | 2 +- 9 files changed, 10 insertions(+), 10 deletions(-) diff --git a/clientapi/auth/storage/accounts/sqlite3/constraint_wasm.go b/clientapi/auth/storage/accounts/sqlite3/constraint_wasm.go index b5077c070..0dd5b1fea 100644 --- a/clientapi/auth/storage/accounts/sqlite3/constraint_wasm.go +++ b/clientapi/auth/storage/accounts/sqlite3/constraint_wasm.go @@ -18,4 +18,4 @@ package sqlite3 func isConstraintError(err error) bool { return false -} \ No newline at end of file +} diff --git a/clientapi/threepid/invites.go b/clientapi/threepid/invites.go index 8fed99565..8f173bf8a 100644 --- a/clientapi/threepid/invites.go +++ b/clientapi/threepid/invites.go @@ -279,7 +279,7 @@ func queryIDServerPubKey(ctx context.Context, idServerName string, keyID string) } var pubKeyRes struct { - PublicKey gomatrixserverlib.Base64String `json:"public_key"` + PublicKey gomatrixserverlib.Base64Bytes `json:"public_key"` } if resp.StatusCode != http.StatusOK { diff --git a/cmd/dendrite-demo-libp2p/mdnslistener.go b/cmd/dendrite-demo-libp2p/mdnslistener.go index 01705cf39..c30aaa331 100644 --- a/cmd/dendrite-demo-libp2p/mdnslistener.go +++ b/cmd/dendrite-demo-libp2p/mdnslistener.go @@ -43,7 +43,7 @@ func (n *mDNSListener) HandlePeerFound(p peer.AddrInfo) { KeyID: "ed25519:p2pdemo", }: { VerifyKey: gomatrixserverlib.VerifyKey{ - Key: gomatrixserverlib.Base64String(raw), + Key: gomatrixserverlib.Base64Bytes(raw), }, ValidUntilTS: math.MaxUint64 >> 1, ExpiredTS: gomatrixserverlib.PublicKeyNotExpired, diff --git a/cmd/dendritejs/keyfetcher.go b/cmd/dendritejs/keyfetcher.go index 0073b7e6b..cef045372 100644 --- a/cmd/dendritejs/keyfetcher.go +++ b/cmd/dendritejs/keyfetcher.go @@ -62,7 +62,7 @@ func (f *libp2pKeyFetcher) FetchKeys( if err != nil { return nil, fmt.Errorf("Failed to extract raw bytes from public key: %w", err) } - b64Key := gomatrixserverlib.Base64String(pubKeyBytes) + b64Key := gomatrixserverlib.Base64Bytes(pubKeyBytes) res[req] = gomatrixserverlib.PublicKeyLookupResult{ VerifyKey: gomatrixserverlib.VerifyKey{ Key: b64Key, diff --git a/federationapi/routing/keys.go b/federationapi/routing/keys.go index 79cc1f82c..a1dd0fd09 100644 --- a/federationapi/routing/keys.go +++ b/federationapi/routing/keys.go @@ -44,7 +44,7 @@ func localKeys(cfg *config.Dendrite, validUntil time.Time) (*gomatrixserverlib.S keys.VerifyKeys = map[gomatrixserverlib.KeyID]gomatrixserverlib.VerifyKey{ cfg.Matrix.KeyID: { - Key: gomatrixserverlib.Base64String(publicKey), + Key: gomatrixserverlib.Base64Bytes(publicKey), }, } diff --git a/go.mod b/go.mod index 5510e8e0e..e40fbc507 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200604085359-baf0c20ac96f + github.com/matrix-org/gomatrixserverlib v0.0.0-20200604102917-067fd074d64b github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible diff --git a/go.sum b/go.sum index 5df03cbaa..72f14ca02 100644 --- a/go.sum +++ b/go.sum @@ -356,8 +356,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 h1:Yb+Wlf github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bhrnp3Ky1qgx/fzCtCALOoGYylh2tpS9K4= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200604085359-baf0c20ac96f h1:G0B8Yu/5RVg4PiNhr9Adw3g7RLmQ95wnOL7bz6tUKrU= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200604085359-baf0c20ac96f/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200604102917-067fd074d64b h1:VuAs44UxWAGQVtZRAdjHocw+jkZ12oZPaWA1qaUDkz4= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200604102917-067fd074d64b/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f/go.mod h1:y0oDTjZDv5SM9a2rp3bl+CU+bvTRINQsdb7YlDql5Go= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= diff --git a/serverkeyapi/storage/postgres/keydb.go b/serverkeyapi/storage/postgres/keydb.go index da3a4d370..6282e13bb 100644 --- a/serverkeyapi/storage/postgres/keydb.go +++ b/serverkeyapi/storage/postgres/keydb.go @@ -60,7 +60,7 @@ func NewDatabase( } value := gomatrixserverlib.PublicKeyLookupResult{ VerifyKey: gomatrixserverlib.VerifyKey{ - Key: gomatrixserverlib.Base64String(serverKey), + Key: gomatrixserverlib.Base64Bytes(serverKey), }, ValidUntilTS: gomatrixserverlib.AsTimestamp(time.Now().Add(100 * 365 * 24 * time.Hour)), ExpiredTS: gomatrixserverlib.PublicKeyNotExpired, diff --git a/serverkeyapi/storage/sqlite3/keydb.go b/serverkeyapi/storage/sqlite3/keydb.go index e8cb78bd6..ab423cff6 100644 --- a/serverkeyapi/storage/sqlite3/keydb.go +++ b/serverkeyapi/storage/sqlite3/keydb.go @@ -65,7 +65,7 @@ func NewDatabase( } value := gomatrixserverlib.PublicKeyLookupResult{ VerifyKey: gomatrixserverlib.VerifyKey{ - Key: gomatrixserverlib.Base64String(serverKey), + Key: gomatrixserverlib.Base64Bytes(serverKey), }, ValidUntilTS: gomatrixserverlib.AsTimestamp(time.Now().Add(100 * 365 * 24 * time.Hour)), ExpiredTS: gomatrixserverlib.PublicKeyNotExpired, From f7025d349946aae75160491d0efcc87100adb406 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 4 Jun 2020 13:30:08 +0100 Subject: [PATCH 131/200] Update gomatrixserverlib --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e40fbc507..a875ef7c9 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200604102917-067fd074d64b + github.com/matrix-org/gomatrixserverlib v0.0.0-20200604122934-a06b83e7f324 github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible diff --git a/go.sum b/go.sum index 72f14ca02..94101251b 100644 --- a/go.sum +++ b/go.sum @@ -356,8 +356,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 h1:Yb+Wlf github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bhrnp3Ky1qgx/fzCtCALOoGYylh2tpS9K4= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200604102917-067fd074d64b h1:VuAs44UxWAGQVtZRAdjHocw+jkZ12oZPaWA1qaUDkz4= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200604102917-067fd074d64b/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200604122934-a06b83e7f324 h1:TjHTrOkX+G99F1NitCwNOPa62beQMAQklBuEBkDP8Hc= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200604122934-a06b83e7f324/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f/go.mod h1:y0oDTjZDv5SM9a2rp3bl+CU+bvTRINQsdb7YlDql5Go= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= From f4c676ccddcd661b7167e8f249f71635a0b03491 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Thu, 4 Jun 2020 14:27:10 +0100 Subject: [PATCH 132/200] Refactor how federationsender gets created (#1095) * Refactor how federationsender gets created * s/httpint/inthttp/ for alphabetical niceness with internal package --- cmd/dendrite-client-api-server/main.go | 2 +- cmd/dendrite-demo-libp2p/main.go | 2 +- cmd/dendrite-federation-api-server/main.go | 2 +- cmd/dendrite-monolith-server/main.go | 14 +-- cmd/dendrite-public-rooms-api-server/main.go | 2 +- cmd/dendrite-room-server/main.go | 2 +- cmd/dendritejs/main.go | 2 +- federationsender/api/api.go | 68 +++++++++-- federationsender/api/perform.go | 112 ------------------ federationsender/api/query.go | 63 ----------- federationsender/federationsender.go | 9 +- federationsender/internal/api.go | 90 --------------- federationsender/inthttp/client.go | 113 +++++++++++++++++++ federationsender/inthttp/server.go | 95 ++++++++++++++++ internal/basecomponent/base.go | 15 +-- 15 files changed, 288 insertions(+), 303 deletions(-) delete mode 100644 federationsender/api/perform.go delete mode 100644 federationsender/api/query.go create mode 100644 federationsender/inthttp/client.go create mode 100644 federationsender/inthttp/server.go diff --git a/cmd/dendrite-client-api-server/main.go b/cmd/dendrite-client-api-server/main.go index f919243de..280e9cdef 100644 --- a/cmd/dendrite-client-api-server/main.go +++ b/cmd/dendrite-client-api-server/main.go @@ -37,7 +37,7 @@ func main() { asQuery := base.CreateHTTPAppServiceAPIs() rsAPI := base.CreateHTTPRoomserverAPIs() - fsAPI := base.CreateHTTPFederationSenderAPIs() + fsAPI := base.FederationSenderHTTPClient() rsAPI.SetFederationSenderAPI(fsAPI) eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New(), deviceDB) diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index c00316ec7..86909685a 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -178,7 +178,7 @@ func main() { base.Base.PublicAPIMux, base.Base.InternalAPIMux, &cfg, - base.Base.EnableHTTPAPIs, + base.Base.UseHTTPAPIs, ) // Expose the matrix APIs directly rather than putting them under a /api path. diff --git a/cmd/dendrite-federation-api-server/main.go b/cmd/dendrite-federation-api-server/main.go index af63b549a..3c42532c0 100644 --- a/cmd/dendrite-federation-api-server/main.go +++ b/cmd/dendrite-federation-api-server/main.go @@ -34,7 +34,7 @@ func main() { serverKeyAPI := base.CreateHTTPServerKeyAPIs() keyRing := serverKeyAPI.KeyRing() - fsAPI := base.CreateHTTPFederationSenderAPIs() + fsAPI := base.FederationSenderHTTPClient() rsAPI := base.CreateHTTPRoomserverAPIs() asAPI := base.CreateHTTPAppServiceAPIs() diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 41907cc0c..eedd4412e 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -73,7 +73,7 @@ func main() { serverKeyAPI := serverkeyapi.SetupServerKeyAPIComponent( base, federation, ) - if base.EnableHTTPAPIs { + if base.UseHTTPAPIs { serverKeyAPI = base.CreateHTTPServerKeyAPIs() } keyRing := serverKeyAPI.KeyRing() @@ -82,29 +82,29 @@ func main() { base, keyRing, federation, ) rsAPI := rsComponent - if base.EnableHTTPAPIs { + if base.UseHTTPAPIs { rsAPI = base.CreateHTTPRoomserverAPIs() } eduInputAPI := eduserver.SetupEDUServerComponent( base, cache.New(), deviceDB, ) - if base.EnableHTTPAPIs { + if base.UseHTTPAPIs { eduInputAPI = base.CreateHTTPEDUServerAPIs() } asAPI := appservice.SetupAppServiceAPIComponent( base, accountDB, deviceDB, federation, rsAPI, transactions.New(), ) - if base.EnableHTTPAPIs { + if base.UseHTTPAPIs { asAPI = base.CreateHTTPAppServiceAPIs() } fsAPI := federationsender.SetupFederationSenderComponent( base, federation, rsAPI, keyRing, ) - if base.EnableHTTPAPIs { - fsAPI = base.CreateHTTPFederationSenderAPIs() + if base.UseHTTPAPIs { + fsAPI = base.FederationSenderHTTPClient() } rsComponent.SetFederationSenderAPI(fsAPI) @@ -132,7 +132,7 @@ func main() { base.PublicAPIMux, base.InternalAPIMux, cfg, - base.EnableHTTPAPIs, + base.UseHTTPAPIs, ) // Expose the matrix APIs directly rather than putting them under a /api path. diff --git a/cmd/dendrite-public-rooms-api-server/main.go b/cmd/dendrite-public-rooms-api-server/main.go index d506e32d1..c2df1b3f8 100644 --- a/cmd/dendrite-public-rooms-api-server/main.go +++ b/cmd/dendrite-public-rooms-api-server/main.go @@ -28,7 +28,7 @@ func main() { deviceDB := base.CreateDeviceDB() - fsAPI := base.CreateHTTPFederationSenderAPIs() + fsAPI := base.FederationSenderHTTPClient() rsAPI := base.CreateHTTPRoomserverAPIs() rsAPI.SetFederationSenderAPI(fsAPI) diff --git a/cmd/dendrite-room-server/main.go b/cmd/dendrite-room-server/main.go index b86b74972..f4da560a6 100644 --- a/cmd/dendrite-room-server/main.go +++ b/cmd/dendrite-room-server/main.go @@ -28,7 +28,7 @@ func main() { serverKeyAPI := base.CreateHTTPServerKeyAPIs() keyRing := serverKeyAPI.KeyRing() - fsAPI := base.CreateHTTPFederationSenderAPIs() + fsAPI := base.FederationSenderHTTPClient() rsAPI := roomserver.SetupRoomServerComponent(base, keyRing, federation) rsAPI.SetFederationSenderAPI(fsAPI) diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index c1aef44d9..4c1b81979 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -235,7 +235,7 @@ func main() { base.PublicAPIMux, base.InternalAPIMux, cfg, - base.EnableHTTPAPIs, + base.UseHTTPAPIs, ) // Expose the matrix APIs via libp2p-js - for federation traffic diff --git a/federationsender/api/api.go b/federationsender/api/api.go index 4eb20cb67..d8251b54d 100644 --- a/federationsender/api/api.go +++ b/federationsender/api/api.go @@ -2,8 +2,9 @@ package api import ( "context" - "errors" - "net/http" + + "github.com/matrix-org/dendrite/federationsender/types" + "github.com/matrix-org/gomatrixserverlib" ) // FederationSenderInternalAPI is used to query information from the federation sender. @@ -50,16 +51,59 @@ type FederationSenderInternalAPI interface { ) error } -// NewFederationSenderInternalAPIHTTP creates a FederationSenderInternalAPI implemented by talking to a HTTP POST API. -// If httpClient is nil an error is returned -func NewFederationSenderInternalAPIHTTP(federationSenderURL string, httpClient *http.Client) (FederationSenderInternalAPI, error) { - if httpClient == nil { - return nil, errors.New("NewFederationSenderInternalAPIHTTP: httpClient is ") - } - return &httpFederationSenderInternalAPI{federationSenderURL, httpClient}, nil +type PerformDirectoryLookupRequest struct { + RoomAlias string `json:"room_alias"` + ServerName gomatrixserverlib.ServerName `json:"server_name"` } -type httpFederationSenderInternalAPI struct { - federationSenderURL string - httpClient *http.Client +type PerformDirectoryLookupResponse struct { + RoomID string `json:"room_id"` + ServerNames []gomatrixserverlib.ServerName `json:"server_names"` +} + +type PerformJoinRequest struct { + RoomID string `json:"room_id"` + UserID string `json:"user_id"` + // The sorted list of servers to try. Servers will be tried sequentially, after de-duplication. + ServerNames types.ServerNames `json:"server_names"` + Content map[string]interface{} `json:"content"` +} + +type PerformJoinResponse struct { +} + +type PerformLeaveRequest struct { + RoomID string `json:"room_id"` + UserID string `json:"user_id"` + ServerNames types.ServerNames `json:"server_names"` +} + +type PerformLeaveResponse struct { +} + +type PerformServersAliveRequest struct { + Servers []gomatrixserverlib.ServerName +} + +type PerformServersAliveResponse struct { +} + +// QueryJoinedHostsInRoomRequest is a request to QueryJoinedHostsInRoom +type QueryJoinedHostsInRoomRequest struct { + RoomID string `json:"room_id"` +} + +// QueryJoinedHostsInRoomResponse is a response to QueryJoinedHostsInRoom +type QueryJoinedHostsInRoomResponse struct { + JoinedHosts []types.JoinedHost `json:"joined_hosts"` +} + +// QueryJoinedHostServerNamesRequest is a request to QueryJoinedHostServerNames +type QueryJoinedHostServerNamesInRoomRequest struct { + RoomID string `json:"room_id"` +} + +// QueryJoinedHostServerNamesResponse is a response to QueryJoinedHostServerNames +type QueryJoinedHostServerNamesInRoomResponse struct { + ServerNames []gomatrixserverlib.ServerName `json:"server_names"` } diff --git a/federationsender/api/perform.go b/federationsender/api/perform.go deleted file mode 100644 index 5e4d7fe97..000000000 --- a/federationsender/api/perform.go +++ /dev/null @@ -1,112 +0,0 @@ -package api - -import ( - "context" - - "github.com/matrix-org/dendrite/federationsender/types" - internalHTTP "github.com/matrix-org/dendrite/internal/http" - "github.com/matrix-org/gomatrixserverlib" - "github.com/opentracing/opentracing-go" -) - -const ( - // FederationSenderPerformJoinRequestPath is the HTTP path for the PerformJoinRequest API. - FederationSenderPerformDirectoryLookupRequestPath = "/federationsender/performDirectoryLookup" - - // FederationSenderPerformJoinRequestPath is the HTTP path for the PerformJoinRequest API. - FederationSenderPerformJoinRequestPath = "/federationsender/performJoinRequest" - - // FederationSenderPerformLeaveRequestPath is the HTTP path for the PerformLeaveRequest API. - FederationSenderPerformLeaveRequestPath = "/federationsender/performLeaveRequest" - - // FederationSenderPerformServersAlivePath is the HTTP path for the PerformServersAlive API. - FederationSenderPerformServersAlivePath = "/federationsender/performServersAlive" -) - -type PerformDirectoryLookupRequest struct { - RoomAlias string `json:"room_alias"` - ServerName gomatrixserverlib.ServerName `json:"server_name"` -} - -type PerformDirectoryLookupResponse struct { - RoomID string `json:"room_id"` - ServerNames []gomatrixserverlib.ServerName `json:"server_names"` -} - -// Handle an instruction to make_join & send_join with a remote server. -func (h *httpFederationSenderInternalAPI) PerformDirectoryLookup( - ctx context.Context, - request *PerformDirectoryLookupRequest, - response *PerformDirectoryLookupResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "PerformDirectoryLookup") - defer span.Finish() - - apiURL := h.federationSenderURL + FederationSenderPerformDirectoryLookupRequestPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} - -type PerformJoinRequest struct { - RoomID string `json:"room_id"` - UserID string `json:"user_id"` - // The sorted list of servers to try. Servers will be tried sequentially, after de-duplication. - ServerNames types.ServerNames `json:"server_names"` - Content map[string]interface{} `json:"content"` -} - -type PerformJoinResponse struct { -} - -// Handle an instruction to make_join & send_join with a remote server. -func (h *httpFederationSenderInternalAPI) PerformJoin( - ctx context.Context, - request *PerformJoinRequest, - response *PerformJoinResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "PerformJoinRequest") - defer span.Finish() - - apiURL := h.federationSenderURL + FederationSenderPerformJoinRequestPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} - -type PerformLeaveRequest struct { - RoomID string `json:"room_id"` - UserID string `json:"user_id"` - ServerNames types.ServerNames `json:"server_names"` -} - -type PerformLeaveResponse struct { -} - -// Handle an instruction to make_leave & send_leave with a remote server. -func (h *httpFederationSenderInternalAPI) PerformLeave( - ctx context.Context, - request *PerformLeaveRequest, - response *PerformLeaveResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "PerformLeaveRequest") - defer span.Finish() - - apiURL := h.federationSenderURL + FederationSenderPerformLeaveRequestPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} - -type PerformServersAliveRequest struct { - Servers []gomatrixserverlib.ServerName -} - -type PerformServersAliveResponse struct { -} - -func (h *httpFederationSenderInternalAPI) PerformServersAlive( - ctx context.Context, - request *PerformServersAliveRequest, - response *PerformServersAliveResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "PerformServersAlive") - defer span.Finish() - - apiURL := h.federationSenderURL + FederationSenderPerformServersAlivePath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} diff --git a/federationsender/api/query.go b/federationsender/api/query.go deleted file mode 100644 index 4c0f757b2..000000000 --- a/federationsender/api/query.go +++ /dev/null @@ -1,63 +0,0 @@ -package api - -import ( - "context" - - "github.com/matrix-org/dendrite/federationsender/types" - internalHTTP "github.com/matrix-org/dendrite/internal/http" - "github.com/matrix-org/gomatrixserverlib" - - "github.com/opentracing/opentracing-go" -) - -// FederationSenderQueryJoinedHostsInRoomPath is the HTTP path for the QueryJoinedHostsInRoom API. -const FederationSenderQueryJoinedHostsInRoomPath = "/federationsender/queryJoinedHostsInRoom" - -// FederationSenderQueryJoinedHostServerNamesInRoomPath is the HTTP path for the QueryJoinedHostServerNamesInRoom API. -const FederationSenderQueryJoinedHostServerNamesInRoomPath = "/federationsender/queryJoinedHostServerNamesInRoom" - -// QueryJoinedHostsInRoomRequest is a request to QueryJoinedHostsInRoom -type QueryJoinedHostsInRoomRequest struct { - RoomID string `json:"room_id"` -} - -// QueryJoinedHostsInRoomResponse is a response to QueryJoinedHostsInRoom -type QueryJoinedHostsInRoomResponse struct { - JoinedHosts []types.JoinedHost `json:"joined_hosts"` -} - -// QueryJoinedHostsInRoom implements FederationSenderInternalAPI -func (h *httpFederationSenderInternalAPI) QueryJoinedHostsInRoom( - ctx context.Context, - request *QueryJoinedHostsInRoomRequest, - response *QueryJoinedHostsInRoomResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "QueryJoinedHostsInRoom") - defer span.Finish() - - apiURL := h.federationSenderURL + FederationSenderQueryJoinedHostsInRoomPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} - -// QueryJoinedHostServerNamesRequest is a request to QueryJoinedHostServerNames -type QueryJoinedHostServerNamesInRoomRequest struct { - RoomID string `json:"room_id"` -} - -// QueryJoinedHostServerNamesResponse is a response to QueryJoinedHostServerNames -type QueryJoinedHostServerNamesInRoomResponse struct { - ServerNames []gomatrixserverlib.ServerName `json:"server_names"` -} - -// QueryJoinedHostServerNamesInRoom implements FederationSenderInternalAPI -func (h *httpFederationSenderInternalAPI) QueryJoinedHostServerNamesInRoom( - ctx context.Context, - request *QueryJoinedHostServerNamesInRoomRequest, - response *QueryJoinedHostServerNamesInRoomResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "QueryJoinedHostServerNamesInRoom") - defer span.Finish() - - apiURL := h.federationSenderURL + FederationSenderQueryJoinedHostServerNamesInRoomPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} diff --git a/federationsender/federationsender.go b/federationsender/federationsender.go index 9e5cc8dd1..e25c27237 100644 --- a/federationsender/federationsender.go +++ b/federationsender/federationsender.go @@ -18,6 +18,7 @@ import ( "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/federationsender/consumers" "github.com/matrix-org/dendrite/federationsender/internal" + "github.com/matrix-org/dendrite/federationsender/inthttp" "github.com/matrix-org/dendrite/federationsender/producers" "github.com/matrix-org/dendrite/federationsender/queue" "github.com/matrix-org/dendrite/federationsender/storage" @@ -65,12 +66,8 @@ func SetupFederationSenderComponent( logrus.WithError(err).Panic("failed to start typing server consumer") } - queryAPI := internal.NewFederationSenderInternalAPI( - federationSenderDB, base.Cfg, roomserverProducer, federation, keyRing, - statistics, queues, - ) - - queryAPI.SetupHTTP(base.InternalAPIMux) + queryAPI := internal.NewFederationSenderInternalAPI(federationSenderDB, base.Cfg, roomserverProducer, federation, keyRing, statistics, queues) + inthttp.AddRoutes(queryAPI, base.InternalAPIMux) return queryAPI } diff --git a/federationsender/internal/api.go b/federationsender/internal/api.go index edf8fb4e9..c2ea0d411 100644 --- a/federationsender/internal/api.go +++ b/federationsender/internal/api.go @@ -1,19 +1,13 @@ package internal import ( - "encoding/json" - "net/http" - - "github.com/gorilla/mux" "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/federationsender/producers" "github.com/matrix-org/dendrite/federationsender/queue" "github.com/matrix-org/dendrite/federationsender/storage" "github.com/matrix-org/dendrite/federationsender/types" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" ) // FederationSenderInternalAPI is an implementation of api.FederationSenderInternalAPI @@ -46,87 +40,3 @@ func NewFederationSenderInternalAPI( queues: queues, } } - -// SetupHTTP adds the FederationSenderInternalAPI handlers to the http.ServeMux. -func (f *FederationSenderInternalAPI) SetupHTTP(internalAPIMux *mux.Router) { - internalAPIMux.Handle( - api.FederationSenderQueryJoinedHostsInRoomPath, - internal.MakeInternalAPI("QueryJoinedHostsInRoom", func(req *http.Request) util.JSONResponse { - var request api.QueryJoinedHostsInRoomRequest - var response api.QueryJoinedHostsInRoomResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := f.QueryJoinedHostsInRoom(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - internalAPIMux.Handle( - api.FederationSenderQueryJoinedHostServerNamesInRoomPath, - internal.MakeInternalAPI("QueryJoinedHostServerNamesInRoom", func(req *http.Request) util.JSONResponse { - var request api.QueryJoinedHostServerNamesInRoomRequest - var response api.QueryJoinedHostServerNamesInRoomResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := f.QueryJoinedHostServerNamesInRoom(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - internalAPIMux.Handle(api.FederationSenderPerformJoinRequestPath, - internal.MakeInternalAPI("PerformJoinRequest", func(req *http.Request) util.JSONResponse { - var request api.PerformJoinRequest - var response api.PerformJoinResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.MessageResponse(http.StatusBadRequest, err.Error()) - } - if err := f.PerformJoin(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - internalAPIMux.Handle(api.FederationSenderPerformLeaveRequestPath, - internal.MakeInternalAPI("PerformLeaveRequest", func(req *http.Request) util.JSONResponse { - var request api.PerformLeaveRequest - var response api.PerformLeaveResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.MessageResponse(http.StatusBadRequest, err.Error()) - } - if err := f.PerformLeave(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - internalAPIMux.Handle(api.FederationSenderPerformDirectoryLookupRequestPath, - internal.MakeInternalAPI("PerformDirectoryLookupRequest", func(req *http.Request) util.JSONResponse { - var request api.PerformDirectoryLookupRequest - var response api.PerformDirectoryLookupResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.MessageResponse(http.StatusBadRequest, err.Error()) - } - if err := f.PerformDirectoryLookup(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - internalAPIMux.Handle(api.FederationSenderPerformServersAlivePath, - internal.MakeInternalAPI("PerformServersAliveRequest", func(req *http.Request) util.JSONResponse { - var request api.PerformServersAliveRequest - var response api.PerformServersAliveResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.MessageResponse(http.StatusBadRequest, err.Error()) - } - if err := f.PerformServersAlive(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) -} diff --git a/federationsender/inthttp/client.go b/federationsender/inthttp/client.go new file mode 100644 index 000000000..8b3a106d8 --- /dev/null +++ b/federationsender/inthttp/client.go @@ -0,0 +1,113 @@ +package inthttp + +import ( + "context" + "errors" + "net/http" + + "github.com/matrix-org/dendrite/federationsender/api" + internalHTTP "github.com/matrix-org/dendrite/internal/http" + "github.com/opentracing/opentracing-go" +) + +// HTTP paths for the internal HTTP API +const ( + FederationSenderQueryJoinedHostsInRoomPath = "/federationsender/queryJoinedHostsInRoom" + FederationSenderQueryJoinedHostServerNamesInRoomPath = "/federationsender/queryJoinedHostServerNamesInRoom" + + FederationSenderPerformDirectoryLookupRequestPath = "/federationsender/performDirectoryLookup" + FederationSenderPerformJoinRequestPath = "/federationsender/performJoinRequest" + FederationSenderPerformLeaveRequestPath = "/federationsender/performLeaveRequest" + FederationSenderPerformServersAlivePath = "/federationsender/performServersAlive" +) + +// NewFederationSenderClient creates a FederationSenderInternalAPI implemented by talking to a HTTP POST API. +// If httpClient is nil an error is returned +func NewFederationSenderClient(federationSenderURL string, httpClient *http.Client) (api.FederationSenderInternalAPI, error) { + if httpClient == nil { + return nil, errors.New("NewFederationSenderInternalAPIHTTP: httpClient is ") + } + return &httpFederationSenderInternalAPI{federationSenderURL, httpClient}, nil +} + +type httpFederationSenderInternalAPI struct { + federationSenderURL string + httpClient *http.Client +} + +// Handle an instruction to make_leave & send_leave with a remote server. +func (h *httpFederationSenderInternalAPI) PerformLeave( + ctx context.Context, + request *api.PerformLeaveRequest, + response *api.PerformLeaveResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "PerformLeaveRequest") + defer span.Finish() + + apiURL := h.federationSenderURL + FederationSenderPerformLeaveRequestPath + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +func (h *httpFederationSenderInternalAPI) PerformServersAlive( + ctx context.Context, + request *api.PerformServersAliveRequest, + response *api.PerformServersAliveResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "PerformServersAlive") + defer span.Finish() + + apiURL := h.federationSenderURL + FederationSenderPerformServersAlivePath + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +// QueryJoinedHostServerNamesInRoom implements FederationSenderInternalAPI +func (h *httpFederationSenderInternalAPI) QueryJoinedHostServerNamesInRoom( + ctx context.Context, + request *api.QueryJoinedHostServerNamesInRoomRequest, + response *api.QueryJoinedHostServerNamesInRoomResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "QueryJoinedHostServerNamesInRoom") + defer span.Finish() + + apiURL := h.federationSenderURL + FederationSenderQueryJoinedHostServerNamesInRoomPath + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +// QueryJoinedHostsInRoom implements FederationSenderInternalAPI +func (h *httpFederationSenderInternalAPI) QueryJoinedHostsInRoom( + ctx context.Context, + request *api.QueryJoinedHostsInRoomRequest, + response *api.QueryJoinedHostsInRoomResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "QueryJoinedHostsInRoom") + defer span.Finish() + + apiURL := h.federationSenderURL + FederationSenderQueryJoinedHostsInRoomPath + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +// Handle an instruction to make_join & send_join with a remote server. +func (h *httpFederationSenderInternalAPI) PerformJoin( + ctx context.Context, + request *api.PerformJoinRequest, + response *api.PerformJoinResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "PerformJoinRequest") + defer span.Finish() + + apiURL := h.federationSenderURL + FederationSenderPerformJoinRequestPath + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +// Handle an instruction to make_join & send_join with a remote server. +func (h *httpFederationSenderInternalAPI) PerformDirectoryLookup( + ctx context.Context, + request *api.PerformDirectoryLookupRequest, + response *api.PerformDirectoryLookupResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "PerformDirectoryLookup") + defer span.Finish() + + apiURL := h.federationSenderURL + FederationSenderPerformDirectoryLookupRequestPath + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} diff --git a/federationsender/inthttp/server.go b/federationsender/inthttp/server.go new file mode 100644 index 000000000..c94a14295 --- /dev/null +++ b/federationsender/inthttp/server.go @@ -0,0 +1,95 @@ +package inthttp + +import ( + "encoding/json" + "net/http" + + "github.com/gorilla/mux" + "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/util" +) + +// AddRoutes adds the FederationSenderInternalAPI handlers to the http.ServeMux. +func AddRoutes(intAPI api.FederationSenderInternalAPI, internalAPIMux *mux.Router) { + internalAPIMux.Handle( + FederationSenderQueryJoinedHostsInRoomPath, + internal.MakeInternalAPI("QueryJoinedHostsInRoom", func(req *http.Request) util.JSONResponse { + var request api.QueryJoinedHostsInRoomRequest + var response api.QueryJoinedHostsInRoomResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := intAPI.QueryJoinedHostsInRoom(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + internalAPIMux.Handle( + FederationSenderQueryJoinedHostServerNamesInRoomPath, + internal.MakeInternalAPI("QueryJoinedHostServerNamesInRoom", func(req *http.Request) util.JSONResponse { + var request api.QueryJoinedHostServerNamesInRoomRequest + var response api.QueryJoinedHostServerNamesInRoomResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := intAPI.QueryJoinedHostServerNamesInRoom(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + internalAPIMux.Handle(FederationSenderPerformJoinRequestPath, + internal.MakeInternalAPI("PerformJoinRequest", func(req *http.Request) util.JSONResponse { + var request api.PerformJoinRequest + var response api.PerformJoinResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := intAPI.PerformJoin(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + internalAPIMux.Handle(FederationSenderPerformLeaveRequestPath, + internal.MakeInternalAPI("PerformLeaveRequest", func(req *http.Request) util.JSONResponse { + var request api.PerformLeaveRequest + var response api.PerformLeaveResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := intAPI.PerformLeave(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + internalAPIMux.Handle(FederationSenderPerformDirectoryLookupRequestPath, + internal.MakeInternalAPI("PerformDirectoryLookupRequest", func(req *http.Request) util.JSONResponse { + var request api.PerformDirectoryLookupRequest + var response api.PerformDirectoryLookupResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := intAPI.PerformDirectoryLookup(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + internalAPIMux.Handle(FederationSenderPerformServersAlivePath, + internal.MakeInternalAPI("PerformServersAliveRequest", func(req *http.Request) util.JSONResponse { + var request api.PerformServersAliveRequest + var response api.PerformServersAliveResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := intAPI.PerformServersAlive(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) +} diff --git a/internal/basecomponent/base.go b/internal/basecomponent/base.go index 42d236f85..a1a908760 100644 --- a/internal/basecomponent/base.go +++ b/internal/basecomponent/base.go @@ -38,6 +38,7 @@ import ( appserviceAPI "github.com/matrix-org/dendrite/appservice/api" eduServerAPI "github.com/matrix-org/dendrite/eduserver/api" federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/federationsender/inthttp" "github.com/matrix-org/dendrite/internal/config" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" serverKeyAPI "github.com/matrix-org/dendrite/serverkeyapi/api" @@ -58,7 +59,7 @@ type BaseDendrite struct { // PublicAPIMux should be used to register new public matrix api endpoints PublicAPIMux *mux.Router InternalAPIMux *mux.Router - EnableHTTPAPIs bool + UseHTTPAPIs bool httpClient *http.Client Cfg *config.Dendrite ImmutableCache caching.ImmutableCache @@ -72,7 +73,7 @@ const HTTPClientTimeout = time.Second * 30 // NewBaseDendrite creates a new instance to be used by a component. // The componentName is used for logging purposes, and should be a friendly name // of the compontent running, e.g. "SyncAPI" -func NewBaseDendrite(cfg *config.Dendrite, componentName string, enableHTTPAPIs bool) *BaseDendrite { +func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs bool) *BaseDendrite { internal.SetupStdLogging() internal.SetupHookLogging(cfg.Logging, componentName) internal.SetupPprof() @@ -118,7 +119,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, enableHTTPAPIs return &BaseDendrite{ componentName: componentName, - EnableHTTPAPIs: enableHTTPAPIs, + UseHTTPAPIs: useHTTPAPIs, tracerCloser: closer, Cfg: cfg, ImmutableCache: cache, @@ -165,10 +166,10 @@ func (b *BaseDendrite) CreateHTTPEDUServerAPIs() eduServerAPI.EDUServerInputAPI return e } -// CreateHTTPFederationSenderAPIs returns FederationSenderInternalAPI for hitting +// FederationSenderHTTPClient returns FederationSenderInternalAPI for hitting // the federation sender over HTTP -func (b *BaseDendrite) CreateHTTPFederationSenderAPIs() federationSenderAPI.FederationSenderInternalAPI { - f, err := federationSenderAPI.NewFederationSenderInternalAPIHTTP(b.Cfg.FederationSenderURL(), b.httpClient) +func (b *BaseDendrite) FederationSenderHTTPClient() federationSenderAPI.FederationSenderInternalAPI { + f, err := inthttp.NewFederationSenderClient(b.Cfg.FederationSenderURL(), b.httpClient) if err != nil { logrus.WithError(err).Panic("NewFederationSenderInternalAPIHTTP failed", b.httpClient) } @@ -241,7 +242,7 @@ func (b *BaseDendrite) SetupAndServeHTTP(bindaddr string, listenaddr string) { b.PublicAPIMux, b.InternalAPIMux, b.Cfg, - b.EnableHTTPAPIs, + b.UseHTTPAPIs, ) logrus.Infof("Starting %s server on %s", b.componentName, serv.Addr) From d785ad82b996989381657292fbd2c28b9fbb7df6 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 4 Jun 2020 15:40:23 +0100 Subject: [PATCH 133/200] Add key validity fetching to server key API (#1094) * Add key validity checks * Store fetched keys * Don't double-cache key results * Perform server key API operations using new context * Revert "Perform server key API operations using new context" This reverts commit 02172223f5cb7850b0852c6ed6836ad82508ea76. * Perform server key API operations using new context --- serverkeyapi/api/api.go | 15 +++++++++++++-- serverkeyapi/internal/api.go | 18 ++++++++++++++++-- serverkeyapi/internal/http.go | 16 ++-------------- 3 files changed, 31 insertions(+), 18 deletions(-) diff --git a/serverkeyapi/api/api.go b/serverkeyapi/api/api.go index f108d437b..a43f9c0a1 100644 --- a/serverkeyapi/api/api.go +++ b/serverkeyapi/api/api.go @@ -4,6 +4,7 @@ import ( "context" "errors" "net/http" + "time" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/gomatrixserverlib" @@ -69,9 +70,12 @@ func (s *httpServerKeyInternalAPI) FetcherName() string { } func (s *httpServerKeyInternalAPI) StoreKeys( - ctx context.Context, + _ context.Context, results map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, ) error { + // Run in a background context - we don't want to stop this work just + // because the caller gives up waiting. + ctx := context.Background() request := InputPublicKeysRequest{ Keys: make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult), } @@ -84,9 +88,12 @@ func (s *httpServerKeyInternalAPI) StoreKeys( } func (s *httpServerKeyInternalAPI) FetchKeys( - ctx context.Context, + _ context.Context, requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp, ) (map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, error) { + // Run in a background context - we don't want to stop this work just + // because the caller gives up waiting. + ctx := context.Background() result := make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult) request := QueryPublicKeysRequest{ Requests: make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp), @@ -94,8 +101,12 @@ func (s *httpServerKeyInternalAPI) FetchKeys( response := QueryPublicKeysResponse{ Results: make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult), } + now := gomatrixserverlib.AsTimestamp(time.Now()) for req, ts := range requests { if res, ok := s.immutableCache.GetServerKey(req); ok { + if now > res.ValidUntilTS && res.ExpiredTS == gomatrixserverlib.PublicKeyNotExpired { + continue + } result[req] = res continue } diff --git a/serverkeyapi/internal/api.go b/serverkeyapi/internal/api.go index cbae59d9a..c63b23f04 100644 --- a/serverkeyapi/internal/api.go +++ b/serverkeyapi/internal/api.go @@ -3,6 +3,7 @@ package internal import ( "context" "fmt" + "time" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/serverkeyapi/api" @@ -24,25 +25,35 @@ func (s *ServerKeyAPI) KeyRing() *gomatrixserverlib.KeyRing { } func (s *ServerKeyAPI) StoreKeys( - ctx context.Context, + _ context.Context, results map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, ) error { + // Run in a background context - we don't want to stop this work just + // because the caller gives up waiting. + ctx := context.Background() // Store any keys that we were given in our database. return s.OurKeyRing.KeyDatabase.StoreKeys(ctx, results) } func (s *ServerKeyAPI) FetchKeys( - ctx context.Context, + _ context.Context, requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp, ) (map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, error) { + // Run in a background context - we don't want to stop this work just + // because the caller gives up waiting. + ctx := context.Background() results := map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult{} // First consult our local database and see if we have the requested // keys. These might come from a cache, depending on the database // implementation used. + now := gomatrixserverlib.AsTimestamp(time.Now()) if dbResults, err := s.OurKeyRing.KeyDatabase.FetchKeys(ctx, requests); err == nil { // We successfully got some keys. Add them to the results and // remove them from the request list. for req, res := range dbResults { + if now > res.ValidUntilTS && res.ExpiredTS == gomatrixserverlib.PublicKeyNotExpired { + continue + } results[req] = res delete(requests, req) } @@ -61,6 +72,9 @@ func (s *ServerKeyAPI) FetchKeys( results[req] = res delete(requests, req) } + if err = s.OurKeyRing.KeyDatabase.StoreKeys(ctx, fetcherResults); err != nil { + return nil, fmt.Errorf("server key API failed to store retrieved keys: %w", err) + } } } // If we failed to fetch any keys then we should report an error. diff --git a/serverkeyapi/internal/http.go b/serverkeyapi/internal/http.go index 303275712..eef66b76c 100644 --- a/serverkeyapi/internal/http.go +++ b/serverkeyapi/internal/http.go @@ -14,28 +14,16 @@ import ( func (s *ServerKeyAPI) SetupHTTP(internalAPIMux *mux.Router) { internalAPIMux.Handle(api.ServerKeyQueryPublicKeyPath, internal.MakeInternalAPI("queryPublicKeys", func(req *http.Request) util.JSONResponse { - result := map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult{} request := api.QueryPublicKeysRequest{} response := api.QueryPublicKeysResponse{} if err := json.NewDecoder(req.Body).Decode(&request); err != nil { return util.MessageResponse(http.StatusBadRequest, err.Error()) } - lookup := make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp) - for req, timestamp := range request.Requests { - if res, ok := s.ImmutableCache.GetServerKey(req); ok { - result[req] = res - continue - } - lookup[req] = timestamp - } - keys, err := s.FetchKeys(req.Context(), lookup) + keys, err := s.FetchKeys(req.Context(), request.Requests) if err != nil { return util.ErrorResponse(err) } - for req, res := range keys { - result[req] = res - } - response.Results = result + response.Results = keys return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) From 9834ac97db042b0c78fbd72652aa534129ca3afe Mon Sep 17 00:00:00 2001 From: Kegsay Date: Thu, 4 Jun 2020 15:43:07 +0100 Subject: [PATCH 134/200] Convert everything but serverkeyapi to inthttp (#1096) * Convert roomserver to new inthttp format * Convert eduserver to new inthttp format * Convert appservice to new inthttp format --- appservice/api/query.go | 57 +-- appservice/appservice.go | 7 +- appservice/inthttp/client.go | 63 ++++ appservice/inthttp/server.go | 43 +++ appservice/query/query.go | 37 -- cmd/dendrite-appservice-server/main.go | 2 +- cmd/dendrite-client-api-server/main.go | 4 +- cmd/dendrite-federation-api-server/main.go | 4 +- cmd/dendrite-federation-sender-server/main.go | 2 +- cmd/dendrite-monolith-server/main.go | 6 +- cmd/dendrite-public-rooms-api-server/main.go | 2 +- cmd/dendrite-sync-api-server/main.go | 2 +- cmd/roomserver-integration-tests/main.go | 5 +- eduserver/api/input.go | 49 --- eduserver/eduserver.go | 3 +- eduserver/input/input.go | 34 -- eduserver/inthttp/client.go | 56 +++ eduserver/inthttp/server.go | 41 +++ internal/basecomponent/base.go | 34 +- roomserver/api/alias.go | 87 ----- roomserver/api/http.go | 41 --- roomserver/api/input.go | 20 - roomserver/api/perform.go | 36 -- roomserver/api/query.go | 205 ----------- roomserver/internal/api.go | 288 --------------- roomserver/inthttp/client.go | 341 ++++++++++++++++++ roomserver/inthttp/server.go | 293 +++++++++++++++ roomserver/roomserver.go | 7 +- syncapi/api/query.go | 123 ------- 29 files changed, 879 insertions(+), 1013 deletions(-) create mode 100644 appservice/inthttp/client.go create mode 100644 appservice/inthttp/server.go create mode 100644 eduserver/inthttp/client.go create mode 100644 eduserver/inthttp/server.go delete mode 100644 roomserver/api/http.go create mode 100644 roomserver/inthttp/client.go create mode 100644 roomserver/inthttp/server.go delete mode 100644 syncapi/api/query.go diff --git a/appservice/api/query.go b/appservice/api/query.go index d36e138cb..6fcb20890 100644 --- a/appservice/api/query.go +++ b/appservice/api/query.go @@ -20,16 +20,12 @@ package api import ( "context" "database/sql" - "errors" - "net/http" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/dendrite/internal" - internalHTTP "github.com/matrix-org/dendrite/internal/http" - opentracing "github.com/opentracing/opentracing-go" ) // RoomAliasExistsRequest is a request to an application service @@ -83,60 +79,9 @@ type AppServiceQueryAPI interface { ) error } -// AppServiceRoomAliasExistsPath is the HTTP path for the RoomAliasExists API -const AppServiceRoomAliasExistsPath = "/appservice/RoomAliasExists" - -// AppServiceUserIDExistsPath is the HTTP path for the UserIDExists API -const AppServiceUserIDExistsPath = "/appservice/UserIDExists" - -// httpAppServiceQueryAPI contains the URL to an appservice query API and a -// reference to a httpClient used to reach it -type httpAppServiceQueryAPI struct { - appserviceURL string - httpClient *http.Client -} - -// NewAppServiceQueryAPIHTTP creates a AppServiceQueryAPI implemented by talking -// to a HTTP POST API. -// If httpClient is nil an error is returned -func NewAppServiceQueryAPIHTTP( - appserviceURL string, - httpClient *http.Client, -) (AppServiceQueryAPI, error) { - if httpClient == nil { - return nil, errors.New("NewRoomserverAliasAPIHTTP: httpClient is ") - } - return &httpAppServiceQueryAPI{appserviceURL, httpClient}, nil -} - -// RoomAliasExists implements AppServiceQueryAPI -func (h *httpAppServiceQueryAPI) RoomAliasExists( - ctx context.Context, - request *RoomAliasExistsRequest, - response *RoomAliasExistsResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "appserviceRoomAliasExists") - defer span.Finish() - - apiURL := h.appserviceURL + AppServiceRoomAliasExistsPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} - -// UserIDExists implements AppServiceQueryAPI -func (h *httpAppServiceQueryAPI) UserIDExists( - ctx context.Context, - request *UserIDExistsRequest, - response *UserIDExistsResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "appserviceUserIDExists") - defer span.Finish() - - apiURL := h.appserviceURL + AppServiceUserIDExistsPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} - // RetrieveUserProfile is a wrapper that queries both the local database and // application services for a given user's profile +// TODO: Remove this, it's called from federationapi and clientapi but is a pure function func RetrieveUserProfile( ctx context.Context, userID string, diff --git a/appservice/appservice.go b/appservice/appservice.go index 68cf52e79..b5ffba5eb 100644 --- a/appservice/appservice.go +++ b/appservice/appservice.go @@ -23,6 +23,7 @@ import ( appserviceAPI "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/appservice/consumers" + "github.com/matrix-org/dendrite/appservice/inthttp" "github.com/matrix-org/dendrite/appservice/query" "github.com/matrix-org/dendrite/appservice/routing" "github.com/matrix-org/dendrite/appservice/storage" @@ -77,14 +78,14 @@ func SetupAppServiceAPIComponent( // Create appserivce query API with an HTTP client that will be used for all // outbound and inbound requests (inbound only for the internal API) - appserviceQueryAPI := query.AppServiceQueryAPI{ + appserviceQueryAPI := &query.AppServiceQueryAPI{ HTTPClient: &http.Client{ Timeout: time.Second * 30, }, Cfg: base.Cfg, } - appserviceQueryAPI.SetupHTTP(base.InternalAPIMux) + inthttp.AddRoutes(appserviceQueryAPI, base.InternalAPIMux) consumer := consumers.NewOutputRoomEventConsumer( base.Cfg, base.KafkaConsumer, accountsDB, appserviceDB, @@ -105,7 +106,7 @@ func SetupAppServiceAPIComponent( accountsDB, federation, transactionsCache, ) - return &appserviceQueryAPI + return appserviceQueryAPI } // generateAppServiceAccounts creates a dummy account based off the diff --git a/appservice/inthttp/client.go b/appservice/inthttp/client.go new file mode 100644 index 000000000..acbd1211b --- /dev/null +++ b/appservice/inthttp/client.go @@ -0,0 +1,63 @@ +package inthttp + +import ( + "context" + "errors" + "net/http" + + "github.com/matrix-org/dendrite/appservice/api" + internalHTTP "github.com/matrix-org/dendrite/internal/http" + "github.com/opentracing/opentracing-go" +) + +// HTTP paths for the internal HTTP APIs +const ( + AppServiceRoomAliasExistsPath = "/appservice/RoomAliasExists" + AppServiceUserIDExistsPath = "/appservice/UserIDExists" +) + +// httpAppServiceQueryAPI contains the URL to an appservice query API and a +// reference to a httpClient used to reach it +type httpAppServiceQueryAPI struct { + appserviceURL string + httpClient *http.Client +} + +// NewAppserviceClient creates a AppServiceQueryAPI implemented by talking +// to a HTTP POST API. +// If httpClient is nil an error is returned +func NewAppserviceClient( + appserviceURL string, + httpClient *http.Client, +) (api.AppServiceQueryAPI, error) { + if httpClient == nil { + return nil, errors.New("NewRoomserverAliasAPIHTTP: httpClient is ") + } + return &httpAppServiceQueryAPI{appserviceURL, httpClient}, nil +} + +// RoomAliasExists implements AppServiceQueryAPI +func (h *httpAppServiceQueryAPI) RoomAliasExists( + ctx context.Context, + request *api.RoomAliasExistsRequest, + response *api.RoomAliasExistsResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "appserviceRoomAliasExists") + defer span.Finish() + + apiURL := h.appserviceURL + AppServiceRoomAliasExistsPath + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +// UserIDExists implements AppServiceQueryAPI +func (h *httpAppServiceQueryAPI) UserIDExists( + ctx context.Context, + request *api.UserIDExistsRequest, + response *api.UserIDExistsResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "appserviceUserIDExists") + defer span.Finish() + + apiURL := h.appserviceURL + AppServiceUserIDExistsPath + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} diff --git a/appservice/inthttp/server.go b/appservice/inthttp/server.go new file mode 100644 index 000000000..1900635a8 --- /dev/null +++ b/appservice/inthttp/server.go @@ -0,0 +1,43 @@ +package inthttp + +import ( + "encoding/json" + "net/http" + + "github.com/gorilla/mux" + "github.com/matrix-org/dendrite/appservice/api" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/util" +) + +// AddRoutes adds the AppServiceQueryAPI handlers to the http.ServeMux. +func AddRoutes(a api.AppServiceQueryAPI, internalAPIMux *mux.Router) { + internalAPIMux.Handle( + AppServiceRoomAliasExistsPath, + internal.MakeInternalAPI("appserviceRoomAliasExists", func(req *http.Request) util.JSONResponse { + var request api.RoomAliasExistsRequest + var response api.RoomAliasExistsResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := a.RoomAliasExists(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + internalAPIMux.Handle( + AppServiceUserIDExistsPath, + internal.MakeInternalAPI("appserviceUserIDExists", func(req *http.Request) util.JSONResponse { + var request api.UserIDExistsRequest + var response api.UserIDExistsResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := a.UserIDExists(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) +} diff --git a/appservice/query/query.go b/appservice/query/query.go index 812ca9f49..fa3844f68 100644 --- a/appservice/query/query.go +++ b/appservice/query/query.go @@ -18,16 +18,12 @@ package query import ( "context" - "encoding/json" "net/http" "net/url" "time" - "github.com/gorilla/mux" "github.com/matrix-org/dendrite/appservice/api" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" - "github.com/matrix-org/util" opentracing "github.com/opentracing/opentracing-go" log "github.com/sirupsen/logrus" ) @@ -180,36 +176,3 @@ func makeHTTPClient() *http.Client { Timeout: time.Second * 30, } } - -// SetupHTTP adds the AppServiceQueryPAI handlers to the http.ServeMux. This -// handles and muxes incoming api requests the to internal AppServiceQueryAPI. -func (a *AppServiceQueryAPI) SetupHTTP(internalAPIMux *mux.Router) { - internalAPIMux.Handle( - api.AppServiceRoomAliasExistsPath, - internal.MakeInternalAPI("appserviceRoomAliasExists", func(req *http.Request) util.JSONResponse { - var request api.RoomAliasExistsRequest - var response api.RoomAliasExistsResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := a.RoomAliasExists(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - internalAPIMux.Handle( - api.AppServiceUserIDExistsPath, - internal.MakeInternalAPI("appserviceUserIDExists", func(req *http.Request) util.JSONResponse { - var request api.UserIDExistsRequest - var response api.UserIDExistsResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := a.UserIDExists(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) -} diff --git a/cmd/dendrite-appservice-server/main.go b/cmd/dendrite-appservice-server/main.go index a508be3b4..c16505fd8 100644 --- a/cmd/dendrite-appservice-server/main.go +++ b/cmd/dendrite-appservice-server/main.go @@ -28,7 +28,7 @@ func main() { accountDB := base.CreateAccountsDB() deviceDB := base.CreateDeviceDB() federation := base.CreateFederationClient() - rsAPI := base.CreateHTTPRoomserverAPIs() + rsAPI := base.RoomserverHTTPClient() cache := transactions.New() appservice.SetupAppServiceAPIComponent( diff --git a/cmd/dendrite-client-api-server/main.go b/cmd/dendrite-client-api-server/main.go index 280e9cdef..ba8e6af7c 100644 --- a/cmd/dendrite-client-api-server/main.go +++ b/cmd/dendrite-client-api-server/main.go @@ -35,8 +35,8 @@ func main() { serverKeyAPI := base.CreateHTTPServerKeyAPIs() keyRing := serverKeyAPI.KeyRing() - asQuery := base.CreateHTTPAppServiceAPIs() - rsAPI := base.CreateHTTPRoomserverAPIs() + asQuery := base.AppserviceHTTPClient() + rsAPI := base.RoomserverHTTPClient() fsAPI := base.FederationSenderHTTPClient() rsAPI.SetFederationSenderAPI(fsAPI) eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New(), deviceDB) diff --git a/cmd/dendrite-federation-api-server/main.go b/cmd/dendrite-federation-api-server/main.go index 3c42532c0..aca6ba579 100644 --- a/cmd/dendrite-federation-api-server/main.go +++ b/cmd/dendrite-federation-api-server/main.go @@ -36,8 +36,8 @@ func main() { fsAPI := base.FederationSenderHTTPClient() - rsAPI := base.CreateHTTPRoomserverAPIs() - asAPI := base.CreateHTTPAppServiceAPIs() + rsAPI := base.RoomserverHTTPClient() + asAPI := base.AppserviceHTTPClient() rsAPI.SetFederationSenderAPI(fsAPI) eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New(), deviceDB) eduProducer := producers.NewEDUServerProducer(eduInputAPI) diff --git a/cmd/dendrite-federation-sender-server/main.go b/cmd/dendrite-federation-sender-server/main.go index 2b1e0ae8f..14524dcad 100644 --- a/cmd/dendrite-federation-sender-server/main.go +++ b/cmd/dendrite-federation-sender-server/main.go @@ -29,7 +29,7 @@ func main() { serverKeyAPI := base.CreateHTTPServerKeyAPIs() keyRing := serverKeyAPI.KeyRing() - rsAPI := base.CreateHTTPRoomserverAPIs() + rsAPI := base.RoomserverHTTPClient() fsAPI := federationsender.SetupFederationSenderComponent( base, federation, rsAPI, keyRing, ) diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index eedd4412e..ba9189256 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -83,21 +83,21 @@ func main() { ) rsAPI := rsComponent if base.UseHTTPAPIs { - rsAPI = base.CreateHTTPRoomserverAPIs() + rsAPI = base.RoomserverHTTPClient() } eduInputAPI := eduserver.SetupEDUServerComponent( base, cache.New(), deviceDB, ) if base.UseHTTPAPIs { - eduInputAPI = base.CreateHTTPEDUServerAPIs() + eduInputAPI = base.EDUServerClient() } asAPI := appservice.SetupAppServiceAPIComponent( base, accountDB, deviceDB, federation, rsAPI, transactions.New(), ) if base.UseHTTPAPIs { - asAPI = base.CreateHTTPAppServiceAPIs() + asAPI = base.AppserviceHTTPClient() } fsAPI := federationsender.SetupFederationSenderComponent( diff --git a/cmd/dendrite-public-rooms-api-server/main.go b/cmd/dendrite-public-rooms-api-server/main.go index c2df1b3f8..18a813baf 100644 --- a/cmd/dendrite-public-rooms-api-server/main.go +++ b/cmd/dendrite-public-rooms-api-server/main.go @@ -29,7 +29,7 @@ func main() { deviceDB := base.CreateDeviceDB() fsAPI := base.FederationSenderHTTPClient() - rsAPI := base.CreateHTTPRoomserverAPIs() + rsAPI := base.RoomserverHTTPClient() rsAPI.SetFederationSenderAPI(fsAPI) publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties(), cfg.Matrix.ServerName) diff --git a/cmd/dendrite-sync-api-server/main.go b/cmd/dendrite-sync-api-server/main.go index 5a1a80dfd..eeab115a9 100644 --- a/cmd/dendrite-sync-api-server/main.go +++ b/cmd/dendrite-sync-api-server/main.go @@ -28,7 +28,7 @@ func main() { accountDB := base.CreateAccountsDB() federation := base.CreateFederationClient() - rsAPI := base.CreateHTTPRoomserverAPIs() + rsAPI := base.RoomserverHTTPClient() syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, rsAPI, federation, cfg) diff --git a/cmd/roomserver-integration-tests/main.go b/cmd/roomserver-integration-tests/main.go index 126053bac..2433f9ff0 100644 --- a/cmd/roomserver-integration-tests/main.go +++ b/cmd/roomserver-integration-tests/main.go @@ -31,6 +31,7 @@ import ( "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/test" "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/inthttp" "github.com/matrix-org/gomatrixserverlib" ) @@ -209,7 +210,7 @@ func writeToRoomServer(input []string, roomserverURL string) error { return err } } - x, err := api.NewRoomserverInternalAPIHTTP(roomserverURL, &http.Client{Timeout: timeoutHTTP}, nil) + x, err := inthttp.NewRoomserverClient(roomserverURL, &http.Client{Timeout: timeoutHTTP}, nil) if err != nil { return err } @@ -276,7 +277,7 @@ func testRoomserver(input []string, wantOutput []string, checkQueries func(api.R cmd.Args = []string{"dendrite-room-server", "--config", filepath.Join(dir, test.ConfigFile)} gotOutput, err := runAndReadFromTopic(cmd, cfg.RoomServerURL()+"/metrics", doInput, outputTopic, len(wantOutput), func() { - queryAPI, _ := api.NewRoomserverInternalAPIHTTP("http://"+string(cfg.Listen.RoomServer), &http.Client{Timeout: timeoutHTTP}, cache) + queryAPI, _ := inthttp.NewRoomserverClient("http://"+string(cfg.Listen.RoomServer), &http.Client{Timeout: timeoutHTTP}, cache) checkQueries(queryAPI) }) if err != nil { diff --git a/eduserver/api/input.go b/eduserver/api/input.go index fa7f30cb6..0d0d21f33 100644 --- a/eduserver/api/input.go +++ b/eduserver/api/input.go @@ -19,12 +19,8 @@ package api import ( "context" - "errors" - "net/http" - internalHTTP "github.com/matrix-org/dendrite/internal/http" "github.com/matrix-org/gomatrixserverlib" - opentracing "github.com/opentracing/opentracing-go" ) // InputTypingEvent is an event for notifying the typing server about typing updates. @@ -77,48 +73,3 @@ type EDUServerInputAPI interface { response *InputSendToDeviceEventResponse, ) error } - -// EDUServerInputTypingEventPath is the HTTP path for the InputTypingEvent API. -const EDUServerInputTypingEventPath = "/eduserver/input" - -// EDUServerInputSendToDeviceEventPath is the HTTP path for the InputSendToDeviceEvent API. -const EDUServerInputSendToDeviceEventPath = "/eduserver/sendToDevice" - -// NewEDUServerInputAPIHTTP creates a EDUServerInputAPI implemented by talking to a HTTP POST API. -func NewEDUServerInputAPIHTTP(eduServerURL string, httpClient *http.Client) (EDUServerInputAPI, error) { - if httpClient == nil { - return nil, errors.New("NewTypingServerInputAPIHTTP: httpClient is ") - } - return &httpEDUServerInputAPI{eduServerURL, httpClient}, nil -} - -type httpEDUServerInputAPI struct { - eduServerURL string - httpClient *http.Client -} - -// InputTypingEvent implements EDUServerInputAPI -func (h *httpEDUServerInputAPI) InputTypingEvent( - ctx context.Context, - request *InputTypingEventRequest, - response *InputTypingEventResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "InputTypingEvent") - defer span.Finish() - - apiURL := h.eduServerURL + EDUServerInputTypingEventPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} - -// InputSendToDeviceEvent implements EDUServerInputAPI -func (h *httpEDUServerInputAPI) InputSendToDeviceEvent( - ctx context.Context, - request *InputSendToDeviceEventRequest, - response *InputSendToDeviceEventResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "InputSendToDeviceEvent") - defer span.Finish() - - apiURL := h.eduServerURL + EDUServerInputSendToDeviceEventPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} diff --git a/eduserver/eduserver.go b/eduserver/eduserver.go index 6f664eb67..59decc8fb 100644 --- a/eduserver/eduserver.go +++ b/eduserver/eduserver.go @@ -21,6 +21,7 @@ import ( "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/eduserver/input" + "github.com/matrix-org/dendrite/eduserver/inthttp" "github.com/matrix-org/dendrite/internal/basecomponent" ) @@ -42,7 +43,7 @@ func SetupEDUServerComponent( ServerName: base.Cfg.Matrix.ServerName, } - inputAPI.SetupHTTP(base.InternalAPIMux) + inthttp.AddRoutes(inputAPI, base.InternalAPIMux) return inputAPI } diff --git a/eduserver/input/input.go b/eduserver/input/input.go index 4e3051950..0bbf5b844 100644 --- a/eduserver/input/input.go +++ b/eduserver/input/input.go @@ -19,17 +19,13 @@ package input import ( "context" "encoding/json" - "net/http" "time" "github.com/Shopify/sarama" - "github.com/gorilla/mux" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/eduserver/cache" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" "github.com/sirupsen/logrus" ) @@ -170,33 +166,3 @@ func (t *EDUServerInputAPI) sendToDeviceEvent(ise *api.InputSendToDeviceEvent) e return nil } - -// SetupHTTP adds the EDUServerInputAPI handlers to the http.ServeMux. -func (t *EDUServerInputAPI) SetupHTTP(internalAPIMux *mux.Router) { - internalAPIMux.Handle(api.EDUServerInputTypingEventPath, - internal.MakeInternalAPI("inputTypingEvents", func(req *http.Request) util.JSONResponse { - var request api.InputTypingEventRequest - var response api.InputTypingEventResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.MessageResponse(http.StatusBadRequest, err.Error()) - } - if err := t.InputTypingEvent(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - internalAPIMux.Handle(api.EDUServerInputSendToDeviceEventPath, - internal.MakeInternalAPI("inputSendToDeviceEvents", func(req *http.Request) util.JSONResponse { - var request api.InputSendToDeviceEventRequest - var response api.InputSendToDeviceEventResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.MessageResponse(http.StatusBadRequest, err.Error()) - } - if err := t.InputSendToDeviceEvent(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) -} diff --git a/eduserver/inthttp/client.go b/eduserver/inthttp/client.go new file mode 100644 index 000000000..149e4fb04 --- /dev/null +++ b/eduserver/inthttp/client.go @@ -0,0 +1,56 @@ +package inthttp + +import ( + "context" + "errors" + "net/http" + + "github.com/matrix-org/dendrite/eduserver/api" + internalHTTP "github.com/matrix-org/dendrite/internal/http" + "github.com/opentracing/opentracing-go" +) + +// HTTP paths for the internal HTTP APIs +const ( + EDUServerInputTypingEventPath = "/eduserver/input" + EDUServerInputSendToDeviceEventPath = "/eduserver/sendToDevice" +) + +// NewEDUServerClient creates a EDUServerInputAPI implemented by talking to a HTTP POST API. +func NewEDUServerClient(eduServerURL string, httpClient *http.Client) (api.EDUServerInputAPI, error) { + if httpClient == nil { + return nil, errors.New("NewEDUServerClient: httpClient is ") + } + return &httpEDUServerInputAPI{eduServerURL, httpClient}, nil +} + +type httpEDUServerInputAPI struct { + eduServerURL string + httpClient *http.Client +} + +// InputTypingEvent implements EDUServerInputAPI +func (h *httpEDUServerInputAPI) InputTypingEvent( + ctx context.Context, + request *api.InputTypingEventRequest, + response *api.InputTypingEventResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "InputTypingEvent") + defer span.Finish() + + apiURL := h.eduServerURL + EDUServerInputTypingEventPath + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +// InputSendToDeviceEvent implements EDUServerInputAPI +func (h *httpEDUServerInputAPI) InputSendToDeviceEvent( + ctx context.Context, + request *api.InputSendToDeviceEventRequest, + response *api.InputSendToDeviceEventResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "InputSendToDeviceEvent") + defer span.Finish() + + apiURL := h.eduServerURL + EDUServerInputSendToDeviceEventPath + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} diff --git a/eduserver/inthttp/server.go b/eduserver/inthttp/server.go new file mode 100644 index 000000000..6c7d21a4e --- /dev/null +++ b/eduserver/inthttp/server.go @@ -0,0 +1,41 @@ +package inthttp + +import ( + "encoding/json" + "net/http" + + "github.com/gorilla/mux" + "github.com/matrix-org/dendrite/eduserver/api" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/util" +) + +// AddRoutes adds the EDUServerInputAPI handlers to the http.ServeMux. +func AddRoutes(t api.EDUServerInputAPI, internalAPIMux *mux.Router) { + internalAPIMux.Handle(EDUServerInputTypingEventPath, + internal.MakeInternalAPI("inputTypingEvents", func(req *http.Request) util.JSONResponse { + var request api.InputTypingEventRequest + var response api.InputTypingEventResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := t.InputTypingEvent(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + internalAPIMux.Handle(EDUServerInputSendToDeviceEventPath, + internal.MakeInternalAPI("inputSendToDeviceEvents", func(req *http.Request) util.JSONResponse { + var request api.InputSendToDeviceEventRequest + var response api.InputSendToDeviceEventResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := t.InputSendToDeviceEvent(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) +} diff --git a/internal/basecomponent/base.go b/internal/basecomponent/base.go index a1a908760..cd297b82e 100644 --- a/internal/basecomponent/base.go +++ b/internal/basecomponent/base.go @@ -36,11 +36,14 @@ import ( "github.com/gorilla/mux" appserviceAPI "github.com/matrix-org/dendrite/appservice/api" + asinthttp "github.com/matrix-org/dendrite/appservice/inthttp" eduServerAPI "github.com/matrix-org/dendrite/eduserver/api" + eduinthttp "github.com/matrix-org/dendrite/eduserver/inthttp" federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" - "github.com/matrix-org/dendrite/federationsender/inthttp" + fsinthttp "github.com/matrix-org/dendrite/federationsender/inthttp" "github.com/matrix-org/dendrite/internal/config" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + rsinthttp "github.com/matrix-org/dendrite/roomserver/inthttp" serverKeyAPI "github.com/matrix-org/dendrite/serverkeyapi/api" "github.com/sirupsen/logrus" @@ -136,32 +139,29 @@ func (b *BaseDendrite) Close() error { return b.tracerCloser.Close() } -// CreateHTTPAppServiceAPIs returns the QueryAPI for hitting the appservice -// component over HTTP. -func (b *BaseDendrite) CreateHTTPAppServiceAPIs() appserviceAPI.AppServiceQueryAPI { - a, err := appserviceAPI.NewAppServiceQueryAPIHTTP(b.Cfg.AppServiceURL(), b.httpClient) +// AppserviceHTTPClient returns the AppServiceQueryAPI for hitting the appservice component over HTTP. +func (b *BaseDendrite) AppserviceHTTPClient() appserviceAPI.AppServiceQueryAPI { + a, err := asinthttp.NewAppserviceClient(b.Cfg.AppServiceURL(), b.httpClient) if err != nil { logrus.WithError(err).Panic("CreateHTTPAppServiceAPIs failed") } return a } -// CreateHTTPRoomserverAPIs returns the AliasAPI, InputAPI and QueryAPI for hitting -// the roomserver over HTTP. -func (b *BaseDendrite) CreateHTTPRoomserverAPIs() roomserverAPI.RoomserverInternalAPI { - rsAPI, err := roomserverAPI.NewRoomserverInternalAPIHTTP(b.Cfg.RoomServerURL(), b.httpClient, b.ImmutableCache) +// RoomserverHTTPClient returns RoomserverInternalAPI for hitting the roomserver over HTTP. +func (b *BaseDendrite) RoomserverHTTPClient() roomserverAPI.RoomserverInternalAPI { + rsAPI, err := rsinthttp.NewRoomserverClient(b.Cfg.RoomServerURL(), b.httpClient, b.ImmutableCache) if err != nil { - logrus.WithError(err).Panic("NewRoomserverInternalAPIHTTP failed", b.httpClient) + logrus.WithError(err).Panic("RoomserverHTTPClient failed", b.httpClient) } return rsAPI } -// CreateHTTPEDUServerAPIs returns eduInputAPI for hitting the EDU -// server over HTTP -func (b *BaseDendrite) CreateHTTPEDUServerAPIs() eduServerAPI.EDUServerInputAPI { - e, err := eduServerAPI.NewEDUServerInputAPIHTTP(b.Cfg.EDUServerURL(), b.httpClient) +// EDUServerClient returns EDUServerInputAPI for hitting the EDU server over HTTP +func (b *BaseDendrite) EDUServerClient() eduServerAPI.EDUServerInputAPI { + e, err := eduinthttp.NewEDUServerClient(b.Cfg.EDUServerURL(), b.httpClient) if err != nil { - logrus.WithError(err).Panic("NewEDUServerInputAPIHTTP failed", b.httpClient) + logrus.WithError(err).Panic("EDUServerClient failed", b.httpClient) } return e } @@ -169,9 +169,9 @@ func (b *BaseDendrite) CreateHTTPEDUServerAPIs() eduServerAPI.EDUServerInputAPI // FederationSenderHTTPClient returns FederationSenderInternalAPI for hitting // the federation sender over HTTP func (b *BaseDendrite) FederationSenderHTTPClient() federationSenderAPI.FederationSenderInternalAPI { - f, err := inthttp.NewFederationSenderClient(b.Cfg.FederationSenderURL(), b.httpClient) + f, err := fsinthttp.NewFederationSenderClient(b.Cfg.FederationSenderURL(), b.httpClient) if err != nil { - logrus.WithError(err).Panic("NewFederationSenderInternalAPIHTTP failed", b.httpClient) + logrus.WithError(err).Panic("FederationSenderHTTPClient failed", b.httpClient) } return f } diff --git a/roomserver/api/alias.go b/roomserver/api/alias.go index 54d2c633b..61fdc6116 100644 --- a/roomserver/api/alias.go +++ b/roomserver/api/alias.go @@ -14,13 +14,6 @@ package api -import ( - "context" - - internalHTTP "github.com/matrix-org/dendrite/internal/http" - opentracing "github.com/opentracing/opentracing-go" -) - // SetRoomAliasRequest is a request to SetRoomAlias type SetRoomAliasRequest struct { // ID of the user setting the alias @@ -83,83 +76,3 @@ type RemoveRoomAliasRequest struct { // RemoveRoomAliasResponse is a response to RemoveRoomAlias type RemoveRoomAliasResponse struct{} - -// RoomserverSetRoomAliasPath is the HTTP path for the SetRoomAlias API. -const RoomserverSetRoomAliasPath = "/roomserver/setRoomAlias" - -// RoomserverGetRoomIDForAliasPath is the HTTP path for the GetRoomIDForAlias API. -const RoomserverGetRoomIDForAliasPath = "/roomserver/GetRoomIDForAlias" - -// RoomserverGetAliasesForRoomIDPath is the HTTP path for the GetAliasesForRoomID API. -const RoomserverGetAliasesForRoomIDPath = "/roomserver/GetAliasesForRoomID" - -// RoomserverGetCreatorIDForAliasPath is the HTTP path for the GetCreatorIDForAlias API. -const RoomserverGetCreatorIDForAliasPath = "/roomserver/GetCreatorIDForAlias" - -// RoomserverRemoveRoomAliasPath is the HTTP path for the RemoveRoomAlias API. -const RoomserverRemoveRoomAliasPath = "/roomserver/removeRoomAlias" - -// SetRoomAlias implements RoomserverAliasAPI -func (h *httpRoomserverInternalAPI) SetRoomAlias( - ctx context.Context, - request *SetRoomAliasRequest, - response *SetRoomAliasResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "SetRoomAlias") - defer span.Finish() - - apiURL := h.roomserverURL + RoomserverSetRoomAliasPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} - -// GetRoomIDForAlias implements RoomserverAliasAPI -func (h *httpRoomserverInternalAPI) GetRoomIDForAlias( - ctx context.Context, - request *GetRoomIDForAliasRequest, - response *GetRoomIDForAliasResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "GetRoomIDForAlias") - defer span.Finish() - - apiURL := h.roomserverURL + RoomserverGetRoomIDForAliasPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} - -// GetAliasesForRoomID implements RoomserverAliasAPI -func (h *httpRoomserverInternalAPI) GetAliasesForRoomID( - ctx context.Context, - request *GetAliasesForRoomIDRequest, - response *GetAliasesForRoomIDResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "GetAliasesForRoomID") - defer span.Finish() - - apiURL := h.roomserverURL + RoomserverGetAliasesForRoomIDPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} - -// GetCreatorIDForAlias implements RoomserverAliasAPI -func (h *httpRoomserverInternalAPI) GetCreatorIDForAlias( - ctx context.Context, - request *GetCreatorIDForAliasRequest, - response *GetCreatorIDForAliasResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "GetCreatorIDForAlias") - defer span.Finish() - - apiURL := h.roomserverURL + RoomserverGetCreatorIDForAliasPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} - -// RemoveRoomAlias implements RoomserverAliasAPI -func (h *httpRoomserverInternalAPI) RemoveRoomAlias( - ctx context.Context, - request *RemoveRoomAliasRequest, - response *RemoveRoomAliasResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "RemoveRoomAlias") - defer span.Finish() - - apiURL := h.roomserverURL + RoomserverRemoveRoomAliasPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} diff --git a/roomserver/api/http.go b/roomserver/api/http.go deleted file mode 100644 index 8a4d72e4e..000000000 --- a/roomserver/api/http.go +++ /dev/null @@ -1,41 +0,0 @@ -package api - -import ( - "errors" - "net/http" - - fsInputAPI "github.com/matrix-org/dendrite/federationsender/api" - "github.com/matrix-org/dendrite/internal/caching" -) - -type httpRoomserverInternalAPI struct { - roomserverURL string - httpClient *http.Client - fsAPI fsInputAPI.FederationSenderInternalAPI - immutableCache caching.ImmutableCache -} - -// NewRoomserverInputAPIHTTP creates a RoomserverInputAPI implemented by talking to a HTTP POST API. -// If httpClient is nil an error is returned -func NewRoomserverInternalAPIHTTP( - roomserverURL string, - httpClient *http.Client, - //fsInputAPI fsAPI.FederationSenderInternalAPI, - immutableCache caching.ImmutableCache, -) (RoomserverInternalAPI, error) { - if httpClient == nil { - return nil, errors.New("NewRoomserverInternalAPIHTTP: httpClient is ") - } - return &httpRoomserverInternalAPI{ - roomserverURL: roomserverURL, - httpClient: httpClient, - immutableCache: immutableCache, - }, nil -} - -// SetFederationSenderInputAPI passes in a federation sender input API reference -// so that we can avoid the chicken-and-egg problem of both the roomserver input API -// and the federation sender input API being interdependent. -func (h *httpRoomserverInternalAPI) SetFederationSenderAPI(fsAPI fsInputAPI.FederationSenderInternalAPI) { - h.fsAPI = fsAPI -} diff --git a/roomserver/api/input.go b/roomserver/api/input.go index d35ead764..6c3c89413 100644 --- a/roomserver/api/input.go +++ b/roomserver/api/input.go @@ -16,11 +16,7 @@ package api import ( - "context" - - internalHTTP "github.com/matrix-org/dendrite/internal/http" "github.com/matrix-org/gomatrixserverlib" - opentracing "github.com/opentracing/opentracing-go" ) const ( @@ -101,19 +97,3 @@ type InputRoomEventsRequest struct { type InputRoomEventsResponse struct { EventID string `json:"event_id"` } - -// RoomserverInputRoomEventsPath is the HTTP path for the InputRoomEvents API. -const RoomserverInputRoomEventsPath = "/roomserver/inputRoomEvents" - -// InputRoomEvents implements RoomserverInputAPI -func (h *httpRoomserverInternalAPI) InputRoomEvents( - ctx context.Context, - request *InputRoomEventsRequest, - response *InputRoomEventsResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "InputRoomEvents") - defer span.Finish() - - apiURL := h.roomserverURL + RoomserverInputRoomEventsPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} diff --git a/roomserver/api/perform.go b/roomserver/api/perform.go index f5afd67bf..1cf54144e 100644 --- a/roomserver/api/perform.go +++ b/roomserver/api/perform.go @@ -1,19 +1,7 @@ package api import ( - "context" - - internalHTTP "github.com/matrix-org/dendrite/internal/http" "github.com/matrix-org/gomatrixserverlib" - "github.com/opentracing/opentracing-go" -) - -const ( - // RoomserverPerformJoinPath is the HTTP path for the PerformJoin API. - RoomserverPerformJoinPath = "/roomserver/performJoin" - - // RoomserverPerformLeavePath is the HTTP path for the PerformLeave API. - RoomserverPerformLeavePath = "/roomserver/performLeave" ) type PerformJoinRequest struct { @@ -27,18 +15,6 @@ type PerformJoinResponse struct { RoomID string `json:"room_id"` } -func (h *httpRoomserverInternalAPI) PerformJoin( - ctx context.Context, - request *PerformJoinRequest, - response *PerformJoinResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "PerformJoin") - defer span.Finish() - - apiURL := h.roomserverURL + RoomserverPerformJoinPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} - type PerformLeaveRequest struct { RoomID string `json:"room_id"` UserID string `json:"user_id"` @@ -46,15 +22,3 @@ type PerformLeaveRequest struct { type PerformLeaveResponse struct { } - -func (h *httpRoomserverInternalAPI) PerformLeave( - ctx context.Context, - request *PerformLeaveRequest, - response *PerformLeaveResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "PerformLeave") - defer span.Finish() - - apiURL := h.roomserverURL + RoomserverPerformLeavePath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} diff --git a/roomserver/api/query.go b/roomserver/api/query.go index 916ecb362..dc005c77c 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -17,12 +17,8 @@ package api import ( - "context" - - internalHTTP "github.com/matrix-org/dendrite/internal/http" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" - opentracing "github.com/opentracing/opentracing-go" ) // QueryLatestEventsAndStateRequest is a request to QueryLatestEventsAndState @@ -271,204 +267,3 @@ type QueryRoomVersionForRoomRequest struct { type QueryRoomVersionForRoomResponse struct { RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"` } - -// RoomserverQueryLatestEventsAndStatePath is the HTTP path for the QueryLatestEventsAndState API. -const RoomserverQueryLatestEventsAndStatePath = "/roomserver/queryLatestEventsAndState" - -// RoomserverQueryStateAfterEventsPath is the HTTP path for the QueryStateAfterEvents API. -const RoomserverQueryStateAfterEventsPath = "/roomserver/queryStateAfterEvents" - -// RoomserverQueryEventsByIDPath is the HTTP path for the QueryEventsByID API. -const RoomserverQueryEventsByIDPath = "/roomserver/queryEventsByID" - -// RoomserverQueryMembershipForUserPath is the HTTP path for the QueryMembershipForUser API. -const RoomserverQueryMembershipForUserPath = "/roomserver/queryMembershipForUser" - -// RoomserverQueryMembershipsForRoomPath is the HTTP path for the QueryMembershipsForRoom API -const RoomserverQueryMembershipsForRoomPath = "/roomserver/queryMembershipsForRoom" - -// RoomserverQueryInvitesForUserPath is the HTTP path for the QueryInvitesForUser API -const RoomserverQueryInvitesForUserPath = "/roomserver/queryInvitesForUser" - -// RoomserverQueryServerAllowedToSeeEventPath is the HTTP path for the QueryServerAllowedToSeeEvent API -const RoomserverQueryServerAllowedToSeeEventPath = "/roomserver/queryServerAllowedToSeeEvent" - -// RoomserverQueryMissingEventsPath is the HTTP path for the QueryMissingEvents API -const RoomserverQueryMissingEventsPath = "/roomserver/queryMissingEvents" - -// RoomserverQueryStateAndAuthChainPath is the HTTP path for the QueryStateAndAuthChain API -const RoomserverQueryStateAndAuthChainPath = "/roomserver/queryStateAndAuthChain" - -// RoomserverQueryBackfillPath is the HTTP path for the QueryBackfillPath API -const RoomserverQueryBackfillPath = "/roomserver/queryBackfill" - -// RoomserverQueryRoomVersionCapabilitiesPath is the HTTP path for the QueryRoomVersionCapabilities API -const RoomserverQueryRoomVersionCapabilitiesPath = "/roomserver/queryRoomVersionCapabilities" - -// RoomserverQueryRoomVersionForRoomPath is the HTTP path for the QueryRoomVersionForRoom API -const RoomserverQueryRoomVersionForRoomPath = "/roomserver/queryRoomVersionForRoom" - -// QueryLatestEventsAndState implements RoomserverQueryAPI -func (h *httpRoomserverInternalAPI) QueryLatestEventsAndState( - ctx context.Context, - request *QueryLatestEventsAndStateRequest, - response *QueryLatestEventsAndStateResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "QueryLatestEventsAndState") - defer span.Finish() - - apiURL := h.roomserverURL + RoomserverQueryLatestEventsAndStatePath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} - -// QueryStateAfterEvents implements RoomserverQueryAPI -func (h *httpRoomserverInternalAPI) QueryStateAfterEvents( - ctx context.Context, - request *QueryStateAfterEventsRequest, - response *QueryStateAfterEventsResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "QueryStateAfterEvents") - defer span.Finish() - - apiURL := h.roomserverURL + RoomserverQueryStateAfterEventsPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} - -// QueryEventsByID implements RoomserverQueryAPI -func (h *httpRoomserverInternalAPI) QueryEventsByID( - ctx context.Context, - request *QueryEventsByIDRequest, - response *QueryEventsByIDResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "QueryEventsByID") - defer span.Finish() - - apiURL := h.roomserverURL + RoomserverQueryEventsByIDPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} - -// QueryMembershipForUser implements RoomserverQueryAPI -func (h *httpRoomserverInternalAPI) QueryMembershipForUser( - ctx context.Context, - request *QueryMembershipForUserRequest, - response *QueryMembershipForUserResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "QueryMembershipForUser") - defer span.Finish() - - apiURL := h.roomserverURL + RoomserverQueryMembershipForUserPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} - -// QueryMembershipsForRoom implements RoomserverQueryAPI -func (h *httpRoomserverInternalAPI) QueryMembershipsForRoom( - ctx context.Context, - request *QueryMembershipsForRoomRequest, - response *QueryMembershipsForRoomResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "QueryMembershipsForRoom") - defer span.Finish() - - apiURL := h.roomserverURL + RoomserverQueryMembershipsForRoomPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} - -// QueryInvitesForUser implements RoomserverQueryAPI -func (h *httpRoomserverInternalAPI) QueryInvitesForUser( - ctx context.Context, - request *QueryInvitesForUserRequest, - response *QueryInvitesForUserResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "QueryInvitesForUser") - defer span.Finish() - - apiURL := h.roomserverURL + RoomserverQueryInvitesForUserPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} - -// QueryServerAllowedToSeeEvent implements RoomserverQueryAPI -func (h *httpRoomserverInternalAPI) QueryServerAllowedToSeeEvent( - ctx context.Context, - request *QueryServerAllowedToSeeEventRequest, - response *QueryServerAllowedToSeeEventResponse, -) (err error) { - span, ctx := opentracing.StartSpanFromContext(ctx, "QueryServerAllowedToSeeEvent") - defer span.Finish() - - apiURL := h.roomserverURL + RoomserverQueryServerAllowedToSeeEventPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} - -// QueryMissingEvents implements RoomServerQueryAPI -func (h *httpRoomserverInternalAPI) QueryMissingEvents( - ctx context.Context, - request *QueryMissingEventsRequest, - response *QueryMissingEventsResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "QueryMissingEvents") - defer span.Finish() - - apiURL := h.roomserverURL + RoomserverQueryMissingEventsPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} - -// QueryStateAndAuthChain implements RoomserverQueryAPI -func (h *httpRoomserverInternalAPI) QueryStateAndAuthChain( - ctx context.Context, - request *QueryStateAndAuthChainRequest, - response *QueryStateAndAuthChainResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "QueryStateAndAuthChain") - defer span.Finish() - - apiURL := h.roomserverURL + RoomserverQueryStateAndAuthChainPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} - -// QueryBackfill implements RoomServerQueryAPI -func (h *httpRoomserverInternalAPI) QueryBackfill( - ctx context.Context, - request *QueryBackfillRequest, - response *QueryBackfillResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "QueryBackfill") - defer span.Finish() - - apiURL := h.roomserverURL + RoomserverQueryBackfillPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} - -// QueryRoomVersionCapabilities implements RoomServerQueryAPI -func (h *httpRoomserverInternalAPI) QueryRoomVersionCapabilities( - ctx context.Context, - request *QueryRoomVersionCapabilitiesRequest, - response *QueryRoomVersionCapabilitiesResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "QueryRoomVersionCapabilities") - defer span.Finish() - - apiURL := h.roomserverURL + RoomserverQueryRoomVersionCapabilitiesPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} - -// QueryRoomVersionForRoom implements RoomServerQueryAPI -func (h *httpRoomserverInternalAPI) QueryRoomVersionForRoom( - ctx context.Context, - request *QueryRoomVersionForRoomRequest, - response *QueryRoomVersionForRoomResponse, -) error { - if roomVersion, ok := h.immutableCache.GetRoomVersion(request.RoomID); ok { - response.RoomVersion = roomVersion - return nil - } - - span, ctx := opentracing.StartSpanFromContext(ctx, "QueryRoomVersionForRoom") - defer span.Finish() - - apiURL := h.roomserverURL + RoomserverQueryRoomVersionForRoomPath - err := internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) - if err == nil { - h.immutableCache.StoreRoomVersion(request.RoomID, response.RoomVersion) - } - return err -} diff --git a/roomserver/internal/api.go b/roomserver/internal/api.go index 248e457d1..3a7b0d76a 100644 --- a/roomserver/internal/api.go +++ b/roomserver/internal/api.go @@ -1,20 +1,14 @@ package internal import ( - "encoding/json" - "net/http" "sync" "github.com/Shopify/sarama" - "github.com/gorilla/mux" fsAPI "github.com/matrix-org/dendrite/federationsender/api" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/config" - "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" ) // RoomserverInternalAPI is an implementation of api.RoomserverInternalAPI @@ -30,285 +24,3 @@ type RoomserverInternalAPI struct { mutex sync.Mutex // Protects calls to processRoomEvent fsAPI fsAPI.FederationSenderInternalAPI } - -// SetupHTTP adds the RoomserverInternalAPI handlers to the http.ServeMux. -// nolint: gocyclo -func (r *RoomserverInternalAPI) SetupHTTP(internalAPIMux *mux.Router) { - internalAPIMux.Handle(api.RoomserverInputRoomEventsPath, - internal.MakeInternalAPI("inputRoomEvents", func(req *http.Request) util.JSONResponse { - var request api.InputRoomEventsRequest - var response api.InputRoomEventsResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.MessageResponse(http.StatusBadRequest, err.Error()) - } - if err := r.InputRoomEvents(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - internalAPIMux.Handle(api.RoomserverPerformJoinPath, - internal.MakeInternalAPI("performJoin", func(req *http.Request) util.JSONResponse { - var request api.PerformJoinRequest - var response api.PerformJoinResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.MessageResponse(http.StatusBadRequest, err.Error()) - } - if err := r.PerformJoin(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - internalAPIMux.Handle(api.RoomserverPerformLeavePath, - internal.MakeInternalAPI("performLeave", func(req *http.Request) util.JSONResponse { - var request api.PerformLeaveRequest - var response api.PerformLeaveResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.MessageResponse(http.StatusBadRequest, err.Error()) - } - if err := r.PerformLeave(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - internalAPIMux.Handle( - api.RoomserverQueryLatestEventsAndStatePath, - internal.MakeInternalAPI("queryLatestEventsAndState", func(req *http.Request) util.JSONResponse { - var request api.QueryLatestEventsAndStateRequest - var response api.QueryLatestEventsAndStateResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryLatestEventsAndState(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - internalAPIMux.Handle( - api.RoomserverQueryStateAfterEventsPath, - internal.MakeInternalAPI("queryStateAfterEvents", func(req *http.Request) util.JSONResponse { - var request api.QueryStateAfterEventsRequest - var response api.QueryStateAfterEventsResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryStateAfterEvents(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - internalAPIMux.Handle( - api.RoomserverQueryEventsByIDPath, - internal.MakeInternalAPI("queryEventsByID", func(req *http.Request) util.JSONResponse { - var request api.QueryEventsByIDRequest - var response api.QueryEventsByIDResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryEventsByID(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - internalAPIMux.Handle( - api.RoomserverQueryMembershipForUserPath, - internal.MakeInternalAPI("QueryMembershipForUser", func(req *http.Request) util.JSONResponse { - var request api.QueryMembershipForUserRequest - var response api.QueryMembershipForUserResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryMembershipForUser(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - internalAPIMux.Handle( - api.RoomserverQueryMembershipsForRoomPath, - internal.MakeInternalAPI("queryMembershipsForRoom", func(req *http.Request) util.JSONResponse { - var request api.QueryMembershipsForRoomRequest - var response api.QueryMembershipsForRoomResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryMembershipsForRoom(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - internalAPIMux.Handle( - api.RoomserverQueryInvitesForUserPath, - internal.MakeInternalAPI("queryInvitesForUser", func(req *http.Request) util.JSONResponse { - var request api.QueryInvitesForUserRequest - var response api.QueryInvitesForUserResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryInvitesForUser(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - internalAPIMux.Handle( - api.RoomserverQueryServerAllowedToSeeEventPath, - internal.MakeInternalAPI("queryServerAllowedToSeeEvent", func(req *http.Request) util.JSONResponse { - var request api.QueryServerAllowedToSeeEventRequest - var response api.QueryServerAllowedToSeeEventResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryServerAllowedToSeeEvent(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - internalAPIMux.Handle( - api.RoomserverQueryMissingEventsPath, - internal.MakeInternalAPI("queryMissingEvents", func(req *http.Request) util.JSONResponse { - var request api.QueryMissingEventsRequest - var response api.QueryMissingEventsResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryMissingEvents(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - internalAPIMux.Handle( - api.RoomserverQueryStateAndAuthChainPath, - internal.MakeInternalAPI("queryStateAndAuthChain", func(req *http.Request) util.JSONResponse { - var request api.QueryStateAndAuthChainRequest - var response api.QueryStateAndAuthChainResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryStateAndAuthChain(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - internalAPIMux.Handle( - api.RoomserverQueryBackfillPath, - internal.MakeInternalAPI("QueryBackfill", func(req *http.Request) util.JSONResponse { - var request api.QueryBackfillRequest - var response api.QueryBackfillResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryBackfill(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - internalAPIMux.Handle( - api.RoomserverQueryRoomVersionCapabilitiesPath, - internal.MakeInternalAPI("QueryRoomVersionCapabilities", func(req *http.Request) util.JSONResponse { - var request api.QueryRoomVersionCapabilitiesRequest - var response api.QueryRoomVersionCapabilitiesResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryRoomVersionCapabilities(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - internalAPIMux.Handle( - api.RoomserverQueryRoomVersionForRoomPath, - internal.MakeInternalAPI("QueryRoomVersionForRoom", func(req *http.Request) util.JSONResponse { - var request api.QueryRoomVersionForRoomRequest - var response api.QueryRoomVersionForRoomResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryRoomVersionForRoom(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - internalAPIMux.Handle( - api.RoomserverSetRoomAliasPath, - internal.MakeInternalAPI("setRoomAlias", func(req *http.Request) util.JSONResponse { - var request api.SetRoomAliasRequest - var response api.SetRoomAliasResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.SetRoomAlias(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - internalAPIMux.Handle( - api.RoomserverGetRoomIDForAliasPath, - internal.MakeInternalAPI("GetRoomIDForAlias", func(req *http.Request) util.JSONResponse { - var request api.GetRoomIDForAliasRequest - var response api.GetRoomIDForAliasResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.GetRoomIDForAlias(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - internalAPIMux.Handle( - api.RoomserverGetCreatorIDForAliasPath, - internal.MakeInternalAPI("GetCreatorIDForAlias", func(req *http.Request) util.JSONResponse { - var request api.GetCreatorIDForAliasRequest - var response api.GetCreatorIDForAliasResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.GetCreatorIDForAlias(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - internalAPIMux.Handle( - api.RoomserverGetAliasesForRoomIDPath, - internal.MakeInternalAPI("getAliasesForRoomID", func(req *http.Request) util.JSONResponse { - var request api.GetAliasesForRoomIDRequest - var response api.GetAliasesForRoomIDResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.GetAliasesForRoomID(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) - internalAPIMux.Handle( - api.RoomserverRemoveRoomAliasPath, - internal.MakeInternalAPI("removeRoomAlias", func(req *http.Request) util.JSONResponse { - var request api.RemoveRoomAliasRequest - var response api.RemoveRoomAliasResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.RemoveRoomAlias(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) -} diff --git a/roomserver/inthttp/client.go b/roomserver/inthttp/client.go new file mode 100644 index 000000000..f8f922fbd --- /dev/null +++ b/roomserver/inthttp/client.go @@ -0,0 +1,341 @@ +package inthttp + +import ( + "context" + "errors" + "net/http" + + fsInputAPI "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/internal/caching" + internalHTTP "github.com/matrix-org/dendrite/internal/http" + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/opentracing/opentracing-go" +) + +const ( + // Alias operations + RoomserverSetRoomAliasPath = "/roomserver/setRoomAlias" + RoomserverGetRoomIDForAliasPath = "/roomserver/GetRoomIDForAlias" + RoomserverGetAliasesForRoomIDPath = "/roomserver/GetAliasesForRoomID" + RoomserverGetCreatorIDForAliasPath = "/roomserver/GetCreatorIDForAlias" + RoomserverRemoveRoomAliasPath = "/roomserver/removeRoomAlias" + + // Input operations + RoomserverInputRoomEventsPath = "/roomserver/inputRoomEvents" + + // Perform operations + RoomserverPerformJoinPath = "/roomserver/performJoin" + RoomserverPerformLeavePath = "/roomserver/performLeave" + + // Query operations + RoomserverQueryLatestEventsAndStatePath = "/roomserver/queryLatestEventsAndState" + RoomserverQueryStateAfterEventsPath = "/roomserver/queryStateAfterEvents" + RoomserverQueryEventsByIDPath = "/roomserver/queryEventsByID" + RoomserverQueryMembershipForUserPath = "/roomserver/queryMembershipForUser" + RoomserverQueryMembershipsForRoomPath = "/roomserver/queryMembershipsForRoom" + RoomserverQueryInvitesForUserPath = "/roomserver/queryInvitesForUser" + RoomserverQueryServerAllowedToSeeEventPath = "/roomserver/queryServerAllowedToSeeEvent" + RoomserverQueryMissingEventsPath = "/roomserver/queryMissingEvents" + RoomserverQueryStateAndAuthChainPath = "/roomserver/queryStateAndAuthChain" + RoomserverQueryBackfillPath = "/roomserver/queryBackfill" + RoomserverQueryRoomVersionCapabilitiesPath = "/roomserver/queryRoomVersionCapabilities" + RoomserverQueryRoomVersionForRoomPath = "/roomserver/queryRoomVersionForRoom" +) + +type httpRoomserverInternalAPI struct { + roomserverURL string + httpClient *http.Client + fsAPI fsInputAPI.FederationSenderInternalAPI + immutableCache caching.ImmutableCache +} + +// NewRoomserverClient creates a RoomserverInputAPI implemented by talking to a HTTP POST API. +// If httpClient is nil an error is returned +func NewRoomserverClient( + roomserverURL string, + httpClient *http.Client, + immutableCache caching.ImmutableCache, +) (api.RoomserverInternalAPI, error) { + if httpClient == nil { + return nil, errors.New("NewRoomserverInternalAPIHTTP: httpClient is ") + } + return &httpRoomserverInternalAPI{ + roomserverURL: roomserverURL, + httpClient: httpClient, + immutableCache: immutableCache, + }, nil +} + +// SetFederationSenderInputAPI passes in a federation sender input API reference +// so that we can avoid the chicken-and-egg problem of both the roomserver input API +// and the federation sender input API being interdependent. +func (h *httpRoomserverInternalAPI) SetFederationSenderAPI(fsAPI fsInputAPI.FederationSenderInternalAPI) { + h.fsAPI = fsAPI +} + +// SetRoomAlias implements RoomserverAliasAPI +func (h *httpRoomserverInternalAPI) SetRoomAlias( + ctx context.Context, + request *api.SetRoomAliasRequest, + response *api.SetRoomAliasResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "SetRoomAlias") + defer span.Finish() + + apiURL := h.roomserverURL + RoomserverSetRoomAliasPath + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +// GetRoomIDForAlias implements RoomserverAliasAPI +func (h *httpRoomserverInternalAPI) GetRoomIDForAlias( + ctx context.Context, + request *api.GetRoomIDForAliasRequest, + response *api.GetRoomIDForAliasResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "GetRoomIDForAlias") + defer span.Finish() + + apiURL := h.roomserverURL + RoomserverGetRoomIDForAliasPath + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +// GetAliasesForRoomID implements RoomserverAliasAPI +func (h *httpRoomserverInternalAPI) GetAliasesForRoomID( + ctx context.Context, + request *api.GetAliasesForRoomIDRequest, + response *api.GetAliasesForRoomIDResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "GetAliasesForRoomID") + defer span.Finish() + + apiURL := h.roomserverURL + RoomserverGetAliasesForRoomIDPath + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +// GetCreatorIDForAlias implements RoomserverAliasAPI +func (h *httpRoomserverInternalAPI) GetCreatorIDForAlias( + ctx context.Context, + request *api.GetCreatorIDForAliasRequest, + response *api.GetCreatorIDForAliasResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "GetCreatorIDForAlias") + defer span.Finish() + + apiURL := h.roomserverURL + RoomserverGetCreatorIDForAliasPath + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +// RemoveRoomAlias implements RoomserverAliasAPI +func (h *httpRoomserverInternalAPI) RemoveRoomAlias( + ctx context.Context, + request *api.RemoveRoomAliasRequest, + response *api.RemoveRoomAliasResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "RemoveRoomAlias") + defer span.Finish() + + apiURL := h.roomserverURL + RoomserverRemoveRoomAliasPath + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +// InputRoomEvents implements RoomserverInputAPI +func (h *httpRoomserverInternalAPI) InputRoomEvents( + ctx context.Context, + request *api.InputRoomEventsRequest, + response *api.InputRoomEventsResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "InputRoomEvents") + defer span.Finish() + + apiURL := h.roomserverURL + RoomserverInputRoomEventsPath + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +func (h *httpRoomserverInternalAPI) PerformJoin( + ctx context.Context, + request *api.PerformJoinRequest, + response *api.PerformJoinResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "PerformJoin") + defer span.Finish() + + apiURL := h.roomserverURL + RoomserverPerformJoinPath + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +func (h *httpRoomserverInternalAPI) PerformLeave( + ctx context.Context, + request *api.PerformLeaveRequest, + response *api.PerformLeaveResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "PerformLeave") + defer span.Finish() + + apiURL := h.roomserverURL + RoomserverPerformLeavePath + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +// QueryLatestEventsAndState implements RoomserverQueryAPI +func (h *httpRoomserverInternalAPI) QueryLatestEventsAndState( + ctx context.Context, + request *api.QueryLatestEventsAndStateRequest, + response *api.QueryLatestEventsAndStateResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "QueryLatestEventsAndState") + defer span.Finish() + + apiURL := h.roomserverURL + RoomserverQueryLatestEventsAndStatePath + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +// QueryStateAfterEvents implements RoomserverQueryAPI +func (h *httpRoomserverInternalAPI) QueryStateAfterEvents( + ctx context.Context, + request *api.QueryStateAfterEventsRequest, + response *api.QueryStateAfterEventsResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "QueryStateAfterEvents") + defer span.Finish() + + apiURL := h.roomserverURL + RoomserverQueryStateAfterEventsPath + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +// QueryEventsByID implements RoomserverQueryAPI +func (h *httpRoomserverInternalAPI) QueryEventsByID( + ctx context.Context, + request *api.QueryEventsByIDRequest, + response *api.QueryEventsByIDResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "QueryEventsByID") + defer span.Finish() + + apiURL := h.roomserverURL + RoomserverQueryEventsByIDPath + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +// QueryMembershipForUser implements RoomserverQueryAPI +func (h *httpRoomserverInternalAPI) QueryMembershipForUser( + ctx context.Context, + request *api.QueryMembershipForUserRequest, + response *api.QueryMembershipForUserResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "QueryMembershipForUser") + defer span.Finish() + + apiURL := h.roomserverURL + RoomserverQueryMembershipForUserPath + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +// QueryMembershipsForRoom implements RoomserverQueryAPI +func (h *httpRoomserverInternalAPI) QueryMembershipsForRoom( + ctx context.Context, + request *api.QueryMembershipsForRoomRequest, + response *api.QueryMembershipsForRoomResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "QueryMembershipsForRoom") + defer span.Finish() + + apiURL := h.roomserverURL + RoomserverQueryMembershipsForRoomPath + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +// QueryInvitesForUser implements RoomserverQueryAPI +func (h *httpRoomserverInternalAPI) QueryInvitesForUser( + ctx context.Context, + request *api.QueryInvitesForUserRequest, + response *api.QueryInvitesForUserResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "QueryInvitesForUser") + defer span.Finish() + + apiURL := h.roomserverURL + RoomserverQueryInvitesForUserPath + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +// QueryServerAllowedToSeeEvent implements RoomserverQueryAPI +func (h *httpRoomserverInternalAPI) QueryServerAllowedToSeeEvent( + ctx context.Context, + request *api.QueryServerAllowedToSeeEventRequest, + response *api.QueryServerAllowedToSeeEventResponse, +) (err error) { + span, ctx := opentracing.StartSpanFromContext(ctx, "QueryServerAllowedToSeeEvent") + defer span.Finish() + + apiURL := h.roomserverURL + RoomserverQueryServerAllowedToSeeEventPath + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +// QueryMissingEvents implements RoomServerQueryAPI +func (h *httpRoomserverInternalAPI) QueryMissingEvents( + ctx context.Context, + request *api.QueryMissingEventsRequest, + response *api.QueryMissingEventsResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "QueryMissingEvents") + defer span.Finish() + + apiURL := h.roomserverURL + RoomserverQueryMissingEventsPath + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +// QueryStateAndAuthChain implements RoomserverQueryAPI +func (h *httpRoomserverInternalAPI) QueryStateAndAuthChain( + ctx context.Context, + request *api.QueryStateAndAuthChainRequest, + response *api.QueryStateAndAuthChainResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "QueryStateAndAuthChain") + defer span.Finish() + + apiURL := h.roomserverURL + RoomserverQueryStateAndAuthChainPath + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +// QueryBackfill implements RoomServerQueryAPI +func (h *httpRoomserverInternalAPI) QueryBackfill( + ctx context.Context, + request *api.QueryBackfillRequest, + response *api.QueryBackfillResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "QueryBackfill") + defer span.Finish() + + apiURL := h.roomserverURL + RoomserverQueryBackfillPath + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +// QueryRoomVersionCapabilities implements RoomServerQueryAPI +func (h *httpRoomserverInternalAPI) QueryRoomVersionCapabilities( + ctx context.Context, + request *api.QueryRoomVersionCapabilitiesRequest, + response *api.QueryRoomVersionCapabilitiesResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "QueryRoomVersionCapabilities") + defer span.Finish() + + apiURL := h.roomserverURL + RoomserverQueryRoomVersionCapabilitiesPath + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +// QueryRoomVersionForRoom implements RoomServerQueryAPI +func (h *httpRoomserverInternalAPI) QueryRoomVersionForRoom( + ctx context.Context, + request *api.QueryRoomVersionForRoomRequest, + response *api.QueryRoomVersionForRoomResponse, +) error { + if roomVersion, ok := h.immutableCache.GetRoomVersion(request.RoomID); ok { + response.RoomVersion = roomVersion + return nil + } + + span, ctx := opentracing.StartSpanFromContext(ctx, "QueryRoomVersionForRoom") + defer span.Finish() + + apiURL := h.roomserverURL + RoomserverQueryRoomVersionForRoomPath + err := internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + if err == nil { + h.immutableCache.StoreRoomVersion(request.RoomID, response.RoomVersion) + } + return err +} diff --git a/roomserver/inthttp/server.go b/roomserver/inthttp/server.go new file mode 100644 index 000000000..9a58a30b0 --- /dev/null +++ b/roomserver/inthttp/server.go @@ -0,0 +1,293 @@ +package inthttp + +import ( + "encoding/json" + "net/http" + + "github.com/gorilla/mux" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/util" +) + +// AddRoutes adds the RoomserverInternalAPI handlers to the http.ServeMux. +// nolint: gocyclo +func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { + internalAPIMux.Handle(RoomserverInputRoomEventsPath, + internal.MakeInternalAPI("inputRoomEvents", func(req *http.Request) util.JSONResponse { + var request api.InputRoomEventsRequest + var response api.InputRoomEventsResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := r.InputRoomEvents(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + internalAPIMux.Handle(RoomserverPerformJoinPath, + internal.MakeInternalAPI("performJoin", func(req *http.Request) util.JSONResponse { + var request api.PerformJoinRequest + var response api.PerformJoinResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := r.PerformJoin(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + internalAPIMux.Handle(RoomserverPerformLeavePath, + internal.MakeInternalAPI("performLeave", func(req *http.Request) util.JSONResponse { + var request api.PerformLeaveRequest + var response api.PerformLeaveResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := r.PerformLeave(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + internalAPIMux.Handle( + RoomserverQueryLatestEventsAndStatePath, + internal.MakeInternalAPI("queryLatestEventsAndState", func(req *http.Request) util.JSONResponse { + var request api.QueryLatestEventsAndStateRequest + var response api.QueryLatestEventsAndStateResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryLatestEventsAndState(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + internalAPIMux.Handle( + RoomserverQueryStateAfterEventsPath, + internal.MakeInternalAPI("queryStateAfterEvents", func(req *http.Request) util.JSONResponse { + var request api.QueryStateAfterEventsRequest + var response api.QueryStateAfterEventsResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryStateAfterEvents(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + internalAPIMux.Handle( + RoomserverQueryEventsByIDPath, + internal.MakeInternalAPI("queryEventsByID", func(req *http.Request) util.JSONResponse { + var request api.QueryEventsByIDRequest + var response api.QueryEventsByIDResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryEventsByID(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + internalAPIMux.Handle( + RoomserverQueryMembershipForUserPath, + internal.MakeInternalAPI("QueryMembershipForUser", func(req *http.Request) util.JSONResponse { + var request api.QueryMembershipForUserRequest + var response api.QueryMembershipForUserResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryMembershipForUser(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + internalAPIMux.Handle( + RoomserverQueryMembershipsForRoomPath, + internal.MakeInternalAPI("queryMembershipsForRoom", func(req *http.Request) util.JSONResponse { + var request api.QueryMembershipsForRoomRequest + var response api.QueryMembershipsForRoomResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryMembershipsForRoom(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + internalAPIMux.Handle( + RoomserverQueryInvitesForUserPath, + internal.MakeInternalAPI("queryInvitesForUser", func(req *http.Request) util.JSONResponse { + var request api.QueryInvitesForUserRequest + var response api.QueryInvitesForUserResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryInvitesForUser(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + internalAPIMux.Handle( + RoomserverQueryServerAllowedToSeeEventPath, + internal.MakeInternalAPI("queryServerAllowedToSeeEvent", func(req *http.Request) util.JSONResponse { + var request api.QueryServerAllowedToSeeEventRequest + var response api.QueryServerAllowedToSeeEventResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryServerAllowedToSeeEvent(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + internalAPIMux.Handle( + RoomserverQueryMissingEventsPath, + internal.MakeInternalAPI("queryMissingEvents", func(req *http.Request) util.JSONResponse { + var request api.QueryMissingEventsRequest + var response api.QueryMissingEventsResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryMissingEvents(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + internalAPIMux.Handle( + RoomserverQueryStateAndAuthChainPath, + internal.MakeInternalAPI("queryStateAndAuthChain", func(req *http.Request) util.JSONResponse { + var request api.QueryStateAndAuthChainRequest + var response api.QueryStateAndAuthChainResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryStateAndAuthChain(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + internalAPIMux.Handle( + RoomserverQueryBackfillPath, + internal.MakeInternalAPI("QueryBackfill", func(req *http.Request) util.JSONResponse { + var request api.QueryBackfillRequest + var response api.QueryBackfillResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryBackfill(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + internalAPIMux.Handle( + RoomserverQueryRoomVersionCapabilitiesPath, + internal.MakeInternalAPI("QueryRoomVersionCapabilities", func(req *http.Request) util.JSONResponse { + var request api.QueryRoomVersionCapabilitiesRequest + var response api.QueryRoomVersionCapabilitiesResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryRoomVersionCapabilities(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + internalAPIMux.Handle( + RoomserverQueryRoomVersionForRoomPath, + internal.MakeInternalAPI("QueryRoomVersionForRoom", func(req *http.Request) util.JSONResponse { + var request api.QueryRoomVersionForRoomRequest + var response api.QueryRoomVersionForRoomResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.QueryRoomVersionForRoom(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + internalAPIMux.Handle( + RoomserverSetRoomAliasPath, + internal.MakeInternalAPI("setRoomAlias", func(req *http.Request) util.JSONResponse { + var request api.SetRoomAliasRequest + var response api.SetRoomAliasResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.SetRoomAlias(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + internalAPIMux.Handle( + RoomserverGetRoomIDForAliasPath, + internal.MakeInternalAPI("GetRoomIDForAlias", func(req *http.Request) util.JSONResponse { + var request api.GetRoomIDForAliasRequest + var response api.GetRoomIDForAliasResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.GetRoomIDForAlias(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + internalAPIMux.Handle( + RoomserverGetCreatorIDForAliasPath, + internal.MakeInternalAPI("GetCreatorIDForAlias", func(req *http.Request) util.JSONResponse { + var request api.GetCreatorIDForAliasRequest + var response api.GetCreatorIDForAliasResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.GetCreatorIDForAlias(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + internalAPIMux.Handle( + RoomserverGetAliasesForRoomIDPath, + internal.MakeInternalAPI("getAliasesForRoomID", func(req *http.Request) util.JSONResponse { + var request api.GetAliasesForRoomIDRequest + var response api.GetAliasesForRoomIDResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.GetAliasesForRoomID(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + internalAPIMux.Handle( + RoomserverRemoveRoomAliasPath, + internal.MakeInternalAPI("removeRoomAlias", func(req *http.Request) util.JSONResponse { + var request api.RemoveRoomAliasRequest + var response api.RemoveRoomAliasResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.ErrorResponse(err) + } + if err := r.RemoveRoomAlias(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) +} diff --git a/roomserver/roomserver.go b/roomserver/roomserver.go index 82934d50d..ae0b0794a 100644 --- a/roomserver/roomserver.go +++ b/roomserver/roomserver.go @@ -16,6 +16,7 @@ package roomserver import ( "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/inthttp" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/dendrite/internal/basecomponent" @@ -38,7 +39,7 @@ func SetupRoomServerComponent( logrus.WithError(err).Panicf("failed to connect to room server db") } - internalAPI := internal.RoomserverInternalAPI{ + internalAPI := &internal.RoomserverInternalAPI{ DB: roomserverDB, Cfg: base.Cfg, Producer: base.KafkaProducer, @@ -49,7 +50,7 @@ func SetupRoomServerComponent( KeyRing: keyRing, } - internalAPI.SetupHTTP(base.InternalAPIMux) + inthttp.AddRoutes(internalAPI, base.InternalAPIMux) - return &internalAPI + return internalAPI } diff --git a/syncapi/api/query.go b/syncapi/api/query.go deleted file mode 100644 index 1da05fcc8..000000000 --- a/syncapi/api/query.go +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package api - -import ( - "context" - "net/http" - - internalHTTP "github.com/matrix-org/dendrite/internal/http" - "github.com/matrix-org/util" - opentracing "github.com/opentracing/opentracing-go" -) - -const ( - SyncAPIQuerySyncPath = "/syncapi/querySync" - SyncAPIQueryStatePath = "/syncapi/queryState" - SyncAPIQueryStateTypePath = "/syncapi/queryStateType" - SyncAPIQueryMessagesPath = "/syncapi/queryMessages" -) - -func NewSyncQueryAPIHTTP(syncapiURL string, httpClient *http.Client) SyncQueryAPI { - if httpClient == nil { - httpClient = http.DefaultClient - } - return &httpSyncQueryAPI{syncapiURL, httpClient} -} - -type httpSyncQueryAPI struct { - syncapiURL string - httpClient *http.Client -} - -type SyncQueryAPI interface { - QuerySync(ctx context.Context, request *QuerySyncRequest, response *QuerySyncResponse) error - QueryState(ctx context.Context, request *QueryStateRequest, response *QueryStateResponse) error - QueryStateType(ctx context.Context, request *QueryStateTypeRequest, response *QueryStateTypeResponse) error - QueryMessages(ctx context.Context, request *QueryMessagesRequest, response *QueryMessagesResponse) error -} - -type QuerySyncRequest struct{} - -type QueryStateRequest struct { - RoomID string -} - -type QueryStateTypeRequest struct { - RoomID string - EventType string - StateKey string -} - -type QueryMessagesRequest struct { - RoomID string -} - -type QuerySyncResponse util.JSONResponse -type QueryStateResponse util.JSONResponse -type QueryStateTypeResponse util.JSONResponse -type QueryMessagesResponse util.JSONResponse - -// QueryLatestEventsAndState implements SyncQueryAPI -func (h *httpSyncQueryAPI) QuerySync( - ctx context.Context, - request *QuerySyncRequest, - response *QuerySyncResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "QuerySync") - defer span.Finish() - - apiURL := h.syncapiURL + SyncAPIQuerySyncPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} - -// QueryStateAfterEvents implements SyncQueryAPI -func (h *httpSyncQueryAPI) QueryState( - ctx context.Context, - request *QueryStateRequest, - response *QueryStateResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "QueryState") - defer span.Finish() - - apiURL := h.syncapiURL + SyncAPIQueryStatePath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} - -// QueryEventsByID implements SyncQueryAPI -func (h *httpSyncQueryAPI) QueryStateType( - ctx context.Context, - request *QueryStateTypeRequest, - response *QueryStateTypeResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "QueryStateType") - defer span.Finish() - - apiURL := h.syncapiURL + SyncAPIQueryStateTypePath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} - -// QueryMembershipForUser implements SyncQueryAPI -func (h *httpSyncQueryAPI) QueryMessages( - ctx context.Context, - request *QueryMessagesRequest, - response *QueryMessagesResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "QueryMessages") - defer span.Finish() - - apiURL := h.syncapiURL + SyncAPIQueryMessagesPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} From 2bd12f635cef281ab0d497c9e1bafe92247d88d5 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Thu, 4 Jun 2020 16:26:35 +0100 Subject: [PATCH 135/200] Convert serverkeys to inthttp (#1097) --- cmd/dendrite-client-api-server/main.go | 2 +- cmd/dendrite-federation-api-server/main.go | 2 +- cmd/dendrite-federation-sender-server/main.go | 2 +- cmd/dendrite-monolith-server/main.go | 2 +- cmd/dendrite-room-server/main.go | 2 +- internal/basecomponent/base.go | 8 +- serverkeyapi/api/api.go | 98 +------------ serverkeyapi/api/http.go | 57 -------- serverkeyapi/internal/api.go | 6 +- serverkeyapi/inthttp/client.go | 137 ++++++++++++++++++ .../{internal/http.go => inthttp/server.go} | 11 +- serverkeyapi/serverkeyapi.go | 6 +- 12 files changed, 164 insertions(+), 169 deletions(-) delete mode 100644 serverkeyapi/api/http.go create mode 100644 serverkeyapi/inthttp/client.go rename serverkeyapi/{internal/http.go => inthttp/server.go} (82%) diff --git a/cmd/dendrite-client-api-server/main.go b/cmd/dendrite-client-api-server/main.go index ba8e6af7c..2eec84ab8 100644 --- a/cmd/dendrite-client-api-server/main.go +++ b/cmd/dendrite-client-api-server/main.go @@ -32,7 +32,7 @@ func main() { deviceDB := base.CreateDeviceDB() federation := base.CreateFederationClient() - serverKeyAPI := base.CreateHTTPServerKeyAPIs() + serverKeyAPI := base.ServerKeyAPIClient() keyRing := serverKeyAPI.KeyRing() asQuery := base.AppserviceHTTPClient() diff --git a/cmd/dendrite-federation-api-server/main.go b/cmd/dendrite-federation-api-server/main.go index aca6ba579..604dd4e3d 100644 --- a/cmd/dendrite-federation-api-server/main.go +++ b/cmd/dendrite-federation-api-server/main.go @@ -31,7 +31,7 @@ func main() { deviceDB := base.CreateDeviceDB() federation := base.CreateFederationClient() - serverKeyAPI := base.CreateHTTPServerKeyAPIs() + serverKeyAPI := base.ServerKeyAPIClient() keyRing := serverKeyAPI.KeyRing() fsAPI := base.FederationSenderHTTPClient() diff --git a/cmd/dendrite-federation-sender-server/main.go b/cmd/dendrite-federation-sender-server/main.go index 14524dcad..aeaa837d1 100644 --- a/cmd/dendrite-federation-sender-server/main.go +++ b/cmd/dendrite-federation-sender-server/main.go @@ -26,7 +26,7 @@ func main() { federation := base.CreateFederationClient() - serverKeyAPI := base.CreateHTTPServerKeyAPIs() + serverKeyAPI := base.ServerKeyAPIClient() keyRing := serverKeyAPI.KeyRing() rsAPI := base.RoomserverHTTPClient() diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index ba9189256..78919f2b6 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -74,7 +74,7 @@ func main() { base, federation, ) if base.UseHTTPAPIs { - serverKeyAPI = base.CreateHTTPServerKeyAPIs() + serverKeyAPI = base.ServerKeyAPIClient() } keyRing := serverKeyAPI.KeyRing() diff --git a/cmd/dendrite-room-server/main.go b/cmd/dendrite-room-server/main.go index f4da560a6..65a1e9ae5 100644 --- a/cmd/dendrite-room-server/main.go +++ b/cmd/dendrite-room-server/main.go @@ -25,7 +25,7 @@ func main() { defer base.Close() // nolint: errcheck federation := base.CreateFederationClient() - serverKeyAPI := base.CreateHTTPServerKeyAPIs() + serverKeyAPI := base.ServerKeyAPIClient() keyRing := serverKeyAPI.KeyRing() fsAPI := base.FederationSenderHTTPClient() diff --git a/internal/basecomponent/base.go b/internal/basecomponent/base.go index cd297b82e..620b12d6a 100644 --- a/internal/basecomponent/base.go +++ b/internal/basecomponent/base.go @@ -45,6 +45,7 @@ import ( roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" rsinthttp "github.com/matrix-org/dendrite/roomserver/inthttp" serverKeyAPI "github.com/matrix-org/dendrite/serverkeyapi/api" + skinthttp "github.com/matrix-org/dendrite/serverkeyapi/inthttp" "github.com/sirupsen/logrus" _ "net/http/pprof" @@ -176,10 +177,9 @@ func (b *BaseDendrite) FederationSenderHTTPClient() federationSenderAPI.Federati return f } -// CreateHTTPServerKeyAPIs returns ServerKeyInternalAPI for hitting the server key -// API over HTTP -func (b *BaseDendrite) CreateHTTPServerKeyAPIs() serverKeyAPI.ServerKeyInternalAPI { - f, err := serverKeyAPI.NewServerKeyInternalAPIHTTP( +// ServerKeyAPIClient returns ServerKeyInternalAPI for hitting the server key API over HTTP +func (b *BaseDendrite) ServerKeyAPIClient() serverKeyAPI.ServerKeyInternalAPI { + f, err := skinthttp.NewServerKeyClient( b.Cfg.ServerKeyAPIURL(), b.httpClient, b.ImmutableCache, diff --git a/serverkeyapi/api/api.go b/serverkeyapi/api/api.go index a43f9c0a1..7af626345 100644 --- a/serverkeyapi/api/api.go +++ b/serverkeyapi/api/api.go @@ -2,11 +2,7 @@ package api import ( "context" - "errors" - "net/http" - "time" - "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/gomatrixserverlib" ) @@ -28,97 +24,17 @@ type ServerKeyInternalAPI interface { ) error } -// NewRoomserverInputAPIHTTP creates a RoomserverInputAPI implemented by talking to a HTTP POST API. -// If httpClient is nil an error is returned -func NewServerKeyInternalAPIHTTP( - serverKeyAPIURL string, - httpClient *http.Client, - immutableCache caching.ImmutableCache, -) (ServerKeyInternalAPI, error) { - if httpClient == nil { - return nil, errors.New("NewRoomserverInternalAPIHTTP: httpClient is ") - } - return &httpServerKeyInternalAPI{ - serverKeyAPIURL: serverKeyAPIURL, - httpClient: httpClient, - immutableCache: immutableCache, - }, nil +type QueryPublicKeysRequest struct { + Requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp `json:"requests"` } -type httpServerKeyInternalAPI struct { - ServerKeyInternalAPI - - serverKeyAPIURL string - httpClient *http.Client - immutableCache caching.ImmutableCache +type QueryPublicKeysResponse struct { + Results map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult `json:"results"` } -func (s *httpServerKeyInternalAPI) KeyRing() *gomatrixserverlib.KeyRing { - // This is a bit of a cheat - we tell gomatrixserverlib that this API is - // both the key database and the key fetcher. While this does have the - // rather unfortunate effect of preventing gomatrixserverlib from handling - // key fetchers directly, we can at least reimplement this behaviour on - // the other end of the API. - return &gomatrixserverlib.KeyRing{ - KeyDatabase: s, - KeyFetchers: []gomatrixserverlib.KeyFetcher{s}, - } +type InputPublicKeysRequest struct { + Keys map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult `json:"keys"` } -func (s *httpServerKeyInternalAPI) FetcherName() string { - return "httpServerKeyInternalAPI" -} - -func (s *httpServerKeyInternalAPI) StoreKeys( - _ context.Context, - results map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, -) error { - // Run in a background context - we don't want to stop this work just - // because the caller gives up waiting. - ctx := context.Background() - request := InputPublicKeysRequest{ - Keys: make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult), - } - response := InputPublicKeysResponse{} - for req, res := range results { - request.Keys[req] = res - s.immutableCache.StoreServerKey(req, res) - } - return s.InputPublicKeys(ctx, &request, &response) -} - -func (s *httpServerKeyInternalAPI) FetchKeys( - _ context.Context, - requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp, -) (map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, error) { - // Run in a background context - we don't want to stop this work just - // because the caller gives up waiting. - ctx := context.Background() - result := make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult) - request := QueryPublicKeysRequest{ - Requests: make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp), - } - response := QueryPublicKeysResponse{ - Results: make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult), - } - now := gomatrixserverlib.AsTimestamp(time.Now()) - for req, ts := range requests { - if res, ok := s.immutableCache.GetServerKey(req); ok { - if now > res.ValidUntilTS && res.ExpiredTS == gomatrixserverlib.PublicKeyNotExpired { - continue - } - result[req] = res - continue - } - request.Requests[req] = ts - } - err := s.QueryPublicKeys(ctx, &request, &response) - if err != nil { - return nil, err - } - for req, res := range response.Results { - result[req] = res - s.immutableCache.StoreServerKey(req, res) - } - return result, nil +type InputPublicKeysResponse struct { } diff --git a/serverkeyapi/api/http.go b/serverkeyapi/api/http.go deleted file mode 100644 index ab89f7bfc..000000000 --- a/serverkeyapi/api/http.go +++ /dev/null @@ -1,57 +0,0 @@ -package api - -import ( - "context" - - commonHTTP "github.com/matrix-org/dendrite/internal/http" - "github.com/matrix-org/gomatrixserverlib" - - "github.com/opentracing/opentracing-go" -) - -const ( - // ServerKeyInputPublicKeyPath is the HTTP path for the InputPublicKeys API. - ServerKeyInputPublicKeyPath = "/serverkeyapi/inputPublicKey" - - // ServerKeyQueryPublicKeyPath is the HTTP path for the QueryPublicKeys API. - ServerKeyQueryPublicKeyPath = "/serverkeyapi/queryPublicKey" -) - -type InputPublicKeysRequest struct { - Keys map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult `json:"keys"` -} - -type InputPublicKeysResponse struct { -} - -func (h *httpServerKeyInternalAPI) InputPublicKeys( - ctx context.Context, - request *InputPublicKeysRequest, - response *InputPublicKeysResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "InputPublicKey") - defer span.Finish() - - apiURL := h.serverKeyAPIURL + ServerKeyInputPublicKeyPath - return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} - -type QueryPublicKeysRequest struct { - Requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp `json:"requests"` -} - -type QueryPublicKeysResponse struct { - Results map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult `json:"results"` -} - -func (h *httpServerKeyInternalAPI) QueryPublicKeys( - ctx context.Context, - request *QueryPublicKeysRequest, - response *QueryPublicKeysResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "QueryPublicKey") - defer span.Finish() - - apiURL := h.serverKeyAPIURL + ServerKeyQueryPublicKeyPath - return commonHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} diff --git a/serverkeyapi/internal/api.go b/serverkeyapi/internal/api.go index c63b23f04..176983c86 100644 --- a/serverkeyapi/internal/api.go +++ b/serverkeyapi/internal/api.go @@ -5,7 +5,6 @@ import ( "fmt" "time" - "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/serverkeyapi/api" "github.com/matrix-org/gomatrixserverlib" ) @@ -13,9 +12,8 @@ import ( type ServerKeyAPI struct { api.ServerKeyInternalAPI - ImmutableCache caching.ImmutableCache - OurKeyRing gomatrixserverlib.KeyRing - FedClient *gomatrixserverlib.FederationClient + OurKeyRing gomatrixserverlib.KeyRing + FedClient *gomatrixserverlib.FederationClient } func (s *ServerKeyAPI) KeyRing() *gomatrixserverlib.KeyRing { diff --git a/serverkeyapi/inthttp/client.go b/serverkeyapi/inthttp/client.go new file mode 100644 index 000000000..f986634ba --- /dev/null +++ b/serverkeyapi/inthttp/client.go @@ -0,0 +1,137 @@ +package inthttp + +import ( + "context" + "errors" + "net/http" + "time" + + "github.com/matrix-org/dendrite/internal/caching" + internalHTTP "github.com/matrix-org/dendrite/internal/http" + "github.com/matrix-org/dendrite/serverkeyapi/api" + "github.com/matrix-org/gomatrixserverlib" + "github.com/opentracing/opentracing-go" +) + +// HTTP paths for the internal HTTP APIs +const ( + ServerKeyInputPublicKeyPath = "/serverkeyapi/inputPublicKey" + ServerKeyQueryPublicKeyPath = "/serverkeyapi/queryPublicKey" +) + +// NewServerKeyClient creates a ServerKeyInternalAPI implemented by talking to a HTTP POST API. +// If httpClient is nil an error is returned +func NewServerKeyClient( + serverKeyAPIURL string, + httpClient *http.Client, + immutableCache caching.ImmutableCache, +) (api.ServerKeyInternalAPI, error) { + if httpClient == nil { + return nil, errors.New("NewRoomserverInternalAPIHTTP: httpClient is ") + } + return &httpServerKeyInternalAPI{ + serverKeyAPIURL: serverKeyAPIURL, + httpClient: httpClient, + immutableCache: immutableCache, + }, nil +} + +type httpServerKeyInternalAPI struct { + serverKeyAPIURL string + httpClient *http.Client + immutableCache caching.ImmutableCache +} + +func (s *httpServerKeyInternalAPI) KeyRing() *gomatrixserverlib.KeyRing { + // This is a bit of a cheat - we tell gomatrixserverlib that this API is + // both the key database and the key fetcher. While this does have the + // rather unfortunate effect of preventing gomatrixserverlib from handling + // key fetchers directly, we can at least reimplement this behaviour on + // the other end of the API. + return &gomatrixserverlib.KeyRing{ + KeyDatabase: s, + KeyFetchers: []gomatrixserverlib.KeyFetcher{s}, + } +} + +func (s *httpServerKeyInternalAPI) FetcherName() string { + return "httpServerKeyInternalAPI" +} + +func (s *httpServerKeyInternalAPI) StoreKeys( + _ context.Context, + results map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, +) error { + // Run in a background context - we don't want to stop this work just + // because the caller gives up waiting. + ctx := context.Background() + request := api.InputPublicKeysRequest{ + Keys: make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult), + } + response := api.InputPublicKeysResponse{} + for req, res := range results { + request.Keys[req] = res + s.immutableCache.StoreServerKey(req, res) + } + return s.InputPublicKeys(ctx, &request, &response) +} + +func (s *httpServerKeyInternalAPI) FetchKeys( + _ context.Context, + requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp, +) (map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, error) { + // Run in a background context - we don't want to stop this work just + // because the caller gives up waiting. + ctx := context.Background() + result := make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult) + request := api.QueryPublicKeysRequest{ + Requests: make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp), + } + response := api.QueryPublicKeysResponse{ + Results: make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult), + } + now := gomatrixserverlib.AsTimestamp(time.Now()) + for req, ts := range requests { + if res, ok := s.immutableCache.GetServerKey(req); ok { + if now > res.ValidUntilTS && res.ExpiredTS == gomatrixserverlib.PublicKeyNotExpired { + continue + } + result[req] = res + continue + } + request.Requests[req] = ts + } + err := s.QueryPublicKeys(ctx, &request, &response) + if err != nil { + return nil, err + } + for req, res := range response.Results { + result[req] = res + s.immutableCache.StoreServerKey(req, res) + } + return result, nil +} + +func (h *httpServerKeyInternalAPI) InputPublicKeys( + ctx context.Context, + request *api.InputPublicKeysRequest, + response *api.InputPublicKeysResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "InputPublicKey") + defer span.Finish() + + apiURL := h.serverKeyAPIURL + ServerKeyInputPublicKeyPath + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +func (h *httpServerKeyInternalAPI) QueryPublicKeys( + ctx context.Context, + request *api.QueryPublicKeysRequest, + response *api.QueryPublicKeysResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "QueryPublicKey") + defer span.Finish() + + apiURL := h.serverKeyAPIURL + ServerKeyQueryPublicKeyPath + return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} diff --git a/serverkeyapi/internal/http.go b/serverkeyapi/inthttp/server.go similarity index 82% rename from serverkeyapi/internal/http.go rename to serverkeyapi/inthttp/server.go index eef66b76c..d5517e143 100644 --- a/serverkeyapi/internal/http.go +++ b/serverkeyapi/inthttp/server.go @@ -1,4 +1,4 @@ -package internal +package inthttp import ( "encoding/json" @@ -6,13 +6,14 @@ import ( "github.com/gorilla/mux" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/serverkeyapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) -func (s *ServerKeyAPI) SetupHTTP(internalAPIMux *mux.Router) { - internalAPIMux.Handle(api.ServerKeyQueryPublicKeyPath, +func AddRoutes(s api.ServerKeyInternalAPI, internalAPIMux *mux.Router, cache caching.ImmutableCache) { + internalAPIMux.Handle(ServerKeyQueryPublicKeyPath, internal.MakeInternalAPI("queryPublicKeys", func(req *http.Request) util.JSONResponse { request := api.QueryPublicKeysRequest{} response := api.QueryPublicKeysResponse{} @@ -27,7 +28,7 @@ func (s *ServerKeyAPI) SetupHTTP(internalAPIMux *mux.Router) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - internalAPIMux.Handle(api.ServerKeyInputPublicKeyPath, + internalAPIMux.Handle(ServerKeyInputPublicKeyPath, internal.MakeInternalAPI("inputPublicKeys", func(req *http.Request) util.JSONResponse { request := api.InputPublicKeysRequest{} response := api.InputPublicKeysResponse{} @@ -37,7 +38,7 @@ func (s *ServerKeyAPI) SetupHTTP(internalAPIMux *mux.Router) { store := make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult) for req, res := range request.Keys { store[req] = res - s.ImmutableCache.StoreServerKey(req, res) + cache.StoreServerKey(req, res) } if err := s.StoreKeys(req.Context(), store); err != nil { return util.ErrorResponse(err) diff --git a/serverkeyapi/serverkeyapi.go b/serverkeyapi/serverkeyapi.go index 61d807d64..5bf8f67d7 100644 --- a/serverkeyapi/serverkeyapi.go +++ b/serverkeyapi/serverkeyapi.go @@ -7,6 +7,7 @@ import ( "github.com/matrix-org/dendrite/internal/basecomponent" "github.com/matrix-org/dendrite/serverkeyapi/api" "github.com/matrix-org/dendrite/serverkeyapi/internal" + "github.com/matrix-org/dendrite/serverkeyapi/inthttp" "github.com/matrix-org/dendrite/serverkeyapi/storage" "github.com/matrix-org/dendrite/serverkeyapi/storage/cache" "github.com/matrix-org/gomatrixserverlib" @@ -34,8 +35,7 @@ func SetupServerKeyAPIComponent( } internalAPI := internal.ServerKeyAPI{ - ImmutableCache: base.ImmutableCache, - FedClient: fedClient, + FedClient: fedClient, OurKeyRing: gomatrixserverlib.KeyRing{ KeyFetchers: []gomatrixserverlib.KeyFetcher{ &gomatrixserverlib.DirectKeyFetcher{ @@ -77,7 +77,7 @@ func SetupServerKeyAPIComponent( }).Info("Enabled perspective key fetcher") } - internalAPI.SetupHTTP(base.InternalAPIMux) + inthttp.AddRoutes(&internalAPI, base.InternalAPIMux, base.ImmutableCache) return &internalAPI } From 29a20d1da7a7f4a04d1e456d72e49424d45d5f4c Mon Sep 17 00:00:00 2001 From: Kegsay Date: Fri, 5 Jun 2020 09:28:15 +0100 Subject: [PATCH 136/200] General cleanup when making components (#1098) * Remove ParseMonolith/LoadMonolith * cleanup which components need to be made --- cmd/dendrite-appservice-server/main.go | 2 +- cmd/dendrite-client-api-server/main.go | 7 ++--- cmd/dendrite-edu-server/main.go | 2 +- cmd/dendrite-federation-api-server/main.go | 12 ++------ cmd/dendrite-federation-sender-server/main.go | 5 ++-- cmd/dendrite-key-server/main.go | 2 +- cmd/dendrite-media-api-server/main.go | 2 +- cmd/dendrite-monolith-server/main.go | 2 +- cmd/dendrite-public-rooms-api-server/main.go | 4 +-- cmd/dendrite-room-server/main.go | 2 +- cmd/dendrite-server-key-api-server/main.go | 2 +- cmd/dendrite-sync-api-server/main.go | 2 +- internal/basecomponent/flags.go | 23 ++------------- internal/config/config.go | 28 ++----------------- roomserver/inthttp/client.go | 6 +--- 15 files changed, 22 insertions(+), 79 deletions(-) diff --git a/cmd/dendrite-appservice-server/main.go b/cmd/dendrite-appservice-server/main.go index c16505fd8..4fc2a173d 100644 --- a/cmd/dendrite-appservice-server/main.go +++ b/cmd/dendrite-appservice-server/main.go @@ -21,7 +21,7 @@ import ( ) func main() { - cfg := basecomponent.ParseFlags() + cfg := basecomponent.ParseFlags(false) base := basecomponent.NewBaseDendrite(cfg, "AppServiceAPI", true) defer base.Close() // nolint: errcheck diff --git a/cmd/dendrite-client-api-server/main.go b/cmd/dendrite-client-api-server/main.go index 2eec84ab8..8ad4231d1 100644 --- a/cmd/dendrite-client-api-server/main.go +++ b/cmd/dendrite-client-api-server/main.go @@ -16,14 +16,12 @@ package main import ( "github.com/matrix-org/dendrite/clientapi" - "github.com/matrix-org/dendrite/eduserver" - "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/internal/basecomponent" "github.com/matrix-org/dendrite/internal/transactions" ) func main() { - cfg := basecomponent.ParseFlags() + cfg := basecomponent.ParseFlags(false) base := basecomponent.NewBaseDendrite(cfg, "ClientAPI", true) defer base.Close() // nolint: errcheck @@ -38,8 +36,7 @@ func main() { asQuery := base.AppserviceHTTPClient() rsAPI := base.RoomserverHTTPClient() fsAPI := base.FederationSenderHTTPClient() - rsAPI.SetFederationSenderAPI(fsAPI) - eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New(), deviceDB) + eduInputAPI := base.EDUServerClient() clientapi.SetupClientAPIComponent( base, deviceDB, accountDB, federation, keyRing, diff --git a/cmd/dendrite-edu-server/main.go b/cmd/dendrite-edu-server/main.go index ca0460f8d..ceeb13658 100644 --- a/cmd/dendrite-edu-server/main.go +++ b/cmd/dendrite-edu-server/main.go @@ -22,7 +22,7 @@ import ( ) func main() { - cfg := basecomponent.ParseFlags() + cfg := basecomponent.ParseFlags(false) base := basecomponent.NewBaseDendrite(cfg, "EDUServerAPI", true) defer func() { if err := base.Close(); err != nil { diff --git a/cmd/dendrite-federation-api-server/main.go b/cmd/dendrite-federation-api-server/main.go index 604dd4e3d..0c4d8b854 100644 --- a/cmd/dendrite-federation-api-server/main.go +++ b/cmd/dendrite-federation-api-server/main.go @@ -16,31 +16,25 @@ package main import ( "github.com/matrix-org/dendrite/clientapi/producers" - "github.com/matrix-org/dendrite/eduserver" - "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/internal/basecomponent" ) func main() { - cfg := basecomponent.ParseFlags() + cfg := basecomponent.ParseFlags(false) base := basecomponent.NewBaseDendrite(cfg, "FederationAPI", true) defer base.Close() // nolint: errcheck accountDB := base.CreateAccountsDB() deviceDB := base.CreateDeviceDB() federation := base.CreateFederationClient() - serverKeyAPI := base.ServerKeyAPIClient() keyRing := serverKeyAPI.KeyRing() - fsAPI := base.FederationSenderHTTPClient() - rsAPI := base.RoomserverHTTPClient() asAPI := base.AppserviceHTTPClient() - rsAPI.SetFederationSenderAPI(fsAPI) - eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New(), deviceDB) - eduProducer := producers.NewEDUServerProducer(eduInputAPI) + // TODO: this isn't a producer + eduProducer := producers.NewEDUServerProducer(base.EDUServerClient()) federationapi.SetupFederationAPIComponent( base, accountDB, deviceDB, federation, keyRing, diff --git a/cmd/dendrite-federation-sender-server/main.go b/cmd/dendrite-federation-sender-server/main.go index aeaa837d1..651eb257f 100644 --- a/cmd/dendrite-federation-sender-server/main.go +++ b/cmd/dendrite-federation-sender-server/main.go @@ -20,7 +20,7 @@ import ( ) func main() { - cfg := basecomponent.ParseFlags() + cfg := basecomponent.ParseFlags(false) base := basecomponent.NewBaseDendrite(cfg, "FederationSender", true) defer base.Close() // nolint: errcheck @@ -30,10 +30,9 @@ func main() { keyRing := serverKeyAPI.KeyRing() rsAPI := base.RoomserverHTTPClient() - fsAPI := federationsender.SetupFederationSenderComponent( + federationsender.SetupFederationSenderComponent( base, federation, rsAPI, keyRing, ) - rsAPI.SetFederationSenderAPI(fsAPI) base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationSender), string(base.Cfg.Listen.FederationSender)) diff --git a/cmd/dendrite-key-server/main.go b/cmd/dendrite-key-server/main.go index 17fee2c64..a8accbcb3 100644 --- a/cmd/dendrite-key-server/main.go +++ b/cmd/dendrite-key-server/main.go @@ -20,7 +20,7 @@ import ( ) func main() { - cfg := basecomponent.ParseFlags() + cfg := basecomponent.ParseFlags(false) base := basecomponent.NewBaseDendrite(cfg, "KeyServer", true) defer base.Close() // nolint: errcheck diff --git a/cmd/dendrite-media-api-server/main.go b/cmd/dendrite-media-api-server/main.go index 1a51717a7..018559383 100644 --- a/cmd/dendrite-media-api-server/main.go +++ b/cmd/dendrite-media-api-server/main.go @@ -20,7 +20,7 @@ import ( ) func main() { - cfg := basecomponent.ParseFlags() + cfg := basecomponent.ParseFlags(false) base := basecomponent.NewBaseDendrite(cfg, "MediaAPI", true) defer base.Close() // nolint: errcheck diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 78919f2b6..57e8f66f5 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -49,7 +49,7 @@ var ( ) func main() { - cfg := basecomponent.ParseMonolithFlags() + cfg := basecomponent.ParseFlags(true) if *enableHTTPAPIs { // If the HTTP APIs are enabled then we need to update the Listen // statements in the configuration so that we know where to find diff --git a/cmd/dendrite-public-rooms-api-server/main.go b/cmd/dendrite-public-rooms-api-server/main.go index 18a813baf..0cad3670c 100644 --- a/cmd/dendrite-public-rooms-api-server/main.go +++ b/cmd/dendrite-public-rooms-api-server/main.go @@ -22,15 +22,13 @@ import ( ) func main() { - cfg := basecomponent.ParseFlags() + cfg := basecomponent.ParseFlags(false) base := basecomponent.NewBaseDendrite(cfg, "PublicRoomsAPI", true) defer base.Close() // nolint: errcheck deviceDB := base.CreateDeviceDB() - fsAPI := base.FederationSenderHTTPClient() rsAPI := base.RoomserverHTTPClient() - rsAPI.SetFederationSenderAPI(fsAPI) publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties(), cfg.Matrix.ServerName) if err != nil { diff --git a/cmd/dendrite-room-server/main.go b/cmd/dendrite-room-server/main.go index 65a1e9ae5..79745cded 100644 --- a/cmd/dendrite-room-server/main.go +++ b/cmd/dendrite-room-server/main.go @@ -20,7 +20,7 @@ import ( ) func main() { - cfg := basecomponent.ParseFlags() + cfg := basecomponent.ParseFlags(false) base := basecomponent.NewBaseDendrite(cfg, "RoomServerAPI", true) defer base.Close() // nolint: errcheck federation := base.CreateFederationClient() diff --git a/cmd/dendrite-server-key-api-server/main.go b/cmd/dendrite-server-key-api-server/main.go index 075734125..e206978cd 100644 --- a/cmd/dendrite-server-key-api-server/main.go +++ b/cmd/dendrite-server-key-api-server/main.go @@ -20,7 +20,7 @@ import ( ) func main() { - cfg := basecomponent.ParseFlags() + cfg := basecomponent.ParseFlags(false) base := basecomponent.NewBaseDendrite(cfg, "ServerKeyAPI", true) defer base.Close() // nolint: errcheck diff --git a/cmd/dendrite-sync-api-server/main.go b/cmd/dendrite-sync-api-server/main.go index eeab115a9..bb56b4ab0 100644 --- a/cmd/dendrite-sync-api-server/main.go +++ b/cmd/dendrite-sync-api-server/main.go @@ -20,7 +20,7 @@ import ( ) func main() { - cfg := basecomponent.ParseFlags() + cfg := basecomponent.ParseFlags(false) base := basecomponent.NewBaseDendrite(cfg, "SyncAPI", true) defer base.Close() // nolint: errcheck diff --git a/internal/basecomponent/flags.go b/internal/basecomponent/flags.go index ecef81f6b..117df0796 100644 --- a/internal/basecomponent/flags.go +++ b/internal/basecomponent/flags.go @@ -25,33 +25,14 @@ import ( var configPath = flag.String("config", "dendrite.yaml", "The path to the config file. For more information, see the config file in this repository.") // ParseFlags parses the commandline flags and uses them to create a config. -// If running as a monolith use `ParseMonolithFlags` instead. -func ParseFlags() *config.Dendrite { +func ParseFlags(monolith bool) *config.Dendrite { flag.Parse() if *configPath == "" { logrus.Fatal("--config must be supplied") } - cfg, err := config.Load(*configPath) - - if err != nil { - logrus.Fatalf("Invalid config file: %s", err) - } - - return cfg -} - -// ParseMonolithFlags parses the commandline flags and uses them to create a -// config. Should only be used if running a monolith. See `ParseFlags`. -func ParseMonolithFlags() *config.Dendrite { - flag.Parse() - - if *configPath == "" { - logrus.Fatal("--config must be supplied") - } - - cfg, err := config.LoadMonolithic(*configPath) + cfg, err := config.Load(*configPath, monolith) if err != nil { logrus.Fatalf("Invalid config file: %s", err) diff --git a/internal/config/config.go b/internal/config/config.go index 597995855..858646ab0 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -370,11 +370,9 @@ type LogrusHook struct { // It implements the error interface. type configErrors []string -// Load a yaml config file for a server run as multiple processes. +// Load a yaml config file for a server run as multiple processes or as a monolith. // Checks the config to ensure that it is valid. -// The checks are different if the server is run as a monolithic process instead -// of being split into multiple components -func Load(configPath string) (*Dendrite, error) { +func Load(configPath string, monolith bool) (*Dendrite, error) { configData, err := ioutil.ReadFile(configPath) if err != nil { return nil, err @@ -385,27 +383,7 @@ func Load(configPath string) (*Dendrite, error) { } // Pass the current working directory and ioutil.ReadFile so that they can // be mocked in the tests - monolithic := false - return loadConfig(basePath, configData, ioutil.ReadFile, monolithic) -} - -// LoadMonolithic loads a yaml config file for a server run as a single monolith. -// Checks the config to ensure that it is valid. -// The checks are different if the server is run as a monolithic process instead -// of being split into multiple components -func LoadMonolithic(configPath string) (*Dendrite, error) { - configData, err := ioutil.ReadFile(configPath) - if err != nil { - return nil, err - } - basePath, err := filepath.Abs(".") - if err != nil { - return nil, err - } - // Pass the current working directory and ioutil.ReadFile so that they can - // be mocked in the tests - monolithic := true - return loadConfig(basePath, configData, ioutil.ReadFile, monolithic) + return loadConfig(basePath, configData, ioutil.ReadFile, monolith) } func loadConfig( diff --git a/roomserver/inthttp/client.go b/roomserver/inthttp/client.go index f8f922fbd..99db9e1e7 100644 --- a/roomserver/inthttp/client.go +++ b/roomserver/inthttp/client.go @@ -45,7 +45,6 @@ const ( type httpRoomserverInternalAPI struct { roomserverURL string httpClient *http.Client - fsAPI fsInputAPI.FederationSenderInternalAPI immutableCache caching.ImmutableCache } @@ -66,11 +65,8 @@ func NewRoomserverClient( }, nil } -// SetFederationSenderInputAPI passes in a federation sender input API reference -// so that we can avoid the chicken-and-egg problem of both the roomserver input API -// and the federation sender input API being interdependent. +// SetFederationSenderInputAPI no-ops in HTTP client mode as there is no chicken/egg scenario func (h *httpRoomserverInternalAPI) SetFederationSenderAPI(fsAPI fsInputAPI.FederationSenderInternalAPI) { - h.fsAPI = fsAPI } // SetRoomAlias implements RoomserverAliasAPI From 76ff47c0522e03eabd72140bf62e1d0d1d1029e0 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 5 Jun 2020 11:48:52 +0100 Subject: [PATCH 137/200] Use AuthChainProvider to try and speed up federated joins (#1100) * Use MissingAuthEventHandler on performjoin to try and speed up cases where we have missing events * Update gomatrixserverlib * Use supplied room version * Use AuthChainProvider * Tweaks * Update gomatrixserverlib * Signature checks --- federationapi/routing/send.go | 2 +- federationsender/internal/perform/join.go | 92 ++++++++++++++++------- go.mod | 2 +- go.sum | 4 +- 4 files changed, 67 insertions(+), 33 deletions(-) diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index 74b4c0144..a2120d81b 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -675,7 +675,7 @@ func (t *txnReq) lookupMissingStateViaState(roomID, eventID string, roomVersion return nil, err } // Check that the returned state is valid. - if err := state.Check(t.context, t.keys); err != nil { + if err := state.Check(t.context, t.keys, nil); err != nil { return nil, err } return &state, nil diff --git a/federationsender/internal/perform/join.go b/federationsender/internal/perform/join.go index 3c7ef0765..9a505d15b 100644 --- a/federationsender/internal/perform/join.go +++ b/federationsender/internal/perform/join.go @@ -33,38 +33,72 @@ func (r joinContext) CheckSendJoinResponse( ) error { // A list of events that we have retried, if they were not included in // the auth events supplied in the send_join. - retries := map[string]bool{} + retries := map[string][]gomatrixserverlib.Event{} + + // Define a function which we can pass to Check to retrieve missing + // auth events inline. This greatly increases our chances of not having + // to repeat the entire set of checks just for a missing event or two. + missingAuth := func(roomVersion gomatrixserverlib.RoomVersion, eventIDs []string) ([]gomatrixserverlib.Event, error) { + returning := []gomatrixserverlib.Event{} + + // See if we have retry entries for each of the supplied event IDs. + for _, eventID := range eventIDs { + // If we've already satisfied a request for this event ID before then + // just append the results. We won't retry the request. + if retry, ok := retries[eventID]; ok { + if retry == nil { + return nil, fmt.Errorf("missingAuth: not retrying failed event ID %q", eventID) + } + returning = append(returning, retry...) + continue + } + + // Make a note of the fact that we tried to do something with this + // event ID, even if we don't succeed. + retries[event.EventID()] = nil + + // Try to retrieve the event from the server that sent us the send + // join response. + tx, txerr := r.federation.GetEvent(ctx, server, eventID) + if txerr != nil { + return nil, fmt.Errorf("missingAuth r.federation.GetEvent: %w", txerr) + } + + // For each event returned, add it to the set of return events. We + // also will populate the retries, in case someone asks for this + // event ID again. + for _, pdu := range tx.PDUs { + // Try to parse the event. + ev, everr := gomatrixserverlib.NewEventFromUntrustedJSON(pdu, roomVersion) + if everr != nil { + return nil, fmt.Errorf("missingAuth gomatrixserverlib.NewEventFromUntrustedJSON: %w", everr) + } + + // Check the signatures of the event. + if res, err := gomatrixserverlib.VerifyEventSignatures(ctx, []gomatrixserverlib.Event{ev}, r.keyRing); err != nil { + return nil, fmt.Errorf("missingAuth VerifyEventSignatures: %w", err) + } else { + for _, err := range res { + if err != nil { + return nil, fmt.Errorf("missingAuth VerifyEventSignatures: %w", err) + } + } + } + + // If the event is OK then add it to the results and the retry map. + returning = append(returning, ev) + retries[event.EventID()] = append(retries[event.EventID()], ev) + retries[ev.EventID()] = append(retries[ev.EventID()], ev) + } + } + + return returning, nil + } -retryCheck: // TODO: Can we expand Check here to return a list of missing auth // events rather than failing one at a time? - if err := respSendJoin.Check(ctx, r.keyRing, event); err != nil { - switch e := err.(type) { - case gomatrixserverlib.MissingAuthEventError: - // Check that we haven't already retried for this event, prevents - // us from ending up in endless loops - if !retries[e.AuthEventID] { - // Ask the server that we're talking to right now for the event - tx, txerr := r.federation.GetEvent(ctx, server, e.AuthEventID) - if txerr != nil { - return fmt.Errorf("r.federation.GetEvent: %w", txerr) - } - // For each event returned, add it to the auth events. - for _, pdu := range tx.PDUs { - ev, everr := gomatrixserverlib.NewEventFromUntrustedJSON(pdu, respMakeJoin.RoomVersion) - if everr != nil { - return fmt.Errorf("gomatrixserverlib.NewEventFromUntrustedJSON: %w", everr) - } - respSendJoin.AuthEvents = append(respSendJoin.AuthEvents, ev) - } - // Mark the event as retried and then give the check another go. - retries[e.AuthEventID] = true - goto retryCheck - } - return fmt.Errorf("respSendJoin (after retries): %w", e) - default: - return fmt.Errorf("respSendJoin: %w", err) - } + if err := respSendJoin.Check(ctx, r.keyRing, event, missingAuth); err != nil { + return fmt.Errorf("respSendJoin: %w", err) } return nil } diff --git a/go.mod b/go.mod index a875ef7c9..61d7358de 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200604122934-a06b83e7f324 + github.com/matrix-org/gomatrixserverlib v0.0.0-20200605101319-57093b526cb1 github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible diff --git a/go.sum b/go.sum index 94101251b..fa3a65431 100644 --- a/go.sum +++ b/go.sum @@ -356,8 +356,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 h1:Yb+Wlf github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bhrnp3Ky1qgx/fzCtCALOoGYylh2tpS9K4= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200604122934-a06b83e7f324 h1:TjHTrOkX+G99F1NitCwNOPa62beQMAQklBuEBkDP8Hc= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200604122934-a06b83e7f324/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200605101319-57093b526cb1 h1:T9nY1X24uHDkC3uMLwoo2W+JE2o0t3k/lYwqORfKSUg= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200605101319-57093b526cb1/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f/go.mod h1:y0oDTjZDv5SM9a2rp3bl+CU+bvTRINQsdb7YlDql5Go= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= From e7b19d2c70be49f3c995a9bfd9dd93ce767d960f Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 5 Jun 2020 16:42:01 +0100 Subject: [PATCH 138/200] More flexible caching (#1101) --- cmd/roomserver-integration-tests/main.go | 2 +- internal/basecomponent/base.go | 10 +-- internal/caching/cache_roomversions.go | 30 ++++++++ internal/caching/cache_serverkeys.go | 41 ++++++++++ internal/caching/caches.go | 15 ++++ internal/caching/immutablecache.go | 17 ----- internal/caching/immutableinmemorylru.go | 95 ------------------------ internal/caching/impl_inmemorylru.go | 73 ++++++++++++++++++ roomserver/internal/api.go | 2 +- roomserver/internal/query.go | 4 +- roomserver/inthttp/client.go | 18 ++--- roomserver/roomserver.go | 2 +- serverkeyapi/inthttp/client.go | 12 +-- serverkeyapi/inthttp/server.go | 2 +- serverkeyapi/serverkeyapi.go | 4 +- serverkeyapi/storage/cache/keydb.go | 4 +- 16 files changed, 189 insertions(+), 142 deletions(-) create mode 100644 internal/caching/cache_roomversions.go create mode 100644 internal/caching/cache_serverkeys.go create mode 100644 internal/caching/caches.go delete mode 100644 internal/caching/immutablecache.go delete mode 100644 internal/caching/immutableinmemorylru.go create mode 100644 internal/caching/impl_inmemorylru.go diff --git a/cmd/roomserver-integration-tests/main.go b/cmd/roomserver-integration-tests/main.go index 2433f9ff0..43aca0789 100644 --- a/cmd/roomserver-integration-tests/main.go +++ b/cmd/roomserver-integration-tests/main.go @@ -255,7 +255,7 @@ func testRoomserver(input []string, wantOutput []string, checkQueries func(api.R panic(err) } - cache, err := caching.NewImmutableInMemoryLRUCache() + cache, err := caching.NewInMemoryLRUCache() if err != nil { panic(err) } diff --git a/internal/basecomponent/base.go b/internal/basecomponent/base.go index 620b12d6a..3ad1e4af2 100644 --- a/internal/basecomponent/base.go +++ b/internal/basecomponent/base.go @@ -66,7 +66,7 @@ type BaseDendrite struct { UseHTTPAPIs bool httpClient *http.Client Cfg *config.Dendrite - ImmutableCache caching.ImmutableCache + Caches *caching.Caches KafkaConsumer sarama.Consumer KafkaProducer sarama.SyncProducer } @@ -95,7 +95,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo kafkaConsumer, kafkaProducer = setupKafka(cfg) } - cache, err := caching.NewImmutableInMemoryLRUCache() + cache, err := caching.NewInMemoryLRUCache() if err != nil { logrus.WithError(err).Warnf("Failed to create cache") } @@ -126,7 +126,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo UseHTTPAPIs: useHTTPAPIs, tracerCloser: closer, Cfg: cfg, - ImmutableCache: cache, + Caches: cache, PublicAPIMux: httpmux.PathPrefix(httpapis.PublicPathPrefix).Subrouter().UseEncodedPath(), InternalAPIMux: httpmux.PathPrefix(httpapis.InternalPathPrefix).Subrouter().UseEncodedPath(), httpClient: &client, @@ -151,7 +151,7 @@ func (b *BaseDendrite) AppserviceHTTPClient() appserviceAPI.AppServiceQueryAPI { // RoomserverHTTPClient returns RoomserverInternalAPI for hitting the roomserver over HTTP. func (b *BaseDendrite) RoomserverHTTPClient() roomserverAPI.RoomserverInternalAPI { - rsAPI, err := rsinthttp.NewRoomserverClient(b.Cfg.RoomServerURL(), b.httpClient, b.ImmutableCache) + rsAPI, err := rsinthttp.NewRoomserverClient(b.Cfg.RoomServerURL(), b.httpClient, b.Caches) if err != nil { logrus.WithError(err).Panic("RoomserverHTTPClient failed", b.httpClient) } @@ -182,7 +182,7 @@ func (b *BaseDendrite) ServerKeyAPIClient() serverKeyAPI.ServerKeyInternalAPI { f, err := skinthttp.NewServerKeyClient( b.Cfg.ServerKeyAPIURL(), b.httpClient, - b.ImmutableCache, + b.Caches, ) if err != nil { logrus.WithError(err).Panic("NewServerKeyInternalAPIHTTP failed", b.httpClient) diff --git a/internal/caching/cache_roomversions.go b/internal/caching/cache_roomversions.go new file mode 100644 index 000000000..0b46d3d4b --- /dev/null +++ b/internal/caching/cache_roomversions.go @@ -0,0 +1,30 @@ +package caching + +import "github.com/matrix-org/gomatrixserverlib" + +const ( + RoomVersionCacheName = "room_versions" + RoomVersionCacheMaxEntries = 1024 + RoomVersionCacheMutable = false +) + +// RoomVersionsCache contains the subset of functions needed for +// a room version cache. +type RoomVersionCache interface { + GetRoomVersion(roomID string) (roomVersion gomatrixserverlib.RoomVersion, ok bool) + StoreRoomVersion(roomID string, roomVersion gomatrixserverlib.RoomVersion) +} + +func (c Caches) GetRoomVersion(roomID string) (gomatrixserverlib.RoomVersion, bool) { + val, found := c.RoomVersions.Get(roomID) + if found && val != nil { + if roomVersion, ok := val.(gomatrixserverlib.RoomVersion); ok { + return roomVersion, true + } + } + return "", false +} + +func (c Caches) StoreRoomVersion(roomID string, roomVersion gomatrixserverlib.RoomVersion) { + c.RoomVersions.Set(roomID, roomVersion) +} diff --git a/internal/caching/cache_serverkeys.go b/internal/caching/cache_serverkeys.go new file mode 100644 index 000000000..8c71ffbd9 --- /dev/null +++ b/internal/caching/cache_serverkeys.go @@ -0,0 +1,41 @@ +package caching + +import ( + "fmt" + + "github.com/matrix-org/gomatrixserverlib" +) + +const ( + ServerKeyCacheName = "server_key" + ServerKeyCacheMaxEntries = 4096 + ServerKeyCacheMutable = true +) + +// ServerKeyCache contains the subset of functions needed for +// a server key cache. +type ServerKeyCache interface { + GetServerKey(request gomatrixserverlib.PublicKeyLookupRequest) (response gomatrixserverlib.PublicKeyLookupResult, ok bool) + StoreServerKey(request gomatrixserverlib.PublicKeyLookupRequest, response gomatrixserverlib.PublicKeyLookupResult) +} + +func (c Caches) GetServerKey( + request gomatrixserverlib.PublicKeyLookupRequest, +) (gomatrixserverlib.PublicKeyLookupResult, bool) { + key := fmt.Sprintf("%s/%s", request.ServerName, request.KeyID) + val, found := c.ServerKeys.Get(key) + if found && val != nil { + if keyLookupResult, ok := val.(gomatrixserverlib.PublicKeyLookupResult); ok { + return keyLookupResult, true + } + } + return gomatrixserverlib.PublicKeyLookupResult{}, false +} + +func (c Caches) StoreServerKey( + request gomatrixserverlib.PublicKeyLookupRequest, + response gomatrixserverlib.PublicKeyLookupResult, +) { + key := fmt.Sprintf("%s/%s", request.ServerName, request.KeyID) + c.ServerKeys.Set(key, response) +} diff --git a/internal/caching/caches.go b/internal/caching/caches.go new file mode 100644 index 000000000..70f380ba5 --- /dev/null +++ b/internal/caching/caches.go @@ -0,0 +1,15 @@ +package caching + +// Caches contains a set of references to caches. They may be +// different implementations as long as they satisfy the Cache +// interface. +type Caches struct { + RoomVersions Cache // implements RoomVersionCache + ServerKeys Cache // implements ServerKeyCache +} + +// Cache is the interface that an implementation must satisfy. +type Cache interface { + Get(key string) (value interface{}, ok bool) + Set(key string, value interface{}) +} diff --git a/internal/caching/immutablecache.go b/internal/caching/immutablecache.go deleted file mode 100644 index fea05dd1b..000000000 --- a/internal/caching/immutablecache.go +++ /dev/null @@ -1,17 +0,0 @@ -package caching - -import ( - "github.com/matrix-org/gomatrixserverlib" -) - -const ( - RoomVersionMaxCacheEntries = 1024 - ServerKeysMaxCacheEntries = 1024 -) - -type ImmutableCache interface { - GetRoomVersion(roomId string) (gomatrixserverlib.RoomVersion, bool) - StoreRoomVersion(roomId string, roomVersion gomatrixserverlib.RoomVersion) - GetServerKey(request gomatrixserverlib.PublicKeyLookupRequest) (gomatrixserverlib.PublicKeyLookupResult, bool) - StoreServerKey(request gomatrixserverlib.PublicKeyLookupRequest, response gomatrixserverlib.PublicKeyLookupResult) -} diff --git a/internal/caching/immutableinmemorylru.go b/internal/caching/immutableinmemorylru.go deleted file mode 100644 index 36cd56dc2..000000000 --- a/internal/caching/immutableinmemorylru.go +++ /dev/null @@ -1,95 +0,0 @@ -package caching - -import ( - "fmt" - - lru "github.com/hashicorp/golang-lru" - "github.com/matrix-org/gomatrixserverlib" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" -) - -type ImmutableInMemoryLRUCache struct { - roomVersions *lru.Cache - serverKeys *lru.Cache -} - -func NewImmutableInMemoryLRUCache() (*ImmutableInMemoryLRUCache, error) { - roomVersionCache, rvErr := lru.New(RoomVersionMaxCacheEntries) - if rvErr != nil { - return nil, rvErr - } - serverKeysCache, rvErr := lru.New(ServerKeysMaxCacheEntries) - if rvErr != nil { - return nil, rvErr - } - cache := &ImmutableInMemoryLRUCache{ - roomVersions: roomVersionCache, - serverKeys: serverKeysCache, - } - cache.configureMetrics() - return cache, nil -} - -func (c *ImmutableInMemoryLRUCache) configureMetrics() { - promauto.NewGaugeFunc(prometheus.GaugeOpts{ - Namespace: "dendrite", - Subsystem: "caching", - Name: "number_room_version_entries", - Help: "The number of room version entries cached.", - }, func() float64 { - return float64(c.roomVersions.Len()) - }) - - promauto.NewGaugeFunc(prometheus.GaugeOpts{ - Namespace: "dendrite", - Subsystem: "caching", - Name: "number_server_key_entries", - Help: "The number of server key entries cached.", - }, func() float64 { - return float64(c.serverKeys.Len()) - }) -} - -func checkForInvalidMutation(cache *lru.Cache, key string, value interface{}) { - if peek, ok := cache.Peek(key); ok && peek != value { - panic(fmt.Sprintf("invalid use of immutable cache tries to mutate existing value of %q", key)) - } -} - -func (c *ImmutableInMemoryLRUCache) GetRoomVersion(roomID string) (gomatrixserverlib.RoomVersion, bool) { - val, found := c.roomVersions.Get(roomID) - if found && val != nil { - if roomVersion, ok := val.(gomatrixserverlib.RoomVersion); ok { - return roomVersion, true - } - } - return "", false -} - -func (c *ImmutableInMemoryLRUCache) StoreRoomVersion(roomID string, roomVersion gomatrixserverlib.RoomVersion) { - checkForInvalidMutation(c.roomVersions, roomID, roomVersion) - c.roomVersions.Add(roomID, roomVersion) -} - -func (c *ImmutableInMemoryLRUCache) GetServerKey( - request gomatrixserverlib.PublicKeyLookupRequest, -) (gomatrixserverlib.PublicKeyLookupResult, bool) { - key := fmt.Sprintf("%s/%s", request.ServerName, request.KeyID) - val, found := c.serverKeys.Get(key) - if found && val != nil { - if keyLookupResult, ok := val.(gomatrixserverlib.PublicKeyLookupResult); ok { - return keyLookupResult, true - } - } - return gomatrixserverlib.PublicKeyLookupResult{}, false -} - -func (c *ImmutableInMemoryLRUCache) StoreServerKey( - request gomatrixserverlib.PublicKeyLookupRequest, - response gomatrixserverlib.PublicKeyLookupResult, -) { - key := fmt.Sprintf("%s/%s", request.ServerName, request.KeyID) - checkForInvalidMutation(c.roomVersions, key, response) - c.serverKeys.Add(request, response) -} diff --git a/internal/caching/impl_inmemorylru.go b/internal/caching/impl_inmemorylru.go new file mode 100644 index 000000000..f7901d2e5 --- /dev/null +++ b/internal/caching/impl_inmemorylru.go @@ -0,0 +1,73 @@ +package caching + +import ( + "fmt" + + lru "github.com/hashicorp/golang-lru" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +func NewInMemoryLRUCache() (*Caches, error) { + roomVersions, err := NewInMemoryLRUCachePartition( + RoomVersionCacheName, + RoomVersionCacheMutable, + RoomVersionCacheMaxEntries, + ) + if err != nil { + return nil, err + } + serverKeys, err := NewInMemoryLRUCachePartition( + ServerKeyCacheName, + ServerKeyCacheMutable, + ServerKeyCacheMaxEntries, + ) + if err != nil { + return nil, err + } + return &Caches{ + RoomVersions: roomVersions, + ServerKeys: serverKeys, + }, nil +} + +type InMemoryLRUCachePartition struct { + name string + mutable bool + maxEntries int + lru *lru.Cache +} + +func NewInMemoryLRUCachePartition(name string, mutable bool, maxEntries int) (*InMemoryLRUCachePartition, error) { + var err error + cache := InMemoryLRUCachePartition{ + name: name, + mutable: mutable, + maxEntries: maxEntries, + } + cache.lru, err = lru.New(maxEntries) + if err != nil { + return nil, err + } + promauto.NewGaugeFunc(prometheus.GaugeOpts{ + Namespace: "dendrite", + Subsystem: "caching_in_memory_lru", + Name: name, + }, func() float64 { + return float64(cache.lru.Len()) + }) + return &cache, nil +} + +func (c *InMemoryLRUCachePartition) Set(key string, value interface{}) { + if !c.mutable { + if peek, ok := c.lru.Peek(key); ok && peek != value { + panic(fmt.Sprintf("invalid use of immutable cache tries to mutate existing value of %q", key)) + } + } + c.lru.Add(key, value) +} + +func (c *InMemoryLRUCachePartition) Get(key string) (value interface{}, ok bool) { + return c.lru.Get(key) +} diff --git a/roomserver/internal/api.go b/roomserver/internal/api.go index 3a7b0d76a..37a8a39bf 100644 --- a/roomserver/internal/api.go +++ b/roomserver/internal/api.go @@ -16,7 +16,7 @@ type RoomserverInternalAPI struct { DB storage.Database Cfg *config.Dendrite Producer sarama.SyncProducer - ImmutableCache caching.ImmutableCache + Cache caching.RoomVersionCache ServerName gomatrixserverlib.ServerName KeyRing gomatrixserverlib.JSONVerifier FedClient *gomatrixserverlib.FederationClient diff --git a/roomserver/internal/query.go b/roomserver/internal/query.go index fce2ae907..9fb67e7eb 100644 --- a/roomserver/internal/query.go +++ b/roomserver/internal/query.go @@ -951,7 +951,7 @@ func (r *RoomserverInternalAPI) QueryRoomVersionForRoom( request *api.QueryRoomVersionForRoomRequest, response *api.QueryRoomVersionForRoomResponse, ) error { - if roomVersion, ok := r.ImmutableCache.GetRoomVersion(request.RoomID); ok { + if roomVersion, ok := r.Cache.GetRoomVersion(request.RoomID); ok { response.RoomVersion = roomVersion return nil } @@ -961,6 +961,6 @@ func (r *RoomserverInternalAPI) QueryRoomVersionForRoom( return err } response.RoomVersion = roomVersion - r.ImmutableCache.StoreRoomVersion(request.RoomID, response.RoomVersion) + r.Cache.StoreRoomVersion(request.RoomID, response.RoomVersion) return nil } diff --git a/roomserver/inthttp/client.go b/roomserver/inthttp/client.go index 99db9e1e7..5cc2537e3 100644 --- a/roomserver/inthttp/client.go +++ b/roomserver/inthttp/client.go @@ -43,9 +43,9 @@ const ( ) type httpRoomserverInternalAPI struct { - roomserverURL string - httpClient *http.Client - immutableCache caching.ImmutableCache + roomserverURL string + httpClient *http.Client + cache caching.RoomVersionCache } // NewRoomserverClient creates a RoomserverInputAPI implemented by talking to a HTTP POST API. @@ -53,15 +53,15 @@ type httpRoomserverInternalAPI struct { func NewRoomserverClient( roomserverURL string, httpClient *http.Client, - immutableCache caching.ImmutableCache, + cache caching.RoomVersionCache, ) (api.RoomserverInternalAPI, error) { if httpClient == nil { return nil, errors.New("NewRoomserverInternalAPIHTTP: httpClient is ") } return &httpRoomserverInternalAPI{ - roomserverURL: roomserverURL, - httpClient: httpClient, - immutableCache: immutableCache, + roomserverURL: roomserverURL, + httpClient: httpClient, + cache: cache, }, nil } @@ -320,7 +320,7 @@ func (h *httpRoomserverInternalAPI) QueryRoomVersionForRoom( request *api.QueryRoomVersionForRoomRequest, response *api.QueryRoomVersionForRoomResponse, ) error { - if roomVersion, ok := h.immutableCache.GetRoomVersion(request.RoomID); ok { + if roomVersion, ok := h.cache.GetRoomVersion(request.RoomID); ok { response.RoomVersion = roomVersion return nil } @@ -331,7 +331,7 @@ func (h *httpRoomserverInternalAPI) QueryRoomVersionForRoom( apiURL := h.roomserverURL + RoomserverQueryRoomVersionForRoomPath err := internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) if err == nil { - h.immutableCache.StoreRoomVersion(request.RoomID, response.RoomVersion) + h.cache.StoreRoomVersion(request.RoomID, response.RoomVersion) } return err } diff --git a/roomserver/roomserver.go b/roomserver/roomserver.go index ae0b0794a..a55b20bef 100644 --- a/roomserver/roomserver.go +++ b/roomserver/roomserver.go @@ -44,7 +44,7 @@ func SetupRoomServerComponent( Cfg: base.Cfg, Producer: base.KafkaProducer, OutputRoomEventTopic: string(base.Cfg.Kafka.Topics.OutputRoomEvent), - ImmutableCache: base.ImmutableCache, + Cache: base.Caches, ServerName: base.Cfg.Matrix.ServerName, FedClient: fedClient, KeyRing: keyRing, diff --git a/serverkeyapi/inthttp/client.go b/serverkeyapi/inthttp/client.go index f986634ba..f22b0e310 100644 --- a/serverkeyapi/inthttp/client.go +++ b/serverkeyapi/inthttp/client.go @@ -24,7 +24,7 @@ const ( func NewServerKeyClient( serverKeyAPIURL string, httpClient *http.Client, - immutableCache caching.ImmutableCache, + cache caching.ServerKeyCache, ) (api.ServerKeyInternalAPI, error) { if httpClient == nil { return nil, errors.New("NewRoomserverInternalAPIHTTP: httpClient is ") @@ -32,14 +32,14 @@ func NewServerKeyClient( return &httpServerKeyInternalAPI{ serverKeyAPIURL: serverKeyAPIURL, httpClient: httpClient, - immutableCache: immutableCache, + cache: cache, }, nil } type httpServerKeyInternalAPI struct { serverKeyAPIURL string httpClient *http.Client - immutableCache caching.ImmutableCache + cache caching.ServerKeyCache } func (s *httpServerKeyInternalAPI) KeyRing() *gomatrixserverlib.KeyRing { @@ -71,7 +71,7 @@ func (s *httpServerKeyInternalAPI) StoreKeys( response := api.InputPublicKeysResponse{} for req, res := range results { request.Keys[req] = res - s.immutableCache.StoreServerKey(req, res) + s.cache.StoreServerKey(req, res) } return s.InputPublicKeys(ctx, &request, &response) } @@ -92,7 +92,7 @@ func (s *httpServerKeyInternalAPI) FetchKeys( } now := gomatrixserverlib.AsTimestamp(time.Now()) for req, ts := range requests { - if res, ok := s.immutableCache.GetServerKey(req); ok { + if res, ok := s.cache.GetServerKey(req); ok { if now > res.ValidUntilTS && res.ExpiredTS == gomatrixserverlib.PublicKeyNotExpired { continue } @@ -107,7 +107,7 @@ func (s *httpServerKeyInternalAPI) FetchKeys( } for req, res := range response.Results { result[req] = res - s.immutableCache.StoreServerKey(req, res) + s.cache.StoreServerKey(req, res) } return result, nil } diff --git a/serverkeyapi/inthttp/server.go b/serverkeyapi/inthttp/server.go index d5517e143..9efe7d9df 100644 --- a/serverkeyapi/inthttp/server.go +++ b/serverkeyapi/inthttp/server.go @@ -12,7 +12,7 @@ import ( "github.com/matrix-org/util" ) -func AddRoutes(s api.ServerKeyInternalAPI, internalAPIMux *mux.Router, cache caching.ImmutableCache) { +func AddRoutes(s api.ServerKeyInternalAPI, internalAPIMux *mux.Router, cache caching.ServerKeyCache) { internalAPIMux.Handle(ServerKeyQueryPublicKeyPath, internal.MakeInternalAPI("queryPublicKeys", func(req *http.Request) util.JSONResponse { request := api.QueryPublicKeysRequest{} diff --git a/serverkeyapi/serverkeyapi.go b/serverkeyapi/serverkeyapi.go index 5bf8f67d7..ad885270a 100644 --- a/serverkeyapi/serverkeyapi.go +++ b/serverkeyapi/serverkeyapi.go @@ -29,7 +29,7 @@ func SetupServerKeyAPIComponent( logrus.WithError(err).Panicf("failed to connect to server key database") } - serverKeyDB, err := cache.NewKeyDatabase(innerDB, base.ImmutableCache) + serverKeyDB, err := cache.NewKeyDatabase(innerDB, base.Caches) if err != nil { logrus.WithError(err).Panicf("failed to set up caching wrapper for server key database") } @@ -77,7 +77,7 @@ func SetupServerKeyAPIComponent( }).Info("Enabled perspective key fetcher") } - inthttp.AddRoutes(&internalAPI, base.InternalAPIMux, base.ImmutableCache) + inthttp.AddRoutes(&internalAPI, base.InternalAPIMux, base.Caches) return &internalAPI } diff --git a/serverkeyapi/storage/cache/keydb.go b/serverkeyapi/storage/cache/keydb.go index a0cdb9007..b662e4fdb 100644 --- a/serverkeyapi/storage/cache/keydb.go +++ b/serverkeyapi/storage/cache/keydb.go @@ -12,10 +12,10 @@ import ( // the public keys for other matrix servers. type KeyDatabase struct { inner gomatrixserverlib.KeyDatabase - cache caching.ImmutableCache + cache caching.ServerKeyCache } -func NewKeyDatabase(inner gomatrixserverlib.KeyDatabase, cache caching.ImmutableCache) (*KeyDatabase, error) { +func NewKeyDatabase(inner gomatrixserverlib.KeyDatabase, cache caching.ServerKeyCache) (*KeyDatabase, error) { if inner == nil { return nil, errors.New("inner database can't be nil") } From 67784ecb56e56e952049067b6d893bddcc5d113b Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 5 Jun 2020 19:00:30 +0100 Subject: [PATCH 139/200] Nuke old Docker stuff (#1104) * Remove old Docker stuff * Move hub stuff to upper level docker directory * Build monolith images * Update README.md * Update paths * Update README.md --- build/docker/Dockerfile | 9 +- build/docker/{hub => }/Dockerfile.component | 0 build/docker/README.md | 118 ++++------ build/docker/build.sh | 18 -- .../{hub => }/config/dendrite-config.yaml | 0 build/docker/dendrite-docker.yml | 129 ----------- .../docker/{hub => }/docker-compose.deps.yml | 2 +- .../{hub => }/docker-compose.monolith.yml | 0 .../{hub => }/docker-compose.polylith.yml | 0 build/docker/docker-compose.yml | 202 ------------------ build/docker/hub/Dockerfile | 10 - build/docker/hub/README.md | 68 ------ build/docker/{hub => }/images-build.sh | 28 +-- build/docker/{hub => }/images-push.sh | 2 + build/docker/services/client-api-proxy.sh | 9 - build/docker/services/client-api.sh | 5 - build/docker/services/edu-server.sh | 5 - build/docker/services/federation-api-proxy.sh | 7 - build/docker/services/federation-api.sh | 5 - build/docker/services/federation-sender.sh | 5 - build/docker/services/key-server.sh | 5 - build/docker/services/media-api.sh | 5 - build/docker/services/monolith.sh | 5 - build/docker/services/public-rooms-api.sh | 5 - build/docker/services/room-server.sh | 5 - build/docker/services/sync-api.sh | 5 - 26 files changed, 67 insertions(+), 585 deletions(-) rename build/docker/{hub => }/Dockerfile.component (100%) delete mode 100644 build/docker/build.sh rename build/docker/{hub => }/config/dendrite-config.yaml (100%) delete mode 100644 build/docker/dendrite-docker.yml rename build/docker/{hub => }/docker-compose.deps.yml (89%) rename build/docker/{hub => }/docker-compose.monolith.yml (100%) rename build/docker/{hub => }/docker-compose.polylith.yml (100%) delete mode 100644 build/docker/docker-compose.yml delete mode 100644 build/docker/hub/Dockerfile delete mode 100644 build/docker/hub/README.md rename build/docker/{hub => }/images-build.sh (75%) rename build/docker/{hub => }/images-push.sh (92%) delete mode 100644 build/docker/services/client-api-proxy.sh delete mode 100644 build/docker/services/client-api.sh delete mode 100644 build/docker/services/edu-server.sh delete mode 100644 build/docker/services/federation-api-proxy.sh delete mode 100644 build/docker/services/federation-api.sh delete mode 100644 build/docker/services/federation-sender.sh delete mode 100644 build/docker/services/key-server.sh delete mode 100644 build/docker/services/media-api.sh delete mode 100644 build/docker/services/monolith.sh delete mode 100644 build/docker/services/public-rooms-api.sh delete mode 100644 build/docker/services/room-server.sh delete mode 100644 build/docker/services/sync-api.sh diff --git a/build/docker/Dockerfile b/build/docker/Dockerfile index 5810825a4..d8e07681f 100644 --- a/build/docker/Dockerfile +++ b/build/docker/Dockerfile @@ -1,9 +1,10 @@ -FROM docker.io/golang:1.13.7-alpine3.11 +FROM docker.io/golang:1.13.7-alpine3.11 AS builder -RUN mkdir /build +RUN apk --update --no-cache add bash build-base WORKDIR /build -RUN apk --update --no-cache add openssl bash git build-base +COPY . /build -CMD ["bash", "docker/build.sh"] +RUN mkdir -p bin +RUN sh ./build.sh \ No newline at end of file diff --git a/build/docker/hub/Dockerfile.component b/build/docker/Dockerfile.component similarity index 100% rename from build/docker/hub/Dockerfile.component rename to build/docker/Dockerfile.component diff --git a/build/docker/README.md b/build/docker/README.md index 83d0b6a87..45d96d1cb 100644 --- a/build/docker/README.md +++ b/build/docker/README.md @@ -1,100 +1,70 @@ -Development with Docker ---- +# Docker images -With `docker` and `docker-compose` you can easily spin up a development environment -and start working on dendrite. +These are Docker images for Dendrite! -### Requirements +## Dockerfiles -- docker -- docker-compose (version 3+) +The `Dockerfile` builds the base image which contains all of the Dendrite +components. The `Dockerfile.component` file takes the given component, as +specified with `--buildarg component=` from the base image and produce +smaller component-specific images, which are substantially smaller and do +not contain the Go toolchain etc. -### Configuration +## Compose files -Create a directory named `cfg` in the root of the project. Copy the -`dendrite-docker.yaml` file into that directory and rename it to `dendrite.yaml`. -It already contains the defaults used in `docker-compose` for networking so you will -only have to change things like the `server_name` or to toggle `naffka`. +There are three sample `docker-compose` files: -You can run the following `docker-compose` commands either from the top directory -specifying the `docker-compose` file -``` -docker-compose -f docker/docker-compose.yml -``` -or from within the `docker` directory +- `docker-compose.deps.yml` which runs the Postgres and Kafka prerequisites +- `docker-compose.monolith.yml` which runs a monolith Dendrite deployment +- `docker-compose.polylith.yml` which runs a polylith Dendrite deployment + +## Configuration + +The `docker-compose` files refer to the `/etc/dendrite` volume as where the +runtime config should come from. The mounted folder must contain: + +- `dendrite.yaml` configuration file (based on the sample `dendrite-config.yaml` + in the `docker/config` folder in the [Dendrite repository](https://github.com/matrix-org/dendrite) +- `matrix_key.pem` server key, as generated using `cmd/generate-keys` +- `server.crt` certificate file +- `server.key` private key file for the above certificate + +To generate keys: ``` -docker-compose +go run github.com/matrix-org/dendrite/cmd/generate-keys \ + --private-key=matrix_key.pem \ + --tls-cert=server.crt \ + --tls-key=server.key ``` -### Starting a monolith server +## Starting Dendrite -For the monolith server you would need a postgres instance +Once in place, start the dependencies: ``` -docker-compose up postgres +docker-compose -f docker-compose.deps.yml up ``` -and the dendrite component from `bin/dendrite-monolith-server` +Wait a few seconds for Kafka and Postgres to finish starting up, and then start a monolith: ``` -docker-compose up monolith +docker-compose -f docker-compose.monolith.yml up ``` -The monolith will be listening on `http://localhost:8008`. - -You would also have to make the following adjustments to `dendrite.yaml`. - - Set `use_naffka: true` - - Uncomment the `database/naffka` postgres url. - -### Starting a multiprocess server - -The multiprocess server requires kafka, zookeeper and postgres +... or start the polylith components: ``` -docker-compose up kafka zookeeper postgres +docker-compose -f docker-compose.polylith.yml up ``` -and the following dendrite components +## Building the images -``` -docker-compose up client_api media_api sync_api room_server public_rooms_api edu_server -docker-compose up client_api_proxy -``` +The `docker/images-build.sh` script will build the base image, followed by +all of the component images. -The `client-api-proxy` will be listening on `http://localhost:8008`. +The `docker/images-push.sh` script will push them to Docker Hub (subject +to permissions). -You would also have to make the following adjustments to `dendrite.yaml`. - - Set `use_naffka: false` - - Comment out the `database/naffka` postgres url. - -### Starting federation - -``` -docker-compose up federation_api federation_sender -docker-compose up federation_api_proxy -``` - -You can point other Matrix servers to `http://localhost:8448`. - -### Creating a new component - -You can create a new dendrite component by adding an entry to the `docker-compose.yml` -file and creating a startup script for the component in `docker/services`. -For more information refer to the official docker-compose [documentation](https://docs.docker.com/compose/). - -```yaml - new_component: - container_name: dendrite_room_server - hostname: new_component - # Start up script. - entrypoint: ["bash", "./docker/services/new-component.sh"] - # Use the common Dockerfile for all the dendrite components. - build: ./ - volumes: - - ..:/build - depends_on: - - another_component - networks: - - internal -``` +If you wish to build and push your own images, rename `matrixdotorg/dendrite` to +the name of another Docker Hub repository in `images-build.sh` and `images-push.sh`. diff --git a/build/docker/build.sh b/build/docker/build.sh deleted file mode 100644 index a3e3ca24d..000000000 --- a/build/docker/build.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -./build.sh - -# Generate the keys if they don't already exist. -if [ ! -f server.key ] || [ ! -f server.crt ] || [ ! -f matrix_key.pem ]; then - echo "Generating keys ..." - - rm -f server.key server.crt matrix_key.pem - - test -f server.key || openssl req -x509 -newkey rsa:4096 \ - -keyout server.key \ - -out server.crt \ - -days 3650 -nodes \ - -subj /CN=localhost - - test -f matrix_key.pem || /build/bin/generate-keys -private-key matrix_key.pem -fi diff --git a/build/docker/hub/config/dendrite-config.yaml b/build/docker/config/dendrite-config.yaml similarity index 100% rename from build/docker/hub/config/dendrite-config.yaml rename to build/docker/config/dendrite-config.yaml diff --git a/build/docker/dendrite-docker.yml b/build/docker/dendrite-docker.yml deleted file mode 100644 index a72ff3ddc..000000000 --- a/build/docker/dendrite-docker.yml +++ /dev/null @@ -1,129 +0,0 @@ -# The config file format version -# This is used by dendrite to tell if it understands the config format. -# This will change if the structure of the config file changes or if the meaning -# of an existing config key changes. -version: 0 - -# The matrix specific config -matrix: - # The name of the server. This is usually the domain name, e.g 'matrix.org', 'localhost'. - server_name: "example.com" - # The path to the PEM formatted matrix private key. - private_key: "matrix_key.pem" - # The x509 certificates used by the federation listeners for this server - federation_certificates: ["server.crt"] - # The list of identity servers trusted to verify third party identifiers by this server. - # Defaults to no trusted servers. - trusted_third_party_id_servers: - - vector.im - - matrix.org - -# The media repository config -media: - # The base path to where the media files will be stored. May be relative or absolute. - base_path: /var/dendrite/media - - # The maximum file size in bytes that is allowed to be stored on this server. - # Note: if max_file_size_bytes is set to 0, the size is unlimited. - # Note: if max_file_size_bytes is not set, it will default to 10485760 (10MB) - max_file_size_bytes: 10485760 - - # Whether to dynamically generate thumbnails on-the-fly if the requested resolution is not already generated - # NOTE: This is a possible denial-of-service attack vector - use at your own risk - dynamic_thumbnails: false - - # A list of thumbnail sizes to be pre-generated for downloaded remote / uploaded content - # method is one of crop or scale. If omitted, it will default to scale. - # crop scales to fill the requested dimensions and crops the excess. - # scale scales to fit the requested dimensions and one dimension may be smaller than requested. - thumbnail_sizes: - - width: 32 - height: 32 - method: crop - - width: 96 - height: 96 - method: crop - - width: 320 - height: 240 - method: scale - - width: 640 - height: 480 - method: scale - - width: 800 - height: 600 - method: scale - -# The config for the TURN server -turn: - # Whether or not guests can request TURN credentials - turn_allow_guests: true - # How long the authorization should last - turn_user_lifetime: "1h" - # The list of TURN URIs to pass to clients - turn_uris: [] - - # Authorization via Shared Secret - # The shared secret from coturn - turn_shared_secret: "" - - # Authorization via Static Username & Password - # Hardcoded Username and Password - turn_username: "" - turn_password: "" - -# The config for communicating with kafka -kafka: - # Where the kafka servers are running. - addresses: ["kafka:9092"] - # Whether to use naffka instead of kafka. - # Naffka can only be used when running dendrite as a single monolithic server. - # Kafka can be used both with a monolithic server and when running the - # components as separate servers. - # If enabled database.naffka must also be specified. - use_naffka: true - # The names of the kafka topics to use. - topics: - output_room_event: roomserverOutput - output_client_data: clientapiOutput - output_typing_event: eduServerOutput - user_updates: userUpdates - - -# The postgres connection configs for connecting to the databases e.g a postgres:// URI -database: - account: "postgres://dendrite:itsasecret@postgres/dendrite_account?sslmode=disable" - device: "postgres://dendrite:itsasecret@postgres/dendrite_device?sslmode=disable" - media_api: "postgres://dendrite:itsasecret@postgres/dendrite_mediaapi?sslmode=disable" - sync_api: "postgres://dendrite:itsasecret@postgres/dendrite_syncapi?sslmode=disable" - room_server: "postgres://dendrite:itsasecret@postgres/dendrite_roomserver?sslmode=disable" - server_key: "postgres://dendrite:itsasecret@postgres/dendrite_serverkey?sslmode=disable" - federation_sender: "postgres://dendrite:itsasecret@postgres/dendrite_federationsender?sslmode=disable" - public_rooms_api: "postgres://dendrite:itsasecret@postgres/dendrite_publicroomsapi?sslmode=disable" - appservice: "postgres://dendrite:itsasecret@postgres/dendrite_appservice?sslmode=disable" - # If using naffka you need to specify a naffka database - naffka: "postgres://dendrite:itsasecret@postgres/dendrite_naffka?sslmode=disable" - -# The TCP host:port pairs to bind the internal HTTP APIs to. -# These shouldn't be exposed to the public internet. -# These aren't needed when running dendrite as a monolithic server. -listen: - room_server: "room_server:7770" - client_api: "client_api:7771" - federation_api: "federation_api:7772" - sync_api: "sync_api:7773" - media_api: "media_api:7774" - public_rooms_api: "public_rooms_api:7775" - federation_sender: "federation_sender:7776" - edu_server: "typing_server:7777" - -# The configuration for tracing the dendrite components. -tracing: - # Config for the jaeger opentracing reporter. - # See https://godoc.org/github.com/uber/jaeger-client-go/config#Configuration - # for documentation. - jaeger: - disabled: true - -# A list of application service config files to use -application_services: - config_files: [] diff --git a/build/docker/hub/docker-compose.deps.yml b/build/docker/docker-compose.deps.yml similarity index 89% rename from build/docker/hub/docker-compose.deps.yml rename to build/docker/docker-compose.deps.yml index ada625aa3..facfc01b3 100644 --- a/build/docker/hub/docker-compose.deps.yml +++ b/build/docker/docker-compose.deps.yml @@ -5,7 +5,7 @@ services: image: postgres:9.5 restart: always volumes: - - ../postgres/create_db.sh:/docker-entrypoint-initdb.d/20-create_db.sh + - ./postgres/create_db.sh:/docker-entrypoint-initdb.d/20-create_db.sh environment: POSTGRES_PASSWORD: itsasecret POSTGRES_USER: dendrite diff --git a/build/docker/hub/docker-compose.monolith.yml b/build/docker/docker-compose.monolith.yml similarity index 100% rename from build/docker/hub/docker-compose.monolith.yml rename to build/docker/docker-compose.monolith.yml diff --git a/build/docker/hub/docker-compose.polylith.yml b/build/docker/docker-compose.polylith.yml similarity index 100% rename from build/docker/hub/docker-compose.polylith.yml rename to build/docker/docker-compose.polylith.yml diff --git a/build/docker/docker-compose.yml b/build/docker/docker-compose.yml deleted file mode 100644 index c6bb45813..000000000 --- a/build/docker/docker-compose.yml +++ /dev/null @@ -1,202 +0,0 @@ -version: "3.4" -services: - riot: - image: vectorim/riot-web - networks: - - internal - ports: - - "8500:80" - - monolith: - container_name: dendrite_monolith - hostname: monolith - entrypoint: ["bash", "./docker/services/monolith.sh", "--config", "/etc/dendrite/dendrite.yaml"] - build: ./ - volumes: - - ..:/build - - ./build/bin:/build/bin - - ../cfg:/etc/dendrite - networks: - - internal - depends_on: - - postgres - ports: - - "8008:8008" - - "8448:8448" - - client_api_proxy: - container_name: dendrite_client_api_proxy - hostname: client_api_proxy - entrypoint: ["bash", "./docker/services/client-api-proxy.sh"] - build: ./ - volumes: - - ..:/build - networks: - - internal - depends_on: - - postgres - - sync_api - - client_api - - media_api - - public_rooms_api - ports: - - "8008:8008" - - client_api: - container_name: dendrite_client_api - hostname: client_api - entrypoint: ["bash", "./docker/services/client-api.sh"] - build: ./ - volumes: - - ..:/build - depends_on: - - postgres - - room_server - networks: - - internal - - media_api: - container_name: dendrite_media_api - hostname: media_api - entrypoint: ["bash", "./docker/services/media-api.sh"] - build: ./ - volumes: - - ..:/build - depends_on: - - postgres - networks: - - internal - - public_rooms_api: - container_name: dendrite_public_rooms_api - hostname: public_rooms_api - entrypoint: ["bash", "./docker/services/public-rooms-api.sh"] - build: ./ - volumes: - - ..:/build - depends_on: - - postgres - networks: - - internal - - sync_api: - container_name: dendrite_sync_api - hostname: sync_api - entrypoint: ["bash", "./docker/services/sync-api.sh"] - build: ./ - volumes: - - ..:/build - depends_on: - - postgres - networks: - - internal - - room_server: - container_name: dendrite_room_server - hostname: room_server - entrypoint: ["bash", "./docker/services/room-server.sh"] - build: ./ - volumes: - - ..:/build - depends_on: - - postgres - networks: - - internal - - edu_server: - container_name: dendrite_edu_server - hostname: edu_server - entrypoint: ["bash", "./docker/services/edu-server.sh"] - build: ./ - volumes: - - ..:/build - networks: - - internal - - federation_api_proxy: - container_name: dendrite_federation_api_proxy - hostname: federation_api_proxy - entrypoint: ["bash", "./docker/services/federation-api-proxy.sh"] - build: ./ - volumes: - - ..:/build - depends_on: - - postgres - - federation_api - - federation_sender - - media_api - networks: - - internal - ports: - - "8448:8448" - - federation_api: - container_name: dendrite_federation_api - hostname: federation_api - entrypoint: ["bash", "./docker/services/federation-api.sh"] - build: ./ - volumes: - - ..:/build - depends_on: - - postgres - networks: - - internal - - federation_sender: - container_name: dendrite_federation_sender - hostname: federation_sender - entrypoint: ["bash", "./docker/services/federation-sender.sh"] - build: ./ - volumes: - - ..:/build - depends_on: - - postgres - networks: - - internal - - key_server: - container_name: dendrite_key_server - hostname: key_server - entrypoint: ["bash", "./docker/services/key-server.sh"] - build: ./ - volumes: - - ..:/build - networks: - - internal - - postgres: - container_name: dendrite_postgres - hostname: postgres - image: postgres:9.5 - restart: always - volumes: - - ./postgres/create_db.sh:/docker-entrypoint-initdb.d/20-create_db.sh - environment: - POSTGRES_PASSWORD: itsasecret - POSTGRES_USER: dendrite - networks: - - internal - - zookeeper: - container_name: dendrite_zk - hostname: zookeeper - image: zookeeper - networks: - - internal - - kafka: - container_name: dendrite_kafka - hostname: kafka - image: wurstmeister/kafka - environment: - KAFKA_ADVERTISED_HOST_NAME: "kafka" - KAFKA_DELETE_TOPIC_ENABLE: "true" - KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181" - depends_on: - - zookeeper - networks: - - internal - -networks: - internal: - attachable: true diff --git a/build/docker/hub/Dockerfile b/build/docker/hub/Dockerfile deleted file mode 100644 index d8e07681f..000000000 --- a/build/docker/hub/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM docker.io/golang:1.13.7-alpine3.11 AS builder - -RUN apk --update --no-cache add bash build-base - -WORKDIR /build - -COPY . /build - -RUN mkdir -p bin -RUN sh ./build.sh \ No newline at end of file diff --git a/build/docker/hub/README.md b/build/docker/hub/README.md deleted file mode 100644 index fd6411439..000000000 --- a/build/docker/hub/README.md +++ /dev/null @@ -1,68 +0,0 @@ -# Docker Hub images - -These are Docker Hub-friendly images for Dendrite. - -## Dockerfiles - -The `Dockerfile` builds the base image which contains all of the Dendrite -components. The `Dockerfile.*` files take components from the base image and -produce smaller component-specific images, which are substantially smaller -and do not contain the Go toolchain etc. - -## Compose files - -There are three sample `docker-compose` files: - -- `docker-compose.deps.yml` which runs the Postgres and Kafka prerequisites -- `docker-compose.monolith.yml` which runs a monolith Dendrite deployment -- `docker-compose.polylith.yml` which runs a polylith Dendrite deployment - -## Configuration - -The `docker-compose` files refer to the `/etc/dendrite` volume as where the -runtime config should come from. The mounted folder must contain: - -- `dendrite.yaml` configuration file (based on the sample `dendrite-config.yaml` - in the `docker/hub/config` folder in the [Dendrite repository](https://github.com/matrix-org/dendrite) -- `matrix_key.pem` server key, as generated using `cmd/generate-keys` -- `server.crt` certificate file -- `server.key` private key file for the above certificate - -To generate keys: - -``` -go run github.com/matrix-org/dendrite/cmd/generate-keys \ - --private-key=matrix_key.pem \ - --tls-cert=server.crt \ - --tls-key=server.key -``` - -## Starting Dendrite - -Once in place, start the dependencies: - -``` -docker-compose -f docker-compose.deps.yml up -``` - -Wait a few seconds for Kafka and Postgres to finish starting up, and then start a monolith: - -``` -docker-compose -f docker-compose.monolith.yml up -``` - -... or start the polylith components: - -``` -docker-compose -f docker-compose.polylith.yml up -``` - -## Building the images - -The `docker/hub/images-build.sh` script will build all of the component images. - -The `docker/hub/images-push.sh` script will push them to Docker Hub (subject -to permissions). - -If you wish to build and push your own images, rename `matrixdotorg/dendrite` to -the name of another Docker Hub repository in `images-build.sh` and `images-push.sh`. diff --git a/build/docker/hub/images-build.sh b/build/docker/images-build.sh similarity index 75% rename from build/docker/hub/images-build.sh rename to build/docker/images-build.sh index b162a0639..6bc058962 100755 --- a/build/docker/hub/images-build.sh +++ b/build/docker/images-build.sh @@ -2,17 +2,19 @@ cd $(git rev-parse --show-toplevel) -docker build -f build/docker/hub/Dockerfile -t matrixdotorg/dendrite:latest . +docker build -f build/docker/Dockerfile -t matrixdotorg/dendrite:latest . -docker build -t matrixdotorg/dendrite:clientapi --build-arg component=dendrite-client-api-server -f build/docker/hub/Dockerfile.component . -docker build -t matrixdotorg/dendrite:clientproxy --build-arg component=client-api-proxy -f build/docker/hub/Dockerfile.component . -docker build -t matrixdotorg/dendrite:eduserver --build-arg component=dendrite-edu-server -f build/docker/hub/Dockerfile.component . -docker build -t matrixdotorg/dendrite:federationapi --build-arg component=dendrite-federation-api-server -f build/docker/hub/Dockerfile.component . -docker build -t matrixdotorg/dendrite:federationsender --build-arg component=dendrite-federation-sender-server -f build/docker/hub/Dockerfile.component . -docker build -t matrixdotorg/dendrite:federationproxy --build-arg component=federation-api-proxy -f build/docker/hub/Dockerfile.component . -docker build -t matrixdotorg/dendrite:keyserver --build-arg component=dendrite-key-server -f build/docker/hub/Dockerfile.component . -docker build -t matrixdotorg/dendrite:mediaapi --build-arg component=dendrite-media-api-server -f build/docker/hub/Dockerfile.component . -docker build -t matrixdotorg/dendrite:publicroomsapi --build-arg component=dendrite-public-rooms-api-server -f build/docker/hub/Dockerfile.component . -docker build -t matrixdotorg/dendrite:roomserver --build-arg component=dendrite-room-server -f build/docker/hub/Dockerfile.component . -docker build -t matrixdotorg/dendrite:syncapi --build-arg component=dendrite-sync-api-server -f build/docker/hub/Dockerfile.component . -docker build -t matrixdotorg/dendrite:serverkeyapi --build-arg component=dendrite-server-key-api-server -f build/docker/hub/Dockerfile.component . +docker build -t matrixdotorg/dendrite:monolith --build-arg component=dendrite-monolith-server -f build/docker/Dockerfile.component . + +docker build -t matrixdotorg/dendrite:clientapi --build-arg component=dendrite-client-api-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite:clientproxy --build-arg component=client-api-proxy -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite:eduserver --build-arg component=dendrite-edu-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite:federationapi --build-arg component=dendrite-federation-api-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite:federationsender --build-arg component=dendrite-federation-sender-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite:federationproxy --build-arg component=federation-api-proxy -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite:keyserver --build-arg component=dendrite-key-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite:mediaapi --build-arg component=dendrite-media-api-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite:publicroomsapi --build-arg component=dendrite-public-rooms-api-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite:roomserver --build-arg component=dendrite-room-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite:syncapi --build-arg component=dendrite-sync-api-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite:serverkeyapi --build-arg component=dendrite-server-key-api-server -f build/docker/Dockerfile.component . diff --git a/build/docker/hub/images-push.sh b/build/docker/images-push.sh similarity index 92% rename from build/docker/hub/images-push.sh rename to build/docker/images-push.sh index c62072687..b39d98d65 100755 --- a/build/docker/hub/images-push.sh +++ b/build/docker/images-push.sh @@ -1,5 +1,7 @@ #!/bin/bash +docker push matrixdotorg/dendrite:monolith + docker push matrixdotorg/dendrite:clientapi docker push matrixdotorg/dendrite:clientproxy docker push matrixdotorg/dendrite:eduserver diff --git a/build/docker/services/client-api-proxy.sh b/build/docker/services/client-api-proxy.sh deleted file mode 100644 index 931f7abbc..000000000 --- a/build/docker/services/client-api-proxy.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -bash ./docker/build.sh - -./bin/client-api-proxy --bind-address ":8008" \ - --client-api-server-url "http://client_api:7771" \ - --sync-api-server-url "http://sync_api:7773" \ - --media-api-server-url "http://media_api:7774" \ - --public-rooms-api-server-url "http://public_rooms_api:7775" \ diff --git a/build/docker/services/client-api.sh b/build/docker/services/client-api.sh deleted file mode 100644 index 8dc822421..000000000 --- a/build/docker/services/client-api.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -bash ./docker/build.sh - -./bin/dendrite-client-api-server --config=dendrite.yaml diff --git a/build/docker/services/edu-server.sh b/build/docker/services/edu-server.sh deleted file mode 100644 index d40b9fa7e..000000000 --- a/build/docker/services/edu-server.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -bash ./docker/build.sh - -./bin/dendrite-edu-server --config=dendrite.yaml diff --git a/build/docker/services/federation-api-proxy.sh b/build/docker/services/federation-api-proxy.sh deleted file mode 100644 index 6ea75c95a..000000000 --- a/build/docker/services/federation-api-proxy.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -bash ./docker/build.sh - -./bin/federation-api-proxy --bind-address ":8448" \ - --federation-api-url "http://federation_api_server:7772" \ - --media-api-server-url "http://media_api:7774" \ diff --git a/build/docker/services/federation-api.sh b/build/docker/services/federation-api.sh deleted file mode 100644 index 807a7cf83..000000000 --- a/build/docker/services/federation-api.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -bash ./docker/build.sh - -./bin/dendrite-federation-api-server --config dendrite.yaml diff --git a/build/docker/services/federation-sender.sh b/build/docker/services/federation-sender.sh deleted file mode 100644 index ea116ef3c..000000000 --- a/build/docker/services/federation-sender.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -bash ./docker/build.sh - -./bin/dendrite-federation-sender-server --config dendrite.yaml diff --git a/build/docker/services/key-server.sh b/build/docker/services/key-server.sh deleted file mode 100644 index 965fa8543..000000000 --- a/build/docker/services/key-server.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -bash ./docker/build.sh - -./bin/dendrite-key-server --config dendrite.yaml diff --git a/build/docker/services/media-api.sh b/build/docker/services/media-api.sh deleted file mode 100644 index 876b3aa8d..000000000 --- a/build/docker/services/media-api.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -bash ./docker/build.sh - -./bin/dendrite-media-api-server --config dendrite.yaml diff --git a/build/docker/services/monolith.sh b/build/docker/services/monolith.sh deleted file mode 100644 index 2287555cd..000000000 --- a/build/docker/services/monolith.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -bash ./docker/build.sh - -./bin/dendrite-monolith-server --tls-cert=server.crt --tls-key=server.key $@ diff --git a/build/docker/services/public-rooms-api.sh b/build/docker/services/public-rooms-api.sh deleted file mode 100644 index 652afcfec..000000000 --- a/build/docker/services/public-rooms-api.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -bash ./docker/build.sh - -./bin/dendrite-public-rooms-api-server --config dendrite.yaml diff --git a/build/docker/services/room-server.sh b/build/docker/services/room-server.sh deleted file mode 100644 index 473b5f5d3..000000000 --- a/build/docker/services/room-server.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -bash ./docker/build.sh - -./bin/dendrite-room-server --config=dendrite.yaml diff --git a/build/docker/services/sync-api.sh b/build/docker/services/sync-api.sh deleted file mode 100644 index ac6433fa5..000000000 --- a/build/docker/services/sync-api.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -bash ./docker/build.sh - -./bin/dendrite-sync-api-server --config=dendrite.yaml From cdb9a115715bd5a9e84df5bc95060e2dac1f6d89 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 8 Jun 2020 13:56:03 +0100 Subject: [PATCH 140/200] Update gomatrixserverlib --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 61d7358de..1cfdf8037 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200605101319-57093b526cb1 + github.com/matrix-org/gomatrixserverlib v0.0.0-20200608125510-defe251235b1 github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible diff --git a/go.sum b/go.sum index fa3a65431..b32a1f237 100644 --- a/go.sum +++ b/go.sum @@ -356,8 +356,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 h1:Yb+Wlf github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bhrnp3Ky1qgx/fzCtCALOoGYylh2tpS9K4= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200605101319-57093b526cb1 h1:T9nY1X24uHDkC3uMLwoo2W+JE2o0t3k/lYwqORfKSUg= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200605101319-57093b526cb1/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200608125510-defe251235b1 h1:BfrvDrbjoPBvYua/3F/FmrqiZTRGrvtoMRgCVnrufMI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200608125510-defe251235b1/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f/go.mod h1:y0oDTjZDv5SM9a2rp3bl+CU+bvTRINQsdb7YlDql5Go= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= From 4f171c56a832c836b0eb21650ee84d56e451dd6a Mon Sep 17 00:00:00 2001 From: Kegsay Date: Mon, 8 Jun 2020 15:51:07 +0100 Subject: [PATCH 141/200] Split out SetupFooComponent (#1106) * Split out adding HTTP routes from making internal APIs for clarity * Split out more components * Split out more things * Finish converting * internal mux for internal routes --- appservice/appservice.go | 33 +++++++++-------- clientapi/clientapi.go | 9 ++--- cmd/dendrite-appservice-server/main.go | 6 ++-- cmd/dendrite-client-api-server/main.go | 4 +-- cmd/dendrite-demo-libp2p/main.go | 27 +++++++------- cmd/dendrite-edu-server/main.go | 3 +- cmd/dendrite-federation-api-server/main.go | 4 +-- cmd/dendrite-federation-sender-server/main.go | 3 +- cmd/dendrite-key-server/main.go | 2 +- cmd/dendrite-media-api-server/main.go | 2 +- cmd/dendrite-monolith-server/main.go | 36 ++++++++++--------- cmd/dendrite-public-rooms-api-server/main.go | 2 +- cmd/dendrite-room-server/main.go | 3 +- cmd/dendrite-server-key-api-server/main.go | 3 +- cmd/dendrite-sync-api-server/main.go | 2 +- cmd/dendritejs/main.go | 22 ++++++------ eduserver/eduserver.go | 21 +++++------ federationapi/federationapi.go | 13 +++---- federationsender/federationsender.go | 18 ++++++---- keyserver/keyserver.go | 12 +++---- mediaapi/mediaapi.go | 14 ++++---- publicroomsapi/publicroomsapi.go | 8 +++-- roomserver/roomserver.go | 21 +++++------ serverkeyapi/serverkeyapi.go | 33 ++++++++++------- syncapi/syncapi.go | 8 +++-- 25 files changed, 171 insertions(+), 138 deletions(-) diff --git a/appservice/appservice.go b/appservice/appservice.go index b5ffba5eb..dca600bf5 100644 --- a/appservice/appservice.go +++ b/appservice/appservice.go @@ -21,6 +21,7 @@ import ( "sync" "time" + "github.com/gorilla/mux" appserviceAPI "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/appservice/consumers" "github.com/matrix-org/dendrite/appservice/inthttp" @@ -40,15 +41,28 @@ import ( "github.com/sirupsen/logrus" ) -// SetupAppServiceAPIComponent sets up and registers HTTP handlers for the AppServices -// component. -func SetupAppServiceAPIComponent( +// AddPublicRoutes registers HTTP handlers for CS API calls +func AddPublicRoutes(router *mux.Router, cfg *config.Dendrite, rsAPI roomserverAPI.RoomserverInternalAPI, + accountsDB accounts.Database, federation *gomatrixserverlib.FederationClient, txnCache *transactions.Cache) { + + routing.Setup( + router, cfg, rsAPI, + accountsDB, federation, txnCache, + ) +} + +// AddInternalRoutes registers HTTP handlers for internal API calls +func AddInternalRoutes(router *mux.Router, queryAPI appserviceAPI.AppServiceQueryAPI) { + inthttp.AddRoutes(queryAPI, router) +} + +// NewInternalAPI returns a concerete implementation of the internal API. Callers +// can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. +func NewInternalAPI( base *basecomponent.BaseDendrite, accountsDB accounts.Database, deviceDB devices.Database, - federation *gomatrixserverlib.FederationClient, rsAPI roomserverAPI.RoomserverInternalAPI, - transactionsCache *transactions.Cache, ) appserviceAPI.AppServiceQueryAPI { // Create a connection to the appservice postgres DB appserviceDB, err := storage.NewDatabase(string(base.Cfg.Database.AppService), base.Cfg.DbProperties()) @@ -85,8 +99,6 @@ func SetupAppServiceAPIComponent( Cfg: base.Cfg, } - inthttp.AddRoutes(appserviceQueryAPI, base.InternalAPIMux) - consumer := consumers.NewOutputRoomEventConsumer( base.Cfg, base.KafkaConsumer, accountsDB, appserviceDB, rsAPI, workerStates, @@ -99,13 +111,6 @@ func SetupAppServiceAPIComponent( if err := workers.SetupTransactionWorkers(appserviceDB, workerStates); err != nil { logrus.WithError(err).Panicf("failed to start app service transaction workers") } - - // Set up HTTP Endpoints - routing.Setup( - base.PublicAPIMux, base.Cfg, rsAPI, - accountsDB, federation, transactionsCache, - ) - return appserviceQueryAPI } diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go index 0ce44c211..d00be6eb0 100644 --- a/clientapi/clientapi.go +++ b/clientapi/clientapi.go @@ -15,6 +15,7 @@ package clientapi import ( + "github.com/gorilla/mux" appserviceAPI "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" @@ -30,9 +31,9 @@ import ( "github.com/sirupsen/logrus" ) -// SetupClientAPIComponent sets up and registers HTTP handlers for the ClientAPI -// component. -func SetupClientAPIComponent( +// AddPublicRoutes sets up and registers HTTP handlers for the ClientAPI component. +func AddPublicRoutes( + router *mux.Router, base *basecomponent.BaseDendrite, deviceDB devices.Database, accountsDB accounts.Database, @@ -65,7 +66,7 @@ func SetupClientAPIComponent( } routing.Setup( - base.PublicAPIMux, base.Cfg, roomserverProducer, rsAPI, asAPI, + router, base.Cfg, roomserverProducer, rsAPI, asAPI, accountsDB, deviceDB, federation, *keyRing, userUpdateProducer, syncProducer, eduProducer, transactionsCache, fsAPI, ) diff --git a/cmd/dendrite-appservice-server/main.go b/cmd/dendrite-appservice-server/main.go index 4fc2a173d..21fb1b79c 100644 --- a/cmd/dendrite-appservice-server/main.go +++ b/cmd/dendrite-appservice-server/main.go @@ -31,9 +31,9 @@ func main() { rsAPI := base.RoomserverHTTPClient() cache := transactions.New() - appservice.SetupAppServiceAPIComponent( - base, accountDB, deviceDB, federation, rsAPI, cache, - ) + intAPI := appservice.NewInternalAPI(base, accountDB, deviceDB, rsAPI) + appservice.AddInternalRoutes(base.InternalAPIMux, intAPI) + appservice.AddPublicRoutes(base.PublicAPIMux, base.Cfg, rsAPI, accountDB, federation, cache) base.SetupAndServeHTTP(string(base.Cfg.Bind.AppServiceAPI), string(base.Cfg.Listen.AppServiceAPI)) diff --git a/cmd/dendrite-client-api-server/main.go b/cmd/dendrite-client-api-server/main.go index 8ad4231d1..58396205c 100644 --- a/cmd/dendrite-client-api-server/main.go +++ b/cmd/dendrite-client-api-server/main.go @@ -38,8 +38,8 @@ func main() { fsAPI := base.FederationSenderHTTPClient() eduInputAPI := base.EDUServerClient() - clientapi.SetupClientAPIComponent( - base, deviceDB, accountDB, federation, keyRing, + clientapi.AddPublicRoutes( + base.PublicAPIMux, base, deviceDB, accountDB, federation, keyRing, rsAPI, eduInputAPI, asQuery, transactions.New(), fsAPI, ) diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index 86909685a..80e2e800f 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -136,42 +136,41 @@ func main() { deviceDB := base.Base.CreateDeviceDB() federation := createFederationClient(base) - serverKeyAPI := serverkeyapi.SetupServerKeyAPIComponent( - &base.Base, federation, + serverKeyAPI := serverkeyapi.NewInternalAPI( + base.Base.Cfg, federation, base.Base.Caches, ) keyRing := serverKeyAPI.KeyRing() createKeyDB( base, serverKeyAPI, ) - rsAPI := roomserver.SetupRoomServerComponent( + rsAPI := roomserver.NewInternalAPI( &base.Base, keyRing, federation, ) - eduInputAPI := eduserver.SetupEDUServerComponent( + eduInputAPI := eduserver.NewInternalAPI( &base.Base, cache.New(), deviceDB, ) - asAPI := appservice.SetupAppServiceAPIComponent( - &base.Base, accountDB, deviceDB, federation, rsAPI, transactions.New(), - ) - fsAPI := federationsender.SetupFederationSenderComponent( + asAPI := appservice.NewInternalAPI(&base.Base, accountDB, deviceDB, rsAPI) + appservice.AddPublicRoutes(base.Base.PublicAPIMux, &cfg, rsAPI, accountDB, federation, transactions.New()) + fsAPI := federationsender.NewInternalAPI( &base.Base, federation, rsAPI, keyRing, ) rsAPI.SetFederationSenderAPI(fsAPI) - clientapi.SetupClientAPIComponent( - &base.Base, deviceDB, accountDB, + clientapi.AddPublicRoutes( + base.Base.PublicAPIMux, &base.Base, deviceDB, accountDB, federation, keyRing, rsAPI, eduInputAPI, asAPI, transactions.New(), fsAPI, ) eduProducer := producers.NewEDUServerProducer(eduInputAPI) - federationapi.SetupFederationAPIComponent(&base.Base, accountDB, deviceDB, federation, keyRing, rsAPI, asAPI, fsAPI, eduProducer) - mediaapi.SetupMediaAPIComponent(&base.Base, deviceDB) + federationapi.AddPublicRoutes(base.Base.PublicAPIMux, base.Base.Cfg, accountDB, deviceDB, federation, keyRing, rsAPI, asAPI, fsAPI, eduProducer) + mediaapi.AddPublicRoutes(base.Base.PublicAPIMux, base.Base.Cfg, deviceDB) publicRoomsDB, err := storage.NewPublicRoomsServerDatabaseWithPubSub(string(base.Base.Cfg.Database.PublicRoomsAPI), base.LibP2PPubsub, cfg.Matrix.ServerName) if err != nil { logrus.WithError(err).Panicf("failed to connect to public rooms db") } - publicroomsapi.SetupPublicRoomsAPIComponent(&base.Base, deviceDB, publicRoomsDB, rsAPI, federation, nil) // Check this later - syncapi.SetupSyncAPIComponent(&base.Base, deviceDB, accountDB, rsAPI, federation, &cfg) + publicroomsapi.AddPublicRoutes(base.Base.PublicAPIMux, &base.Base, deviceDB, publicRoomsDB, rsAPI, federation, nil) // Check this later + syncapi.AddPublicRoutes(base.Base.PublicAPIMux, &base.Base, deviceDB, accountDB, rsAPI, federation, &cfg) internal.SetupHTTPAPI( http.DefaultServeMux, diff --git a/cmd/dendrite-edu-server/main.go b/cmd/dendrite-edu-server/main.go index ceeb13658..a86f66c89 100644 --- a/cmd/dendrite-edu-server/main.go +++ b/cmd/dendrite-edu-server/main.go @@ -31,7 +31,8 @@ func main() { }() deviceDB := base.CreateDeviceDB() - eduserver.SetupEDUServerComponent(base, cache.New(), deviceDB) + intAPI := eduserver.NewInternalAPI(base, cache.New(), deviceDB) + eduserver.AddInternalRoutes(base.InternalAPIMux, intAPI) base.SetupAndServeHTTP(string(base.Cfg.Bind.EDUServer), string(base.Cfg.Listen.EDUServer)) diff --git a/cmd/dendrite-federation-api-server/main.go b/cmd/dendrite-federation-api-server/main.go index 0c4d8b854..7481299e9 100644 --- a/cmd/dendrite-federation-api-server/main.go +++ b/cmd/dendrite-federation-api-server/main.go @@ -36,8 +36,8 @@ func main() { // TODO: this isn't a producer eduProducer := producers.NewEDUServerProducer(base.EDUServerClient()) - federationapi.SetupFederationAPIComponent( - base, accountDB, deviceDB, federation, keyRing, + federationapi.AddPublicRoutes( + base.PublicAPIMux, base.Cfg, accountDB, deviceDB, federation, keyRing, rsAPI, asAPI, fsAPI, eduProducer, ) diff --git a/cmd/dendrite-federation-sender-server/main.go b/cmd/dendrite-federation-sender-server/main.go index 651eb257f..7b60ad055 100644 --- a/cmd/dendrite-federation-sender-server/main.go +++ b/cmd/dendrite-federation-sender-server/main.go @@ -30,9 +30,10 @@ func main() { keyRing := serverKeyAPI.KeyRing() rsAPI := base.RoomserverHTTPClient() - federationsender.SetupFederationSenderComponent( + fsAPI := federationsender.NewInternalAPI( base, federation, rsAPI, keyRing, ) + federationsender.AddInternalRoutes(base.InternalAPIMux, fsAPI) base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationSender), string(base.Cfg.Listen.FederationSender)) diff --git a/cmd/dendrite-key-server/main.go b/cmd/dendrite-key-server/main.go index a8accbcb3..5dccc3004 100644 --- a/cmd/dendrite-key-server/main.go +++ b/cmd/dendrite-key-server/main.go @@ -27,7 +27,7 @@ func main() { accountDB := base.CreateAccountsDB() deviceDB := base.CreateDeviceDB() - keyserver.SetupKeyServerComponent(base, deviceDB, accountDB) + keyserver.AddPublicRoutes(base.PublicAPIMux, base.Cfg, deviceDB, accountDB) base.SetupAndServeHTTP(string(base.Cfg.Bind.KeyServer), string(base.Cfg.Listen.KeyServer)) diff --git a/cmd/dendrite-media-api-server/main.go b/cmd/dendrite-media-api-server/main.go index 018559383..5c65fad05 100644 --- a/cmd/dendrite-media-api-server/main.go +++ b/cmd/dendrite-media-api-server/main.go @@ -26,7 +26,7 @@ func main() { deviceDB := base.CreateDeviceDB() - mediaapi.SetupMediaAPIComponent(base, deviceDB) + mediaapi.AddPublicRoutes(base.PublicAPIMux, base.Cfg, deviceDB) base.SetupAndServeHTTP(string(base.Cfg.Bind.MediaAPI), string(base.Cfg.Listen.MediaAPI)) diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 57e8f66f5..97fa3a2a6 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -70,62 +70,66 @@ func main() { deviceDB := base.CreateDeviceDB() federation := base.CreateFederationClient() - serverKeyAPI := serverkeyapi.SetupServerKeyAPIComponent( - base, federation, + serverKeyAPI := serverkeyapi.NewInternalAPI( + base.Cfg, federation, base.Caches, ) if base.UseHTTPAPIs { + serverkeyapi.AddInternalRoutes(base.InternalAPIMux, serverKeyAPI, base.Caches) serverKeyAPI = base.ServerKeyAPIClient() } keyRing := serverKeyAPI.KeyRing() - rsComponent := roomserver.SetupRoomServerComponent( + rsComponent := roomserver.NewInternalAPI( base, keyRing, federation, ) rsAPI := rsComponent if base.UseHTTPAPIs { + roomserver.AddInternalRoutes(base.InternalAPIMux, rsAPI) rsAPI = base.RoomserverHTTPClient() } - eduInputAPI := eduserver.SetupEDUServerComponent( + eduInputAPI := eduserver.NewInternalAPI( base, cache.New(), deviceDB, ) if base.UseHTTPAPIs { + eduserver.AddInternalRoutes(base.InternalAPIMux, eduInputAPI) eduInputAPI = base.EDUServerClient() } - asAPI := appservice.SetupAppServiceAPIComponent( - base, accountDB, deviceDB, federation, rsAPI, transactions.New(), - ) + asAPI := appservice.NewInternalAPI(base, accountDB, deviceDB, rsAPI) + appservice.AddPublicRoutes(base.PublicAPIMux, cfg, rsAPI, accountDB, federation, transactions.New()) if base.UseHTTPAPIs { + appservice.AddInternalRoutes(base.InternalAPIMux, asAPI) asAPI = base.AppserviceHTTPClient() } - fsAPI := federationsender.SetupFederationSenderComponent( + fsAPI := federationsender.NewInternalAPI( base, federation, rsAPI, keyRing, ) if base.UseHTTPAPIs { + federationsender.AddInternalRoutes(base.InternalAPIMux, fsAPI) fsAPI = base.FederationSenderHTTPClient() } rsComponent.SetFederationSenderAPI(fsAPI) - clientapi.SetupClientAPIComponent( - base, deviceDB, accountDB, + clientapi.AddPublicRoutes( + base.PublicAPIMux, base, deviceDB, accountDB, federation, keyRing, rsAPI, eduInputAPI, asAPI, transactions.New(), fsAPI, ) - keyserver.SetupKeyServerComponent( - base, deviceDB, accountDB, + keyserver.AddPublicRoutes( + base.PublicAPIMux, base.Cfg, deviceDB, accountDB, ) eduProducer := producers.NewEDUServerProducer(eduInputAPI) - federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, keyRing, rsAPI, asAPI, fsAPI, eduProducer) - mediaapi.SetupMediaAPIComponent(base, deviceDB) + federationapi.AddPublicRoutes(base.PublicAPIMux, base.Cfg, accountDB, deviceDB, federation, keyRing, rsAPI, asAPI, fsAPI, eduProducer) + mediaapi.AddPublicRoutes(base.PublicAPIMux, base.Cfg, deviceDB) publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties(), cfg.Matrix.ServerName) if err != nil { logrus.WithError(err).Panicf("failed to connect to public rooms db") } - publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, rsAPI, federation, nil) - syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, rsAPI, federation, cfg) + publicroomsapi.AddPublicRoutes(base.PublicAPIMux, base, deviceDB, publicRoomsDB, rsAPI, federation, nil) + syncapi.AddPublicRoutes(base.PublicAPIMux, base, deviceDB, accountDB, rsAPI, federation, cfg) internal.SetupHTTPAPI( http.DefaultServeMux, diff --git a/cmd/dendrite-public-rooms-api-server/main.go b/cmd/dendrite-public-rooms-api-server/main.go index 0cad3670c..413d7ecbb 100644 --- a/cmd/dendrite-public-rooms-api-server/main.go +++ b/cmd/dendrite-public-rooms-api-server/main.go @@ -34,7 +34,7 @@ func main() { if err != nil { logrus.WithError(err).Panicf("failed to connect to public rooms db") } - publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, rsAPI, nil, nil) + publicroomsapi.AddPublicRoutes(base.PublicAPIMux, base, deviceDB, publicRoomsDB, rsAPI, nil, nil) base.SetupAndServeHTTP(string(base.Cfg.Bind.PublicRoomsAPI), string(base.Cfg.Listen.PublicRoomsAPI)) diff --git a/cmd/dendrite-room-server/main.go b/cmd/dendrite-room-server/main.go index 79745cded..a2f1e9415 100644 --- a/cmd/dendrite-room-server/main.go +++ b/cmd/dendrite-room-server/main.go @@ -29,8 +29,9 @@ func main() { keyRing := serverKeyAPI.KeyRing() fsAPI := base.FederationSenderHTTPClient() - rsAPI := roomserver.SetupRoomServerComponent(base, keyRing, federation) + rsAPI := roomserver.NewInternalAPI(base, keyRing, federation) rsAPI.SetFederationSenderAPI(fsAPI) + roomserver.AddInternalRoutes(base.InternalAPIMux, rsAPI) base.SetupAndServeHTTP(string(base.Cfg.Bind.RoomServer), string(base.Cfg.Listen.RoomServer)) diff --git a/cmd/dendrite-server-key-api-server/main.go b/cmd/dendrite-server-key-api-server/main.go index e206978cd..b4bfcbffb 100644 --- a/cmd/dendrite-server-key-api-server/main.go +++ b/cmd/dendrite-server-key-api-server/main.go @@ -26,7 +26,8 @@ func main() { federation := base.CreateFederationClient() - serverkeyapi.SetupServerKeyAPIComponent(base, federation) + intAPI := serverkeyapi.NewInternalAPI(base.Cfg, federation, base.Caches) + serverkeyapi.AddInternalRoutes(base.InternalAPIMux, intAPI, base.Caches) base.SetupAndServeHTTP(string(base.Cfg.Bind.ServerKeyAPI), string(base.Cfg.Listen.ServerKeyAPI)) } diff --git a/cmd/dendrite-sync-api-server/main.go b/cmd/dendrite-sync-api-server/main.go index bb56b4ab0..4ad68c5e8 100644 --- a/cmd/dendrite-sync-api-server/main.go +++ b/cmd/dendrite-sync-api-server/main.go @@ -30,7 +30,7 @@ func main() { rsAPI := base.RoomserverHTTPClient() - syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, rsAPI, federation, cfg) + syncapi.AddPublicRoutes(base.PublicAPIMux, base, deviceDB, accountDB, rsAPI, federation, cfg) base.SetupAndServeHTTP(string(base.Cfg.Bind.SyncAPI), string(base.Cfg.Listen.SyncAPI)) diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 4c1b81979..9c6c7c031 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -206,29 +206,29 @@ func main() { KeyDatabase: fetcher, } - rsAPI := roomserver.SetupRoomServerComponent(base, keyRing, federation) - eduInputAPI := eduserver.SetupEDUServerComponent(base, cache.New(), deviceDB) - asQuery := appservice.SetupAppServiceAPIComponent( - base, accountDB, deviceDB, federation, rsAPI, transactions.New(), + rsAPI := roomserver.NewInternalAPI(base, keyRing, federation) + eduInputAPI := eduserver.NewInternalAPI(base, cache.New(), deviceDB) + asQuery := appservice.NewInternalAPI( + base, accountDB, deviceDB, rsAPI, ) - fedSenderAPI := federationsender.SetupFederationSenderComponent(base, federation, rsAPI, &keyRing) + fedSenderAPI := federationsender.NewInternalAPI(base, federation, rsAPI, &keyRing) rsAPI.SetFederationSenderAPI(fedSenderAPI) p2pPublicRoomProvider := NewLibP2PPublicRoomsProvider(node, fedSenderAPI) - clientapi.SetupClientAPIComponent( - base, deviceDB, accountDB, + clientapi.AddPublicRoutes( + base.PublicAPIMux, base, deviceDB, accountDB, federation, &keyRing, rsAPI, eduInputAPI, asQuery, transactions.New(), fedSenderAPI, ) eduProducer := producers.NewEDUServerProducer(eduInputAPI) - federationapi.SetupFederationAPIComponent(base, accountDB, deviceDB, federation, &keyRing, rsAPI, asQuery, fedSenderAPI, eduProducer) - mediaapi.SetupMediaAPIComponent(base, deviceDB) + federationapi.AddPublicRoutes(base.PublicAPIMux, base.Cfg, accountDB, deviceDB, federation, &keyRing, rsAPI, asQuery, fedSenderAPI, eduProducer) + mediaapi.AddPublicRoutes(base.PublicAPIMux, base.Cfg, deviceDB) publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), cfg.Matrix.ServerName) if err != nil { logrus.WithError(err).Panicf("failed to connect to public rooms db") } - publicroomsapi.SetupPublicRoomsAPIComponent(base, deviceDB, publicRoomsDB, rsAPI, federation, p2pPublicRoomProvider) - syncapi.SetupSyncAPIComponent(base, deviceDB, accountDB, rsAPI, federation, cfg) + publicroomsapi.AddPublicRoutes(base.PublicAPIMux, base, deviceDB, publicRoomsDB, rsAPI, federation, p2pPublicRoomProvider) + syncapi.AddPublicRoutes(base.PublicAPIMux, base, deviceDB, accountDB, rsAPI, federation, cfg) internal.SetupHTTPAPI( http.DefaultServeMux, diff --git a/eduserver/eduserver.go b/eduserver/eduserver.go index 59decc8fb..c14be3f67 100644 --- a/eduserver/eduserver.go +++ b/eduserver/eduserver.go @@ -17,6 +17,7 @@ package eduserver import ( + "github.com/gorilla/mux" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/eduserver/cache" @@ -25,16 +26,20 @@ import ( "github.com/matrix-org/dendrite/internal/basecomponent" ) -// SetupEDUServerComponent sets up and registers HTTP handlers for the -// EDUServer component. Returns instances of the various roomserver APIs, -// allowing other components running in the same process to hit the query the -// APIs directly instead of having to use HTTP. -func SetupEDUServerComponent( +// AddInternalRoutes registers HTTP handlers for the internal API. Invokes functions +// on the given input API. +func AddInternalRoutes(internalMux *mux.Router, inputAPI api.EDUServerInputAPI) { + inthttp.AddRoutes(inputAPI, internalMux) +} + +// NewInternalAPI returns a concerete implementation of the internal API. Callers +// can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. +func NewInternalAPI( base *basecomponent.BaseDendrite, eduCache *cache.EDUCache, deviceDB devices.Database, ) api.EDUServerInputAPI { - inputAPI := &input.EDUServerInputAPI{ + return &input.EDUServerInputAPI{ Cache: eduCache, DeviceDB: deviceDB, Producer: base.KafkaProducer, @@ -42,8 +47,4 @@ func SetupEDUServerComponent( OutputSendToDeviceEventTopic: string(base.Cfg.Kafka.Topics.OutputSendToDeviceEvent), ServerName: base.Cfg.Matrix.ServerName, } - - inthttp.AddRoutes(inputAPI, base.InternalAPIMux) - - return inputAPI } diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go index baeaa36bc..7aecd272b 100644 --- a/federationapi/federationapi.go +++ b/federationapi/federationapi.go @@ -15,11 +15,12 @@ package federationapi import ( + "github.com/gorilla/mux" appserviceAPI "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/config" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" // TODO: Are we really wanting to pull in the producer from clientapi @@ -28,10 +29,10 @@ import ( "github.com/matrix-org/gomatrixserverlib" ) -// SetupFederationAPIComponent sets up and registers HTTP handlers for the -// FederationAPI component. -func SetupFederationAPIComponent( - base *basecomponent.BaseDendrite, +// AddPublicRoutes sets up and registers HTTP handlers on the base API muxes for the FederationAPI component. +func AddPublicRoutes( + router *mux.Router, + cfg *config.Dendrite, accountsDB accounts.Database, deviceDB devices.Database, federation *gomatrixserverlib.FederationClient, @@ -44,7 +45,7 @@ func SetupFederationAPIComponent( roomserverProducer := producers.NewRoomserverProducer(rsAPI) routing.Setup( - base.PublicAPIMux, base.Cfg, rsAPI, asAPI, roomserverProducer, + router, cfg, rsAPI, asAPI, roomserverProducer, eduProducer, federationSenderAPI, *keyRing, federation, accountsDB, deviceDB, ) diff --git a/federationsender/federationsender.go b/federationsender/federationsender.go index e25c27237..e3e0ef9d8 100644 --- a/federationsender/federationsender.go +++ b/federationsender/federationsender.go @@ -15,6 +15,7 @@ package federationsender import ( + "github.com/gorilla/mux" "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/federationsender/consumers" "github.com/matrix-org/dendrite/federationsender/internal" @@ -29,9 +30,15 @@ import ( "github.com/sirupsen/logrus" ) -// SetupFederationSenderComponent sets up and registers HTTP handlers for the -// FederationSender component. -func SetupFederationSenderComponent( +// AddInternalRoutes registers HTTP handlers for the internal API. Invokes functions +// on the given input API. +func AddInternalRoutes(router *mux.Router, intAPI api.FederationSenderInternalAPI) { + inthttp.AddRoutes(intAPI, router) +} + +// NewInternalAPI returns a concerete implementation of the internal API. Callers +// can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. +func NewInternalAPI( base *basecomponent.BaseDendrite, federation *gomatrixserverlib.FederationClient, rsAPI roomserverAPI.RoomserverInternalAPI, @@ -66,8 +73,5 @@ func SetupFederationSenderComponent( logrus.WithError(err).Panic("failed to start typing server consumer") } - queryAPI := internal.NewFederationSenderInternalAPI(federationSenderDB, base.Cfg, roomserverProducer, federation, keyRing, statistics, queues) - inthttp.AddRoutes(queryAPI, base.InternalAPIMux) - - return queryAPI + return internal.NewFederationSenderInternalAPI(federationSenderDB, base.Cfg, roomserverProducer, federation, keyRing, statistics, queues) } diff --git a/keyserver/keyserver.go b/keyserver/keyserver.go index 4e785cbc7..1eb730541 100644 --- a/keyserver/keyserver.go +++ b/keyserver/keyserver.go @@ -15,18 +15,18 @@ package keyserver import ( + "github.com/gorilla/mux" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/keyserver/routing" ) -// SetupFederationSenderComponent sets up and registers HTTP handlers for the -// FederationSender component. -func SetupKeyServerComponent( - base *basecomponent.BaseDendrite, +// AddPublicRoutes registers HTTP handlers for CS API calls +func AddPublicRoutes( + router *mux.Router, cfg *config.Dendrite, deviceDB devices.Database, accountsDB accounts.Database, ) { - routing.Setup(base.PublicAPIMux, base.Cfg, accountsDB, deviceDB) + routing.Setup(router, cfg, accountsDB, deviceDB) } diff --git a/mediaapi/mediaapi.go b/mediaapi/mediaapi.go index b5bec3907..d4e260ea4 100644 --- a/mediaapi/mediaapi.go +++ b/mediaapi/mediaapi.go @@ -15,26 +15,26 @@ package mediaapi import ( + "github.com/gorilla/mux" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/mediaapi/routing" "github.com/matrix-org/dendrite/mediaapi/storage" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" ) -// SetupMediaAPIComponent sets up and registers HTTP handlers for the MediaAPI -// component. -func SetupMediaAPIComponent( - base *basecomponent.BaseDendrite, +// AddPublicRoutes sets up and registers HTTP handlers for the MediaAPI component. +func AddPublicRoutes( + router *mux.Router, cfg *config.Dendrite, deviceDB devices.Database, ) { - mediaDB, err := storage.Open(string(base.Cfg.Database.MediaAPI), base.Cfg.DbProperties()) + mediaDB, err := storage.Open(string(cfg.Database.MediaAPI), cfg.DbProperties()) if err != nil { logrus.WithError(err).Panicf("failed to connect to media db") } routing.Setup( - base.PublicAPIMux, base.Cfg, mediaDB, deviceDB, gomatrixserverlib.NewClient(), + router, cfg, mediaDB, deviceDB, gomatrixserverlib.NewClient(), ) } diff --git a/publicroomsapi/publicroomsapi.go b/publicroomsapi/publicroomsapi.go index b53351ffd..280ab9e3d 100644 --- a/publicroomsapi/publicroomsapi.go +++ b/publicroomsapi/publicroomsapi.go @@ -15,6 +15,7 @@ package publicroomsapi import ( + "github.com/gorilla/mux" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/internal/basecomponent" "github.com/matrix-org/dendrite/publicroomsapi/consumers" @@ -26,9 +27,10 @@ import ( "github.com/sirupsen/logrus" ) -// SetupPublicRoomsAPIComponent sets up and registers HTTP handlers for the PublicRoomsAPI +// AddPublicRoutes sets up and registers HTTP handlers for the PublicRoomsAPI // component. -func SetupPublicRoomsAPIComponent( +func AddPublicRoutes( + router *mux.Router, base *basecomponent.BaseDendrite, deviceDB devices.Database, publicRoomsDB storage.Database, @@ -43,5 +45,5 @@ func SetupPublicRoomsAPIComponent( logrus.WithError(err).Panic("failed to start public rooms server consumer") } - routing.Setup(base.PublicAPIMux, deviceDB, publicRoomsDB, rsAPI, fedClient, extRoomsProvider) + routing.Setup(router, deviceDB, publicRoomsDB, rsAPI, fedClient, extRoomsProvider) } diff --git a/roomserver/roomserver.go b/roomserver/roomserver.go index a55b20bef..a9db22d7a 100644 --- a/roomserver/roomserver.go +++ b/roomserver/roomserver.go @@ -15,6 +15,7 @@ package roomserver import ( + "github.com/gorilla/mux" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/inthttp" "github.com/matrix-org/gomatrixserverlib" @@ -25,11 +26,15 @@ import ( "github.com/sirupsen/logrus" ) -// SetupRoomServerComponent sets up and registers HTTP handlers for the -// RoomServer component. Returns instances of the various roomserver APIs, -// allowing other components running in the same process to hit the query the -// APIs directly instead of having to use HTTP. -func SetupRoomServerComponent( +// AddInternalRoutes registers HTTP handlers for the internal API. Invokes functions +// on the given input API. +func AddInternalRoutes(router *mux.Router, intAPI api.RoomserverInternalAPI) { + inthttp.AddRoutes(intAPI, router) +} + +// NewInternalAPI returns a concerete implementation of the internal API. Callers +// can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. +func NewInternalAPI( base *basecomponent.BaseDendrite, keyRing gomatrixserverlib.JSONVerifier, fedClient *gomatrixserverlib.FederationClient, @@ -39,7 +44,7 @@ func SetupRoomServerComponent( logrus.WithError(err).Panicf("failed to connect to room server db") } - internalAPI := &internal.RoomserverInternalAPI{ + return &internal.RoomserverInternalAPI{ DB: roomserverDB, Cfg: base.Cfg, Producer: base.KafkaProducer, @@ -49,8 +54,4 @@ func SetupRoomServerComponent( FedClient: fedClient, KeyRing: keyRing, } - - inthttp.AddRoutes(internalAPI, base.InternalAPIMux) - - return internalAPI } diff --git a/serverkeyapi/serverkeyapi.go b/serverkeyapi/serverkeyapi.go index ad885270a..58ca00b73 100644 --- a/serverkeyapi/serverkeyapi.go +++ b/serverkeyapi/serverkeyapi.go @@ -4,7 +4,9 @@ import ( "crypto/ed25519" "encoding/base64" - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/gorilla/mux" + "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/serverkeyapi/api" "github.com/matrix-org/dendrite/serverkeyapi/internal" "github.com/matrix-org/dendrite/serverkeyapi/inthttp" @@ -14,22 +16,31 @@ import ( "github.com/sirupsen/logrus" ) -func SetupServerKeyAPIComponent( - base *basecomponent.BaseDendrite, +// AddInternalRoutes registers HTTP handlers for the internal API. Invokes functions +// on the given input API. +func AddInternalRoutes(router *mux.Router, intAPI api.ServerKeyInternalAPI, caches *caching.Caches) { + inthttp.AddRoutes(intAPI, router, caches) +} + +// NewInternalAPI returns a concerete implementation of the internal API. Callers +// can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. +func NewInternalAPI( + cfg *config.Dendrite, fedClient *gomatrixserverlib.FederationClient, + caches *caching.Caches, ) api.ServerKeyInternalAPI { innerDB, err := storage.NewDatabase( - string(base.Cfg.Database.ServerKey), - base.Cfg.DbProperties(), - base.Cfg.Matrix.ServerName, - base.Cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey), - base.Cfg.Matrix.KeyID, + string(cfg.Database.ServerKey), + cfg.DbProperties(), + cfg.Matrix.ServerName, + cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey), + cfg.Matrix.KeyID, ) if err != nil { logrus.WithError(err).Panicf("failed to connect to server key database") } - serverKeyDB, err := cache.NewKeyDatabase(innerDB, base.Caches) + serverKeyDB, err := cache.NewKeyDatabase(innerDB, caches) if err != nil { logrus.WithError(err).Panicf("failed to set up caching wrapper for server key database") } @@ -47,7 +58,7 @@ func SetupServerKeyAPIComponent( } var b64e = base64.StdEncoding.WithPadding(base64.NoPadding) - for _, ps := range base.Cfg.Matrix.KeyPerspectives { + for _, ps := range cfg.Matrix.KeyPerspectives { perspective := &gomatrixserverlib.PerspectiveKeyFetcher{ PerspectiveServerName: ps.ServerName, PerspectiveServerKeys: map[gomatrixserverlib.KeyID]ed25519.PublicKey{}, @@ -77,7 +88,5 @@ func SetupServerKeyAPIComponent( }).Info("Enabled perspective key fetcher") } - inthttp.AddRoutes(&internalAPI, base.InternalAPIMux, base.Caches) - return &internalAPI } diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go index 762f4e9d3..6e84dcb6f 100644 --- a/syncapi/syncapi.go +++ b/syncapi/syncapi.go @@ -17,6 +17,7 @@ package syncapi import ( "context" + "github.com/gorilla/mux" "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" @@ -32,9 +33,10 @@ import ( "github.com/matrix-org/dendrite/syncapi/sync" ) -// SetupSyncAPIComponent sets up and registers HTTP handlers for the SyncAPI +// AddPublicRoutes sets up and registers HTTP handlers for the SyncAPI // component. -func SetupSyncAPIComponent( +func AddPublicRoutes( + router *mux.Router, base *basecomponent.BaseDendrite, deviceDB devices.Database, accountsDB accounts.Database, @@ -88,5 +90,5 @@ func SetupSyncAPIComponent( logrus.WithError(err).Panicf("failed to start send-to-device consumer") } - routing.Setup(base.PublicAPIMux, requestPool, syncDB, deviceDB, federation, rsAPI, cfg) + routing.Setup(router, requestPool, syncDB, deviceDB, federation, rsAPI, cfg) } From 85ac8a3f5ba407ece584843a4d77466c1c4f5565 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 9 Jun 2020 12:07:33 +0100 Subject: [PATCH 142/200] Factor out how monolith routes get added (#1107) Previously we had 3 monoliths: - dendrite-monolith-server - dendrite-demo-libp2p - dendritejs which all had their own of setting up public routes. Factor this out into a new `setup.Monolith` struct which gets all dependencies set as fields. This is different to `basecomponent.Base` which doesn't provide any way to set configured deps (e.g public rooms db) Part of a larger process to clean up how we initialise Dendrite. --- clientapi/clientapi.go | 23 +++--- cmd/dendrite-client-api-server/main.go | 2 +- cmd/dendrite-demo-libp2p/main.go | 38 +++++----- cmd/dendrite-monolith-server/main.go | 42 +++++------ cmd/dendrite-public-rooms-api-server/main.go | 2 +- cmd/dendrite-sync-api-server/main.go | 2 +- cmd/dendritejs/main.go | 37 ++++++---- internal/setup/monolith.go | 78 ++++++++++++++++++++ publicroomsapi/publicroomsapi.go | 8 +- syncapi/syncapi.go | 14 ++-- 10 files changed, 170 insertions(+), 76 deletions(-) create mode 100644 internal/setup/monolith.go diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go index d00be6eb0..987815c23 100644 --- a/clientapi/clientapi.go +++ b/clientapi/clientapi.go @@ -15,6 +15,7 @@ package clientapi import ( + "github.com/Shopify/sarama" "github.com/gorilla/mux" appserviceAPI "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" @@ -24,7 +25,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/routing" eduServerAPI "github.com/matrix-org/dendrite/eduserver/api" federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/transactions" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" @@ -34,7 +35,9 @@ import ( // AddPublicRoutes sets up and registers HTTP handlers for the ClientAPI component. func AddPublicRoutes( router *mux.Router, - base *basecomponent.BaseDendrite, + cfg *config.Dendrite, + consumer sarama.Consumer, + producer sarama.SyncProducer, deviceDB devices.Database, accountsDB accounts.Database, federation *gomatrixserverlib.FederationClient, @@ -49,24 +52,24 @@ func AddPublicRoutes( eduProducer := producers.NewEDUServerProducer(eduInputAPI) userUpdateProducer := &producers.UserUpdateProducer{ - Producer: base.KafkaProducer, - Topic: string(base.Cfg.Kafka.Topics.UserUpdates), + Producer: producer, + Topic: string(cfg.Kafka.Topics.UserUpdates), } syncProducer := &producers.SyncAPIProducer{ - Producer: base.KafkaProducer, - Topic: string(base.Cfg.Kafka.Topics.OutputClientData), + Producer: producer, + Topic: string(cfg.Kafka.Topics.OutputClientData), } - consumer := consumers.NewOutputRoomEventConsumer( - base.Cfg, base.KafkaConsumer, accountsDB, rsAPI, + roomEventConsumer := consumers.NewOutputRoomEventConsumer( + cfg, consumer, accountsDB, rsAPI, ) - if err := consumer.Start(); err != nil { + if err := roomEventConsumer.Start(); err != nil { logrus.WithError(err).Panicf("failed to start room server consumer") } routing.Setup( - router, base.Cfg, roomserverProducer, rsAPI, asAPI, + router, cfg, roomserverProducer, rsAPI, asAPI, accountsDB, deviceDB, federation, *keyRing, userUpdateProducer, syncProducer, eduProducer, transactionsCache, fsAPI, ) diff --git a/cmd/dendrite-client-api-server/main.go b/cmd/dendrite-client-api-server/main.go index 58396205c..9cf57ef68 100644 --- a/cmd/dendrite-client-api-server/main.go +++ b/cmd/dendrite-client-api-server/main.go @@ -39,7 +39,7 @@ func main() { eduInputAPI := base.EDUServerClient() clientapi.AddPublicRoutes( - base.PublicAPIMux, base, deviceDB, accountDB, federation, keyRing, + base.PublicAPIMux, base.Cfg, base.KafkaConsumer, base.KafkaProducer, deviceDB, accountDB, federation, keyRing, rsAPI, eduInputAPI, asQuery, transactions.New(), fsAPI, ) diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index 80e2e800f..377e017ef 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -29,20 +29,15 @@ import ( p2phttp "github.com/libp2p/go-libp2p-http" p2pdisc "github.com/libp2p/go-libp2p/p2p/discovery" "github.com/matrix-org/dendrite/appservice" - "github.com/matrix-org/dendrite/clientapi" "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/cmd/dendrite-demo-libp2p/storage" "github.com/matrix-org/dendrite/eduserver" - "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/federationsender" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" - "github.com/matrix-org/dendrite/internal/transactions" - "github.com/matrix-org/dendrite/mediaapi" - "github.com/matrix-org/dendrite/publicroomsapi" + "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/serverkeyapi" - "github.com/matrix-org/dendrite/syncapi" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/dendrite/eduserver/cache" @@ -151,26 +146,35 @@ func main() { &base.Base, cache.New(), deviceDB, ) asAPI := appservice.NewInternalAPI(&base.Base, accountDB, deviceDB, rsAPI) - appservice.AddPublicRoutes(base.Base.PublicAPIMux, &cfg, rsAPI, accountDB, federation, transactions.New()) fsAPI := federationsender.NewInternalAPI( &base.Base, federation, rsAPI, keyRing, ) rsAPI.SetFederationSenderAPI(fsAPI) - - clientapi.AddPublicRoutes( - base.Base.PublicAPIMux, &base.Base, deviceDB, accountDB, - federation, keyRing, rsAPI, - eduInputAPI, asAPI, transactions.New(), fsAPI, - ) eduProducer := producers.NewEDUServerProducer(eduInputAPI) - federationapi.AddPublicRoutes(base.Base.PublicAPIMux, base.Base.Cfg, accountDB, deviceDB, federation, keyRing, rsAPI, asAPI, fsAPI, eduProducer) - mediaapi.AddPublicRoutes(base.Base.PublicAPIMux, base.Base.Cfg, deviceDB) publicRoomsDB, err := storage.NewPublicRoomsServerDatabaseWithPubSub(string(base.Base.Cfg.Database.PublicRoomsAPI), base.LibP2PPubsub, cfg.Matrix.ServerName) if err != nil { logrus.WithError(err).Panicf("failed to connect to public rooms db") } - publicroomsapi.AddPublicRoutes(base.Base.PublicAPIMux, &base.Base, deviceDB, publicRoomsDB, rsAPI, federation, nil) // Check this later - syncapi.AddPublicRoutes(base.Base.PublicAPIMux, &base.Base, deviceDB, accountDB, rsAPI, federation, &cfg) + + monolith := setup.Monolith{ + Config: base.Base.Cfg, + AccountDB: accountDB, + DeviceDB: deviceDB, + FedClient: federation, + KeyRing: keyRing, + KafkaConsumer: base.Base.KafkaConsumer, + KafkaProducer: base.Base.KafkaProducer, + + AppserviceAPI: asAPI, + EDUInternalAPI: eduInputAPI, + EDUProducer: eduProducer, + FederationSenderAPI: fsAPI, + RoomserverAPI: rsAPI, + ServerKeyAPI: serverKeyAPI, + + PublicRoomsDB: publicRoomsDB, + } + monolith.AddAllPublicRoutes(base.Base.PublicAPIMux) internal.SetupHTTPAPI( http.DefaultServeMux, diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 97fa3a2a6..dda3ade5a 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -19,23 +19,17 @@ import ( "net/http" "github.com/matrix-org/dendrite/appservice" - "github.com/matrix-org/dendrite/clientapi" "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/eduserver/cache" - "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/federationsender" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/basecomponent" "github.com/matrix-org/dendrite/internal/config" - "github.com/matrix-org/dendrite/internal/transactions" - "github.com/matrix-org/dendrite/keyserver" - "github.com/matrix-org/dendrite/mediaapi" - "github.com/matrix-org/dendrite/publicroomsapi" + "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/serverkeyapi" - "github.com/matrix-org/dendrite/syncapi" "github.com/sirupsen/logrus" ) @@ -97,7 +91,6 @@ func main() { } asAPI := appservice.NewInternalAPI(base, accountDB, deviceDB, rsAPI) - appservice.AddPublicRoutes(base.PublicAPIMux, cfg, rsAPI, accountDB, federation, transactions.New()) if base.UseHTTPAPIs { appservice.AddInternalRoutes(base.InternalAPIMux, asAPI) asAPI = base.AppserviceHTTPClient() @@ -112,24 +105,31 @@ func main() { } rsComponent.SetFederationSenderAPI(fsAPI) - clientapi.AddPublicRoutes( - base.PublicAPIMux, base, deviceDB, accountDB, - federation, keyRing, rsAPI, - eduInputAPI, asAPI, transactions.New(), fsAPI, - ) - - keyserver.AddPublicRoutes( - base.PublicAPIMux, base.Cfg, deviceDB, accountDB, - ) eduProducer := producers.NewEDUServerProducer(eduInputAPI) - federationapi.AddPublicRoutes(base.PublicAPIMux, base.Cfg, accountDB, deviceDB, federation, keyRing, rsAPI, asAPI, fsAPI, eduProducer) - mediaapi.AddPublicRoutes(base.PublicAPIMux, base.Cfg, deviceDB) publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties(), cfg.Matrix.ServerName) if err != nil { logrus.WithError(err).Panicf("failed to connect to public rooms db") } - publicroomsapi.AddPublicRoutes(base.PublicAPIMux, base, deviceDB, publicRoomsDB, rsAPI, federation, nil) - syncapi.AddPublicRoutes(base.PublicAPIMux, base, deviceDB, accountDB, rsAPI, federation, cfg) + + monolith := setup.Monolith{ + Config: base.Cfg, + AccountDB: accountDB, + DeviceDB: deviceDB, + FedClient: federation, + KeyRing: keyRing, + KafkaConsumer: base.KafkaConsumer, + KafkaProducer: base.KafkaProducer, + + AppserviceAPI: asAPI, + EDUInternalAPI: eduInputAPI, + EDUProducer: eduProducer, + FederationSenderAPI: fsAPI, + RoomserverAPI: rsAPI, + ServerKeyAPI: serverKeyAPI, + + PublicRoomsDB: publicRoomsDB, + } + monolith.AddAllPublicRoutes(base.PublicAPIMux) internal.SetupHTTPAPI( http.DefaultServeMux, diff --git a/cmd/dendrite-public-rooms-api-server/main.go b/cmd/dendrite-public-rooms-api-server/main.go index 413d7ecbb..ff7d7958f 100644 --- a/cmd/dendrite-public-rooms-api-server/main.go +++ b/cmd/dendrite-public-rooms-api-server/main.go @@ -34,7 +34,7 @@ func main() { if err != nil { logrus.WithError(err).Panicf("failed to connect to public rooms db") } - publicroomsapi.AddPublicRoutes(base.PublicAPIMux, base, deviceDB, publicRoomsDB, rsAPI, nil, nil) + publicroomsapi.AddPublicRoutes(base.PublicAPIMux, base.Cfg, base.KafkaConsumer, deviceDB, publicRoomsDB, rsAPI, nil, nil) base.SetupAndServeHTTP(string(base.Cfg.Bind.PublicRoomsAPI), string(base.Cfg.Listen.PublicRoomsAPI)) diff --git a/cmd/dendrite-sync-api-server/main.go b/cmd/dendrite-sync-api-server/main.go index 4ad68c5e8..a5302f74e 100644 --- a/cmd/dendrite-sync-api-server/main.go +++ b/cmd/dendrite-sync-api-server/main.go @@ -30,7 +30,7 @@ func main() { rsAPI := base.RoomserverHTTPClient() - syncapi.AddPublicRoutes(base.PublicAPIMux, base, deviceDB, accountDB, rsAPI, federation, cfg) + syncapi.AddPublicRoutes(base.PublicAPIMux, base.KafkaConsumer, deviceDB, accountDB, rsAPI, federation, cfg) base.SetupAndServeHTTP(string(base.Cfg.Bind.SyncAPI), string(base.Cfg.Listen.SyncAPI)) diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 9c6c7c031..9901c6e5f 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -23,21 +23,16 @@ import ( "syscall/js" "github.com/matrix-org/dendrite/appservice" - "github.com/matrix-org/dendrite/clientapi" "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/eduserver/cache" - "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/federationsender" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/basecomponent" "github.com/matrix-org/dendrite/internal/config" - "github.com/matrix-org/dendrite/internal/transactions" - "github.com/matrix-org/dendrite/mediaapi" - "github.com/matrix-org/dendrite/publicroomsapi" + "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/roomserver" - "github.com/matrix-org/dendrite/syncapi" go_http_js_libp2p "github.com/matrix-org/go-http-js-libp2p" "github.com/matrix-org/gomatrixserverlib" @@ -215,20 +210,32 @@ func main() { rsAPI.SetFederationSenderAPI(fedSenderAPI) p2pPublicRoomProvider := NewLibP2PPublicRoomsProvider(node, fedSenderAPI) - clientapi.AddPublicRoutes( - base.PublicAPIMux, base, deviceDB, accountDB, - federation, &keyRing, rsAPI, - eduInputAPI, asQuery, transactions.New(), fedSenderAPI, - ) eduProducer := producers.NewEDUServerProducer(eduInputAPI) - federationapi.AddPublicRoutes(base.PublicAPIMux, base.Cfg, accountDB, deviceDB, federation, &keyRing, rsAPI, asQuery, fedSenderAPI, eduProducer) - mediaapi.AddPublicRoutes(base.PublicAPIMux, base.Cfg, deviceDB) publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), cfg.Matrix.ServerName) if err != nil { logrus.WithError(err).Panicf("failed to connect to public rooms db") } - publicroomsapi.AddPublicRoutes(base.PublicAPIMux, base, deviceDB, publicRoomsDB, rsAPI, federation, p2pPublicRoomProvider) - syncapi.AddPublicRoutes(base.PublicAPIMux, base, deviceDB, accountDB, rsAPI, federation, cfg) + + monolith := setup.Monolith{ + Config: base.Cfg, + AccountDB: accountDB, + DeviceDB: deviceDB, + FedClient: federation, + KeyRing: &keyRing, + KafkaConsumer: base.KafkaConsumer, + KafkaProducer: base.KafkaProducer, + + AppserviceAPI: asQuery, + EDUInternalAPI: eduInputAPI, + EDUProducer: eduProducer, + FederationSenderAPI: fedSenderAPI, + RoomserverAPI: rsAPI, + //ServerKeyAPI: serverKeyAPI, + + PublicRoomsDB: publicRoomsDB, + ExtPublicRoomsProvider: p2pPublicRoomProvider, + } + monolith.AddAllPublicRoutes(base.PublicAPIMux) internal.SetupHTTPAPI( http.DefaultServeMux, diff --git a/internal/setup/monolith.go b/internal/setup/monolith.go new file mode 100644 index 000000000..d9bb28395 --- /dev/null +++ b/internal/setup/monolith.go @@ -0,0 +1,78 @@ +package setup + +import ( + "github.com/Shopify/sarama" + "github.com/gorilla/mux" + appserviceAPI "github.com/matrix-org/dendrite/appservice/api" + "github.com/matrix-org/dendrite/clientapi" + "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" + "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" + "github.com/matrix-org/dendrite/clientapi/producers" + eduServerAPI "github.com/matrix-org/dendrite/eduserver/api" + "github.com/matrix-org/dendrite/federationapi" + federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/transactions" + "github.com/matrix-org/dendrite/keyserver" + "github.com/matrix-org/dendrite/mediaapi" + "github.com/matrix-org/dendrite/publicroomsapi" + "github.com/matrix-org/dendrite/publicroomsapi/storage" + "github.com/matrix-org/dendrite/publicroomsapi/types" + roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + serverKeyAPI "github.com/matrix-org/dendrite/serverkeyapi/api" + "github.com/matrix-org/dendrite/syncapi" + "github.com/matrix-org/gomatrixserverlib" +) + +// Monolith represents an instantiation of all dependencies required to build +// all components of Dendrite, for use in monolith mode. +type Monolith struct { + Config *config.Dendrite + DeviceDB devices.Database + AccountDB accounts.Database + KeyRing *gomatrixserverlib.KeyRing + FedClient *gomatrixserverlib.FederationClient + KafkaConsumer sarama.Consumer + KafkaProducer sarama.SyncProducer + + AppserviceAPI appserviceAPI.AppServiceQueryAPI + EDUInternalAPI eduServerAPI.EDUServerInputAPI + FederationSenderAPI federationSenderAPI.FederationSenderInternalAPI + RoomserverAPI roomserverAPI.RoomserverInternalAPI + ServerKeyAPI serverKeyAPI.ServerKeyInternalAPI + + // TODO: remove, this isn't even a producer + EDUProducer *producers.EDUServerProducer + // TODO: can we remove this? It's weird that we are required the database + // yet every other component can do that on its own. libp2p-demo uses a custom + // database though annoyingly. + PublicRoomsDB storage.Database + + // Optional + ExtPublicRoomsProvider types.ExternalPublicRoomsProvider +} + +// AddAllPublicRoutes attaches all public paths to the given router +func (m *Monolith) AddAllPublicRoutes(publicMux *mux.Router) { + clientapi.AddPublicRoutes( + publicMux, m.Config, m.KafkaConsumer, m.KafkaProducer, m.DeviceDB, m.AccountDB, + m.FedClient, m.KeyRing, m.RoomserverAPI, + m.EDUInternalAPI, m.AppserviceAPI, transactions.New(), + m.FederationSenderAPI, + ) + + keyserver.AddPublicRoutes(publicMux, m.Config, m.DeviceDB, m.AccountDB) + federationapi.AddPublicRoutes( + publicMux, m.Config, m.AccountDB, m.DeviceDB, m.FedClient, + m.KeyRing, m.RoomserverAPI, m.AppserviceAPI, m.FederationSenderAPI, + m.EDUProducer, + ) + mediaapi.AddPublicRoutes(publicMux, m.Config, m.DeviceDB) + publicroomsapi.AddPublicRoutes( + publicMux, m.Config, m.KafkaConsumer, m.DeviceDB, m.PublicRoomsDB, m.RoomserverAPI, m.FedClient, + m.ExtPublicRoomsProvider, + ) + syncapi.AddPublicRoutes( + publicMux, m.KafkaConsumer, m.DeviceDB, m.AccountDB, m.RoomserverAPI, m.FedClient, m.Config, + ) +} diff --git a/publicroomsapi/publicroomsapi.go b/publicroomsapi/publicroomsapi.go index 280ab9e3d..1f98a4e05 100644 --- a/publicroomsapi/publicroomsapi.go +++ b/publicroomsapi/publicroomsapi.go @@ -15,9 +15,10 @@ package publicroomsapi import ( + "github.com/Shopify/sarama" "github.com/gorilla/mux" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/publicroomsapi/consumers" "github.com/matrix-org/dendrite/publicroomsapi/routing" "github.com/matrix-org/dendrite/publicroomsapi/storage" @@ -31,7 +32,8 @@ import ( // component. func AddPublicRoutes( router *mux.Router, - base *basecomponent.BaseDendrite, + cfg *config.Dendrite, + consumer sarama.Consumer, deviceDB devices.Database, publicRoomsDB storage.Database, rsAPI roomserverAPI.RoomserverInternalAPI, @@ -39,7 +41,7 @@ func AddPublicRoutes( extRoomsProvider types.ExternalPublicRoomsProvider, ) { rsConsumer := consumers.NewOutputRoomEventConsumer( - base.Cfg, base.KafkaConsumer, publicRoomsDB, rsAPI, + cfg, consumer, publicRoomsDB, rsAPI, ) if err := rsConsumer.Start(); err != nil { logrus.WithError(err).Panic("failed to start public rooms server consumer") diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go index 6e84dcb6f..40e652af4 100644 --- a/syncapi/syncapi.go +++ b/syncapi/syncapi.go @@ -17,11 +17,11 @@ package syncapi import ( "context" + "github.com/Shopify/sarama" "github.com/gorilla/mux" "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" - "github.com/matrix-org/dendrite/internal/basecomponent" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" @@ -37,14 +37,14 @@ import ( // component. func AddPublicRoutes( router *mux.Router, - base *basecomponent.BaseDendrite, + consumer sarama.Consumer, deviceDB devices.Database, accountsDB accounts.Database, rsAPI api.RoomserverInternalAPI, federation *gomatrixserverlib.FederationClient, cfg *config.Dendrite, ) { - syncDB, err := storage.NewSyncServerDatasource(string(base.Cfg.Database.SyncAPI), base.Cfg.DbProperties()) + syncDB, err := storage.NewSyncServerDatasource(string(cfg.Database.SyncAPI), cfg.DbProperties()) if err != nil { logrus.WithError(err).Panicf("failed to connect to sync db") } @@ -63,28 +63,28 @@ func AddPublicRoutes( requestPool := sync.NewRequestPool(syncDB, notifier, accountsDB) roomConsumer := consumers.NewOutputRoomEventConsumer( - base.Cfg, base.KafkaConsumer, notifier, syncDB, rsAPI, + cfg, consumer, notifier, syncDB, rsAPI, ) if err = roomConsumer.Start(); err != nil { logrus.WithError(err).Panicf("failed to start room server consumer") } clientConsumer := consumers.NewOutputClientDataConsumer( - base.Cfg, base.KafkaConsumer, notifier, syncDB, + cfg, consumer, notifier, syncDB, ) if err = clientConsumer.Start(); err != nil { logrus.WithError(err).Panicf("failed to start client data consumer") } typingConsumer := consumers.NewOutputTypingEventConsumer( - base.Cfg, base.KafkaConsumer, notifier, syncDB, + cfg, consumer, notifier, syncDB, ) if err = typingConsumer.Start(); err != nil { logrus.WithError(err).Panicf("failed to start typing consumer") } sendToDeviceConsumer := consumers.NewOutputSendToDeviceEventConsumer( - base.Cfg, base.KafkaConsumer, notifier, syncDB, + cfg, consumer, notifier, syncDB, ) if err = sendToDeviceConsumer.Start(); err != nil { logrus.WithError(err).Panicf("failed to start send-to-device consumer") From 98cb0705eab07233cbc78fbe4342e1b1890c2a60 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Wed, 10 Jun 2020 10:34:22 +0100 Subject: [PATCH 143/200] Remove unused UserUpdates producer (#1109) --- clientapi/clientapi.go | 7 +--- clientapi/producers/userupdate.go | 62 ------------------------------- clientapi/routing/profile.go | 18 +-------- clientapi/routing/routing.go | 5 +-- cmd/dendrite-demo-libp2p/main.go | 1 - cmd/dendritejs/main.go | 1 - internal/config/config.go | 3 -- internal/test/config.go | 1 - 8 files changed, 5 insertions(+), 93 deletions(-) delete mode 100644 clientapi/producers/userupdate.go diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go index 987815c23..90db9eead 100644 --- a/clientapi/clientapi.go +++ b/clientapi/clientapi.go @@ -51,11 +51,6 @@ func AddPublicRoutes( roomserverProducer := producers.NewRoomserverProducer(rsAPI) eduProducer := producers.NewEDUServerProducer(eduInputAPI) - userUpdateProducer := &producers.UserUpdateProducer{ - Producer: producer, - Topic: string(cfg.Kafka.Topics.UserUpdates), - } - syncProducer := &producers.SyncAPIProducer{ Producer: producer, Topic: string(cfg.Kafka.Topics.OutputClientData), @@ -70,7 +65,7 @@ func AddPublicRoutes( routing.Setup( router, cfg, roomserverProducer, rsAPI, asAPI, - accountsDB, deviceDB, federation, *keyRing, userUpdateProducer, + accountsDB, deviceDB, federation, *keyRing, syncProducer, eduProducer, transactionsCache, fsAPI, ) } diff --git a/clientapi/producers/userupdate.go b/clientapi/producers/userupdate.go deleted file mode 100644 index 02e1700fd..000000000 --- a/clientapi/producers/userupdate.go +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2017 Vector Creations Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package producers - -import ( - "encoding/json" - - "github.com/Shopify/sarama" -) - -// UserUpdateProducer produces events related to user updates. -type UserUpdateProducer struct { - Topic string - Producer sarama.SyncProducer -} - -// TODO: Move this struct to `internal` so the components that consume the topic -// can use it when parsing incoming messages -type profileUpdate struct { - Updated string `json:"updated"` // Which attribute is updated (can be either `avatar_url` or `displayname`) - OldValue string `json:"old_value"` // The attribute's value before the update - NewValue string `json:"new_value"` // The attribute's value after the update -} - -// SendUpdate sends an update using kafka to notify the roomserver of the -// profile update. Returns an error if the update failed to send. -func (p *UserUpdateProducer) SendUpdate( - userID string, updatedAttribute string, oldValue string, newValue string, -) error { - var update profileUpdate - var m sarama.ProducerMessage - - m.Topic = string(p.Topic) - m.Key = sarama.StringEncoder(userID) - - update = profileUpdate{ - Updated: updatedAttribute, - OldValue: oldValue, - NewValue: newValue, - } - - value, err := json.Marshal(update) - if err != nil { - return err - } - m.Value = sarama.ByteEncoder(value) - - _, _, err = p.Producer.SendMessage(&m) - return err -} diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index dbf6ef1d9..c0fe32a31 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -94,7 +94,7 @@ func GetAvatarURL( // nolint:gocyclo func SetAvatarURL( req *http.Request, accountDB accounts.Database, device *authtypes.Device, - userID string, producer *producers.UserUpdateProducer, cfg *config.Dendrite, + userID string, cfg *config.Dendrite, rsProducer *producers.RoomserverProducer, rsAPI api.RoomserverInternalAPI, ) util.JSONResponse { if userID != device.UserID { @@ -104,8 +104,6 @@ func SetAvatarURL( } } - changedKey := "avatar_url" - var r internal.AvatarURL if resErr := httputil.UnmarshalJSONRequest(req, &r); resErr != nil { return *resErr @@ -174,11 +172,6 @@ func SetAvatarURL( return jsonerror.InternalServerError() } - if err := producer.SendUpdate(userID, changedKey, oldProfile.AvatarURL, r.AvatarURL); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("producer.SendUpdate failed") - return jsonerror.InternalServerError() - } - return util.JSONResponse{ Code: http.StatusOK, JSON: struct{}{}, @@ -216,7 +209,7 @@ func GetDisplayName( // nolint:gocyclo func SetDisplayName( req *http.Request, accountDB accounts.Database, device *authtypes.Device, - userID string, producer *producers.UserUpdateProducer, cfg *config.Dendrite, + userID string, cfg *config.Dendrite, rsProducer *producers.RoomserverProducer, rsAPI api.RoomserverInternalAPI, ) util.JSONResponse { if userID != device.UserID { @@ -226,8 +219,6 @@ func SetDisplayName( } } - changedKey := "displayname" - var r internal.DisplayName if resErr := httputil.UnmarshalJSONRequest(req, &r); resErr != nil { return *resErr @@ -296,11 +287,6 @@ func SetDisplayName( return jsonerror.InternalServerError() } - if err := producer.SendUpdate(userID, changedKey, oldProfile.DisplayName, r.DisplayName); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("producer.SendUpdate failed") - return jsonerror.InternalServerError() - } - return util.JSONResponse{ Code: http.StatusOK, JSON: struct{}{}, diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 83e399ac3..eb558205e 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -55,7 +55,6 @@ func Setup( deviceDB devices.Database, federation *gomatrixserverlib.FederationClient, keyRing gomatrixserverlib.KeyRing, - userUpdateProducer *producers.UserUpdateProducer, syncProducer *producers.SyncAPIProducer, eduProducer *producers.EDUServerProducer, transactionsCache *transactions.Cache, @@ -387,7 +386,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SetAvatarURL(req, accountDB, device, vars["userID"], userUpdateProducer, cfg, producer, rsAPI) + return SetAvatarURL(req, accountDB, device, vars["userID"], cfg, producer, rsAPI) }), ).Methods(http.MethodPut, http.MethodOptions) // Browsers use the OPTIONS HTTP method to check if the CORS policy allows @@ -409,7 +408,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SetDisplayName(req, accountDB, device, vars["userID"], userUpdateProducer, cfg, producer, rsAPI) + return SetDisplayName(req, accountDB, device, vars["userID"], cfg, producer, rsAPI) }), ).Methods(http.MethodPut, http.MethodOptions) // Browsers use the OPTIONS HTTP method to check if the CORS policy allows diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index 377e017ef..ba1af148d 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -109,7 +109,6 @@ func main() { cfg.Kafka.Topics.OutputRoomEvent = "roomserverOutput" cfg.Kafka.Topics.OutputClientData = "clientapiOutput" cfg.Kafka.Topics.OutputTypingEvent = "typingServerOutput" - cfg.Kafka.Topics.UserUpdates = "userUpdates" cfg.Database.Account = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName)) cfg.Database.Device = config.DataSource(fmt.Sprintf("file:%s-device.db", *instanceName)) cfg.Database.MediaAPI = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName)) diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 9901c6e5f..86ae33688 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -169,7 +169,6 @@ func main() { cfg.Database.RoomServer = "file:/idb/dendritejs_roomserver.db" cfg.Database.ServerKey = "file:/idb/dendritejs_serverkey.db" cfg.Database.SyncAPI = "file:/idb/dendritejs_syncapi.db" - cfg.Kafka.Topics.UserUpdates = "user_updates" cfg.Kafka.Topics.OutputTypingEvent = "output_typing_event" cfg.Kafka.Topics.OutputSendToDeviceEvent = "output_send_to_device_event" cfg.Kafka.Topics.OutputClientData = "output_client_data" diff --git a/internal/config/config.go b/internal/config/config.go index 858646ab0..bff4945be 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -154,8 +154,6 @@ type Dendrite struct { OutputTypingEvent Topic `yaml:"output_typing_event"` // Topic for eduserver/api.OutputSendToDeviceEvent events. OutputSendToDeviceEvent Topic `yaml:"output_send_to_device_event"` - // Topic for user updates (profile, presence) - UserUpdates Topic `yaml:"user_updates"` } } `yaml:"kafka"` @@ -591,7 +589,6 @@ func (config *Dendrite) checkKafka(configErrs *configErrors, monolithic bool) { checkNotEmpty(configErrs, "kafka.topics.output_room_event", string(config.Kafka.Topics.OutputRoomEvent)) checkNotEmpty(configErrs, "kafka.topics.output_client_data", string(config.Kafka.Topics.OutputClientData)) checkNotEmpty(configErrs, "kafka.topics.output_typing_event", string(config.Kafka.Topics.OutputTypingEvent)) - checkNotEmpty(configErrs, "kafka.topics.user_updates", string(config.Kafka.Topics.UserUpdates)) } // checkDatabase verifies the parameters database.* are valid. diff --git a/internal/test/config.go b/internal/test/config.go index 06510c8b8..951f65a12 100644 --- a/internal/test/config.go +++ b/internal/test/config.go @@ -84,7 +84,6 @@ func MakeConfig(configDir, kafkaURI, database, host string, startPort int) (*con cfg.Kafka.Topics.OutputRoomEvent = "test.room.output" cfg.Kafka.Topics.OutputClientData = "test.clientapi.output" cfg.Kafka.Topics.OutputTypingEvent = "test.typing.output" - cfg.Kafka.Topics.UserUpdates = "test.user.output" // TODO: Use different databases for the different schemas. // Using the same database for every schema currently works because From 464718c3e60cd4c095357bd8b80859eab9cb3b2d Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 10 Jun 2020 10:54:06 +0100 Subject: [PATCH 144/200] Yggdrasil-based P2P demo (#1108) * Initial work on Yggdrasil demo * Muxing? * Yamux * Updates to yamux * Updates * Comments * Update to use monolith stuff * Update go.mod/go.sum * Set defaults * Tweaks * Update yggdrasil * Update config * MarshalIndent * Change default instance name/port * add -peer switch * gocyclo, for a change * Determinate yamux roles * Fix copyright notices * Remove HTTP API checks as always false, remove unused topic --- cmd/dendrite-demo-yggdrasil/main.go | 201 ++++++++++++++++++ cmd/dendrite-demo-yggdrasil/yggconn/node.go | 129 +++++++++++ .../yggconn/session.go | 115 ++++++++++ go.mod | 3 + go.sum | 43 ++++ 5 files changed, 491 insertions(+) create mode 100644 cmd/dendrite-demo-yggdrasil/main.go create mode 100644 cmd/dendrite-demo-yggdrasil/yggconn/node.go create mode 100644 cmd/dendrite-demo-yggdrasil/yggconn/session.go diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go new file mode 100644 index 000000000..e5192c41c --- /dev/null +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -0,0 +1,201 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "crypto/tls" + "flag" + "fmt" + "net" + "net/http" + "strings" + "time" + + "github.com/matrix-org/dendrite/appservice" + "github.com/matrix-org/dendrite/clientapi/producers" + "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/yggconn" + "github.com/matrix-org/dendrite/eduserver" + "github.com/matrix-org/dendrite/eduserver/cache" + "github.com/matrix-org/dendrite/federationsender" + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/setup" + "github.com/matrix-org/dendrite/publicroomsapi/storage" + "github.com/matrix-org/dendrite/roomserver" + "github.com/matrix-org/dendrite/serverkeyapi" + "github.com/matrix-org/gomatrixserverlib" + + "github.com/sirupsen/logrus" +) + +var ( + instanceName = flag.String("name", "dendrite-p2p-ygg", "the name of this P2P demo instance") + instancePort = flag.Int("port", 8008, "the port that the client API will listen on") + instancePeer = flag.String("peer", "", "an internet Yggdrasil peer to connect to") +) + +type yggroundtripper struct { + inner *http.Transport +} + +func (y *yggroundtripper) RoundTrip(req *http.Request) (*http.Response, error) { + req.URL.Scheme = "http" + return y.inner.RoundTrip(req) +} + +func createFederationClient( + base *basecomponent.BaseDendrite, n *yggconn.Node, +) *gomatrixserverlib.FederationClient { + yggdialer := func(_, address string) (net.Conn, error) { + tokens := strings.Split(address, ":") + return n.Dial("curve25519", tokens[0]) + } + yggdialerctx := func(ctx context.Context, network, address string) (net.Conn, error) { + return yggdialer(network, address) + } + tr := &http.Transport{} + tr.RegisterProtocol( + "matrix", &yggroundtripper{ + inner: &http.Transport{ + DialContext: yggdialerctx, + }, + }, + ) + return gomatrixserverlib.NewFederationClientWithTransport( + base.Cfg.Matrix.ServerName, base.Cfg.Matrix.KeyID, base.Cfg.Matrix.PrivateKey, tr, + ) +} + +// nolint:gocyclo +func main() { + flag.Parse() + + // Build both ends of a HTTP multiplex. + httpServer := &http.Server{ + Addr: ":0", + TLSNextProto: map[string]func(*http.Server, *tls.Conn, http.Handler){}, + ReadTimeout: 30 * time.Second, + WriteTimeout: 30 * time.Second, + IdleTimeout: 60 * time.Second, + BaseContext: func(_ net.Listener) context.Context { + return context.Background() + }, + } + + ygg, err := yggconn.Setup(*instanceName, *instancePeer) + if err != nil { + panic(err) + } + + cfg := &config.Dendrite{} + cfg.SetDefaults() + cfg.Matrix.ServerName = gomatrixserverlib.ServerName(ygg.EncryptionPublicKey()) + cfg.Matrix.PrivateKey = ygg.SigningPrivateKey() + cfg.Matrix.KeyID = gomatrixserverlib.KeyID("ed25519:auto") + cfg.Kafka.UseNaffka = true + cfg.Kafka.Topics.OutputRoomEvent = "roomserverOutput" + cfg.Kafka.Topics.OutputClientData = "clientapiOutput" + cfg.Kafka.Topics.OutputTypingEvent = "typingServerOutput" + cfg.Database.Account = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName)) + cfg.Database.Device = config.DataSource(fmt.Sprintf("file:%s-device.db", *instanceName)) + cfg.Database.MediaAPI = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName)) + cfg.Database.SyncAPI = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName)) + cfg.Database.RoomServer = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName)) + cfg.Database.ServerKey = config.DataSource(fmt.Sprintf("file:%s-serverkey.db", *instanceName)) + cfg.Database.FederationSender = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", *instanceName)) + cfg.Database.AppService = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName)) + cfg.Database.PublicRoomsAPI = config.DataSource(fmt.Sprintf("file:%s-publicroomsa.db", *instanceName)) + cfg.Database.Naffka = config.DataSource(fmt.Sprintf("file:%s-naffka.db", *instanceName)) + if err = cfg.Derive(); err != nil { + panic(err) + } + + base := basecomponent.NewBaseDendrite(cfg, "Monolith", false) + defer base.Close() // nolint: errcheck + + accountDB := base.CreateAccountsDB() + deviceDB := base.CreateDeviceDB() + federation := createFederationClient(base, ygg) + + serverKeyAPI := serverkeyapi.NewInternalAPI( + base.Cfg, federation, base.Caches, + ) + keyRing := serverKeyAPI.KeyRing() + + rsComponent := roomserver.NewInternalAPI( + base, keyRing, federation, + ) + rsAPI := rsComponent + + eduInputAPI := eduserver.NewInternalAPI( + base, cache.New(), deviceDB, + ) + + asAPI := appservice.NewInternalAPI(base, accountDB, deviceDB, rsAPI) + + fsAPI := federationsender.NewInternalAPI( + base, federation, rsAPI, keyRing, + ) + + rsComponent.SetFederationSenderAPI(fsAPI) + + eduProducer := producers.NewEDUServerProducer(eduInputAPI) + publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties(), cfg.Matrix.ServerName) + if err != nil { + logrus.WithError(err).Panicf("failed to connect to public rooms db") + } + + monolith := setup.Monolith{ + Config: base.Cfg, + AccountDB: accountDB, + DeviceDB: deviceDB, + FedClient: federation, + KeyRing: keyRing, + KafkaConsumer: base.KafkaConsumer, + KafkaProducer: base.KafkaProducer, + + AppserviceAPI: asAPI, + EDUInternalAPI: eduInputAPI, + EDUProducer: eduProducer, + FederationSenderAPI: fsAPI, + RoomserverAPI: rsAPI, + ServerKeyAPI: serverKeyAPI, + + PublicRoomsDB: publicRoomsDB, + } + monolith.AddAllPublicRoutes(base.PublicAPIMux) + + internal.SetupHTTPAPI( + http.DefaultServeMux, + base.PublicAPIMux, + base.InternalAPIMux, + cfg, + base.UseHTTPAPIs, + ) + + go func() { + logrus.Info("Listening on ", ygg.EncryptionPublicKey()) + logrus.Fatal(httpServer.Serve(ygg)) + }() + go func() { + httpBindAddr := fmt.Sprintf(":%d", *instancePort) + logrus.Info("Listening on ", httpBindAddr) + logrus.Fatal(http.ListenAndServe(httpBindAddr, nil)) + }() + + select {} +} diff --git a/cmd/dendrite-demo-yggdrasil/yggconn/node.go b/cmd/dendrite-demo-yggdrasil/yggconn/node.go new file mode 100644 index 000000000..517f7f794 --- /dev/null +++ b/cmd/dendrite-demo-yggdrasil/yggconn/node.go @@ -0,0 +1,129 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package yggconn + +import ( + "crypto/ed25519" + "encoding/hex" + "encoding/json" + "fmt" + "io/ioutil" + "log" + "os" + "sync" + + "github.com/libp2p/go-yamux" + yggdrasiladmin "github.com/yggdrasil-network/yggdrasil-go/src/admin" + yggdrasilconfig "github.com/yggdrasil-network/yggdrasil-go/src/config" + yggdrasilmulticast "github.com/yggdrasil-network/yggdrasil-go/src/multicast" + "github.com/yggdrasil-network/yggdrasil-go/src/yggdrasil" + + gologme "github.com/gologme/log" +) + +type Node struct { + core *yggdrasil.Core + config *yggdrasilconfig.NodeConfig + state *yggdrasilconfig.NodeState + admin *yggdrasiladmin.AdminSocket + multicast *yggdrasilmulticast.Multicast + log *gologme.Logger + listener *yggdrasil.Listener + dialer *yggdrasil.Dialer + sessions sync.Map // string -> yamux.Session + incoming chan *yamux.Stream +} + +// nolint:gocyclo +func Setup(instanceName, instancePeer string) (*Node, error) { + n := &Node{ + core: &yggdrasil.Core{}, + config: yggdrasilconfig.GenerateConfig(), + admin: &yggdrasiladmin.AdminSocket{}, + multicast: &yggdrasilmulticast.Multicast{}, + log: gologme.New(os.Stdout, "YGG ", log.Flags()), + incoming: make(chan *yamux.Stream), + } + n.config.AdminListen = fmt.Sprintf("unix://./%s-yggdrasil.sock", instanceName) + n.config.MulticastInterfaces = []string{".*"} + + yggfile := fmt.Sprintf("%s-yggdrasil.conf", instanceName) + if _, err := os.Stat(yggfile); !os.IsNotExist(err) { + yggconf, e := ioutil.ReadFile(yggfile) + if e != nil { + panic(err) + } + if err := json.Unmarshal([]byte(yggconf), &n.config); err != nil { + panic(err) + } + } else { + j, err := json.MarshalIndent(n.config, "", " ") + if err != nil { + panic(err) + } + if e := ioutil.WriteFile(yggfile, j, 0600); e != nil { + n.log.Printf("Couldn't write private key to file '%s': %s\n", yggfile, e) + } + } + + var err error + n.log.EnableLevel("error") + n.log.EnableLevel("warn") + n.log.EnableLevel("info") + n.state, err = n.core.Start(n.config, n.log) + if err != nil { + panic(err) + } + if instancePeer != "" { + if err = n.core.AddPeer(instancePeer, ""); err != nil { + panic(err) + } + } + if err = n.admin.Init(n.core, n.state, n.log, nil); err != nil { + panic(err) + } + if err = n.admin.Start(); err != nil { + panic(err) + } + if err = n.multicast.Init(n.core, n.state, n.log, nil); err != nil { + panic(err) + } + if err = n.multicast.Start(); err != nil { + panic(err) + } + n.admin.SetupAdminHandlers(n.admin) + n.multicast.SetupAdminHandlers(n.admin) + n.listener, err = n.core.ConnListen() + if err != nil { + panic(err) + } + n.dialer, err = n.core.ConnDialer() + if err != nil { + panic(err) + } + + go n.listenFromYgg() + + return n, nil +} + +func (n *Node) EncryptionPublicKey() string { + return n.core.EncryptionPublicKey() +} + +func (n *Node) SigningPrivateKey() ed25519.PrivateKey { + privBytes, _ := hex.DecodeString(n.config.SigningPrivateKey) + return ed25519.PrivateKey(privBytes) +} diff --git a/cmd/dendrite-demo-yggdrasil/yggconn/session.go b/cmd/dendrite-demo-yggdrasil/yggconn/session.go new file mode 100644 index 000000000..4e9a9f043 --- /dev/null +++ b/cmd/dendrite-demo-yggdrasil/yggconn/session.go @@ -0,0 +1,115 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package yggconn + +import ( + "context" + "net" + "strings" + "time" + + "github.com/libp2p/go-yamux" +) + +func (n *Node) yamuxConfig() *yamux.Config { + cfg := yamux.DefaultConfig() + cfg.EnableKeepAlive = true + cfg.KeepAliveInterval = time.Second * 30 + cfg.MaxMessageSize = 65535 + cfg.ReadBufSize = 655350 + return cfg +} + +func (n *Node) listenFromYgg() { + for { + conn, err := n.listener.Accept() + if err != nil { + n.log.Println("n.listener.Accept:", err) + return + } + var session *yamux.Session + if strings.Compare(n.EncryptionPublicKey(), conn.RemoteAddr().String()) < 0 { + session, err = yamux.Client(conn, n.yamuxConfig()) + } else { + session, err = yamux.Server(conn, n.yamuxConfig()) + } + if err != nil { + return + } + go n.listenFromYggConn(session) + } +} + +func (n *Node) listenFromYggConn(session *yamux.Session) { + n.sessions.Store(session.RemoteAddr().String(), session) + defer n.sessions.Delete(session.RemoteAddr()) + + for { + st, err := session.AcceptStream() + if err != nil { + n.log.Println("session.AcceptStream:", err) + return + } + n.incoming <- st + } +} + +// Implements net.Listener +func (n *Node) Accept() (net.Conn, error) { + return <-n.incoming, nil +} + +// Implements net.Listener +func (n *Node) Close() error { + return n.listener.Close() +} + +// Implements net.Listener +func (n *Node) Addr() net.Addr { + return n.listener.Addr() +} + +// Implements http.Transport.Dial +func (n *Node) Dial(network, address string) (net.Conn, error) { + return n.DialContext(context.TODO(), network, address) +} + +// Implements http.Transport.DialContext +func (n *Node) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + s, ok1 := n.sessions.Load(address) + session, ok2 := s.(*yamux.Session) + if !ok1 || !ok2 { + conn, err := n.dialer.DialContext(ctx, network, address) + if err != nil { + n.log.Println("n.dialer.DialContext:", err) + return nil, err + } + if strings.Compare(n.EncryptionPublicKey(), address) < 0 { + session, err = yamux.Client(conn, n.yamuxConfig()) + } else { + session, err = yamux.Server(conn, n.yamuxConfig()) + } + if err != nil { + return nil, err + } + go n.listenFromYggConn(session) + } + st, err := session.OpenStream() + if err != nil { + n.log.Println("session.OpenStream:", err) + return nil, err + } + return st, nil +} diff --git a/go.mod b/go.mod index 1cfdf8037..a93f9a832 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/matrix-org/dendrite require ( github.com/Shopify/sarama v1.26.1 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect + github.com/gologme/log v1.2.0 github.com/gorilla/mux v1.7.3 github.com/hashicorp/golang-lru v0.5.4 github.com/lib/pq v1.2.0 @@ -14,6 +15,7 @@ require ( github.com/libp2p/go-libp2p-kad-dht v0.5.0 github.com/libp2p/go-libp2p-pubsub v0.2.5 github.com/libp2p/go-libp2p-record v0.1.2 + github.com/libp2p/go-yamux v1.3.7 github.com/matrix-org/dugong v0.0.0-20171220115018-ea0a4690a0d5 github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 @@ -32,6 +34,7 @@ require ( github.com/uber-go/atomic v1.3.0 // indirect github.com/uber/jaeger-client-go v2.15.0+incompatible github.com/uber/jaeger-lib v1.5.0 + github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20200530233943-aec82d7a391b go.uber.org/atomic v1.4.0 golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d gopkg.in/h2non/bimg.v1 v1.0.18 diff --git a/go.sum b/go.sum index b32a1f237..00d25a3f6 100644 --- a/go.sum +++ b/go.sum @@ -1,12 +1,15 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= +github.com/Arceliar/phony v0.0.0-20191006174943-d0c68492aca0 h1:p3puK8Sl2xK+2FnnIvY/C0N1aqJo2kbEsdAzU+Tnv48= +github.com/Arceliar/phony v0.0.0-20191006174943-d0c68492aca0/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= github.com/Shopify/sarama v1.26.1 h1:3jnfWKD7gVwbB1KSy/lE0szA9duPuSFLViK0o/d3DgA= github.com/Shopify/sarama v1.26.1/go.mod h1:NbSGBSSndYaIhRcBtY9V0U7AyH+x71bG668AuWys/yU= github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= @@ -34,6 +37,7 @@ github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtE github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cheggaaa/pb/v3 v3.0.4/go.mod h1:7rgWxLrAUcFMkvJuv09+DYi7mMUYi8nO9iOWcvGJPfw= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= @@ -62,6 +66,7 @@ github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/frankban/quicktest v1.0.0/go.mod h1:R98jIehRai+d1/3Hv2//jOVCTJhW1VBavT6B6CuGq2k= @@ -92,6 +97,9 @@ github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U= +github.com/gologme/log v1.2.0 h1:Ya5Ip/KD6FX7uH0S31QO87nCCSucKtF44TLbTtO7V4c= +github.com/gologme/log v1.2.0/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= @@ -114,6 +122,7 @@ github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/U github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -122,6 +131,8 @@ github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uG github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hjson/hjson-go v3.0.1-0.20190209023717-9147687966d9+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio= +github.com/hjson/hjson-go v3.0.2-0.20200316202735-d5d0e8b0617d+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huin/goupnp v1.0.0 h1:wg75sLpL6DZqwHQN6E1Cfk6mtfzS45z8OV+ic+DtHRo= @@ -182,6 +193,7 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= +github.com/kardianos/minwinsvc v0.0.0-20151122163309-cad6b2b879b0/go.mod h1:rUi0/YffDo1oXBOGn1KRq7Fr07LX48XEBecQnmwjsAo= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= @@ -343,6 +355,10 @@ github.com/libp2p/go-yamux v1.2.3 h1:xX8A36vpXb59frIzWFdEgptLMsOANMFq2K7fPRlunYI github.com/libp2p/go-yamux v1.2.3/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= github.com/libp2p/go-yamux v1.3.0 h1:FsYzT16Wq2XqUGJsBbOxoz9g+dFklvNi7jN6YFPfl7U= github.com/libp2p/go-yamux v1.3.0/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= +github.com/libp2p/go-yamux v1.3.7 h1:v40A1eSPJDIZwz2AvrV3cxpTZEGDP11QJbukmEhYyQI= +github.com/libp2p/go-yamux v1.3.7/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/hkmpooHE= +github.com/lxn/walk v0.0.0-20191128110447-55ccb3a9f5c1/go.mod h1:E23UucZGqpuUANJooIbHWCufXvOcT6E7Stq81gU+CSQ= +github.com/lxn/win v0.0.0-20191128105842-2da648fda5b4/go.mod h1:ouWl4wViUNh8tPSIwxTVMuS014WakR1hqvBc2I0bMoA= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -365,9 +381,13 @@ github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/em github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5 h1:tHXDdz1cpzGaovsTB+TVB8q90WEokoVmfMqoVcrLUgw= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= +github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-sqlite3 v2.0.2+incompatible h1:qzw9c2GNT8UFrgWNDhCTqRqYUSmu/Dav/9Z58LGpk7U= github.com/mattn/go-sqlite3 v2.0.2+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= @@ -527,6 +547,8 @@ github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMW github.com/uber/jaeger-lib v1.5.0 h1:OHbgr8l656Ub3Fw5k9SWnBfIEwvoHQ+W2y+Aa9D1Uyo= github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/vishvananda/netlink v1.0.0/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= +github.com/vishvananda/netns v0.0.0-20190625233234-7109fa855b0f/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= @@ -543,6 +565,11 @@ github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7V github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yggdrasil-network/yggdrasil-extras v0.0.0-20200525205615-6c8a4a2e8855/go.mod h1:xQdsh08Io6nV4WRnOVTe6gI8/2iTvfLDQ0CYa5aMt+I= +github.com/yggdrasil-network/yggdrasil-go v0.3.14 h1:vWzYzCQxOruS+J5FkLfXOS0JhCJx1yI9Erj/h2wfZ/E= +github.com/yggdrasil-network/yggdrasil-go v0.3.14/go.mod h1:rkQzLzVHlFdzsEMG+bDdTI+KeWPCZq1HpXRFzwinf6M= +github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20200530233943-aec82d7a391b h1:ELOisSxFXCcptRs4LFub+Hz5fYUvV12wZrTps99Eb3E= +github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20200530233943-aec82d7a391b/go.mod h1:d+Nz6SPeG6kmeSPFL0cvfWfgwEql75fUnZiAONgvyBE= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -566,6 +593,7 @@ golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -587,9 +615,12 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191003171128-d98b1b443823/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -614,16 +645,25 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/p golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191003212358-c178f38b412c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 h1:ywK/j/KkyTHcdyYSZNXGjMwgmDSfjglYZ3vStQ/gSCU= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200301040627-c5d0d7b4ec88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 h1:uYVVQ9WP/Ds2ROhcaGPeIdVq0RIXVLwsHlnvJ+cT1So= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3-0.20191230102452-929e72ca90de h1:aYKJLPSrddB2N7/6OKyFqJ337SXpo61bBuvO5p1+7iY= +golang.org/x/text v0.3.3-0.20191230102452-929e72ca90de/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -635,6 +675,9 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.zx2c4.com/wireguard v0.0.20200122-0.20200214175355-9cbcff10dd3e/go.mod h1:P2HsVp8SKwZEufsnezXZA4GRX/T49/HlU7DGuelXsU4= +golang.zx2c4.com/wireguard v0.0.20200320/go.mod h1:lDian4Sw4poJ04SgHh35nzMVwGSYlPumkdnHcucAQoY= +golang.zx2c4.com/wireguard/windows v0.1.0/go.mod h1:EK7CxrFnicmYJ0ZCF6crBh2/EMMeSxMlqgLlwN0Kv9s= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= From d9d6f4568ce891ae0ae9d2a3449974d3777bd21d Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 10 Jun 2020 11:02:03 +0100 Subject: [PATCH 145/200] Use ServerKeyAPI as keyring (#1110) * Use ServerKeyAPI as keyring * Update fetcher name for debug logs --- serverkeyapi/internal/api.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/serverkeyapi/internal/api.go b/serverkeyapi/internal/api.go index 176983c86..92d6a70b2 100644 --- a/serverkeyapi/internal/api.go +++ b/serverkeyapi/internal/api.go @@ -17,9 +17,13 @@ type ServerKeyAPI struct { } func (s *ServerKeyAPI) KeyRing() *gomatrixserverlib.KeyRing { - // Return a real keyring - one that has the real database and real - // fetchers. - return &s.OurKeyRing + // Return a keyring that forces requests to be proxied through the + // below functions. That way we can enforce things like validity + // and keeping the cache up-to-date. + return &gomatrixserverlib.KeyRing{ + KeyDatabase: s, + KeyFetchers: []gomatrixserverlib.KeyFetcher{s}, + } } func (s *ServerKeyAPI) StoreKeys( @@ -84,5 +88,5 @@ func (s *ServerKeyAPI) FetchKeys( } func (s *ServerKeyAPI) FetcherName() string { - return s.OurKeyRing.KeyDatabase.FetcherName() + return fmt.Sprintf("ServerKeyAPI (wrapping %q)", s.OurKeyRing.KeyDatabase.FetcherName()) } From b7187a9a354530c1846c2a97be701ea484f91c2c Mon Sep 17 00:00:00 2001 From: Kegsay Date: Wed, 10 Jun 2020 12:17:54 +0100 Subject: [PATCH 146/200] Remove clientapi producers which aren't actually producers (#1111) * Remove clientapi producers which aren't actually producers They are actually just convenience wrappers around the internal APIs for roomserver/eduserver. Move their logic to their respective `api` packages and call them directly. * Remove TODO * unbreak ygg --- clientapi/clientapi.go | 7 +- clientapi/routing/createroom.go | 17 ++-- clientapi/routing/membership.go | 17 ++-- clientapi/routing/profile.go | 15 ++-- clientapi/routing/routing.go | 26 +++--- clientapi/routing/sendevent.go | 8 +- clientapi/routing/sendtodevice.go | 8 +- clientapi/routing/sendtyping.go | 8 +- clientapi/threepid/invites.go | 11 ++- cmd/dendrite-demo-libp2p/main.go | 3 - cmd/dendrite-demo-yggdrasil/main.go | 3 - cmd/dendrite-federation-api-server/main.go | 5 +- cmd/dendrite-monolith-server/main.go | 3 - cmd/dendritejs/main.go | 3 - .../eduserver.go => eduserver/api/wrapper.go | 41 ++++------ federationapi/federationapi.go | 10 +-- federationapi/routing/invite.go | 8 +- federationapi/routing/join.go | 8 +- federationapi/routing/leave.go | 9 +-- federationapi/routing/routing.go | 19 +++-- federationapi/routing/send.go | 41 +++++----- federationapi/routing/send_test.go | 16 ++-- federationapi/routing/threepid.go | 14 ++-- internal/setup/monolith.go | 5 +- .../api/wrapper.go | 81 ++++++++----------- 25 files changed, 160 insertions(+), 226 deletions(-) rename clientapi/producers/eduserver.go => eduserver/api/wrapper.go (56%) rename clientapi/producers/roomserver.go => roomserver/api/wrapper.go (50%) diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go index 90db9eead..2780f367c 100644 --- a/clientapi/clientapi.go +++ b/clientapi/clientapi.go @@ -48,9 +48,6 @@ func AddPublicRoutes( transactionsCache *transactions.Cache, fsAPI federationSenderAPI.FederationSenderInternalAPI, ) { - roomserverProducer := producers.NewRoomserverProducer(rsAPI) - eduProducer := producers.NewEDUServerProducer(eduInputAPI) - syncProducer := &producers.SyncAPIProducer{ Producer: producer, Topic: string(cfg.Kafka.Topics.OutputClientData), @@ -64,8 +61,8 @@ func AddPublicRoutes( } routing.Setup( - router, cfg, roomserverProducer, rsAPI, asAPI, + router, cfg, eduInputAPI, rsAPI, asAPI, accountsDB, deviceDB, federation, *keyRing, - syncProducer, eduProducer, transactionsCache, fsAPI, + syncProducer, transactionsCache, fsAPI, ) } diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index 1b4ff184f..89f49d356 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -29,7 +29,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/clientapi/threepid" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" @@ -137,21 +136,21 @@ type fledglingEvent struct { // CreateRoom implements /createRoom func CreateRoom( req *http.Request, device *authtypes.Device, - cfg *config.Dendrite, producer *producers.RoomserverProducer, + cfg *config.Dendrite, accountDB accounts.Database, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, ) util.JSONResponse { // TODO (#267): Check room ID doesn't clash with an existing one, and we // probably shouldn't be using pseudo-random strings, maybe GUIDs? roomID := fmt.Sprintf("!%s:%s", util.RandomString(16), cfg.Matrix.ServerName) - return createRoom(req, device, cfg, roomID, producer, accountDB, rsAPI, asAPI) + return createRoom(req, device, cfg, roomID, accountDB, rsAPI, asAPI) } // createRoom implements /createRoom // nolint: gocyclo func createRoom( req *http.Request, device *authtypes.Device, - cfg *config.Dendrite, roomID string, producer *producers.RoomserverProducer, + cfg *config.Dendrite, roomID string, accountDB accounts.Database, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, ) util.JSONResponse { @@ -344,9 +343,9 @@ func createRoom( } // send events to the room server - _, err = producer.SendEvents(req.Context(), builtEvents, cfg.Matrix.ServerName, nil) + _, err = roomserverAPI.SendEvents(req.Context(), rsAPI, builtEvents, cfg.Matrix.ServerName, nil) if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("producer.SendEvents failed") + util.GetLogger(req.Context()).WithError(err).Error("SendEvents failed") return jsonerror.InternalServerError() } @@ -404,14 +403,14 @@ func createRoom( } } // Send the invite event to the roomserver. - if err = producer.SendInvite( - req.Context(), + if err = roomserverAPI.SendInvite( + req.Context(), rsAPI, inviteEvent.Headered(roomVersion), strippedState, // invite room state cfg.Matrix.ServerName, // send as server nil, // transaction ID ); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("producer.SendEvents failed") + util.GetLogger(req.Context()).WithError(err).Error("SendInvite failed") return jsonerror.InternalServerError() } } diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index 74d92a059..0b2c2bc5e 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -25,7 +25,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/clientapi/threepid" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" @@ -46,7 +45,6 @@ func SendMembership( req *http.Request, accountDB accounts.Database, device *authtypes.Device, roomID string, membership string, cfg *config.Dendrite, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, - producer *producers.RoomserverProducer, ) util.JSONResponse { verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} verRes := api.QueryRoomVersionForRoomResponse{} @@ -71,7 +69,7 @@ func SendMembership( } inviteStored, jsonErrResp := checkAndProcessThreepid( - req, device, &body, cfg, rsAPI, accountDB, producer, + req, device, &body, cfg, rsAPI, accountDB, membership, roomID, evTime, ) if jsonErrResp != nil { @@ -112,8 +110,8 @@ func SendMembership( switch membership { case gomatrixserverlib.Invite: // Invites need to be handled specially - err = producer.SendInvite( - req.Context(), + err = roomserverAPI.SendInvite( + req.Context(), rsAPI, event.Headered(verRes.RoomVersion), nil, // ask the roomserver to draw up invite room state for us cfg.Matrix.ServerName, @@ -130,14 +128,14 @@ func SendMembership( }{roomID} fallthrough default: - _, err = producer.SendEvents( - req.Context(), + _, err = roomserverAPI.SendEvents( + req.Context(), rsAPI, []gomatrixserverlib.HeaderedEvent{event.Headered(verRes.RoomVersion)}, cfg.Matrix.ServerName, nil, ) if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("producer.SendEvents failed") + util.GetLogger(req.Context()).WithError(err).Error("SendEvents failed") return jsonerror.InternalServerError() } } @@ -252,13 +250,12 @@ func checkAndProcessThreepid( cfg *config.Dendrite, rsAPI roomserverAPI.RoomserverInternalAPI, accountDB accounts.Database, - producer *producers.RoomserverProducer, membership, roomID string, evTime time.Time, ) (inviteStored bool, errRes *util.JSONResponse) { inviteStored, err := threepid.CheckAndProcessInvite( - req.Context(), device, body, cfg, rsAPI, accountDB, producer, + req.Context(), device, body, cfg, rsAPI, accountDB, membership, roomID, evTime, ) if err == threepid.ErrMissingParameter { diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index c0fe32a31..642a72887 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -24,7 +24,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/roomserver/api" @@ -94,8 +93,7 @@ func GetAvatarURL( // nolint:gocyclo func SetAvatarURL( req *http.Request, accountDB accounts.Database, device *authtypes.Device, - userID string, cfg *config.Dendrite, - rsProducer *producers.RoomserverProducer, rsAPI api.RoomserverInternalAPI, + userID string, cfg *config.Dendrite, rsAPI api.RoomserverInternalAPI, ) util.JSONResponse { if userID != device.UserID { return util.JSONResponse{ @@ -167,8 +165,8 @@ func SetAvatarURL( return jsonerror.InternalServerError() } - if _, err := rsProducer.SendEvents(req.Context(), events, cfg.Matrix.ServerName, nil); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("rsProducer.SendEvents failed") + if _, err := api.SendEvents(req.Context(), rsAPI, events, cfg.Matrix.ServerName, nil); err != nil { + util.GetLogger(req.Context()).WithError(err).Error("SendEvents failed") return jsonerror.InternalServerError() } @@ -209,8 +207,7 @@ func GetDisplayName( // nolint:gocyclo func SetDisplayName( req *http.Request, accountDB accounts.Database, device *authtypes.Device, - userID string, cfg *config.Dendrite, - rsProducer *producers.RoomserverProducer, rsAPI api.RoomserverInternalAPI, + userID string, cfg *config.Dendrite, rsAPI api.RoomserverInternalAPI, ) util.JSONResponse { if userID != device.UserID { return util.JSONResponse{ @@ -282,8 +279,8 @@ func SetDisplayName( return jsonerror.InternalServerError() } - if _, err := rsProducer.SendEvents(req.Context(), events, cfg.Matrix.ServerName, nil); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("rsProducer.SendEvents failed") + if _, err := api.SendEvents(req.Context(), rsAPI, events, cfg.Matrix.ServerName, nil); err != nil { + util.GetLogger(req.Context()).WithError(err).Error("SendEvents failed") return jsonerror.InternalServerError() } diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index eb558205e..024707759 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -27,6 +27,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" + eduServerAPI "github.com/matrix-org/dendrite/eduserver/api" federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" @@ -48,7 +49,7 @@ const pathPrefixUnstable = "/client/unstable" // nolint: gocyclo func Setup( publicAPIMux *mux.Router, cfg *config.Dendrite, - producer *producers.RoomserverProducer, + eduAPI eduServerAPI.EDUServerInputAPI, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, accountDB accounts.Database, @@ -56,7 +57,6 @@ func Setup( federation *gomatrixserverlib.FederationClient, keyRing gomatrixserverlib.KeyRing, syncProducer *producers.SyncAPIProducer, - eduProducer *producers.EDUServerProducer, transactionsCache *transactions.Cache, federationSender federationSenderAPI.FederationSenderInternalAPI, ) { @@ -89,7 +89,7 @@ func Setup( r0mux.Handle("/createRoom", internal.MakeAuthAPI("createRoom", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - return CreateRoom(req, device, cfg, producer, accountDB, rsAPI, asAPI) + return CreateRoom(req, device, cfg, accountDB, rsAPI, asAPI) }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/join/{roomIDOrAlias}", @@ -125,7 +125,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SendMembership(req, accountDB, device, vars["roomID"], vars["membership"], cfg, rsAPI, asAPI, producer) + return SendMembership(req, accountDB, device, vars["roomID"], vars["membership"], cfg, rsAPI, asAPI) }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/send/{eventType}", @@ -134,7 +134,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, nil, cfg, rsAPI, producer, nil) + return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, nil, cfg, rsAPI, nil) }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/send/{eventType}/{txnID}", @@ -145,7 +145,7 @@ func Setup( } txnID := vars["txnID"] return SendEvent(req, device, vars["roomID"], vars["eventType"], &txnID, - nil, cfg, rsAPI, producer, transactionsCache) + nil, cfg, rsAPI, transactionsCache) }), ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/event/{eventID}", @@ -194,7 +194,7 @@ func Setup( if strings.HasSuffix(eventType, "/") { eventType = eventType[:len(eventType)-1] } - return SendEvent(req, device, vars["roomID"], eventType, nil, &emptyString, cfg, rsAPI, producer, nil) + return SendEvent(req, device, vars["roomID"], eventType, nil, &emptyString, cfg, rsAPI, nil) }), ).Methods(http.MethodPut, http.MethodOptions) @@ -205,7 +205,7 @@ func Setup( return util.ErrorResponse(err) } stateKey := vars["stateKey"] - return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, &stateKey, cfg, rsAPI, producer, nil) + return SendEvent(req, device, vars["roomID"], vars["eventType"], nil, &stateKey, cfg, rsAPI, nil) }), ).Methods(http.MethodPut, http.MethodOptions) @@ -269,7 +269,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SendTyping(req, device, vars["roomID"], vars["userID"], accountDB, eduProducer) + return SendTyping(req, device, vars["roomID"], vars["userID"], accountDB, eduAPI) }), ).Methods(http.MethodPut, http.MethodOptions) @@ -280,7 +280,7 @@ func Setup( return util.ErrorResponse(err) } txnID := vars["txnID"] - return SendToDevice(req, device, eduProducer, transactionsCache, vars["eventType"], &txnID) + return SendToDevice(req, device, eduAPI, transactionsCache, vars["eventType"], &txnID) }), ).Methods(http.MethodPut, http.MethodOptions) @@ -294,7 +294,7 @@ func Setup( return util.ErrorResponse(err) } txnID := vars["txnID"] - return SendToDevice(req, device, eduProducer, transactionsCache, vars["eventType"], &txnID) + return SendToDevice(req, device, eduAPI, transactionsCache, vars["eventType"], &txnID) }), ).Methods(http.MethodPut, http.MethodOptions) @@ -386,7 +386,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SetAvatarURL(req, accountDB, device, vars["userID"], cfg, producer, rsAPI) + return SetAvatarURL(req, accountDB, device, vars["userID"], cfg, rsAPI) }), ).Methods(http.MethodPut, http.MethodOptions) // Browsers use the OPTIONS HTTP method to check if the CORS policy allows @@ -408,7 +408,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SetDisplayName(req, accountDB, device, vars["userID"], cfg, producer, rsAPI) + return SetDisplayName(req, accountDB, device, vars["userID"], cfg, rsAPI) }), ).Methods(http.MethodPut, http.MethodOptions) // Browsers use the OPTIONS HTTP method to check if the CORS policy allows diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go index a8f8893ba..5d5507e85 100644 --- a/clientapi/routing/sendevent.go +++ b/clientapi/routing/sendevent.go @@ -20,7 +20,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/transactions" @@ -46,7 +45,6 @@ func SendEvent( roomID, eventType string, txnID, stateKey *string, cfg *config.Dendrite, rsAPI api.RoomserverInternalAPI, - producer *producers.RoomserverProducer, txnCache *transactions.Cache, ) util.JSONResponse { verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} @@ -80,8 +78,8 @@ func SendEvent( // pass the new event to the roomserver and receive the correct event ID // event ID in case of duplicate transaction is discarded - eventID, err := producer.SendEvents( - req.Context(), + eventID, err := api.SendEvents( + req.Context(), rsAPI, []gomatrixserverlib.HeaderedEvent{ e.Headered(verRes.RoomVersion), }, @@ -89,7 +87,7 @@ func SendEvent( txnAndSessionID, ) if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("producer.SendEvents failed") + util.GetLogger(req.Context()).WithError(err).Error("SendEvents failed") return jsonerror.InternalServerError() } util.GetLogger(req.Context()).WithFields(logrus.Fields{ diff --git a/clientapi/routing/sendtodevice.go b/clientapi/routing/sendtodevice.go index 5d3060d74..dc0a6572b 100644 --- a/clientapi/routing/sendtodevice.go +++ b/clientapi/routing/sendtodevice.go @@ -19,7 +19,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/clientapi/producers" + "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/internal/transactions" "github.com/matrix-org/util" ) @@ -28,7 +28,7 @@ import ( // sends the device events to the EDU Server func SendToDevice( req *http.Request, device *authtypes.Device, - eduProducer *producers.EDUServerProducer, + eduAPI api.EDUServerInputAPI, txnCache *transactions.Cache, eventType string, txnID *string, ) util.JSONResponse { @@ -48,8 +48,8 @@ func SendToDevice( for userID, byUser := range httpReq.Messages { for deviceID, message := range byUser { - if err := eduProducer.SendToDevice( - req.Context(), device.UserID, userID, deviceID, eventType, message, + if err := api.SendToDevice( + req.Context(), eduAPI, device.UserID, userID, deviceID, eventType, message, ); err != nil { util.GetLogger(req.Context()).WithError(err).Error("eduProducer.SendToDevice failed") return jsonerror.InternalServerError() diff --git a/clientapi/routing/sendtyping.go b/clientapi/routing/sendtyping.go index ffaa0e662..2eae16582 100644 --- a/clientapi/routing/sendtyping.go +++ b/clientapi/routing/sendtyping.go @@ -20,8 +20,8 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/clientapi/userutil" + "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/util" ) @@ -35,7 +35,7 @@ type typingContentJSON struct { func SendTyping( req *http.Request, device *authtypes.Device, roomID string, userID string, accountDB accounts.Database, - eduProducer *producers.EDUServerProducer, + eduAPI api.EDUServerInputAPI, ) util.JSONResponse { if device.UserID != userID { return util.JSONResponse{ @@ -69,8 +69,8 @@ func SendTyping( return *resErr } - if err = eduProducer.SendTyping( - req.Context(), userID, roomID, r.Typing, r.Timeout, + if err = api.SendTyping( + req.Context(), eduAPI, userID, roomID, r.Typing, r.Timeout, ); err != nil { util.GetLogger(req.Context()).WithError(err).Error("eduProducer.Send failed") return jsonerror.InternalServerError() diff --git a/clientapi/threepid/invites.go b/clientapi/threepid/invites.go index 8f173bf8a..b0df0dd4d 100644 --- a/clientapi/threepid/invites.go +++ b/clientapi/threepid/invites.go @@ -26,7 +26,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" - "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/roomserver/api" @@ -88,7 +87,7 @@ func CheckAndProcessInvite( ctx context.Context, device *authtypes.Device, body *MembershipRequest, cfg *config.Dendrite, rsAPI api.RoomserverInternalAPI, db accounts.Database, - producer *producers.RoomserverProducer, membership string, roomID string, + membership string, roomID string, evTime time.Time, ) (inviteStoredOnIDServer bool, err error) { if membership != gomatrixserverlib.Invite || (body.Address == "" && body.IDServer == "" && body.Medium == "") { @@ -112,7 +111,7 @@ func CheckAndProcessInvite( // "m.room.third_party_invite" have to be emitted from the data in // storeInviteRes. err = emit3PIDInviteEvent( - ctx, body, storeInviteRes, device, roomID, cfg, rsAPI, producer, evTime, + ctx, body, storeInviteRes, device, roomID, cfg, rsAPI, evTime, ) inviteStoredOnIDServer = err == nil @@ -331,7 +330,7 @@ func emit3PIDInviteEvent( ctx context.Context, body *MembershipRequest, res *idServerStoreInviteResponse, device *authtypes.Device, roomID string, cfg *config.Dendrite, - rsAPI api.RoomserverInternalAPI, producer *producers.RoomserverProducer, + rsAPI api.RoomserverInternalAPI, evTime time.Time, ) error { builder := &gomatrixserverlib.EventBuilder{ @@ -359,8 +358,8 @@ func emit3PIDInviteEvent( return err } - _, err = producer.SendEvents( - ctx, + _, err = api.SendEvents( + ctx, rsAPI, []gomatrixserverlib.HeaderedEvent{ (*event).Headered(queryRes.RoomVersion), }, diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index ba1af148d..b7cbcdc9c 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -29,7 +29,6 @@ import ( p2phttp "github.com/libp2p/go-libp2p-http" p2pdisc "github.com/libp2p/go-libp2p/p2p/discovery" "github.com/matrix-org/dendrite/appservice" - "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/cmd/dendrite-demo-libp2p/storage" "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/federationsender" @@ -149,7 +148,6 @@ func main() { &base.Base, federation, rsAPI, keyRing, ) rsAPI.SetFederationSenderAPI(fsAPI) - eduProducer := producers.NewEDUServerProducer(eduInputAPI) publicRoomsDB, err := storage.NewPublicRoomsServerDatabaseWithPubSub(string(base.Base.Cfg.Database.PublicRoomsAPI), base.LibP2PPubsub, cfg.Matrix.ServerName) if err != nil { logrus.WithError(err).Panicf("failed to connect to public rooms db") @@ -166,7 +164,6 @@ func main() { AppserviceAPI: asAPI, EDUInternalAPI: eduInputAPI, - EDUProducer: eduProducer, FederationSenderAPI: fsAPI, RoomserverAPI: rsAPI, ServerKeyAPI: serverKeyAPI, diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index e5192c41c..62c561290 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -25,7 +25,6 @@ import ( "time" "github.com/matrix-org/dendrite/appservice" - "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/yggconn" "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/eduserver/cache" @@ -153,7 +152,6 @@ func main() { rsComponent.SetFederationSenderAPI(fsAPI) - eduProducer := producers.NewEDUServerProducer(eduInputAPI) publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties(), cfg.Matrix.ServerName) if err != nil { logrus.WithError(err).Panicf("failed to connect to public rooms db") @@ -170,7 +168,6 @@ func main() { AppserviceAPI: asAPI, EDUInternalAPI: eduInputAPI, - EDUProducer: eduProducer, FederationSenderAPI: fsAPI, RoomserverAPI: rsAPI, ServerKeyAPI: serverKeyAPI, diff --git a/cmd/dendrite-federation-api-server/main.go b/cmd/dendrite-federation-api-server/main.go index 7481299e9..fcdc000c4 100644 --- a/cmd/dendrite-federation-api-server/main.go +++ b/cmd/dendrite-federation-api-server/main.go @@ -15,7 +15,6 @@ package main import ( - "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/federationapi" "github.com/matrix-org/dendrite/internal/basecomponent" ) @@ -33,12 +32,10 @@ func main() { fsAPI := base.FederationSenderHTTPClient() rsAPI := base.RoomserverHTTPClient() asAPI := base.AppserviceHTTPClient() - // TODO: this isn't a producer - eduProducer := producers.NewEDUServerProducer(base.EDUServerClient()) federationapi.AddPublicRoutes( base.PublicAPIMux, base.Cfg, accountDB, deviceDB, federation, keyRing, - rsAPI, asAPI, fsAPI, eduProducer, + rsAPI, asAPI, fsAPI, base.EDUServerClient(), ) base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationAPI), string(base.Cfg.Listen.FederationAPI)) diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index dda3ade5a..195a1ac50 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -19,7 +19,6 @@ import ( "net/http" "github.com/matrix-org/dendrite/appservice" - "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/federationsender" @@ -105,7 +104,6 @@ func main() { } rsComponent.SetFederationSenderAPI(fsAPI) - eduProducer := producers.NewEDUServerProducer(eduInputAPI) publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties(), cfg.Matrix.ServerName) if err != nil { logrus.WithError(err).Panicf("failed to connect to public rooms db") @@ -122,7 +120,6 @@ func main() { AppserviceAPI: asAPI, EDUInternalAPI: eduInputAPI, - EDUProducer: eduProducer, FederationSenderAPI: fsAPI, RoomserverAPI: rsAPI, ServerKeyAPI: serverKeyAPI, diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 86ae33688..0512ee5c9 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -23,7 +23,6 @@ import ( "syscall/js" "github.com/matrix-org/dendrite/appservice" - "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/federationsender" @@ -209,7 +208,6 @@ func main() { rsAPI.SetFederationSenderAPI(fedSenderAPI) p2pPublicRoomProvider := NewLibP2PPublicRoomsProvider(node, fedSenderAPI) - eduProducer := producers.NewEDUServerProducer(eduInputAPI) publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), cfg.Matrix.ServerName) if err != nil { logrus.WithError(err).Panicf("failed to connect to public rooms db") @@ -226,7 +224,6 @@ func main() { AppserviceAPI: asQuery, EDUInternalAPI: eduInputAPI, - EDUProducer: eduProducer, FederationSenderAPI: fedSenderAPI, RoomserverAPI: rsAPI, //ServerKeyAPI: serverKeyAPI, diff --git a/clientapi/producers/eduserver.go b/eduserver/api/wrapper.go similarity index 56% rename from clientapi/producers/eduserver.go rename to eduserver/api/wrapper.go index 102c1fad8..c2c4596de 100644 --- a/clientapi/producers/eduserver.go +++ b/eduserver/api/wrapper.go @@ -1,3 +1,5 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -10,35 +12,22 @@ // See the License for the specific language governing permissions and // limitations under the License. -package producers +package api import ( "context" "encoding/json" "time" - "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/gomatrixserverlib" ) -// EDUServerProducer produces events for the EDU server to consume -type EDUServerProducer struct { - InputAPI api.EDUServerInputAPI -} - -// NewEDUServerProducer creates a new EDUServerProducer -func NewEDUServerProducer(inputAPI api.EDUServerInputAPI) *EDUServerProducer { - return &EDUServerProducer{ - InputAPI: inputAPI, - } -} - // SendTyping sends a typing event to EDU server -func (p *EDUServerProducer) SendTyping( - ctx context.Context, userID, roomID string, +func SendTyping( + ctx context.Context, eduAPI EDUServerInputAPI, userID, roomID string, typing bool, timeoutMS int64, ) error { - requestData := api.InputTypingEvent{ + requestData := InputTypingEvent{ UserID: userID, RoomID: roomID, Typing: typing, @@ -46,24 +35,24 @@ func (p *EDUServerProducer) SendTyping( OriginServerTS: gomatrixserverlib.AsTimestamp(time.Now()), } - var response api.InputTypingEventResponse - err := p.InputAPI.InputTypingEvent( - ctx, &api.InputTypingEventRequest{InputTypingEvent: requestData}, &response, + var response InputTypingEventResponse + err := eduAPI.InputTypingEvent( + ctx, &InputTypingEventRequest{InputTypingEvent: requestData}, &response, ) return err } // SendToDevice sends a typing event to EDU server -func (p *EDUServerProducer) SendToDevice( - ctx context.Context, sender, userID, deviceID, eventType string, +func SendToDevice( + ctx context.Context, eduAPI EDUServerInputAPI, sender, userID, deviceID, eventType string, message interface{}, ) error { js, err := json.Marshal(message) if err != nil { return err } - requestData := api.InputSendToDeviceEvent{ + requestData := InputSendToDeviceEvent{ UserID: userID, DeviceID: deviceID, SendToDeviceEvent: gomatrixserverlib.SendToDeviceEvent{ @@ -72,9 +61,9 @@ func (p *EDUServerProducer) SendToDevice( Content: js, }, } - request := api.InputSendToDeviceEventRequest{ + request := InputSendToDeviceEventRequest{ InputSendToDeviceEvent: requestData, } - response := api.InputSendToDeviceEventResponse{} - return p.InputAPI.InputSendToDeviceEvent(ctx, &request, &response) + response := InputSendToDeviceEventResponse{} + return eduAPI.InputSendToDeviceEvent(ctx, &request, &response) } diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go index 7aecd272b..9299b5016 100644 --- a/federationapi/federationapi.go +++ b/federationapi/federationapi.go @@ -19,12 +19,11 @@ import ( appserviceAPI "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" + eduserverAPI "github.com/matrix-org/dendrite/eduserver/api" federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/internal/config" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" - // TODO: Are we really wanting to pull in the producer from clientapi - "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/federationapi/routing" "github.com/matrix-org/gomatrixserverlib" ) @@ -40,13 +39,12 @@ func AddPublicRoutes( rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, federationSenderAPI federationSenderAPI.FederationSenderInternalAPI, - eduProducer *producers.EDUServerProducer, + eduAPI eduserverAPI.EDUServerInputAPI, ) { - roomserverProducer := producers.NewRoomserverProducer(rsAPI) routing.Setup( - router, cfg, rsAPI, asAPI, roomserverProducer, - eduProducer, federationSenderAPI, *keyRing, + router, cfg, rsAPI, asAPI, + eduAPI, federationSenderAPI, *keyRing, federation, accountsDB, deviceDB, ) } diff --git a/federationapi/routing/invite.go b/federationapi/routing/invite.go index 05efe5872..908a04fcb 100644 --- a/federationapi/routing/invite.go +++ b/federationapi/routing/invite.go @@ -20,8 +20,8 @@ import ( "net/http" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/roomserver/api" roomserverVersion "github.com/matrix-org/dendrite/roomserver/version" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -34,7 +34,7 @@ func Invite( roomID string, eventID string, cfg *config.Dendrite, - producer *producers.RoomserverProducer, + rsAPI api.RoomserverInternalAPI, keys gomatrixserverlib.KeyRing, ) util.JSONResponse { inviteReq := gomatrixserverlib.InviteV2Request{} @@ -98,8 +98,8 @@ func Invite( ) // Add the invite event to the roomserver. - if err = producer.SendInvite( - httpReq.Context(), + if err = api.SendInvite( + httpReq.Context(), rsAPI, signedEvent.Headered(inviteReq.RoomVersion()), inviteReq.InviteRoomState(), event.Origin(), diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go index 5b4e8db3e..593aa169f 100644 --- a/federationapi/routing/join.go +++ b/federationapi/routing/join.go @@ -21,7 +21,6 @@ import ( "time" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/roomserver/api" @@ -144,7 +143,6 @@ func SendJoin( request *gomatrixserverlib.FederationRequest, cfg *config.Dendrite, rsAPI api.RoomserverInternalAPI, - producer *producers.RoomserverProducer, keys gomatrixserverlib.KeyRing, roomID, eventID string, ) util.JSONResponse { @@ -267,8 +265,8 @@ func SendJoin( // We are responsible for notifying other servers that the user has joined // the room, so set SendAsServer to cfg.Matrix.ServerName if !alreadyJoined { - _, err = producer.SendEvents( - httpReq.Context(), + _, err = api.SendEvents( + httpReq.Context(), rsAPI, []gomatrixserverlib.HeaderedEvent{ event.Headered(stateAndAuthChainResponse.RoomVersion), }, @@ -276,7 +274,7 @@ func SendJoin( nil, ) if err != nil { - util.GetLogger(httpReq.Context()).WithError(err).Error("producer.SendEvents failed") + util.GetLogger(httpReq.Context()).WithError(err).Error("SendEvents failed") return jsonerror.InternalServerError() } } diff --git a/federationapi/routing/leave.go b/federationapi/routing/leave.go index 62ca11457..f998be45a 100644 --- a/federationapi/routing/leave.go +++ b/federationapi/routing/leave.go @@ -17,7 +17,6 @@ import ( "time" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/roomserver/api" @@ -113,13 +112,13 @@ func SendLeave( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, cfg *config.Dendrite, - producer *producers.RoomserverProducer, + rsAPI api.RoomserverInternalAPI, keys gomatrixserverlib.KeyRing, roomID, eventID string, ) util.JSONResponse { verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} verRes := api.QueryRoomVersionForRoomResponse{} - if err := producer.RsAPI.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil { + if err := rsAPI.QueryRoomVersionForRoom(httpReq.Context(), &verReq, &verRes); err != nil { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.UnsupportedRoomVersion(err.Error()), @@ -194,8 +193,8 @@ func SendLeave( // Send the events to the room server. // We are responsible for notifying other servers that the user has left // the room, so set SendAsServer to cfg.Matrix.ServerName - _, err = producer.SendEvents( - httpReq.Context(), + _, err = api.SendEvents( + httpReq.Context(), rsAPI, []gomatrixserverlib.HeaderedEvent{ event.Headered(verRes.RoomVersion), }, diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index e6c6df658..754dcdfb0 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -21,7 +21,7 @@ import ( appserviceAPI "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" - "github.com/matrix-org/dendrite/clientapi/producers" + eduserverAPI "github.com/matrix-org/dendrite/eduserver/api" federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" @@ -49,8 +49,7 @@ func Setup( cfg *config.Dendrite, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, - producer *producers.RoomserverProducer, - eduProducer *producers.EDUServerProducer, + eduAPI eduserverAPI.EDUServerInputAPI, fsAPI federationSenderAPI.FederationSenderInternalAPI, keys gomatrixserverlib.KeyRing, federation *gomatrixserverlib.FederationClient, @@ -82,7 +81,7 @@ func Setup( func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return Send( httpReq, request, gomatrixserverlib.TransactionID(vars["txnID"]), - cfg, rsAPI, producer, eduProducer, keys, federation, + cfg, rsAPI, eduAPI, keys, federation, ) }, )).Methods(http.MethodPut, http.MethodOptions) @@ -92,14 +91,14 @@ func Setup( func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return Invite( httpReq, request, vars["roomID"], vars["eventID"], - cfg, producer, keys, + cfg, rsAPI, keys, ) }, )).Methods(http.MethodPut, http.MethodOptions) v1fedmux.Handle("/3pid/onbind", internal.MakeExternalAPI("3pid_onbind", func(req *http.Request) util.JSONResponse { - return CreateInvitesFrom3PIDInvites(req, rsAPI, asAPI, cfg, producer, federation, accountDB) + return CreateInvitesFrom3PIDInvites(req, rsAPI, asAPI, cfg, federation, accountDB) }, )).Methods(http.MethodPost, http.MethodOptions) @@ -107,7 +106,7 @@ func Setup( "exchange_third_party_invite", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return ExchangeThirdPartyInvite( - httpReq, request, vars["roomID"], rsAPI, cfg, federation, producer, + httpReq, request, vars["roomID"], rsAPI, cfg, federation, ) }, )).Methods(http.MethodPut, http.MethodOptions) @@ -206,7 +205,7 @@ func Setup( roomID := vars["roomID"] eventID := vars["eventID"] res := SendJoin( - httpReq, request, cfg, rsAPI, producer, keys, roomID, eventID, + httpReq, request, cfg, rsAPI, keys, roomID, eventID, ) return util.JSONResponse{ Headers: res.Headers, @@ -224,7 +223,7 @@ func Setup( roomID := vars["roomID"] eventID := vars["eventID"] return SendJoin( - httpReq, request, cfg, rsAPI, producer, keys, roomID, eventID, + httpReq, request, cfg, rsAPI, keys, roomID, eventID, ) }, )).Methods(http.MethodPut) @@ -246,7 +245,7 @@ func Setup( roomID := vars["roomID"] eventID := vars["eventID"] return SendLeave( - httpReq, request, cfg, producer, keys, roomID, eventID, + httpReq, request, cfg, rsAPI, keys, roomID, eventID, ) }, )).Methods(http.MethodPut) diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index a2120d81b..a057750f6 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -21,7 +21,7 @@ import ( "net/http" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/clientapi/producers" + eduserverAPI "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" @@ -36,20 +36,18 @@ func Send( txnID gomatrixserverlib.TransactionID, cfg *config.Dendrite, rsAPI api.RoomserverInternalAPI, - producer *producers.RoomserverProducer, - eduProducer *producers.EDUServerProducer, + eduAPI eduserverAPI.EDUServerInputAPI, keys gomatrixserverlib.KeyRing, federation *gomatrixserverlib.FederationClient, ) util.JSONResponse { t := txnReq{ - context: httpReq.Context(), - rsAPI: rsAPI, - producer: producer, - eduProducer: eduProducer, - keys: keys, - federation: federation, - haveEvents: make(map[string]*gomatrixserverlib.HeaderedEvent), - newEvents: make(map[string]bool), + context: httpReq.Context(), + rsAPI: rsAPI, + eduAPI: eduAPI, + keys: keys, + federation: federation, + haveEvents: make(map[string]*gomatrixserverlib.HeaderedEvent), + newEvents: make(map[string]bool), } var txnEvents struct { @@ -91,12 +89,11 @@ func Send( type txnReq struct { gomatrixserverlib.Transaction - context context.Context - rsAPI api.RoomserverInternalAPI - producer *producers.RoomserverProducer - eduProducer *producers.EDUServerProducer - keys gomatrixserverlib.JSONVerifier - federation txnFederationClient + context context.Context + rsAPI api.RoomserverInternalAPI + eduAPI eduserverAPI.EDUServerInputAPI + keys gomatrixserverlib.JSONVerifier + federation txnFederationClient // local cache of events for auth checks, etc - this may include events // which the roomserver is unaware of. haveEvents map[string]*gomatrixserverlib.HeaderedEvent @@ -262,7 +259,7 @@ func (t *txnReq) processEDUs(edus []gomatrixserverlib.EDU) { util.GetLogger(t.context).WithError(err).Error("Failed to unmarshal typing event") continue } - if err := t.eduProducer.SendTyping(t.context, typingPayload.UserID, typingPayload.RoomID, typingPayload.Typing, 30*1000); err != nil { + if err := eduserverAPI.SendTyping(t.context, t.eduAPI, typingPayload.UserID, typingPayload.RoomID, typingPayload.Typing, 30*1000); err != nil { util.GetLogger(t.context).WithError(err).Error("Failed to send typing event to edu server") } case gomatrixserverlib.MDirectToDevice: @@ -275,7 +272,7 @@ func (t *txnReq) processEDUs(edus []gomatrixserverlib.EDU) { for userID, byUser := range directPayload.Messages { for deviceID, message := range byUser { // TODO: check that the user and the device actually exist here - if err := t.eduProducer.SendToDevice(t.context, directPayload.Sender, userID, deviceID, directPayload.Type, message); err != nil { + if err := eduserverAPI.SendToDevice(t.context, t.eduAPI, directPayload.Sender, userID, deviceID, directPayload.Type, message); err != nil { util.GetLogger(t.context).WithError(err).WithFields(logrus.Fields{ "sender": directPayload.Sender, "user_id": userID, @@ -325,8 +322,8 @@ func (t *txnReq) processEvent(e gomatrixserverlib.Event, isInboundTxn bool) erro } // pass the event to the roomserver - _, err := t.producer.SendEvents( - t.context, + _, err := api.SendEvents( + t.context, t.rsAPI, []gomatrixserverlib.HeaderedEvent{ e.Headered(stateResp.RoomVersion), }, @@ -408,7 +405,7 @@ func (t *txnReq) processEventWithMissingState(e gomatrixserverlib.Event, roomVer // pass the event along with the state to the roomserver using a background context so we don't // needlessly expire - return t.producer.SendEventWithState(context.Background(), resolvedState, e.Headered(roomVersion), t.haveEventIDs()) + return api.SendEventWithState(context.Background(), t.rsAPI, resolvedState, e.Headered(roomVersion), t.haveEventIDs()) } // lookupStateAfterEvent returns the room state after `eventID`, which is the state before eventID with the state of `eventID` (if it's a state event) diff --git a/federationapi/routing/send_test.go b/federationapi/routing/send_test.go index 3e28a3476..b81f1c003 100644 --- a/federationapi/routing/send_test.go +++ b/federationapi/routing/send_test.go @@ -8,7 +8,6 @@ import ( "testing" "time" - "github.com/matrix-org/dendrite/clientapi/producers" eduAPI "github.com/matrix-org/dendrite/eduserver/api" fsAPI "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/roomserver/api" @@ -339,14 +338,13 @@ func (c *txnFedClient) LookupMissingEvents(ctx context.Context, s gomatrixserver func mustCreateTransaction(rsAPI api.RoomserverInternalAPI, fedClient txnFederationClient, pdus []json.RawMessage) *txnReq { t := &txnReq{ - context: context.Background(), - rsAPI: rsAPI, - producer: producers.NewRoomserverProducer(rsAPI), - eduProducer: producers.NewEDUServerProducer(&testEDUProducer{}), - keys: &testNopJSONVerifier{}, - federation: fedClient, - haveEvents: make(map[string]*gomatrixserverlib.HeaderedEvent), - newEvents: make(map[string]bool), + context: context.Background(), + rsAPI: rsAPI, + eduAPI: &testEDUProducer{}, + keys: &testNopJSONVerifier{}, + federation: fedClient, + haveEvents: make(map[string]*gomatrixserverlib.HeaderedEvent), + newEvents: make(map[string]bool), } t.PDUs = pdus t.Origin = testOrigin diff --git a/federationapi/routing/threepid.go b/federationapi/routing/threepid.go index 8053cedde..8f3193870 100644 --- a/federationapi/routing/threepid.go +++ b/federationapi/routing/threepid.go @@ -25,7 +25,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/roomserver/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" @@ -60,7 +59,7 @@ var ( func CreateInvitesFrom3PIDInvites( req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, cfg *config.Dendrite, - producer *producers.RoomserverProducer, federation *gomatrixserverlib.FederationClient, + federation *gomatrixserverlib.FederationClient, accountDB accounts.Database, ) util.JSONResponse { var body invites @@ -92,8 +91,8 @@ func CreateInvitesFrom3PIDInvites( } // Send all the events - if _, err := producer.SendEvents(req.Context(), evs, cfg.Matrix.ServerName, nil); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("producer.SendEvents failed") + if _, err := api.SendEvents(req.Context(), rsAPI, evs, cfg.Matrix.ServerName, nil); err != nil { + util.GetLogger(req.Context()).WithError(err).Error("SendEvents failed") return jsonerror.InternalServerError() } @@ -111,7 +110,6 @@ func ExchangeThirdPartyInvite( rsAPI roomserverAPI.RoomserverInternalAPI, cfg *config.Dendrite, federation *gomatrixserverlib.FederationClient, - producer *producers.RoomserverProducer, ) util.JSONResponse { var builder gomatrixserverlib.EventBuilder if err := json.Unmarshal(request.Content(), &builder); err != nil { @@ -176,15 +174,15 @@ func ExchangeThirdPartyInvite( } // Send the event to the roomserver - if _, err = producer.SendEvents( - httpReq.Context(), + if _, err = api.SendEvents( + httpReq.Context(), rsAPI, []gomatrixserverlib.HeaderedEvent{ signedEvent.Event.Headered(verRes.RoomVersion), }, cfg.Matrix.ServerName, nil, ); err != nil { - util.GetLogger(httpReq.Context()).WithError(err).Error("producer.SendEvents failed") + util.GetLogger(httpReq.Context()).WithError(err).Error("SendEvents failed") return jsonerror.InternalServerError() } diff --git a/internal/setup/monolith.go b/internal/setup/monolith.go index d9bb28395..35fcd3119 100644 --- a/internal/setup/monolith.go +++ b/internal/setup/monolith.go @@ -7,7 +7,6 @@ import ( "github.com/matrix-org/dendrite/clientapi" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" - "github.com/matrix-org/dendrite/clientapi/producers" eduServerAPI "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/federationapi" federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" @@ -41,8 +40,6 @@ type Monolith struct { RoomserverAPI roomserverAPI.RoomserverInternalAPI ServerKeyAPI serverKeyAPI.ServerKeyInternalAPI - // TODO: remove, this isn't even a producer - EDUProducer *producers.EDUServerProducer // TODO: can we remove this? It's weird that we are required the database // yet every other component can do that on its own. libp2p-demo uses a custom // database though annoyingly. @@ -65,7 +62,7 @@ func (m *Monolith) AddAllPublicRoutes(publicMux *mux.Router) { federationapi.AddPublicRoutes( publicMux, m.Config, m.AccountDB, m.DeviceDB, m.FedClient, m.KeyRing, m.RoomserverAPI, m.AppserviceAPI, m.FederationSenderAPI, - m.EDUProducer, + m.EDUInternalAPI, ) mediaapi.AddPublicRoutes(publicMux, m.Config, m.DeviceDB) publicroomsapi.AddPublicRoutes( diff --git a/clientapi/producers/roomserver.go b/roomserver/api/wrapper.go similarity index 50% rename from clientapi/producers/roomserver.go rename to roomserver/api/wrapper.go index f0733db9c..97940e0c2 100644 --- a/clientapi/producers/roomserver.go +++ b/roomserver/api/wrapper.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,63 +12,51 @@ // See the License for the specific language governing permissions and // limitations under the License. -package producers +package api import ( "context" - "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" ) -// RoomserverProducer produces events for the roomserver to consume. -type RoomserverProducer struct { - RsAPI api.RoomserverInternalAPI -} - -// NewRoomserverProducer creates a new RoomserverProducer -func NewRoomserverProducer(rsAPI api.RoomserverInternalAPI) *RoomserverProducer { - return &RoomserverProducer{ - RsAPI: rsAPI, - } -} - -// SendEvents writes the given events to the roomserver input log. The events are written with KindNew. -func (c *RoomserverProducer) SendEvents( - ctx context.Context, events []gomatrixserverlib.HeaderedEvent, sendAsServer gomatrixserverlib.ServerName, - txnID *api.TransactionID, +// SendEvents to the roomserver The events are written with KindNew. +func SendEvents( + ctx context.Context, rsAPI RoomserverInternalAPI, events []gomatrixserverlib.HeaderedEvent, + sendAsServer gomatrixserverlib.ServerName, txnID *TransactionID, ) (string, error) { - ires := make([]api.InputRoomEvent, len(events)) + ires := make([]InputRoomEvent, len(events)) for i, event := range events { - ires[i] = api.InputRoomEvent{ - Kind: api.KindNew, + ires[i] = InputRoomEvent{ + Kind: KindNew, Event: event, AuthEventIDs: event.AuthEventIDs(), SendAsServer: string(sendAsServer), TransactionID: txnID, } } - return c.SendInputRoomEvents(ctx, ires) + return SendInputRoomEvents(ctx, rsAPI, ires) } -// SendEventWithState writes an event with KindNew to the roomserver input log +// SendEventWithState writes an event with KindNew to the roomserver // with the state at the event as KindOutlier before it. Will not send any event that is // marked as `true` in haveEventIDs -func (c *RoomserverProducer) SendEventWithState( - ctx context.Context, state *gomatrixserverlib.RespState, event gomatrixserverlib.HeaderedEvent, haveEventIDs map[string]bool, +func SendEventWithState( + ctx context.Context, rsAPI RoomserverInternalAPI, state *gomatrixserverlib.RespState, + event gomatrixserverlib.HeaderedEvent, haveEventIDs map[string]bool, ) error { outliers, err := state.Events() if err != nil { return err } - var ires []api.InputRoomEvent + var ires []InputRoomEvent for _, outlier := range outliers { if haveEventIDs[outlier.EventID()] { continue } - ires = append(ires, api.InputRoomEvent{ - Kind: api.KindOutlier, + ires = append(ires, InputRoomEvent{ + Kind: KindOutlier, Event: outlier.Headered(event.RoomVersion), AuthEventIDs: outlier.AuthEventIDs(), }) @@ -79,39 +67,40 @@ func (c *RoomserverProducer) SendEventWithState( stateEventIDs[i] = state.StateEvents[i].EventID() } - ires = append(ires, api.InputRoomEvent{ - Kind: api.KindNew, + ires = append(ires, InputRoomEvent{ + Kind: KindNew, Event: event, AuthEventIDs: event.AuthEventIDs(), HasState: true, StateEventIDs: stateEventIDs, }) - _, err = c.SendInputRoomEvents(ctx, ires) + _, err = SendInputRoomEvents(ctx, rsAPI, ires) return err } -// SendInputRoomEvents writes the given input room events to the roomserver input API. -func (c *RoomserverProducer) SendInputRoomEvents( - ctx context.Context, ires []api.InputRoomEvent, +// SendInputRoomEvents to the roomserver. +func SendInputRoomEvents( + ctx context.Context, rsAPI RoomserverInternalAPI, ires []InputRoomEvent, ) (eventID string, err error) { - request := api.InputRoomEventsRequest{InputRoomEvents: ires} - var response api.InputRoomEventsResponse - err = c.RsAPI.InputRoomEvents(ctx, &request, &response) + request := InputRoomEventsRequest{InputRoomEvents: ires} + var response InputRoomEventsResponse + err = rsAPI.InputRoomEvents(ctx, &request, &response) eventID = response.EventID return } -// SendInvite writes the invite event to the roomserver input API. +// SendInvite event to the roomserver. // This should only be needed for invite events that occur outside of a known room. // If we are in the room then the event should be sent using the SendEvents method. -func (c *RoomserverProducer) SendInvite( - ctx context.Context, inviteEvent gomatrixserverlib.HeaderedEvent, +func SendInvite( + ctx context.Context, + rsAPI RoomserverInternalAPI, inviteEvent gomatrixserverlib.HeaderedEvent, inviteRoomState []gomatrixserverlib.InviteV2StrippedState, - sendAsServer gomatrixserverlib.ServerName, txnID *api.TransactionID, + sendAsServer gomatrixserverlib.ServerName, txnID *TransactionID, ) error { - request := api.InputRoomEventsRequest{ - InputInviteEvents: []api.InputInviteEvent{{ + request := InputRoomEventsRequest{ + InputInviteEvents: []InputInviteEvent{{ Event: inviteEvent, InviteRoomState: inviteRoomState, RoomVersion: inviteEvent.RoomVersion, @@ -119,6 +108,6 @@ func (c *RoomserverProducer) SendInvite( TransactionID: txnID, }}, } - var response api.InputRoomEventsResponse - return c.RsAPI.InputRoomEvents(ctx, &request, &response) + var response InputRoomEventsResponse + return rsAPI.InputRoomEvents(ctx, &request, &response) } From 90a0aa9b3e14dc5b6aa8a5be6b38112ec164502b Mon Sep 17 00:00:00 2001 From: Kegsay Date: Wed, 10 Jun 2020 13:28:32 +0100 Subject: [PATCH 147/200] Remove public appservice routes (#1112) --- appservice/appservice.go | 13 ------ appservice/routing/routing.go | 65 -------------------------- cmd/dendrite-appservice-server/main.go | 4 -- 3 files changed, 82 deletions(-) delete mode 100644 appservice/routing/routing.go diff --git a/appservice/appservice.go b/appservice/appservice.go index dca600bf5..b7adf755b 100644 --- a/appservice/appservice.go +++ b/appservice/appservice.go @@ -26,7 +26,6 @@ import ( "github.com/matrix-org/dendrite/appservice/consumers" "github.com/matrix-org/dendrite/appservice/inthttp" "github.com/matrix-org/dendrite/appservice/query" - "github.com/matrix-org/dendrite/appservice/routing" "github.com/matrix-org/dendrite/appservice/storage" "github.com/matrix-org/dendrite/appservice/types" "github.com/matrix-org/dendrite/appservice/workers" @@ -35,22 +34,10 @@ import ( "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/basecomponent" "github.com/matrix-org/dendrite/internal/config" - "github.com/matrix-org/dendrite/internal/transactions" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" ) -// AddPublicRoutes registers HTTP handlers for CS API calls -func AddPublicRoutes(router *mux.Router, cfg *config.Dendrite, rsAPI roomserverAPI.RoomserverInternalAPI, - accountsDB accounts.Database, federation *gomatrixserverlib.FederationClient, txnCache *transactions.Cache) { - - routing.Setup( - router, cfg, rsAPI, - accountsDB, federation, txnCache, - ) -} - // AddInternalRoutes registers HTTP handlers for internal API calls func AddInternalRoutes(router *mux.Router, queryAPI appserviceAPI.AppServiceQueryAPI) { inthttp.AddRoutes(queryAPI, router) diff --git a/appservice/routing/routing.go b/appservice/routing/routing.go deleted file mode 100644 index e5ed7ab40..000000000 --- a/appservice/routing/routing.go +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2018 Vector Creations Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package routing - -import ( - "net/http" - - "github.com/gorilla/mux" - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" - "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/dendrite/internal/config" - "github.com/matrix-org/dendrite/internal/transactions" - "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" -) - -const pathPrefixApp = "/app/v1" - -// Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client -// to clients which need to make outbound HTTP requests. -// -// Due to Setup being used to call many other functions, a gocyclo nolint is -// applied: -// nolint: gocyclo -func Setup( - apiMux *mux.Router, cfg *config.Dendrite, // nolint: unparam - rsAPI api.RoomserverInternalAPI, // nolint: unparam - accountDB accounts.Database, // nolint: unparam - federation *gomatrixserverlib.FederationClient, // nolint: unparam - transactionsCache *transactions.Cache, // nolint: unparam -) { - appMux := apiMux.PathPrefix(pathPrefixApp).Subrouter() - - appMux.Handle("/alias", - internal.MakeExternalAPI("alias", func(req *http.Request) util.JSONResponse { - // TODO: Implement - return util.JSONResponse{ - Code: http.StatusOK, - JSON: nil, - } - }), - ).Methods(http.MethodGet, http.MethodOptions) - appMux.Handle("/user", - internal.MakeExternalAPI("user", func(req *http.Request) util.JSONResponse { - // TODO: Implement - return util.JSONResponse{ - Code: http.StatusOK, - JSON: nil, - } - }), - ).Methods(http.MethodGet, http.MethodOptions) -} diff --git a/cmd/dendrite-appservice-server/main.go b/cmd/dendrite-appservice-server/main.go index 21fb1b79c..fe00a117d 100644 --- a/cmd/dendrite-appservice-server/main.go +++ b/cmd/dendrite-appservice-server/main.go @@ -17,7 +17,6 @@ package main import ( "github.com/matrix-org/dendrite/appservice" "github.com/matrix-org/dendrite/internal/basecomponent" - "github.com/matrix-org/dendrite/internal/transactions" ) func main() { @@ -27,13 +26,10 @@ func main() { defer base.Close() // nolint: errcheck accountDB := base.CreateAccountsDB() deviceDB := base.CreateDeviceDB() - federation := base.CreateFederationClient() rsAPI := base.RoomserverHTTPClient() - cache := transactions.New() intAPI := appservice.NewInternalAPI(base, accountDB, deviceDB, rsAPI) appservice.AddInternalRoutes(base.InternalAPIMux, intAPI) - appservice.AddPublicRoutes(base.PublicAPIMux, base.Cfg, rsAPI, accountDB, federation, cache) base.SetupAndServeHTTP(string(base.Cfg.Bind.AppServiceAPI), string(base.Cfg.Listen.AppServiceAPI)) From 3b4be900000cc79bba4a0d98295b4afe9afea116 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 10 Jun 2020 16:29:02 +0100 Subject: [PATCH 148/200] Improvements to Yggdrasil demo (#1114) * Improvements to Yggdrasil demo * Fix missing copyright * Fix tie-break --- cmd/dendrite-demo-yggdrasil/convert/25519.go | 53 ++++++++++++++ .../convert/25519_test.go | 51 ++++++++++++++ cmd/dendrite-demo-yggdrasil/main.go | 25 ++++--- .../signing/fetcher.go | 69 +++++++++++++++++++ cmd/dendrite-demo-yggdrasil/yggconn/node.go | 28 ++++++-- .../yggconn/session.go | 8 +-- 6 files changed, 217 insertions(+), 17 deletions(-) create mode 100644 cmd/dendrite-demo-yggdrasil/convert/25519.go create mode 100644 cmd/dendrite-demo-yggdrasil/convert/25519_test.go create mode 100644 cmd/dendrite-demo-yggdrasil/signing/fetcher.go diff --git a/cmd/dendrite-demo-yggdrasil/convert/25519.go b/cmd/dendrite-demo-yggdrasil/convert/25519.go new file mode 100644 index 000000000..97f053ec0 --- /dev/null +++ b/cmd/dendrite-demo-yggdrasil/convert/25519.go @@ -0,0 +1,53 @@ +// Copyright 2019 Google LLC +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd +// +// Original code from https://github.com/FiloSottile/age/blob/bbab440e198a4d67ba78591176c7853e62d29e04/internal/age/ssh.go + +package convert + +import ( + "crypto/ed25519" + "crypto/sha512" + "math/big" + + "golang.org/x/crypto/curve25519" +) + +var curve25519P, _ = new(big.Int).SetString("57896044618658097711785492504343953926634992332820282019728792003956564819949", 10) + +func Ed25519PrivateKeyToCurve25519(pk ed25519.PrivateKey) []byte { + h := sha512.New() + _, _ = h.Write(pk.Seed()) + out := h.Sum(nil) + return out[:curve25519.ScalarSize] +} + +func Ed25519PublicKeyToCurve25519(pk ed25519.PublicKey) []byte { + // ed25519.PublicKey is a little endian representation of the y-coordinate, + // with the most significant bit set based on the sign of the x-coordinate. + bigEndianY := make([]byte, ed25519.PublicKeySize) + for i, b := range pk { + bigEndianY[ed25519.PublicKeySize-i-1] = b + } + bigEndianY[0] &= 0b0111_1111 + + // The Montgomery u-coordinate is derived through the bilinear map + // u = (1 + y) / (1 - y) + // See https://blog.filippo.io/using-ed25519-keys-for-encryption. + y := new(big.Int).SetBytes(bigEndianY) + denom := big.NewInt(1) + denom.ModInverse(denom.Sub(denom, y), curve25519P) // 1 / (1 - y) + u := y.Mul(y.Add(y, big.NewInt(1)), denom) + u.Mod(u, curve25519P) + + out := make([]byte, curve25519.PointSize) + uBytes := u.Bytes() + for i, b := range uBytes { + out[len(uBytes)-i-1] = b + } + + return out +} diff --git a/cmd/dendrite-demo-yggdrasil/convert/25519_test.go b/cmd/dendrite-demo-yggdrasil/convert/25519_test.go new file mode 100644 index 000000000..22177b8b4 --- /dev/null +++ b/cmd/dendrite-demo-yggdrasil/convert/25519_test.go @@ -0,0 +1,51 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package convert + +import ( + "bytes" + "crypto/ed25519" + "encoding/hex" + "testing" + + "golang.org/x/crypto/curve25519" +) + +func TestKeyConversion(t *testing.T) { + edPub, edPriv, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + t.Log("Signing public:", hex.EncodeToString(edPub)) + t.Log("Signing private:", hex.EncodeToString(edPriv)) + + cuPriv := Ed25519PrivateKeyToCurve25519(edPriv) + t.Log("Encryption private:", hex.EncodeToString(cuPriv)) + + cuPub := Ed25519PublicKeyToCurve25519(edPub) + t.Log("Converted encryption public:", hex.EncodeToString(cuPub)) + + var realPub, realPriv [32]byte + copy(realPriv[:32], cuPriv[:32]) + curve25519.ScalarBaseMult(&realPub, &realPriv) + t.Log("Scalar-multed encryption public:", hex.EncodeToString(realPub[:])) + + if !bytes.Equal(realPriv[:], cuPriv[:]) { + t.Fatal("Private keys should be equal (this means the test is broken)") + } + if !bytes.Equal(realPub[:], cuPub[:]) { + t.Fatal("Public keys should be equal") + } +} diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index 62c561290..94e94d8d3 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -16,7 +16,9 @@ package main import ( "context" + "crypto/ed25519" "crypto/tls" + "encoding/hex" "flag" "fmt" "net" @@ -25,6 +27,8 @@ import ( "time" "github.com/matrix-org/dendrite/appservice" + "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/convert" + "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/yggconn" "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/eduserver/cache" @@ -35,7 +39,6 @@ import ( "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/roomserver" - "github.com/matrix-org/dendrite/serverkeyapi" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" @@ -61,7 +64,13 @@ func createFederationClient( ) *gomatrixserverlib.FederationClient { yggdialer := func(_, address string) (net.Conn, error) { tokens := strings.Split(address, ":") - return n.Dial("curve25519", tokens[0]) + raw, err := hex.DecodeString(tokens[0]) + if err != nil { + return nil, fmt.Errorf("hex.DecodeString: %w", err) + } + converted := convert.Ed25519PublicKeyToCurve25519(ed25519.PublicKey(raw)) + convhex := hex.EncodeToString(converted) + return n.Dial("curve25519", convhex) } yggdialerctx := func(ctx context.Context, network, address string) (net.Conn, error) { return yggdialer(network, address) @@ -102,9 +111,9 @@ func main() { cfg := &config.Dendrite{} cfg.SetDefaults() - cfg.Matrix.ServerName = gomatrixserverlib.ServerName(ygg.EncryptionPublicKey()) + cfg.Matrix.ServerName = gomatrixserverlib.ServerName(ygg.DerivedServerName()) cfg.Matrix.PrivateKey = ygg.SigningPrivateKey() - cfg.Matrix.KeyID = gomatrixserverlib.KeyID("ed25519:auto") + cfg.Matrix.KeyID = gomatrixserverlib.KeyID(signing.KeyID) cfg.Kafka.UseNaffka = true cfg.Kafka.Topics.OutputRoomEvent = "roomserverOutput" cfg.Kafka.Topics.OutputClientData = "clientapiOutput" @@ -130,9 +139,7 @@ func main() { deviceDB := base.CreateDeviceDB() federation := createFederationClient(base, ygg) - serverKeyAPI := serverkeyapi.NewInternalAPI( - base.Cfg, federation, base.Caches, - ) + serverKeyAPI := &signing.YggdrasilKeys{} keyRing := serverKeyAPI.KeyRing() rsComponent := roomserver.NewInternalAPI( @@ -170,7 +177,7 @@ func main() { EDUInternalAPI: eduInputAPI, FederationSenderAPI: fsAPI, RoomserverAPI: rsAPI, - ServerKeyAPI: serverKeyAPI, + //ServerKeyAPI: serverKeyAPI, PublicRoomsDB: publicRoomsDB, } @@ -185,7 +192,7 @@ func main() { ) go func() { - logrus.Info("Listening on ", ygg.EncryptionPublicKey()) + logrus.Info("Listening on ", ygg.DerivedServerName()) logrus.Fatal(httpServer.Serve(ygg)) }() go func() { diff --git a/cmd/dendrite-demo-yggdrasil/signing/fetcher.go b/cmd/dendrite-demo-yggdrasil/signing/fetcher.go new file mode 100644 index 000000000..bcec0cbec --- /dev/null +++ b/cmd/dendrite-demo-yggdrasil/signing/fetcher.go @@ -0,0 +1,69 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package signing + +import ( + "context" + "encoding/hex" + "fmt" + "time" + + "github.com/matrix-org/gomatrixserverlib" +) + +const KeyID = "ed25519:dendrite-demo-yggdrasil" + +type YggdrasilKeys struct { +} + +func (f *YggdrasilKeys) KeyRing() *gomatrixserverlib.KeyRing { + return &gomatrixserverlib.KeyRing{ + KeyDatabase: f, + } +} + +func (f *YggdrasilKeys) FetchKeys( + ctx context.Context, + requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp, +) (map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, error) { + res := make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult) + for req := range requests { + if req.KeyID != KeyID { + return nil, fmt.Errorf("FetchKeys: cannot fetch key with ID %s, should be %s", req.KeyID, KeyID) + } + + hexkey, err := hex.DecodeString(string(req.ServerName)) + if err != nil { + return nil, fmt.Errorf("FetchKeys: can't decode server name %q: %w", req.ServerName, err) + } + + res[req] = gomatrixserverlib.PublicKeyLookupResult{ + VerifyKey: gomatrixserverlib.VerifyKey{ + Key: hexkey, + }, + ExpiredTS: gomatrixserverlib.PublicKeyNotExpired, + ValidUntilTS: gomatrixserverlib.AsTimestamp(time.Now().Add(24 * time.Hour * 365)), + } + } + return res, nil +} + +func (f *YggdrasilKeys) FetcherName() string { + return "YggdrasilKeys" +} + +func (f *YggdrasilKeys) StoreKeys(ctx context.Context, results map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult) error { + return nil +} diff --git a/cmd/dendrite-demo-yggdrasil/yggconn/node.go b/cmd/dendrite-demo-yggdrasil/yggconn/node.go index 517f7f794..14bbf84d7 100644 --- a/cmd/dendrite-demo-yggdrasil/yggconn/node.go +++ b/cmd/dendrite-demo-yggdrasil/yggconn/node.go @@ -24,6 +24,8 @@ import ( "os" "sync" + "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/convert" + "github.com/libp2p/go-yamux" yggdrasiladmin "github.com/yggdrasil-network/yggdrasil-go/src/admin" yggdrasilconfig "github.com/yggdrasil-network/yggdrasil-go/src/config" @@ -56,8 +58,6 @@ func Setup(instanceName, instancePeer string) (*Node, error) { log: gologme.New(os.Stdout, "YGG ", log.Flags()), incoming: make(chan *yamux.Stream), } - n.config.AdminListen = fmt.Sprintf("unix://./%s-yggdrasil.sock", instanceName) - n.config.MulticastInterfaces = []string{".*"} yggfile := fmt.Sprintf("%s-yggdrasil.conf", instanceName) if _, err := os.Stat(yggfile); !os.IsNotExist(err) { @@ -69,6 +69,11 @@ func Setup(instanceName, instancePeer string) (*Node, error) { panic(err) } } else { + n.config.AdminListen = fmt.Sprintf("unix://./%s-yggdrasil.sock", instanceName) + n.config.MulticastInterfaces = []string{".*"} + n.config.EncryptionPrivateKey = hex.EncodeToString(n.EncryptionPrivateKey()) + n.config.EncryptionPublicKey = hex.EncodeToString(n.EncryptionPublicKey()) + j, err := json.MarshalIndent(n.config, "", " ") if err != nil { panic(err) @@ -119,8 +124,23 @@ func Setup(instanceName, instancePeer string) (*Node, error) { return n, nil } -func (n *Node) EncryptionPublicKey() string { - return n.core.EncryptionPublicKey() +func (n *Node) DerivedServerName() string { + return hex.EncodeToString(n.SigningPublicKey()) +} + +func (n *Node) EncryptionPublicKey() []byte { + edkey := n.SigningPublicKey() + return convert.Ed25519PublicKeyToCurve25519(edkey) +} + +func (n *Node) EncryptionPrivateKey() []byte { + edkey := n.SigningPrivateKey() + return convert.Ed25519PrivateKeyToCurve25519(edkey) +} + +func (n *Node) SigningPublicKey() ed25519.PublicKey { + pubBytes, _ := hex.DecodeString(n.config.SigningPublicKey) + return ed25519.PublicKey(pubBytes) } func (n *Node) SigningPrivateKey() ed25519.PrivateKey { diff --git a/cmd/dendrite-demo-yggdrasil/yggconn/session.go b/cmd/dendrite-demo-yggdrasil/yggconn/session.go index 4e9a9f043..a9bb15f7e 100644 --- a/cmd/dendrite-demo-yggdrasil/yggconn/session.go +++ b/cmd/dendrite-demo-yggdrasil/yggconn/session.go @@ -40,10 +40,10 @@ func (n *Node) listenFromYgg() { return } var session *yamux.Session - if strings.Compare(n.EncryptionPublicKey(), conn.RemoteAddr().String()) < 0 { - session, err = yamux.Client(conn, n.yamuxConfig()) - } else { + if strings.Compare(conn.RemoteAddr().String(), n.DerivedServerName()) < 0 { session, err = yamux.Server(conn, n.yamuxConfig()) + } else { + session, err = yamux.Client(conn, n.yamuxConfig()) } if err != nil { return @@ -96,7 +96,7 @@ func (n *Node) DialContext(ctx context.Context, network, address string) (net.Co n.log.Println("n.dialer.DialContext:", err) return nil, err } - if strings.Compare(n.EncryptionPublicKey(), address) < 0 { + if strings.Compare(address, n.DerivedServerName()) > 0 { session, err = yamux.Client(conn, n.yamuxConfig()) } else { session, err = yamux.Server(conn, n.yamuxConfig()) From 399b6ae334fe2559d8d141ba42ff6ff960887993 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Wed, 10 Jun 2020 16:54:43 +0100 Subject: [PATCH 149/200] Remove federationsender producer, which in fact was not a producer (#1115) * Remove federationsender producer, which in fact was not a producer * Set the signing struct --- federationsender/federationsender.go | 13 ++- federationsender/internal/api.go | 10 +- federationsender/internal/perform.go | 10 +- federationsender/producers/roomserver.go | 108 --------------------- federationsender/queue/destinationqueue.go | 13 ++- federationsender/queue/queue.go | 23 ++++- 6 files changed, 40 insertions(+), 137 deletions(-) delete mode 100644 federationsender/producers/roomserver.go diff --git a/federationsender/federationsender.go b/federationsender/federationsender.go index e3e0ef9d8..621920ef5 100644 --- a/federationsender/federationsender.go +++ b/federationsender/federationsender.go @@ -20,7 +20,6 @@ import ( "github.com/matrix-org/dendrite/federationsender/consumers" "github.com/matrix-org/dendrite/federationsender/internal" "github.com/matrix-org/dendrite/federationsender/inthttp" - "github.com/matrix-org/dendrite/federationsender/producers" "github.com/matrix-org/dendrite/federationsender/queue" "github.com/matrix-org/dendrite/federationsender/storage" "github.com/matrix-org/dendrite/federationsender/types" @@ -49,13 +48,13 @@ func NewInternalAPI( logrus.WithError(err).Panic("failed to connect to federation sender db") } - roomserverProducer := producers.NewRoomserverProducer( - rsAPI, base.Cfg.Matrix.ServerName, base.Cfg.Matrix.KeyID, base.Cfg.Matrix.PrivateKey, - ) - statistics := &types.Statistics{} queues := queue.NewOutgoingQueues( - base.Cfg.Matrix.ServerName, federation, roomserverProducer, statistics, + base.Cfg.Matrix.ServerName, federation, rsAPI, statistics, &queue.SigningInfo{ + KeyID: base.Cfg.Matrix.KeyID, + PrivateKey: base.Cfg.Matrix.PrivateKey, + ServerName: base.Cfg.Matrix.ServerName, + }, ) rsConsumer := consumers.NewOutputRoomEventConsumer( @@ -73,5 +72,5 @@ func NewInternalAPI( logrus.WithError(err).Panic("failed to start typing server consumer") } - return internal.NewFederationSenderInternalAPI(federationSenderDB, base.Cfg, roomserverProducer, federation, keyRing, statistics, queues) + return internal.NewFederationSenderInternalAPI(federationSenderDB, base.Cfg, rsAPI, federation, keyRing, statistics, queues) } diff --git a/federationsender/internal/api.go b/federationsender/internal/api.go index c2ea0d411..0dca32fc9 100644 --- a/federationsender/internal/api.go +++ b/federationsender/internal/api.go @@ -1,22 +1,20 @@ package internal import ( - "github.com/matrix-org/dendrite/federationsender/api" - "github.com/matrix-org/dendrite/federationsender/producers" "github.com/matrix-org/dendrite/federationsender/queue" "github.com/matrix-org/dendrite/federationsender/storage" "github.com/matrix-org/dendrite/federationsender/types" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" ) // FederationSenderInternalAPI is an implementation of api.FederationSenderInternalAPI type FederationSenderInternalAPI struct { - api.FederationSenderInternalAPI db storage.Database cfg *config.Dendrite statistics *types.Statistics - producer *producers.RoomserverProducer + rsAPI api.RoomserverInternalAPI federation *gomatrixserverlib.FederationClient keyRing *gomatrixserverlib.KeyRing queues *queue.OutgoingQueues @@ -24,7 +22,7 @@ type FederationSenderInternalAPI struct { func NewFederationSenderInternalAPI( db storage.Database, cfg *config.Dendrite, - producer *producers.RoomserverProducer, + rsAPI api.RoomserverInternalAPI, federation *gomatrixserverlib.FederationClient, keyRing *gomatrixserverlib.KeyRing, statistics *types.Statistics, @@ -33,7 +31,7 @@ func NewFederationSenderInternalAPI( return &FederationSenderInternalAPI{ db: db, cfg: cfg, - producer: producer, + rsAPI: rsAPI, federation: federation, keyRing: keyRing, statistics: statistics, diff --git a/federationsender/internal/perform.go b/federationsender/internal/perform.go index c601e9604..7ced4af86 100644 --- a/federationsender/internal/perform.go +++ b/federationsender/internal/perform.go @@ -7,6 +7,7 @@ import ( "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/federationsender/internal/perform" + roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/version" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -175,10 +176,11 @@ func (r *FederationSenderInternalAPI) performJoinUsingServer( // If we successfully performed a send_join above then the other // server now thinks we're a part of the room. Send the newly // returned state to the roomserver to update our local view. - if err = r.producer.SendEventWithState( - ctx, - respSendJoin.ToRespState(), - event.Headered(respMakeJoin.RoomVersion), + respState := respSendJoin.ToRespState() + if err = roomserverAPI.SendEventWithState( + ctx, r.rsAPI, + &respState, + event.Headered(respMakeJoin.RoomVersion), nil, ); err != nil { return fmt.Errorf("r.producer.SendEventWithState: %w", err) } diff --git a/federationsender/producers/roomserver.go b/federationsender/producers/roomserver.go deleted file mode 100644 index 76fedf537..000000000 --- a/federationsender/producers/roomserver.go +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package producers - -import ( - "context" - "crypto/ed25519" - - "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/gomatrixserverlib" -) - -// RoomserverProducer produces events for the roomserver to consume. -type RoomserverProducer struct { - InputAPI api.RoomserverInternalAPI - serverName gomatrixserverlib.ServerName - keyID gomatrixserverlib.KeyID - privateKey ed25519.PrivateKey -} - -// NewRoomserverProducer creates a new RoomserverProducer -func NewRoomserverProducer( - rsAPI api.RoomserverInternalAPI, serverName gomatrixserverlib.ServerName, - keyID gomatrixserverlib.KeyID, privateKey ed25519.PrivateKey, -) *RoomserverProducer { - return &RoomserverProducer{ - InputAPI: rsAPI, - serverName: serverName, - keyID: keyID, - privateKey: privateKey, - } -} - -// SendInviteResponse drops an invite response back into the roomserver so that users -// already in the room will be notified of the new invite. The invite response is signed -// by the remote side. -func (c *RoomserverProducer) SendInviteResponse( - ctx context.Context, res gomatrixserverlib.RespInviteV2, roomVersion gomatrixserverlib.RoomVersion, -) (string, error) { - ev := res.Event.Sign(string(c.serverName), c.keyID, c.privateKey).Headered(roomVersion) - ire := api.InputRoomEvent{ - Kind: api.KindNew, - Event: ev, - AuthEventIDs: ev.AuthEventIDs(), - SendAsServer: string(c.serverName), - TransactionID: nil, - } - return c.SendInputRoomEvents(ctx, []api.InputRoomEvent{ire}) -} - -// SendEventWithState writes an event with KindNew to the roomserver input log -// with the state at the event as KindOutlier before it. -func (c *RoomserverProducer) SendEventWithState( - ctx context.Context, state gomatrixserverlib.RespState, event gomatrixserverlib.HeaderedEvent, -) error { - outliers, err := state.Events() - if err != nil { - return err - } - - var ires []api.InputRoomEvent - for _, outlier := range outliers { - ires = append(ires, api.InputRoomEvent{ - Kind: api.KindOutlier, - Event: outlier.Headered(event.RoomVersion), - AuthEventIDs: outlier.AuthEventIDs(), - }) - } - - stateEventIDs := make([]string, len(state.StateEvents)) - for i := range state.StateEvents { - stateEventIDs[i] = state.StateEvents[i].EventID() - } - - ires = append(ires, api.InputRoomEvent{ - Kind: api.KindNew, - Event: event, - AuthEventIDs: event.AuthEventIDs(), - HasState: true, - StateEventIDs: stateEventIDs, - }) - - _, err = c.SendInputRoomEvents(ctx, ires) - return err -} - -// SendInputRoomEvents writes the given input room events to the roomserver input API. -func (c *RoomserverProducer) SendInputRoomEvents( - ctx context.Context, ires []api.InputRoomEvent, -) (eventID string, err error) { - request := api.InputRoomEventsRequest{InputRoomEvents: ires} - var response api.InputRoomEventsResponse - err = c.InputAPI.InputRoomEvents(ctx, &request, &response) - eventID = response.EventID - return -} diff --git a/federationsender/queue/destinationqueue.go b/federationsender/queue/destinationqueue.go index bf9042f45..4449f9e63 100644 --- a/federationsender/queue/destinationqueue.go +++ b/federationsender/queue/destinationqueue.go @@ -20,8 +20,8 @@ import ( "fmt" "time" - "github.com/matrix-org/dendrite/federationsender/producers" "github.com/matrix-org/dendrite/federationsender/types" + "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" @@ -34,7 +34,8 @@ import ( // ensures that only one request is in flight to a given destination // at a time. type destinationQueue struct { - rsProducer *producers.RoomserverProducer // roomserver producer + signing *SigningInfo + rsAPI api.RoomserverInternalAPI client *gomatrixserverlib.FederationClient // federation client origin gomatrixserverlib.ServerName // origin of requests destination gomatrixserverlib.ServerName // destination of requests @@ -370,11 +371,9 @@ func (oq *destinationQueue) nextInvites( return done, err } - if _, err = oq.rsProducer.SendInviteResponse( - context.TODO(), - inviteRes, - roomVersion, - ); err != nil { + invEv := inviteRes.Event.Sign(string(oq.signing.ServerName), oq.signing.KeyID, oq.signing.PrivateKey).Headered(roomVersion) + _, err = api.SendEvents(context.TODO(), oq.rsAPI, []gomatrixserverlib.HeaderedEvent{invEv}, oq.signing.ServerName, nil) + if err != nil { log.WithFields(log.Fields{ "event_id": ev.EventID(), "state_key": ev.StateKey(), diff --git a/federationsender/queue/queue.go b/federationsender/queue/queue.go index 386a3397f..5b8fc3c56 100644 --- a/federationsender/queue/queue.go +++ b/federationsender/queue/queue.go @@ -15,11 +15,12 @@ package queue import ( + "crypto/ed25519" "fmt" "sync" - "github.com/matrix-org/dendrite/federationsender/producers" "github.com/matrix-org/dendrite/federationsender/types" + "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" log "github.com/sirupsen/logrus" @@ -28,10 +29,11 @@ import ( // OutgoingQueues is a collection of queues for sending transactions to other // matrix servers type OutgoingQueues struct { - rsProducer *producers.RoomserverProducer + rsAPI api.RoomserverInternalAPI origin gomatrixserverlib.ServerName client *gomatrixserverlib.FederationClient statistics *types.Statistics + signing *SigningInfo queuesMutex sync.Mutex // protects the below queues map[gomatrixserverlib.ServerName]*destinationQueue } @@ -40,18 +42,28 @@ type OutgoingQueues struct { func NewOutgoingQueues( origin gomatrixserverlib.ServerName, client *gomatrixserverlib.FederationClient, - rsProducer *producers.RoomserverProducer, + rsAPI api.RoomserverInternalAPI, statistics *types.Statistics, + signing *SigningInfo, ) *OutgoingQueues { return &OutgoingQueues{ - rsProducer: rsProducer, + rsAPI: rsAPI, origin: origin, client: client, statistics: statistics, + signing: signing, queues: map[gomatrixserverlib.ServerName]*destinationQueue{}, } } +// TODO: Move this somewhere useful for other components as we often need to ferry these 3 variables +// around together +type SigningInfo struct { + ServerName gomatrixserverlib.ServerName + KeyID gomatrixserverlib.KeyID + PrivateKey ed25519.PrivateKey +} + func (oqs *OutgoingQueues) getQueueIfExists(destination gomatrixserverlib.ServerName) *destinationQueue { oqs.queuesMutex.Lock() defer oqs.queuesMutex.Unlock() @@ -64,7 +76,7 @@ func (oqs *OutgoingQueues) getQueue(destination gomatrixserverlib.ServerName) *d oq := oqs.queues[destination] if oq == nil { oq = &destinationQueue{ - rsProducer: oqs.rsProducer, + rsAPI: oqs.rsAPI, origin: oqs.origin, destination: destination, client: oqs.client, @@ -73,6 +85,7 @@ func (oqs *OutgoingQueues) getQueue(destination gomatrixserverlib.ServerName) *d incomingEDUs: make(chan *gomatrixserverlib.EDU, 128), incomingInvites: make(chan *gomatrixserverlib.InviteV2Request, 128), retryServerCh: make(chan bool), + signing: oqs.signing, } oqs.queues[destination] = oq } From 63a24e81c49f8956de50a876c45c32775bd2acc6 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 10 Jun 2020 17:18:37 +0100 Subject: [PATCH 150/200] Yggdrasil demo tweaks --- cmd/dendrite-demo-yggdrasil/main.go | 6 +++--- .../yggconn/session.go | 21 +++++++++++++------ 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index 94e94d8d3..8bf1f7561 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -96,9 +96,9 @@ func main() { httpServer := &http.Server{ Addr: ":0", TLSNextProto: map[string]func(*http.Server, *tls.Conn, http.Handler){}, - ReadTimeout: 30 * time.Second, - WriteTimeout: 30 * time.Second, - IdleTimeout: 60 * time.Second, + ReadTimeout: 5 * time.Second, + WriteTimeout: 5 * time.Second, + IdleTimeout: 15 * time.Second, BaseContext: func(_ net.Listener) context.Context { return context.Background() }, diff --git a/cmd/dendrite-demo-yggdrasil/yggconn/session.go b/cmd/dendrite-demo-yggdrasil/yggconn/session.go index a9bb15f7e..32ae014a0 100644 --- a/cmd/dendrite-demo-yggdrasil/yggconn/session.go +++ b/cmd/dendrite-demo-yggdrasil/yggconn/session.go @@ -25,8 +25,8 @@ import ( func (n *Node) yamuxConfig() *yamux.Config { cfg := yamux.DefaultConfig() - cfg.EnableKeepAlive = true - cfg.KeepAliveInterval = time.Second * 30 + cfg.EnableKeepAlive = false + cfg.ConnectionWriteTimeout = time.Second * 5 cfg.MaxMessageSize = 65535 cfg.ReadBufSize = 655350 return cfg @@ -40,6 +40,8 @@ func (n *Node) listenFromYgg() { return } var session *yamux.Session + // If the remote address is lower than ours then we'll be the + // server. Otherwse we'll be the client. if strings.Compare(conn.RemoteAddr().String(), n.DerivedServerName()) < 0 { session, err = yamux.Server(conn, n.yamuxConfig()) } else { @@ -55,6 +57,11 @@ func (n *Node) listenFromYgg() { func (n *Node) listenFromYggConn(session *yamux.Session) { n.sessions.Store(session.RemoteAddr().String(), session) defer n.sessions.Delete(session.RemoteAddr()) + defer func() { + if err := session.Close(); err != nil { + n.log.Println("session.Close:", err) + } + }() for { st, err := session.AcceptStream() @@ -90,16 +97,18 @@ func (n *Node) Dial(network, address string) (net.Conn, error) { func (n *Node) DialContext(ctx context.Context, network, address string) (net.Conn, error) { s, ok1 := n.sessions.Load(address) session, ok2 := s.(*yamux.Session) - if !ok1 || !ok2 { + if !ok1 || !ok2 || (ok1 && ok2 && session.IsClosed()) { conn, err := n.dialer.DialContext(ctx, network, address) if err != nil { n.log.Println("n.dialer.DialContext:", err) return nil, err } - if strings.Compare(address, n.DerivedServerName()) > 0 { - session, err = yamux.Client(conn, n.yamuxConfig()) - } else { + // If the remote address is lower than ours then we will be the + // server. Otherwise we'll be the client. + if strings.Compare(conn.RemoteAddr().String(), n.DerivedServerName()) < 0 { session, err = yamux.Server(conn, n.yamuxConfig()) + } else { + session, err = yamux.Client(conn, n.yamuxConfig()) } if err != nil { return nil, err From b7d0ca68554a41c4ca04370fc9343fc4b2b99257 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 11 Jun 2020 09:50:54 +0100 Subject: [PATCH 151/200] Update Yggdrasil demo timeouts again --- cmd/dendrite-demo-yggdrasil/main.go | 10 ++++++---- cmd/dendrite-demo-yggdrasil/yggconn/session.go | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index 8bf1f7561..b1530e883 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -79,7 +79,9 @@ func createFederationClient( tr.RegisterProtocol( "matrix", &yggroundtripper{ inner: &http.Transport{ - DialContext: yggdialerctx, + ResponseHeaderTimeout: 15 * time.Second, + IdleConnTimeout: 60 * time.Second, + DialContext: yggdialerctx, }, }, ) @@ -96,9 +98,9 @@ func main() { httpServer := &http.Server{ Addr: ":0", TLSNextProto: map[string]func(*http.Server, *tls.Conn, http.Handler){}, - ReadTimeout: 5 * time.Second, - WriteTimeout: 5 * time.Second, - IdleTimeout: 15 * time.Second, + ReadTimeout: 15 * time.Second, + WriteTimeout: 45 * time.Second, + IdleTimeout: 60 * time.Second, BaseContext: func(_ net.Listener) context.Context { return context.Background() }, diff --git a/cmd/dendrite-demo-yggdrasil/yggconn/session.go b/cmd/dendrite-demo-yggdrasil/yggconn/session.go index 32ae014a0..8d96a9cbc 100644 --- a/cmd/dendrite-demo-yggdrasil/yggconn/session.go +++ b/cmd/dendrite-demo-yggdrasil/yggconn/session.go @@ -26,7 +26,7 @@ import ( func (n *Node) yamuxConfig() *yamux.Config { cfg := yamux.DefaultConfig() cfg.EnableKeepAlive = false - cfg.ConnectionWriteTimeout = time.Second * 5 + cfg.ConnectionWriteTimeout = time.Second * 15 cfg.MaxMessageSize = 65535 cfg.ReadBufSize = 655350 return cfg From ce2517b9fb70403d1dbeb52194ccd55122f8ed14 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 11 Jun 2020 10:16:46 +0100 Subject: [PATCH 152/200] Support embedding Riot Web (#1113) * Embed * Support for embedding is better now * HTTP on localhost for Ygg demo * Move embed into Yggdrasil demo --- .gitignore | 3 + .../embed/embed_other.go | 7 +++ .../embed/embed_riotweb.go | 61 +++++++++++++++++++ cmd/dendrite-demo-yggdrasil/main.go | 5 +- go.mod | 1 + 5 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 cmd/dendrite-demo-yggdrasil/embed/embed_other.go create mode 100644 cmd/dendrite-demo-yggdrasil/embed/embed_riotweb.go diff --git a/.gitignore b/.gitignore index 1de8887ce..34c5b6805 100644 --- a/.gitignore +++ b/.gitignore @@ -49,3 +49,6 @@ dendrite.yaml # Log files *.log* + +# Generated code +cmd/dendrite-demo-yggdrasil/embed/fs*.go diff --git a/cmd/dendrite-demo-yggdrasil/embed/embed_other.go b/cmd/dendrite-demo-yggdrasil/embed/embed_other.go new file mode 100644 index 000000000..a9108fad3 --- /dev/null +++ b/cmd/dendrite-demo-yggdrasil/embed/embed_other.go @@ -0,0 +1,7 @@ +// +build !riotweb + +package embed + +func Embed(_ int, _ string) { + +} diff --git a/cmd/dendrite-demo-yggdrasil/embed/embed_riotweb.go b/cmd/dendrite-demo-yggdrasil/embed/embed_riotweb.go new file mode 100644 index 000000000..360d0bc5b --- /dev/null +++ b/cmd/dendrite-demo-yggdrasil/embed/embed_riotweb.go @@ -0,0 +1,61 @@ +// +build riotweb + +package embed + +import ( + "fmt" + "io" + "net/http" + + "github.com/tidwall/sjson" +) + +// From within the Riot Web directory: +// go run github.com/mjibson/esc -o /path/to/dendrite/internal/embed/fs_riotweb.go -private -pkg embed . + +func Embed(listenPort int, serverName string) { + url := fmt.Sprintf("http://localhost:%d", listenPort) + embeddedFS := _escFS(false) + embeddedServ := http.FileServer(embeddedFS) + + http.DefaultServeMux.Handle("/", embeddedServ) + http.DefaultServeMux.HandleFunc("/config.json", func(w http.ResponseWriter, _ *http.Request) { + configFile, err := embeddedFS.Open("/config.sample.json") + if err != nil { + w.WriteHeader(500) + io.WriteString(w, "Couldn't open the file: "+err.Error()) + return + } + configFileInfo, err := configFile.Stat() + if err != nil { + w.WriteHeader(500) + io.WriteString(w, "Couldn't stat the file: "+err.Error()) + return + } + buf := make([]byte, configFileInfo.Size()) + n, err := configFile.Read(buf) + if err != nil { + w.WriteHeader(500) + io.WriteString(w, "Couldn't read the file: "+err.Error()) + return + } + if int64(n) != configFileInfo.Size() { + w.WriteHeader(500) + io.WriteString(w, "The returned file size didn't match what we expected") + return + } + js, _ := sjson.SetBytes(buf, "default_server_config.m\\.homeserver.base_url", url) + js, _ = sjson.SetBytes(js, "default_server_config.m\\.homeserver.server_name", serverName) + js, _ = sjson.SetBytes(js, "brand", fmt.Sprintf("Riot %s", serverName)) + js, _ = sjson.SetBytes(js, "disable_guests", true) + js, _ = sjson.SetBytes(js, "disable_3pid_login", true) + js, _ = sjson.DeleteBytes(js, "welcomeUserId") + _, _ = w.Write(js) + }) + + fmt.Println("*-------------------------------*") + fmt.Println("| This build includes Riot Web! |") + fmt.Println("*-------------------------------*") + fmt.Println("Point your browser to:", url) + fmt.Println() +} diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index b1530e883..6ef56d325 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -28,6 +28,7 @@ import ( "github.com/matrix-org/dendrite/appservice" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/convert" + "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/embed" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/yggconn" "github.com/matrix-org/dendrite/eduserver" @@ -166,6 +167,8 @@ func main() { logrus.WithError(err).Panicf("failed to connect to public rooms db") } + embed.Embed(*instancePort, "Yggdrasil Demo") + monolith := setup.Monolith{ Config: base.Cfg, AccountDB: accountDB, @@ -198,7 +201,7 @@ func main() { logrus.Fatal(httpServer.Serve(ygg)) }() go func() { - httpBindAddr := fmt.Sprintf(":%d", *instancePort) + httpBindAddr := fmt.Sprintf("localhost:%d", *instancePort) logrus.Info("Listening on ", httpBindAddr) logrus.Fatal(http.ListenAndServe(httpBindAddr, nil)) }() diff --git a/go.mod b/go.mod index a93f9a832..a05bb71e2 100644 --- a/go.mod +++ b/go.mod @@ -31,6 +31,7 @@ require ( github.com/prometheus/client_golang v1.4.1 github.com/sirupsen/logrus v1.6.0 github.com/tidwall/gjson v1.6.0 + github.com/tidwall/sjson v1.0.3 github.com/uber-go/atomic v1.3.0 // indirect github.com/uber/jaeger-client-go v2.15.0+incompatible github.com/uber/jaeger-lib v1.5.0 From 89d61c487751e4beca71d396d719758ae4a98332 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 11 Jun 2020 13:08:11 +0100 Subject: [PATCH 153/200] Fix tiebreak in Yggdrasil demo (uses wrong key) --- cmd/dendrite-demo-yggdrasil/yggconn/node.go | 4 ++++ cmd/dendrite-demo-yggdrasil/yggconn/session.go | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/cmd/dendrite-demo-yggdrasil/yggconn/node.go b/cmd/dendrite-demo-yggdrasil/yggconn/node.go index 14bbf84d7..a625f8d8d 100644 --- a/cmd/dendrite-demo-yggdrasil/yggconn/node.go +++ b/cmd/dendrite-demo-yggdrasil/yggconn/node.go @@ -128,6 +128,10 @@ func (n *Node) DerivedServerName() string { return hex.EncodeToString(n.SigningPublicKey()) } +func (n *Node) DerivedSessionName() string { + return hex.EncodeToString(n.EncryptionPublicKey()) +} + func (n *Node) EncryptionPublicKey() []byte { edkey := n.SigningPublicKey() return convert.Ed25519PublicKeyToCurve25519(edkey) diff --git a/cmd/dendrite-demo-yggdrasil/yggconn/session.go b/cmd/dendrite-demo-yggdrasil/yggconn/session.go index 8d96a9cbc..c50b6b73c 100644 --- a/cmd/dendrite-demo-yggdrasil/yggconn/session.go +++ b/cmd/dendrite-demo-yggdrasil/yggconn/session.go @@ -42,7 +42,7 @@ func (n *Node) listenFromYgg() { var session *yamux.Session // If the remote address is lower than ours then we'll be the // server. Otherwse we'll be the client. - if strings.Compare(conn.RemoteAddr().String(), n.DerivedServerName()) < 0 { + if strings.Compare(conn.RemoteAddr().String(), n.DerivedSessionName()) < 0 { session, err = yamux.Server(conn, n.yamuxConfig()) } else { session, err = yamux.Client(conn, n.yamuxConfig()) @@ -105,7 +105,7 @@ func (n *Node) DialContext(ctx context.Context, network, address string) (net.Co } // If the remote address is lower than ours then we will be the // server. Otherwise we'll be the client. - if strings.Compare(conn.RemoteAddr().String(), n.DerivedServerName()) < 0 { + if strings.Compare(conn.RemoteAddr().String(), n.DerivedSessionName()) < 0 { session, err = yamux.Server(conn, n.yamuxConfig()) } else { session, err = yamux.Client(conn, n.yamuxConfig()) From 25cd2dd1c925fa0c1eeb27a3cd71e668344102ad Mon Sep 17 00:00:00 2001 From: Kegsay Date: Thu, 11 Jun 2020 15:07:16 +0100 Subject: [PATCH 154/200] Remove unused internal APIs (#1117) --- federationapi/routing/send_test.go | 9 -------- federationapi/routing/state.go | 13 +----------- federationsender/api/api.go | 22 ++----------------- federationsender/internal/query.go | 10 --------- federationsender/inthttp/client.go | 14 ------------ federationsender/inthttp/server.go | 14 ------------ roomserver/api/api.go | 7 ------ roomserver/api/query.go | 17 --------------- roomserver/internal/query.go | 34 ------------------------------ roomserver/inthttp/client.go | 14 ------------ roomserver/inthttp/server.go | 14 ------------ 11 files changed, 3 insertions(+), 165 deletions(-) diff --git a/federationapi/routing/send_test.go b/federationapi/routing/send_test.go index b81f1c003..9081a8700 100644 --- a/federationapi/routing/send_test.go +++ b/federationapi/routing/send_test.go @@ -181,15 +181,6 @@ func (t *testRoomserverAPI) QueryMembershipsForRoom( return fmt.Errorf("not implemented") } -// Query a list of invite event senders for a user in a room. -func (t *testRoomserverAPI) QueryInvitesForUser( - ctx context.Context, - request *api.QueryInvitesForUserRequest, - response *api.QueryInvitesForUserResponse, -) error { - return fmt.Errorf("not implemented") -} - // Query whether a server is allowed to see an event func (t *testRoomserverAPI) QueryServerAllowedToSeeEvent( ctx context.Context, diff --git a/federationapi/routing/state.go b/federationapi/routing/state.go index f90c494c3..04a18904b 100644 --- a/federationapi/routing/state.go +++ b/federationapi/routing/state.go @@ -107,15 +107,13 @@ func getState( return nil, &util.JSONResponse{Code: http.StatusNotFound, JSON: nil} } - authEventIDs := getIDsFromEventRef(event.AuthEvents()) - var response api.QueryStateAndAuthChainResponse err := rsAPI.QueryStateAndAuthChain( ctx, &api.QueryStateAndAuthChainRequest{ RoomID: roomID, PrevEventIDs: []string{eventID}, - AuthEventIDs: authEventIDs, + AuthEventIDs: event.AuthEventIDs(), }, &response, ) @@ -134,15 +132,6 @@ func getState( }, nil } -func getIDsFromEventRef(events []gomatrixserverlib.EventReference) []string { - IDs := make([]string, len(events)) - for i := range events { - IDs[i] = events[i].EventID - } - - return IDs -} - func getIDsFromEvent(events []gomatrixserverlib.Event) []string { IDs := make([]string, len(events)) for i := range events { diff --git a/federationsender/api/api.go b/federationsender/api/api.go index d8251b54d..02c762582 100644 --- a/federationsender/api/api.go +++ b/federationsender/api/api.go @@ -15,14 +15,6 @@ type FederationSenderInternalAPI interface { request *PerformDirectoryLookupRequest, response *PerformDirectoryLookupResponse, ) error - // Query the joined hosts and the membership events accounting for their participation in a room. - // Note that if a server has multiple users in the room, it will have multiple entries in the returned slice. - // See `QueryJoinedHostServerNamesInRoom` for a de-duplicated version. - QueryJoinedHostsInRoom( - ctx context.Context, - request *QueryJoinedHostsInRoomRequest, - response *QueryJoinedHostsInRoomResponse, - ) error // Query the server names of the joined hosts in a room. // Unlike QueryJoinedHostsInRoom, this function returns a de-duplicated slice // containing only the server names (without information for membership events). @@ -88,22 +80,12 @@ type PerformServersAliveRequest struct { type PerformServersAliveResponse struct { } -// QueryJoinedHostsInRoomRequest is a request to QueryJoinedHostsInRoom -type QueryJoinedHostsInRoomRequest struct { - RoomID string `json:"room_id"` -} - -// QueryJoinedHostsInRoomResponse is a response to QueryJoinedHostsInRoom -type QueryJoinedHostsInRoomResponse struct { - JoinedHosts []types.JoinedHost `json:"joined_hosts"` -} - -// QueryJoinedHostServerNamesRequest is a request to QueryJoinedHostServerNames +// QueryJoinedHostServerNamesInRoomRequest is a request to QueryJoinedHostServerNames type QueryJoinedHostServerNamesInRoomRequest struct { RoomID string `json:"room_id"` } -// QueryJoinedHostServerNamesResponse is a response to QueryJoinedHostServerNames +// QueryJoinedHostServerNamesInRoomResponse is a response to QueryJoinedHostServerNames type QueryJoinedHostServerNamesInRoomResponse struct { ServerNames []gomatrixserverlib.ServerName `json:"server_names"` } diff --git a/federationsender/internal/query.go b/federationsender/internal/query.go index 88dd50a62..253400a2d 100644 --- a/federationsender/internal/query.go +++ b/federationsender/internal/query.go @@ -7,16 +7,6 @@ import ( "github.com/matrix-org/gomatrixserverlib" ) -// QueryJoinedHostsInRoom implements api.FederationSenderInternalAPI -func (f *FederationSenderInternalAPI) QueryJoinedHostsInRoom( - ctx context.Context, - request *api.QueryJoinedHostsInRoomRequest, - response *api.QueryJoinedHostsInRoomResponse, -) (err error) { - response.JoinedHosts, err = f.db.GetJoinedHosts(ctx, request.RoomID) - return -} - // QueryJoinedHostServerNamesInRoom implements api.FederationSenderInternalAPI func (f *FederationSenderInternalAPI) QueryJoinedHostServerNamesInRoom( ctx context.Context, diff --git a/federationsender/inthttp/client.go b/federationsender/inthttp/client.go index 8b3a106d8..1e243c607 100644 --- a/federationsender/inthttp/client.go +++ b/federationsender/inthttp/client.go @@ -12,7 +12,6 @@ import ( // HTTP paths for the internal HTTP API const ( - FederationSenderQueryJoinedHostsInRoomPath = "/federationsender/queryJoinedHostsInRoom" FederationSenderQueryJoinedHostServerNamesInRoomPath = "/federationsender/queryJoinedHostServerNamesInRoom" FederationSenderPerformDirectoryLookupRequestPath = "/federationsender/performDirectoryLookup" @@ -73,19 +72,6 @@ func (h *httpFederationSenderInternalAPI) QueryJoinedHostServerNamesInRoom( return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } -// QueryJoinedHostsInRoom implements FederationSenderInternalAPI -func (h *httpFederationSenderInternalAPI) QueryJoinedHostsInRoom( - ctx context.Context, - request *api.QueryJoinedHostsInRoomRequest, - response *api.QueryJoinedHostsInRoomResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "QueryJoinedHostsInRoom") - defer span.Finish() - - apiURL := h.federationSenderURL + FederationSenderQueryJoinedHostsInRoomPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} - // Handle an instruction to make_join & send_join with a remote server. func (h *httpFederationSenderInternalAPI) PerformJoin( ctx context.Context, diff --git a/federationsender/inthttp/server.go b/federationsender/inthttp/server.go index c94a14295..a9076661d 100644 --- a/federationsender/inthttp/server.go +++ b/federationsender/inthttp/server.go @@ -12,20 +12,6 @@ import ( // AddRoutes adds the FederationSenderInternalAPI handlers to the http.ServeMux. func AddRoutes(intAPI api.FederationSenderInternalAPI, internalAPIMux *mux.Router) { - internalAPIMux.Handle( - FederationSenderQueryJoinedHostsInRoomPath, - internal.MakeInternalAPI("QueryJoinedHostsInRoom", func(req *http.Request) util.JSONResponse { - var request api.QueryJoinedHostsInRoomRequest - var response api.QueryJoinedHostsInRoomResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := intAPI.QueryJoinedHostsInRoom(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) internalAPIMux.Handle( FederationSenderQueryJoinedHostServerNamesInRoomPath, internal.MakeInternalAPI("QueryJoinedHostServerNamesInRoom", func(req *http.Request) util.JSONResponse { diff --git a/roomserver/api/api.go b/roomserver/api/api.go index aefe55bcd..3a2ad059b 100644 --- a/roomserver/api/api.go +++ b/roomserver/api/api.go @@ -65,13 +65,6 @@ type RoomserverInternalAPI interface { response *QueryMembershipsForRoomResponse, ) error - // Query a list of invite event senders for a user in a room. - QueryInvitesForUser( - ctx context.Context, - request *QueryInvitesForUserRequest, - response *QueryInvitesForUserResponse, - ) error - // Query whether a server is allowed to see an event QueryServerAllowedToSeeEvent( ctx context.Context, diff --git a/roomserver/api/query.go b/roomserver/api/query.go index dc005c77c..c9a46ae9e 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -140,23 +140,6 @@ type QueryMembershipsForRoomResponse struct { HasBeenInRoom bool `json:"has_been_in_room"` } -// QueryInvitesForUserRequest is a request to QueryInvitesForUser -type QueryInvitesForUserRequest struct { - // The room ID to look up invites in. - RoomID string `json:"room_id"` - // The User ID to look up invites for. - TargetUserID string `json:"target_user_id"` -} - -// QueryInvitesForUserResponse is a response to QueryInvitesForUser -// This is used when accepting an invite or rejecting a invite to tell which -// remote matrix servers to contact. -type QueryInvitesForUserResponse struct { - // A list of matrix user IDs for each sender of an active invite targeting - // the requested user ID. - InviteSenderUserIDs []string `json:"invite_sender_user_ids"` -} - // QueryServerAllowedToSeeEventRequest is a request to QueryServerAllowedToSeeEvent type QueryServerAllowedToSeeEventRequest struct { // The event ID to look up invites in. diff --git a/roomserver/internal/query.go b/roomserver/internal/query.go index 9fb67e7eb..aea933884 100644 --- a/roomserver/internal/query.go +++ b/roomserver/internal/query.go @@ -353,40 +353,6 @@ func getMembershipsAtState( return events, nil } -// QueryInvitesForUser implements api.RoomserverInternalAPI -func (r *RoomserverInternalAPI) QueryInvitesForUser( - ctx context.Context, - request *api.QueryInvitesForUserRequest, - response *api.QueryInvitesForUserResponse, -) error { - roomNID, err := r.DB.RoomNID(ctx, request.RoomID) - if err != nil { - return err - } - - targetUserNIDs, err := r.DB.EventStateKeyNIDs(ctx, []string{request.TargetUserID}) - if err != nil { - return err - } - targetUserNID := targetUserNIDs[request.TargetUserID] - - senderUserNIDs, err := r.DB.GetInvitesForUser(ctx, roomNID, targetUserNID) - if err != nil { - return err - } - - senderUserIDs, err := r.DB.EventStateKeys(ctx, senderUserNIDs) - if err != nil { - return err - } - - for _, senderUserID := range senderUserIDs { - response.InviteSenderUserIDs = append(response.InviteSenderUserIDs, senderUserID) - } - - return nil -} - // QueryServerAllowedToSeeEvent implements api.RoomserverInternalAPI func (r *RoomserverInternalAPI) QueryServerAllowedToSeeEvent( ctx context.Context, diff --git a/roomserver/inthttp/client.go b/roomserver/inthttp/client.go index 5cc2537e3..6f5e882e7 100644 --- a/roomserver/inthttp/client.go +++ b/roomserver/inthttp/client.go @@ -33,7 +33,6 @@ const ( RoomserverQueryEventsByIDPath = "/roomserver/queryEventsByID" RoomserverQueryMembershipForUserPath = "/roomserver/queryMembershipForUser" RoomserverQueryMembershipsForRoomPath = "/roomserver/queryMembershipsForRoom" - RoomserverQueryInvitesForUserPath = "/roomserver/queryInvitesForUser" RoomserverQueryServerAllowedToSeeEventPath = "/roomserver/queryServerAllowedToSeeEvent" RoomserverQueryMissingEventsPath = "/roomserver/queryMissingEvents" RoomserverQueryStateAndAuthChainPath = "/roomserver/queryStateAndAuthChain" @@ -236,19 +235,6 @@ func (h *httpRoomserverInternalAPI) QueryMembershipsForRoom( return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } -// QueryInvitesForUser implements RoomserverQueryAPI -func (h *httpRoomserverInternalAPI) QueryInvitesForUser( - ctx context.Context, - request *api.QueryInvitesForUserRequest, - response *api.QueryInvitesForUserResponse, -) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "QueryInvitesForUser") - defer span.Finish() - - apiURL := h.roomserverURL + RoomserverQueryInvitesForUserPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) -} - // QueryServerAllowedToSeeEvent implements RoomserverQueryAPI func (h *httpRoomserverInternalAPI) QueryServerAllowedToSeeEvent( ctx context.Context, diff --git a/roomserver/inthttp/server.go b/roomserver/inthttp/server.go index 9a58a30b0..3a13ce371 100644 --- a/roomserver/inthttp/server.go +++ b/roomserver/inthttp/server.go @@ -122,20 +122,6 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) - internalAPIMux.Handle( - RoomserverQueryInvitesForUserPath, - internal.MakeInternalAPI("queryInvitesForUser", func(req *http.Request) util.JSONResponse { - var request api.QueryInvitesForUserRequest - var response api.QueryInvitesForUserResponse - if err := json.NewDecoder(req.Body).Decode(&request); err != nil { - return util.ErrorResponse(err) - } - if err := r.QueryInvitesForUser(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) - } - return util.JSONResponse{Code: http.StatusOK, JSON: &response} - }), - ) internalAPIMux.Handle( RoomserverQueryServerAllowedToSeeEventPath, internal.MakeInternalAPI("queryServerAllowedToSeeEvent", func(req *http.Request) util.JSONResponse { From ec7718e7f842fa0fc5198489c904de21003db4c2 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Thu, 11 Jun 2020 19:50:40 +0100 Subject: [PATCH 155/200] Roomserver API changes (#1118) * s/QueryBackfill/PerformBackfill/g * OutputEvent now includes AddStateEvents which contain the full event of extra state events * Only include adds not the current event * Get adding state right --- appservice/consumers/roomserver.go | 51 +---------- clientapi/consumers/roomserver.go | 64 ++------------ federationapi/routing/backfill.go | 8 +- federationapi/routing/send_test.go | 6 +- federationsender/consumers/roomserver.go | 6 +- publicroomsapi/consumers/roomserver.go | 17 +--- roomserver/api/api.go | 6 +- roomserver/api/output.go | 27 ++++++ roomserver/api/perform.go | 29 +++++++ roomserver/api/query.go | 29 ------- roomserver/internal/input_latest_events.go | 66 +++++++++++---- ...{query_backfill.go => perform_backfill.go} | 0 roomserver/internal/query.go | 12 +-- roomserver/inthttp/client.go | 18 ++-- roomserver/inthttp/server.go | 10 +-- syncapi/consumers/roomserver.go | 84 +------------------ syncapi/routing/messages.go | 6 +- 17 files changed, 152 insertions(+), 287 deletions(-) rename roomserver/internal/{query_backfill.go => perform_backfill.go} (100%) diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go index bb4df7906..1657fe542 100644 --- a/appservice/consumers/roomserver.go +++ b/appservice/consumers/roomserver.go @@ -91,60 +91,13 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { return nil } - ev := output.NewRoomEvent.Event - log.WithFields(log.Fields{ - "event_id": ev.EventID(), - "room_id": ev.RoomID(), - "type": ev.Type(), - }).Info("appservice received an event from roomserver") - - missingEvents, err := s.lookupMissingStateEvents(output.NewRoomEvent.AddsStateEventIDs, ev) - if err != nil { - return err - } - events := append(missingEvents, ev) + events := []gomatrixserverlib.HeaderedEvent{output.NewRoomEvent.Event} + events = append(events, output.NewRoomEvent.AddStateEvents...) // Send event to any relevant application services return s.filterRoomserverEvents(context.TODO(), events) } -// lookupMissingStateEvents looks up the state events that are added by a new event, -// and returns any not already present. -func (s *OutputRoomEventConsumer) lookupMissingStateEvents( - addsStateEventIDs []string, event gomatrixserverlib.HeaderedEvent, -) ([]gomatrixserverlib.HeaderedEvent, error) { - // Fast path if there aren't any new state events. - if len(addsStateEventIDs) == 0 { - return []gomatrixserverlib.HeaderedEvent{}, nil - } - - // Fast path if the only state event added is the event itself. - if len(addsStateEventIDs) == 1 && addsStateEventIDs[0] == event.EventID() { - return []gomatrixserverlib.HeaderedEvent{}, nil - } - - result := []gomatrixserverlib.HeaderedEvent{} - missing := []string{} - for _, id := range addsStateEventIDs { - if id != event.EventID() { - // If the event isn't the current one, add it to the list of events - // to retrieve from the roomserver - missing = append(missing, id) - } - } - - // Request the missing events from the roomserver - eventReq := api.QueryEventsByIDRequest{EventIDs: missing} - var eventResp api.QueryEventsByIDResponse - if err := s.rsAPI.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil { - return nil, err - } - - result = append(result, eventResp.Events...) - - return result, nil -} - // filterRoomserverEvents takes in events and decides whether any of them need // to be passed on to an external application service. It does this by checking // each namespace of each registered application service, and if there is a diff --git a/clientapi/consumers/roomserver.go b/clientapi/consumers/roomserver.go index bd8ac1dcd..caa028ba3 100644 --- a/clientapi/consumers/roomserver.go +++ b/clientapi/consumers/roomserver.go @@ -84,63 +84,9 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { return nil } - ev := output.NewRoomEvent.Event - log.WithFields(log.Fields{ - "event_id": ev.EventID(), - "room_id": ev.RoomID(), - "type": ev.Type(), - }).Info("received event from roomserver") - - events, err := s.lookupStateEvents(output.NewRoomEvent.AddsStateEventIDs, ev.Event) - if err != nil { - return err - } - - return s.db.UpdateMemberships(context.TODO(), events, output.NewRoomEvent.RemovesStateEventIDs) -} - -// lookupStateEvents looks up the state events that are added by a new event. -func (s *OutputRoomEventConsumer) lookupStateEvents( - addsStateEventIDs []string, event gomatrixserverlib.Event, -) ([]gomatrixserverlib.Event, error) { - // Fast path if there aren't any new state events. - if len(addsStateEventIDs) == 0 { - // If the event is a membership update (e.g. for a profile update), it won't - // show up in AddsStateEventIDs, so we need to add it manually - if event.Type() == "m.room.member" { - return []gomatrixserverlib.Event{event}, nil - } - return nil, nil - } - - // Fast path if the only state event added is the event itself. - if len(addsStateEventIDs) == 1 && addsStateEventIDs[0] == event.EventID() { - return []gomatrixserverlib.Event{event}, nil - } - - result := []gomatrixserverlib.Event{} - missing := []string{} - for _, id := range addsStateEventIDs { - // Append the current event in the results if its ID is in the events list - if id == event.EventID() { - result = append(result, event) - } else { - // If the event isn't the current one, add it to the list of events - // to retrieve from the roomserver - missing = append(missing, id) - } - } - - // Request the missing events from the roomserver - eventReq := api.QueryEventsByIDRequest{EventIDs: missing} - var eventResp api.QueryEventsByIDResponse - if err := s.rsAPI.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil { - return nil, err - } - - for _, headeredEvent := range eventResp.Events { - result = append(result, headeredEvent.Event) - } - - return result, nil + return s.db.UpdateMemberships( + context.TODO(), + gomatrixserverlib.UnwrapEventHeaders(output.NewRoomEvent.AddsState()), + output.NewRoomEvent.RemovesStateEventIDs, + ) } diff --git a/federationapi/routing/backfill.go b/federationapi/routing/backfill.go index 10bc62630..f906c73c9 100644 --- a/federationapi/routing/backfill.go +++ b/federationapi/routing/backfill.go @@ -37,7 +37,7 @@ func Backfill( roomID string, cfg *config.Dendrite, ) util.JSONResponse { - var res api.QueryBackfillResponse + var res api.PerformBackfillResponse var eIDs []string var limit string var exists bool @@ -68,7 +68,7 @@ func Backfill( } // Populate the request. - req := api.QueryBackfillRequest{ + req := api.PerformBackfillRequest{ RoomID: roomID, // we don't know who the successors are for these events, which won't // be a problem because we don't use that information when servicing /backfill requests, @@ -87,8 +87,8 @@ func Backfill( } // Query the roomserver. - if err = rsAPI.QueryBackfill(httpReq.Context(), &req, &res); err != nil { - util.GetLogger(httpReq.Context()).WithError(err).Error("query.QueryBackfill failed") + if err = rsAPI.PerformBackfill(httpReq.Context(), &req, &res); err != nil { + util.GetLogger(httpReq.Context()).WithError(err).Error("query.PerformBackfill failed") return jsonerror.InternalServerError() } diff --git a/federationapi/routing/send_test.go b/federationapi/routing/send_test.go index 9081a8700..adae7c221 100644 --- a/federationapi/routing/send_test.go +++ b/federationapi/routing/send_test.go @@ -211,10 +211,10 @@ func (t *testRoomserverAPI) QueryStateAndAuthChain( } // Query a given amount (or less) of events prior to a given set of events. -func (t *testRoomserverAPI) QueryBackfill( +func (t *testRoomserverAPI) PerformBackfill( ctx context.Context, - request *api.QueryBackfillRequest, - response *api.QueryBackfillResponse, + request *api.PerformBackfillRequest, + response *api.PerformBackfillResponse, ) error { return fmt.Errorf("not implemented") } diff --git a/federationsender/consumers/roomserver.go b/federationsender/consumers/roomserver.go index 5f8a555bd..a15937f9e 100644 --- a/federationsender/consumers/roomserver.go +++ b/federationsender/consumers/roomserver.go @@ -131,11 +131,7 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { // processMessage updates the list of currently joined hosts in the room // and then sends the event to the hosts that were joined before the event. func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent) error { - addsStateEvents, err := s.lookupStateEvents(ore.AddsStateEventIDs, ore.Event.Event) - if err != nil { - return err - } - addsJoinedHosts, err := joinedHostsFromEvents(addsStateEvents) + addsJoinedHosts, err := joinedHostsFromEvents(gomatrixserverlib.UnwrapEventHeaders(ore.AddsState())) if err != nil { return err } diff --git a/publicroomsapi/consumers/roomserver.go b/publicroomsapi/consumers/roomserver.go index c513d3b24..ba187cb11 100644 --- a/publicroomsapi/consumers/roomserver.go +++ b/publicroomsapi/consumers/roomserver.go @@ -78,20 +78,6 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { return nil } - ev := output.NewRoomEvent.Event - log.WithFields(log.Fields{ - "event_id": ev.EventID(), - "room_id": ev.RoomID(), - "type": ev.Type(), - }).Info("received event from roomserver") - - addQueryReq := api.QueryEventsByIDRequest{EventIDs: output.NewRoomEvent.AddsStateEventIDs} - var addQueryRes api.QueryEventsByIDResponse - if err := s.rsAPI.QueryEventsByID(context.TODO(), &addQueryReq, &addQueryRes); err != nil { - log.Warn(err) - return err - } - remQueryReq := api.QueryEventsByIDRequest{EventIDs: output.NewRoomEvent.RemovesStateEventIDs} var remQueryRes api.QueryEventsByIDResponse if err := s.rsAPI.QueryEventsByID(context.TODO(), &remQueryReq, &remQueryRes); err != nil { @@ -100,9 +86,10 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { } var addQueryEvents, remQueryEvents []gomatrixserverlib.Event - for _, headeredEvent := range addQueryRes.Events { + for _, headeredEvent := range output.NewRoomEvent.AddsState() { addQueryEvents = append(addQueryEvents, headeredEvent.Event) } + addQueryEvents = append(addQueryEvents, output.NewRoomEvent.Event.Unwrap()) for _, headeredEvent := range remQueryRes.Events { remQueryEvents = append(remQueryEvents, headeredEvent.Event) } diff --git a/roomserver/api/api.go b/roomserver/api/api.go index 3a2ad059b..967f58baf 100644 --- a/roomserver/api/api.go +++ b/roomserver/api/api.go @@ -89,10 +89,10 @@ type RoomserverInternalAPI interface { ) error // Query a given amount (or less) of events prior to a given set of events. - QueryBackfill( + PerformBackfill( ctx context.Context, - request *QueryBackfillRequest, - response *QueryBackfillResponse, + request *PerformBackfillRequest, + response *PerformBackfillResponse, ) error // Asks for the default room version as preferred by the server. diff --git a/roomserver/api/output.go b/roomserver/api/output.go index 92a468a96..2bbd97af8 100644 --- a/roomserver/api/output.go +++ b/roomserver/api/output.go @@ -63,6 +63,13 @@ type OutputNewRoomEvent struct { // Together with RemovesStateEventIDs this allows the receiver to keep an up to date // view of the current state of the room. AddsStateEventIDs []string `json:"adds_state_event_ids"` + // All extra newly added state events. This is only set if there are *extra* events + // other than `Event`. This can happen when forks get merged because state resolution + // may decide a bunch of state events on one branch are now valid, so they will be + // present in this list. This is useful when trying to maintain the current state of a room + // as to do so you need to include both these events and `Event`. + AddStateEvents []gomatrixserverlib.HeaderedEvent `json:"adds_state_events"` + // The state event IDs that were removed from the state of the room by this event. RemovesStateEventIDs []string `json:"removes_state_event_ids"` // The ID of the event that was output before this event. @@ -112,6 +119,26 @@ type OutputNewRoomEvent struct { TransactionID *TransactionID `json:"transaction_id"` } +// AddsState returns all added state events from this event. +// +// This function is needed because `AddStateEvents` will not include a copy of +// the original event to save space, so you cannot use that slice alone. +// Instead, use this function which will add the original event if it is present +// in `AddsStateEventIDs`. +func (ore *OutputNewRoomEvent) AddsState() []gomatrixserverlib.HeaderedEvent { + includeOutputEvent := false + for _, id := range ore.AddsStateEventIDs { + if id == ore.Event.EventID() { + includeOutputEvent = true + break + } + } + if !includeOutputEvent { + return ore.AddStateEvents + } + return append(ore.AddStateEvents, ore.Event) +} + // An OutputNewInviteEvent is written whenever an invite becomes active. // Invite events can be received outside of an existing room so have to be // tracked separately from the room events themselves. diff --git a/roomserver/api/perform.go b/roomserver/api/perform.go index 1cf54144e..3e5cae1b6 100644 --- a/roomserver/api/perform.go +++ b/roomserver/api/perform.go @@ -2,6 +2,7 @@ package api import ( "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" ) type PerformJoinRequest struct { @@ -22,3 +23,31 @@ type PerformLeaveRequest struct { type PerformLeaveResponse struct { } + +// PerformBackfillRequest is a request to PerformBackfill. +type PerformBackfillRequest struct { + // The room to backfill + RoomID string `json:"room_id"` + // A map of backwards extremity event ID to a list of its prev_event IDs. + BackwardsExtremities map[string][]string `json:"backwards_extremities"` + // The maximum number of events to retrieve. + Limit int `json:"limit"` + // The server interested in the events. + ServerName gomatrixserverlib.ServerName `json:"server_name"` +} + +// PrevEventIDs returns the prev_event IDs of all backwards extremities, de-duplicated in a lexicographically sorted order. +func (r *PerformBackfillRequest) PrevEventIDs() []string { + var prevEventIDs []string + for _, pes := range r.BackwardsExtremities { + prevEventIDs = append(prevEventIDs, pes...) + } + prevEventIDs = util.UniqueStrings(prevEventIDs) + return prevEventIDs +} + +// PerformBackfillResponse is a response to PerformBackfill. +type PerformBackfillResponse struct { + // Missing events, arbritrary order. + Events []gomatrixserverlib.HeaderedEvent `json:"events"` +} diff --git a/roomserver/api/query.go b/roomserver/api/query.go index c9a46ae9e..b1525342e 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -18,7 +18,6 @@ package api import ( "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" ) // QueryLatestEventsAndStateRequest is a request to QueryLatestEventsAndState @@ -204,34 +203,6 @@ type QueryStateAndAuthChainResponse struct { AuthChainEvents []gomatrixserverlib.HeaderedEvent `json:"auth_chain_events"` } -// QueryBackfillRequest is a request to QueryBackfill. -type QueryBackfillRequest struct { - // The room to backfill - RoomID string `json:"room_id"` - // A map of backwards extremity event ID to a list of its prev_event IDs. - BackwardsExtremities map[string][]string `json:"backwards_extremities"` - // The maximum number of events to retrieve. - Limit int `json:"limit"` - // The server interested in the events. - ServerName gomatrixserverlib.ServerName `json:"server_name"` -} - -// PrevEventIDs returns the prev_event IDs of all backwards extremities, de-duplicated in a lexicographically sorted order. -func (r *QueryBackfillRequest) PrevEventIDs() []string { - var prevEventIDs []string - for _, pes := range r.BackwardsExtremities { - prevEventIDs = append(prevEventIDs, pes...) - } - prevEventIDs = util.UniqueStrings(prevEventIDs) - return prevEventIDs -} - -// QueryBackfillResponse is a response to QueryBackfill. -type QueryBackfillResponse struct { - // Missing events, arbritrary order. - Events []gomatrixserverlib.HeaderedEvent `json:"events"` -} - // QueryRoomVersionCapabilitiesRequest asks for the default room version type QueryRoomVersionCapabilitiesRequest struct{} diff --git a/roomserver/internal/input_latest_events.go b/roomserver/internal/input_latest_events.go index aea85ca95..e69307ada 100644 --- a/roomserver/internal/input_latest_events.go +++ b/roomserver/internal/input_latest_events.go @@ -19,6 +19,7 @@ package internal import ( "bytes" "context" + "fmt" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/roomserver/api" @@ -310,24 +311,11 @@ func (u *latestEventsUpdater) makeOutputNewRoomEvent() (*api.OutputEvent, error) TransactionID: u.transactionID, } - var stateEventNIDs []types.EventNID - for _, entry := range u.added { - stateEventNIDs = append(stateEventNIDs, entry.EventNID) - } - for _, entry := range u.removed { - stateEventNIDs = append(stateEventNIDs, entry.EventNID) - } - for _, entry := range u.stateBeforeEventRemoves { - stateEventNIDs = append(stateEventNIDs, entry.EventNID) - } - for _, entry := range u.stateBeforeEventAdds { - stateEventNIDs = append(stateEventNIDs, entry.EventNID) - } - stateEventNIDs = stateEventNIDs[:util.SortAndUnique(eventNIDSorter(stateEventNIDs))] - eventIDMap, err := u.api.DB.EventIDs(u.ctx, stateEventNIDs) + eventIDMap, err := u.stateEventMap() if err != nil { return nil, err } + for _, entry := range u.added { ore.AddsStateEventIDs = append(ore.AddsStateEventIDs, eventIDMap[entry.EventNID]) } @@ -342,12 +330,60 @@ func (u *latestEventsUpdater) makeOutputNewRoomEvent() (*api.OutputEvent, error) } ore.SendAsServer = u.sendAsServer + // include extra state events if they were added as nearly every downstream component will care about it + // and we'd rather not have them all hit QueryEventsByID at the same time! + if len(ore.AddsStateEventIDs) > 0 { + ore.AddStateEvents, err = u.extraEventsForIDs(roomVersion, ore.AddsStateEventIDs) + if err != nil { + return nil, fmt.Errorf("failed to load add_state_events from db: %w", err) + } + } + return &api.OutputEvent{ Type: api.OutputTypeNewRoomEvent, NewRoomEvent: &ore, }, nil } +// extraEventsForIDs returns the full events for the event IDs given, but does not include the current event being +// updated. +func (u *latestEventsUpdater) extraEventsForIDs(roomVersion gomatrixserverlib.RoomVersion, eventIDs []string) ([]gomatrixserverlib.HeaderedEvent, error) { + var extraEventIDs []string + for _, e := range eventIDs { + if e == u.event.EventID() { + continue + } + extraEventIDs = append(extraEventIDs, e) + } + if len(extraEventIDs) == 0 { + return nil, nil + } + extraEvents, err := u.api.DB.EventsFromIDs(u.ctx, extraEventIDs) + if err != nil { + return nil, err + } + var h []gomatrixserverlib.HeaderedEvent + for _, e := range extraEvents { + h = append(h, e.Headered(roomVersion)) + } + return h, nil +} + +// retrieve an event nid -> event ID map for all events that need updating +func (u *latestEventsUpdater) stateEventMap() (map[types.EventNID]string, error) { + var stateEventNIDs []types.EventNID + var allStateEntries []types.StateEntry + allStateEntries = append(allStateEntries, u.added...) + allStateEntries = append(allStateEntries, u.removed...) + allStateEntries = append(allStateEntries, u.stateBeforeEventRemoves...) + allStateEntries = append(allStateEntries, u.stateBeforeEventAdds...) + for _, entry := range allStateEntries { + stateEventNIDs = append(stateEventNIDs, entry.EventNID) + } + stateEventNIDs = stateEventNIDs[:util.SortAndUnique(eventNIDSorter(stateEventNIDs))] + return u.api.DB.EventIDs(u.ctx, stateEventNIDs) +} + type eventNIDSorter []types.EventNID func (s eventNIDSorter) Len() int { return len(s) } diff --git a/roomserver/internal/query_backfill.go b/roomserver/internal/perform_backfill.go similarity index 100% rename from roomserver/internal/query_backfill.go rename to roomserver/internal/perform_backfill.go diff --git a/roomserver/internal/query.go b/roomserver/internal/query.go index aea933884..375ddc23c 100644 --- a/roomserver/internal/query.go +++ b/roomserver/internal/query.go @@ -441,11 +441,11 @@ func (r *RoomserverInternalAPI) QueryMissingEvents( return err } -// QueryBackfill implements api.RoomServerQueryAPI -func (r *RoomserverInternalAPI) QueryBackfill( +// PerformBackfill implements api.RoomServerQueryAPI +func (r *RoomserverInternalAPI) PerformBackfill( ctx context.Context, - request *api.QueryBackfillRequest, - response *api.QueryBackfillResponse, + request *api.PerformBackfillRequest, + response *api.PerformBackfillResponse, ) error { // if we are requesting the backfill then we need to do a federation hit // TODO: we could be more sensible and fetch as many events we already have then request the rest @@ -489,7 +489,7 @@ func (r *RoomserverInternalAPI) QueryBackfill( return err } -func (r *RoomserverInternalAPI) backfillViaFederation(ctx context.Context, req *api.QueryBackfillRequest, res *api.QueryBackfillResponse) error { +func (r *RoomserverInternalAPI) backfillViaFederation(ctx context.Context, req *api.PerformBackfillRequest, res *api.PerformBackfillResponse) error { roomVer, err := r.DB.GetRoomVersionForRoom(ctx, req.RoomID) if err != nil { return fmt.Errorf("backfillViaFederation: unknown room version for room %s : %w", req.RoomID, err) @@ -647,7 +647,7 @@ func (r *RoomserverInternalAPI) scanEventTree( var pre string // TODO: add tests for this function to ensure it meets the contract that callers expect (and doc what that is supposed to be) - // Currently, callers like QueryBackfill will call scanEventTree with a pre-populated `visited` map, assuming that by doing + // Currently, callers like PerformBackfill will call scanEventTree with a pre-populated `visited` map, assuming that by doing // so means that the events in that map will NOT be returned from this function. That is not currently true, resulting in // duplicate events being sent in response to /backfill requests. initialIgnoreList := make(map[string]bool, len(visited)) diff --git a/roomserver/inthttp/client.go b/roomserver/inthttp/client.go index 6f5e882e7..1244300d4 100644 --- a/roomserver/inthttp/client.go +++ b/roomserver/inthttp/client.go @@ -24,8 +24,9 @@ const ( RoomserverInputRoomEventsPath = "/roomserver/inputRoomEvents" // Perform operations - RoomserverPerformJoinPath = "/roomserver/performJoin" - RoomserverPerformLeavePath = "/roomserver/performLeave" + RoomserverPerformJoinPath = "/roomserver/performJoin" + RoomserverPerformLeavePath = "/roomserver/performLeave" + RoomserverPerformBackfillPath = "/roomserver/performBackfill" // Query operations RoomserverQueryLatestEventsAndStatePath = "/roomserver/queryLatestEventsAndState" @@ -36,7 +37,6 @@ const ( RoomserverQueryServerAllowedToSeeEventPath = "/roomserver/queryServerAllowedToSeeEvent" RoomserverQueryMissingEventsPath = "/roomserver/queryMissingEvents" RoomserverQueryStateAndAuthChainPath = "/roomserver/queryStateAndAuthChain" - RoomserverQueryBackfillPath = "/roomserver/queryBackfill" RoomserverQueryRoomVersionCapabilitiesPath = "/roomserver/queryRoomVersionCapabilities" RoomserverQueryRoomVersionForRoomPath = "/roomserver/queryRoomVersionForRoom" ) @@ -274,16 +274,16 @@ func (h *httpRoomserverInternalAPI) QueryStateAndAuthChain( return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } -// QueryBackfill implements RoomServerQueryAPI -func (h *httpRoomserverInternalAPI) QueryBackfill( +// PerformBackfill implements RoomServerQueryAPI +func (h *httpRoomserverInternalAPI) PerformBackfill( ctx context.Context, - request *api.QueryBackfillRequest, - response *api.QueryBackfillResponse, + request *api.PerformBackfillRequest, + response *api.PerformBackfillResponse, ) error { - span, ctx := opentracing.StartSpanFromContext(ctx, "QueryBackfill") + span, ctx := opentracing.StartSpanFromContext(ctx, "PerformBackfill") defer span.Finish() - apiURL := h.roomserverURL + RoomserverQueryBackfillPath + apiURL := h.roomserverURL + RoomserverPerformBackfillPath return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } diff --git a/roomserver/inthttp/server.go b/roomserver/inthttp/server.go index 3a13ce371..8ac815f3e 100644 --- a/roomserver/inthttp/server.go +++ b/roomserver/inthttp/server.go @@ -165,14 +165,14 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { }), ) internalAPIMux.Handle( - RoomserverQueryBackfillPath, - internal.MakeInternalAPI("QueryBackfill", func(req *http.Request) util.JSONResponse { - var request api.QueryBackfillRequest - var response api.QueryBackfillResponse + RoomserverPerformBackfillPath, + internal.MakeInternalAPI("PerformBackfill", func(req *http.Request) util.JSONResponse { + var request api.PerformBackfillRequest + var response api.PerformBackfillResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { return util.ErrorResponse(err) } - if err := r.QueryBackfill(req.Context(), &request, &response); err != nil { + if err := r.PerformBackfill(req.Context(), &request, &response); err != nil { return util.ErrorResponse(err) } return util.JSONResponse{Code: http.StatusOK, JSON: &response} diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index 055f76603..135976823 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -17,7 +17,6 @@ package consumers import ( "context" "encoding/json" - "fmt" "github.com/Shopify/sarama" "github.com/matrix-org/dendrite/internal" @@ -105,17 +104,9 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent( "room_version": ev.RoomVersion, }).Info("received event from roomserver") - addsStateEvents, err := s.lookupStateEvents(msg.AddsStateEventIDs, ev) - if err != nil { - log.WithFields(log.Fields{ - "event": string(ev.JSON()), - log.ErrorKey: err, - "add": msg.AddsStateEventIDs, - "del": msg.RemovesStateEventIDs, - }).Panicf("roomserver output log: state event lookup failure") - } + addsStateEvents := msg.AddsState() - ev, err = s.updateStateEvent(ev) + ev, err := s.updateStateEvent(ev) if err != nil { return err } @@ -185,63 +176,6 @@ func (s *OutputRoomEventConsumer) onRetireInviteEvent( return nil } -// lookupStateEvents looks up the state events that are added by a new event. -func (s *OutputRoomEventConsumer) lookupStateEvents( - addsStateEventIDs []string, event gomatrixserverlib.HeaderedEvent, -) ([]gomatrixserverlib.HeaderedEvent, error) { - // Fast path if there aren't any new state events. - if len(addsStateEventIDs) == 0 { - return nil, nil - } - - // Fast path if the only state event added is the event itself. - if len(addsStateEventIDs) == 1 && addsStateEventIDs[0] == event.EventID() { - return []gomatrixserverlib.HeaderedEvent{event}, nil - } - - // Check if this is re-adding a state events that we previously processed - // If we have previously received a state event it may still be in - // our event database. - result, err := s.db.Events(context.TODO(), addsStateEventIDs) - if err != nil { - return nil, err - } - missing := missingEventsFrom(result, addsStateEventIDs) - - // Check if event itself is being added. - for _, eventID := range missing { - if eventID == event.EventID() { - result = append(result, event) - break - } - } - missing = missingEventsFrom(result, addsStateEventIDs) - - if len(missing) == 0 { - return result, nil - } - - // At this point the missing events are neither the event itself nor are - // they present in our local database. Our only option is to fetch them - // from the roomserver using the query API. - eventReq := api.QueryEventsByIDRequest{EventIDs: missing} - var eventResp api.QueryEventsByIDResponse - if err := s.rsAPI.QueryEventsByID(context.TODO(), &eventReq, &eventResp); err != nil { - return nil, err - } - - result = append(result, eventResp.Events...) - missing = missingEventsFrom(result, addsStateEventIDs) - - if len(missing) != 0 { - return nil, fmt.Errorf( - "missing %d state events IDs at event %q", len(missing), event.EventID(), - ) - } - - return result, nil -} - func (s *OutputRoomEventConsumer) updateStateEvent(event gomatrixserverlib.HeaderedEvent) (gomatrixserverlib.HeaderedEvent, error) { var stateKey string if event.StateKey() == nil { @@ -270,17 +204,3 @@ func (s *OutputRoomEventConsumer) updateStateEvent(event gomatrixserverlib.Heade event.Event, err = event.SetUnsigned(prev) return event, err } - -func missingEventsFrom(events []gomatrixserverlib.HeaderedEvent, required []string) []string { - have := map[string]bool{} - for _, event := range events { - have[event.EventID()] = true - } - var missing []string - for _, eventID := range required { - if !have[eventID] { - missing = append(missing, eventID) - } - } - return missing -} diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index 8c8976345..de5429db4 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -375,15 +375,15 @@ func (e eventsByDepth) Less(i, j int) bool { // Returns an error if there was an issue with retrieving the list of servers in // the room or sending the request. func (r *messagesReq) backfill(roomID string, backwardsExtremities map[string][]string, limit int) ([]gomatrixserverlib.HeaderedEvent, error) { - var res api.QueryBackfillResponse - err := r.rsAPI.QueryBackfill(context.Background(), &api.QueryBackfillRequest{ + var res api.PerformBackfillResponse + err := r.rsAPI.PerformBackfill(context.Background(), &api.PerformBackfillRequest{ RoomID: roomID, BackwardsExtremities: backwardsExtremities, Limit: limit, ServerName: r.cfg.Matrix.ServerName, }, &res) if err != nil { - return nil, fmt.Errorf("QueryBackfill failed: %w", err) + return nil, fmt.Errorf("PerformBackfill failed: %w", err) } util.GetLogger(r.ctx).WithField("new_events", len(res.Events)).Info("Storing new events from backfill") From 079d8fe8fb521f76fee3bff5b47482d5fb911257 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 12 Jun 2020 11:07:26 +0100 Subject: [PATCH 156/200] More key tweaks (#1116) --- internal/caching/cache_serverkeys.go | 9 +++++++++ internal/caching/caches.go | 1 + internal/caching/impl_inmemorylru.go | 7 +++++++ serverkeyapi/internal/api.go | 13 ++++++++++--- serverkeyapi/inthttp/client.go | 7 +------ serverkeyapi/inthttp/server.go | 8 +------- 6 files changed, 29 insertions(+), 16 deletions(-) diff --git a/internal/caching/cache_serverkeys.go b/internal/caching/cache_serverkeys.go index 8c71ffbd9..b5e315758 100644 --- a/internal/caching/cache_serverkeys.go +++ b/internal/caching/cache_serverkeys.go @@ -2,6 +2,7 @@ package caching import ( "fmt" + "time" "github.com/matrix-org/gomatrixserverlib" ) @@ -23,9 +24,17 @@ func (c Caches) GetServerKey( request gomatrixserverlib.PublicKeyLookupRequest, ) (gomatrixserverlib.PublicKeyLookupResult, bool) { key := fmt.Sprintf("%s/%s", request.ServerName, request.KeyID) + now := gomatrixserverlib.AsTimestamp(time.Now()) val, found := c.ServerKeys.Get(key) if found && val != nil { if keyLookupResult, ok := val.(gomatrixserverlib.PublicKeyLookupResult); ok { + if !keyLookupResult.WasValidAt(now, true) { + // We appear to be past the key validity so don't return this + // with the results. This ensures that the cache doesn't return + // values that are not useful to us. + c.ServerKeys.Unset(key) + return gomatrixserverlib.PublicKeyLookupResult{}, false + } return keyLookupResult, true } } diff --git a/internal/caching/caches.go b/internal/caching/caches.go index 70f380ba5..419623e27 100644 --- a/internal/caching/caches.go +++ b/internal/caching/caches.go @@ -12,4 +12,5 @@ type Caches struct { type Cache interface { Get(key string) (value interface{}, ok bool) Set(key string, value interface{}) + Unset(key string) } diff --git a/internal/caching/impl_inmemorylru.go b/internal/caching/impl_inmemorylru.go index f7901d2e5..158deca49 100644 --- a/internal/caching/impl_inmemorylru.go +++ b/internal/caching/impl_inmemorylru.go @@ -68,6 +68,13 @@ func (c *InMemoryLRUCachePartition) Set(key string, value interface{}) { c.lru.Add(key, value) } +func (c *InMemoryLRUCachePartition) Unset(key string) { + if !c.mutable { + panic(fmt.Sprintf("invalid use of immutable cache tries to unset value of %q", key)) + } + c.lru.Remove(key) +} + func (c *InMemoryLRUCachePartition) Get(key string) (value interface{}, ok bool) { return c.lru.Get(key) } diff --git a/serverkeyapi/internal/api.go b/serverkeyapi/internal/api.go index 92d6a70b2..7a35aa8e7 100644 --- a/serverkeyapi/internal/api.go +++ b/serverkeyapi/internal/api.go @@ -22,7 +22,7 @@ func (s *ServerKeyAPI) KeyRing() *gomatrixserverlib.KeyRing { // and keeping the cache up-to-date. return &gomatrixserverlib.KeyRing{ KeyDatabase: s, - KeyFetchers: []gomatrixserverlib.KeyFetcher{s}, + KeyFetchers: []gomatrixserverlib.KeyFetcher{}, } } @@ -45,15 +45,17 @@ func (s *ServerKeyAPI) FetchKeys( // because the caller gives up waiting. ctx := context.Background() results := map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult{} + now := gomatrixserverlib.AsTimestamp(time.Now()) // First consult our local database and see if we have the requested // keys. These might come from a cache, depending on the database // implementation used. - now := gomatrixserverlib.AsTimestamp(time.Now()) if dbResults, err := s.OurKeyRing.KeyDatabase.FetchKeys(ctx, requests); err == nil { // We successfully got some keys. Add them to the results and // remove them from the request list. for req, res := range dbResults { - if now > res.ValidUntilTS && res.ExpiredTS == gomatrixserverlib.PublicKeyNotExpired { + if !res.WasValidAt(now, true) { + // We appear to be past the key validity. Don't return this + // key with the results. continue } results[req] = res @@ -71,6 +73,11 @@ func (s *ServerKeyAPI) FetchKeys( // We successfully got some keys. Add them to the results and // remove them from the request list. for req, res := range fetcherResults { + if !res.WasValidAt(now, true) { + // We appear to be past the key validity. Don't return this + // key with the results. + continue + } results[req] = res delete(requests, req) } diff --git a/serverkeyapi/inthttp/client.go b/serverkeyapi/inthttp/client.go index f22b0e310..2587160d4 100644 --- a/serverkeyapi/inthttp/client.go +++ b/serverkeyapi/inthttp/client.go @@ -4,7 +4,6 @@ import ( "context" "errors" "net/http" - "time" "github.com/matrix-org/dendrite/internal/caching" internalHTTP "github.com/matrix-org/dendrite/internal/http" @@ -50,7 +49,7 @@ func (s *httpServerKeyInternalAPI) KeyRing() *gomatrixserverlib.KeyRing { // the other end of the API. return &gomatrixserverlib.KeyRing{ KeyDatabase: s, - KeyFetchers: []gomatrixserverlib.KeyFetcher{s}, + KeyFetchers: []gomatrixserverlib.KeyFetcher{}, } } @@ -90,12 +89,8 @@ func (s *httpServerKeyInternalAPI) FetchKeys( response := api.QueryPublicKeysResponse{ Results: make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult), } - now := gomatrixserverlib.AsTimestamp(time.Now()) for req, ts := range requests { if res, ok := s.cache.GetServerKey(req); ok { - if now > res.ValidUntilTS && res.ExpiredTS == gomatrixserverlib.PublicKeyNotExpired { - continue - } result[req] = res continue } diff --git a/serverkeyapi/inthttp/server.go b/serverkeyapi/inthttp/server.go index 9efe7d9df..fd4b72c7e 100644 --- a/serverkeyapi/inthttp/server.go +++ b/serverkeyapi/inthttp/server.go @@ -8,7 +8,6 @@ import ( "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/serverkeyapi/api" - "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -35,12 +34,7 @@ func AddRoutes(s api.ServerKeyInternalAPI, internalAPIMux *mux.Router, cache cac if err := json.NewDecoder(req.Body).Decode(&request); err != nil { return util.MessageResponse(http.StatusBadRequest, err.Error()) } - store := make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult) - for req, res := range request.Keys { - store[req] = res - cache.StoreServerKey(req, res) - } - if err := s.StoreKeys(req.Context(), store); err != nil { + if err := s.StoreKeys(req.Context(), request.Keys); err != nil { return util.ErrorResponse(err) } return util.JSONResponse{Code: http.StatusOK, JSON: &response} From 4675e1ddb6a48fe1425032dc4f3cef56cbde7243 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Fri, 12 Jun 2020 12:10:08 +0100 Subject: [PATCH 157/200] Add trace logging to RoomserverInternalAPI (#1120) This is a wrapper around whatever impl we have which then logs the function name/request/response/error. Also tweak when we log on kafka streams: only log on the producer side not the consumer side: we've never had issues with comms and having 1 message rather than N would be nice. --- clientapi/producers/syncapi.go | 9 +- cmd/dendrite-monolith-server/main.go | 13 +- eduserver/input/input.go | 16 +- federationsender/consumers/roomserver.go | 5 - roomserver/api/api_trace.go | 218 +++++++++++++++++++++++ roomserver/internal/input.go | 16 ++ syncapi/consumers/roomserver.go | 5 - 7 files changed, 261 insertions(+), 21 deletions(-) create mode 100644 roomserver/api/api_trace.go diff --git a/clientapi/producers/syncapi.go b/clientapi/producers/syncapi.go index 244a61dc2..375b1eee4 100644 --- a/clientapi/producers/syncapi.go +++ b/clientapi/producers/syncapi.go @@ -17,9 +17,9 @@ package producers import ( "encoding/json" - "github.com/matrix-org/dendrite/internal" - "github.com/Shopify/sarama" + "github.com/matrix-org/dendrite/internal" + log "github.com/sirupsen/logrus" ) // SyncAPIProducer produces events for the sync API server to consume @@ -44,6 +44,11 @@ func (p *SyncAPIProducer) SendData(userID string, roomID string, dataType string m.Topic = string(p.Topic) m.Key = sarama.StringEncoder(userID) m.Value = sarama.ByteEncoder(value) + log.WithFields(log.Fields{ + "user_id": userID, + "room_id": roomID, + "data_type": dataType, + }).Infof("Producing to topic '%s'", p.Topic) _, _, err = p.Producer.SendMessage(&m) return err diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 195a1ac50..2d538027e 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -17,6 +17,7 @@ package main import ( "flag" "net/http" + "os" "github.com/matrix-org/dendrite/appservice" "github.com/matrix-org/dendrite/eduserver" @@ -28,6 +29,7 @@ import ( "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/roomserver" + "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/serverkeyapi" "github.com/sirupsen/logrus" @@ -39,6 +41,7 @@ var ( certFile = flag.String("tls-cert", "", "The PEM formatted X509 certificate to use for TLS") keyFile = flag.String("tls-key", "", "The PEM private key to use for TLS") enableHTTPAPIs = flag.Bool("api", false, "Use HTTP APIs instead of short-circuiting (warning: exposes API endpoints!)") + traceInternal = os.Getenv("DENDRITE_TRACE_INTERNAL") == "1" ) func main() { @@ -72,14 +75,18 @@ func main() { } keyRing := serverKeyAPI.KeyRing() - rsComponent := roomserver.NewInternalAPI( + rsAPI := roomserver.NewInternalAPI( base, keyRing, federation, ) - rsAPI := rsComponent if base.UseHTTPAPIs { roomserver.AddInternalRoutes(base.InternalAPIMux, rsAPI) rsAPI = base.RoomserverHTTPClient() } + if traceInternal { + rsAPI = &api.RoomserverInternalAPITrace{ + Impl: rsAPI, + } + } eduInputAPI := eduserver.NewInternalAPI( base, cache.New(), deviceDB, @@ -102,7 +109,7 @@ func main() { federationsender.AddInternalRoutes(base.InternalAPIMux, fsAPI) fsAPI = base.FederationSenderHTTPClient() } - rsComponent.SetFederationSenderAPI(fsAPI) + rsAPI.SetFederationSenderAPI(fsAPI) publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties(), cfg.Matrix.ServerName) if err != nil { diff --git a/eduserver/input/input.go b/eduserver/input/input.go index 0bbf5b844..6eafce42f 100644 --- a/eduserver/input/input.go +++ b/eduserver/input/input.go @@ -97,6 +97,11 @@ func (t *EDUServerInputAPI) sendTypingEvent(ite *api.InputTypingEvent) error { if err != nil { return err } + logrus.WithFields(logrus.Fields{ + "room_id": ite.RoomID, + "user_id": ite.UserID, + "typing": ite.Typing, + }).Infof("Producing to topic '%s'", t.OutputTypingEventTopic) m := &sarama.ProducerMessage{ Topic: string(t.OutputTypingEventTopic), @@ -132,6 +137,11 @@ func (t *EDUServerInputAPI) sendToDeviceEvent(ise *api.InputSendToDeviceEvent) e devices = append(devices, ise.DeviceID) } + logrus.WithFields(logrus.Fields{ + "user_id": ise.UserID, + "num_devices": len(devices), + "type": ise.Type, + }).Infof("Producing to topic '%s'", t.OutputSendToDeviceEventTopic) for _, device := range devices { ote := &api.OutputSendToDeviceEvent{ UserID: ise.UserID, @@ -139,12 +149,6 @@ func (t *EDUServerInputAPI) sendToDeviceEvent(ise *api.InputSendToDeviceEvent) e SendToDeviceEvent: ise.SendToDeviceEvent, } - logrus.WithFields(logrus.Fields{ - "user_id": ise.UserID, - "device_id": ise.DeviceID, - "event_type": ise.Type, - }).Info("handling send-to-device message") - eventJSON, err := json.Marshal(ote) if err != nil { logrus.WithError(err).Error("sendToDevice failed json.Marshal") diff --git a/federationsender/consumers/roomserver.go b/federationsender/consumers/roomserver.go index a15937f9e..299c7b37a 100644 --- a/federationsender/consumers/roomserver.go +++ b/federationsender/consumers/roomserver.go @@ -86,11 +86,6 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { switch output.Type { case api.OutputTypeNewRoomEvent: ev := &output.NewRoomEvent.Event - log.WithFields(log.Fields{ - "event_id": ev.EventID(), - "room_id": ev.RoomID(), - "send_as_server": output.NewRoomEvent.SendAsServer, - }).Info("received room event from roomserver") if err := s.processMessage(*output.NewRoomEvent); err != nil { // panic rather than continue with an inconsistent database diff --git a/roomserver/api/api_trace.go b/roomserver/api/api_trace.go new file mode 100644 index 000000000..a478eeb9a --- /dev/null +++ b/roomserver/api/api_trace.go @@ -0,0 +1,218 @@ +package api + +import ( + "context" + "encoding/json" + "fmt" + + fsAPI "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/util" +) + +// RoomserverInternalAPITrace wraps a RoomserverInternalAPI and logs the +// complete request/response/error +type RoomserverInternalAPITrace struct { + Impl RoomserverInternalAPI +} + +func (t *RoomserverInternalAPITrace) SetFederationSenderAPI(fsAPI fsAPI.FederationSenderInternalAPI) { + t.Impl.SetFederationSenderAPI(fsAPI) +} + +func (t *RoomserverInternalAPITrace) InputRoomEvents( + ctx context.Context, + req *InputRoomEventsRequest, + res *InputRoomEventsResponse, +) error { + err := t.Impl.InputRoomEvents(ctx, req, res) + util.GetLogger(ctx).WithError(err).Infof("InputRoomEvents req=%+v res=%+v", js(req), js(res)) + return err +} + +func (t *RoomserverInternalAPITrace) PerformJoin( + ctx context.Context, + req *PerformJoinRequest, + res *PerformJoinResponse, +) error { + err := t.Impl.PerformJoin(ctx, req, res) + util.GetLogger(ctx).WithError(err).Infof("PerformJoin req=%+v res=%+v", js(req), js(res)) + return err +} + +func (t *RoomserverInternalAPITrace) PerformLeave( + ctx context.Context, + req *PerformLeaveRequest, + res *PerformLeaveResponse, +) error { + err := t.Impl.PerformLeave(ctx, req, res) + util.GetLogger(ctx).WithError(err).Infof("PerformLeave req=%+v res=%+v", js(req), js(res)) + return err +} + +func (t *RoomserverInternalAPITrace) QueryLatestEventsAndState( + ctx context.Context, + req *QueryLatestEventsAndStateRequest, + res *QueryLatestEventsAndStateResponse, +) error { + err := t.Impl.QueryLatestEventsAndState(ctx, req, res) + util.GetLogger(ctx).WithError(err).Infof("QueryLatestEventsAndState req=%+v res=%+v", js(req), js(res)) + return err +} + +func (t *RoomserverInternalAPITrace) QueryStateAfterEvents( + ctx context.Context, + req *QueryStateAfterEventsRequest, + res *QueryStateAfterEventsResponse, +) error { + err := t.Impl.QueryStateAfterEvents(ctx, req, res) + util.GetLogger(ctx).WithError(err).Infof("QueryStateAfterEvents req=%+v res=%+v", js(req), js(res)) + return err +} + +func (t *RoomserverInternalAPITrace) QueryEventsByID( + ctx context.Context, + req *QueryEventsByIDRequest, + res *QueryEventsByIDResponse, +) error { + err := t.Impl.QueryEventsByID(ctx, req, res) + util.GetLogger(ctx).WithError(err).Infof("QueryEventsByID req=%+v res=%+v", js(req), js(res)) + return err +} + +func (t *RoomserverInternalAPITrace) QueryMembershipForUser( + ctx context.Context, + req *QueryMembershipForUserRequest, + res *QueryMembershipForUserResponse, +) error { + err := t.Impl.QueryMembershipForUser(ctx, req, res) + util.GetLogger(ctx).WithError(err).Infof("QueryMembershipForUser req=%+v res=%+v", js(req), js(res)) + return err +} + +func (t *RoomserverInternalAPITrace) QueryMembershipsForRoom( + ctx context.Context, + req *QueryMembershipsForRoomRequest, + res *QueryMembershipsForRoomResponse, +) error { + err := t.Impl.QueryMembershipsForRoom(ctx, req, res) + util.GetLogger(ctx).WithError(err).Infof("QueryMembershipsForRoom req=%+v res=%+v", js(req), js(res)) + return err +} + +func (t *RoomserverInternalAPITrace) QueryServerAllowedToSeeEvent( + ctx context.Context, + req *QueryServerAllowedToSeeEventRequest, + res *QueryServerAllowedToSeeEventResponse, +) error { + err := t.Impl.QueryServerAllowedToSeeEvent(ctx, req, res) + util.GetLogger(ctx).WithError(err).Infof("QueryServerAllowedToSeeEvent req=%+v res=%+v", js(req), js(res)) + return err +} + +func (t *RoomserverInternalAPITrace) QueryMissingEvents( + ctx context.Context, + req *QueryMissingEventsRequest, + res *QueryMissingEventsResponse, +) error { + err := t.Impl.QueryMissingEvents(ctx, req, res) + util.GetLogger(ctx).WithError(err).Infof("QueryMissingEvents req=%+v res=%+v", js(req), js(res)) + return err +} + +func (t *RoomserverInternalAPITrace) QueryStateAndAuthChain( + ctx context.Context, + req *QueryStateAndAuthChainRequest, + res *QueryStateAndAuthChainResponse, +) error { + err := t.Impl.QueryStateAndAuthChain(ctx, req, res) + util.GetLogger(ctx).WithError(err).Infof("QueryStateAndAuthChain req=%+v res=%+v", js(req), js(res)) + return err +} + +func (t *RoomserverInternalAPITrace) PerformBackfill( + ctx context.Context, + req *PerformBackfillRequest, + res *PerformBackfillResponse, +) error { + err := t.Impl.PerformBackfill(ctx, req, res) + util.GetLogger(ctx).WithError(err).Infof("PerformBackfill req=%+v res=%+v", js(req), js(res)) + return err +} + +func (t *RoomserverInternalAPITrace) QueryRoomVersionCapabilities( + ctx context.Context, + req *QueryRoomVersionCapabilitiesRequest, + res *QueryRoomVersionCapabilitiesResponse, +) error { + err := t.Impl.QueryRoomVersionCapabilities(ctx, req, res) + util.GetLogger(ctx).WithError(err).Infof("QueryRoomVersionCapabilities req=%+v res=%+v", js(req), js(res)) + return err +} + +func (t *RoomserverInternalAPITrace) QueryRoomVersionForRoom( + ctx context.Context, + req *QueryRoomVersionForRoomRequest, + res *QueryRoomVersionForRoomResponse, +) error { + err := t.Impl.QueryRoomVersionForRoom(ctx, req, res) + util.GetLogger(ctx).WithError(err).Infof("QueryRoomVersionForRoom req=%+v res=%+v", js(req), js(res)) + return err +} + +func (t *RoomserverInternalAPITrace) SetRoomAlias( + ctx context.Context, + req *SetRoomAliasRequest, + res *SetRoomAliasResponse, +) error { + err := t.Impl.SetRoomAlias(ctx, req, res) + util.GetLogger(ctx).WithError(err).Infof("SetRoomAlias req=%+v res=%+v", js(req), js(res)) + return err +} + +func (t *RoomserverInternalAPITrace) GetRoomIDForAlias( + ctx context.Context, + req *GetRoomIDForAliasRequest, + res *GetRoomIDForAliasResponse, +) error { + err := t.Impl.GetRoomIDForAlias(ctx, req, res) + util.GetLogger(ctx).WithError(err).Infof("GetRoomIDForAlias req=%+v res=%+v", js(req), js(res)) + return err +} + +func (t *RoomserverInternalAPITrace) GetAliasesForRoomID( + ctx context.Context, + req *GetAliasesForRoomIDRequest, + res *GetAliasesForRoomIDResponse, +) error { + err := t.Impl.GetAliasesForRoomID(ctx, req, res) + util.GetLogger(ctx).WithError(err).Infof("GetAliasesForRoomID req=%+v res=%+v", js(req), js(res)) + return err +} + +func (t *RoomserverInternalAPITrace) GetCreatorIDForAlias( + ctx context.Context, + req *GetCreatorIDForAliasRequest, + res *GetCreatorIDForAliasResponse, +) error { + err := t.Impl.GetCreatorIDForAlias(ctx, req, res) + util.GetLogger(ctx).WithError(err).Infof("GetCreatorIDForAlias req=%+v res=%+v", js(req), js(res)) + return err +} + +func (t *RoomserverInternalAPITrace) RemoveRoomAlias( + ctx context.Context, + req *RemoveRoomAliasRequest, + res *RemoveRoomAliasResponse, +) error { + err := t.Impl.RemoveRoomAlias(ctx, req, res) + util.GetLogger(ctx).WithError(err).Infof("RemoveRoomAlias req=%+v res=%+v", js(req), js(res)) + return err +} + +func js(thing interface{}) string { + b, err := json.Marshal(thing) + if err != nil { + return fmt.Sprintf("Marshal error:%s", err) + } + return string(b) +} diff --git a/roomserver/internal/input.go b/roomserver/internal/input.go index 932b4df46..e863af953 100644 --- a/roomserver/internal/input.go +++ b/roomserver/internal/input.go @@ -21,6 +21,7 @@ import ( "github.com/Shopify/sarama" "github.com/matrix-org/dendrite/roomserver/api" + log "github.com/sirupsen/logrus" fsAPI "github.com/matrix-org/dendrite/federationsender/api" ) @@ -40,6 +41,21 @@ func (r *RoomserverInternalAPI) WriteOutputEvents(roomID string, updates []api.O if err != nil { return err } + logger := log.WithFields(log.Fields{ + "room_id": roomID, + "type": updates[i].Type, + }) + if updates[i].NewRoomEvent != nil { + logger = logger.WithFields(log.Fields{ + "event_type": updates[i].NewRoomEvent.Event.Type(), + "event_id": updates[i].NewRoomEvent.Event.EventID(), + "adds_state": len(updates[i].NewRoomEvent.AddsStateEventIDs), + "removes_state": len(updates[i].NewRoomEvent.RemovesStateEventIDs), + "send_as_server": updates[i].NewRoomEvent.SendAsServer, + "sender": updates[i].NewRoomEvent.Event.Sender(), + }) + } + logger.Infof("Producing to topic '%s'", r.OutputRoomEventTopic) messages[i] = &sarama.ProducerMessage{ Topic: r.OutputRoomEventTopic, Key: sarama.StringEncoder(roomID), diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index 135976823..98be5bb73 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -98,11 +98,6 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent( ctx context.Context, msg api.OutputNewRoomEvent, ) error { ev := msg.Event - log.WithFields(log.Fields{ - "event_id": ev.EventID(), - "room_id": ev.RoomID(), - "room_version": ev.RoomVersion, - }).Info("received event from roomserver") addsStateEvents := msg.AddsState() From ecd7accbad724f26248498a9035a1fbc69e2f08d Mon Sep 17 00:00:00 2001 From: Kegsay Date: Fri, 12 Jun 2020 14:55:57 +0100 Subject: [PATCH 158/200] Rehuffle where things are in the internal package (#1122) renamed: internal/eventcontent.go -> internal/eventutil/eventcontent.go renamed: internal/events.go -> internal/eventutil/events.go renamed: internal/types.go -> internal/eventutil/types.go renamed: internal/http/http.go -> internal/httputil/http.go renamed: internal/httpapi.go -> internal/httputil/httpapi.go renamed: internal/httpapi_test.go -> internal/httputil/httpapi_test.go renamed: internal/httpapis/paths.go -> internal/httputil/paths.go renamed: internal/routing.go -> internal/httputil/routing.go renamed: internal/basecomponent/base.go -> internal/setup/base.go renamed: internal/basecomponent/flags.go -> internal/setup/flags.go renamed: internal/partition_offset_table.go -> internal/sqlutil/partition_offset_table.go renamed: internal/postgres.go -> internal/sqlutil/postgres.go renamed: internal/postgres_wasm.go -> internal/sqlutil/postgres_wasm.go renamed: internal/sql.go -> internal/sqlutil/sql.go --- appservice/api/query.go | 5 +- appservice/appservice.go | 8 +- appservice/inthttp/client.go | 6 +- appservice/inthttp/server.go | 6 +- appservice/storage/postgres/storage.go | 3 +- appservice/storage/sqlite3/storage.go | 3 +- appservice/storage/storage.go | 4 +- appservice/storage/storage_wasm.go | 4 +- .../accounts/postgres/account_data_table.go | 1 - .../accounts/postgres/membership_table.go | 3 +- .../auth/storage/accounts/postgres/storage.go | 23 +- .../accounts/postgres/threepid_table.go | 6 +- .../accounts/sqlite3/membership_table.go | 3 +- .../auth/storage/accounts/sqlite3/storage.go | 19 +- .../accounts/sqlite3/threepid_table.go | 5 +- clientapi/auth/storage/accounts/storage.go | 4 +- .../auth/storage/accounts/storage_wasm.go | 4 +- .../storage/devices/postgres/devices_table.go | 11 +- .../auth/storage/devices/postgres/storage.go | 15 +- .../storage/devices/sqlite3/devices_table.go | 16 +- .../auth/storage/devices/sqlite3/storage.go | 15 +- clientapi/auth/storage/devices/storage.go | 4 +- .../auth/storage/devices/storage_wasm.go | 4 +- clientapi/producers/syncapi.go | 4 +- clientapi/routing/createroom.go | 16 +- clientapi/routing/membership.go | 8 +- clientapi/routing/profile.go | 24 +-- clientapi/routing/register.go | 9 +- clientapi/routing/routing.go | 198 +++++++++--------- clientapi/routing/sendevent.go | 8 +- clientapi/threepid/invites.go | 4 +- cmd/dendrite-appservice-server/main.go | 6 +- cmd/dendrite-client-api-server/main.go | 6 +- cmd/dendrite-demo-libp2p/main.go | 4 +- cmd/dendrite-demo-libp2p/p2pdendrite.go | 6 +- cmd/dendrite-demo-yggdrasil/main.go | 9 +- cmd/dendrite-edu-server/main.go | 6 +- cmd/dendrite-federation-api-server/main.go | 6 +- cmd/dendrite-federation-sender-server/main.go | 6 +- cmd/dendrite-key-server/main.go | 6 +- cmd/dendrite-media-api-server/main.go | 6 +- cmd/dendrite-monolith-server/main.go | 13 +- cmd/dendrite-public-rooms-api-server/main.go | 6 +- cmd/dendrite-room-server/main.go | 6 +- cmd/dendrite-server-key-api-server/main.go | 6 +- cmd/dendrite-sync-api-server/main.go | 6 +- cmd/dendritejs/main.go | 7 +- eduserver/eduserver.go | 4 +- eduserver/inthttp/client.go | 6 +- eduserver/inthttp/server.go | 6 +- federationapi/routing/join.go | 8 +- federationapi/routing/leave.go | 8 +- federationapi/routing/profile.go | 8 +- federationapi/routing/routing.go | 44 ++-- federationsender/federationsender.go | 4 +- federationsender/inthttp/client.go | 12 +- federationsender/inthttp/server.go | 12 +- .../storage/postgres/joined_hosts_table.go | 7 +- .../storage/postgres/room_table.go | 8 +- federationsender/storage/postgres/storage.go | 7 +- .../storage/sqlite3/joined_hosts_table.go | 7 +- .../storage/sqlite3/room_table.go | 8 +- federationsender/storage/sqlite3/storage.go | 7 +- federationsender/storage/storage.go | 4 +- federationsender/storage/storage_wasm.go | 4 +- go.mod | 1 + go.sum | 7 + internal/consumers.go | 11 +- internal/{ => eventutil}/eventcontent.go | 4 +- internal/{ => eventutil}/events.go | 4 +- internal/{ => eventutil}/types.go | 4 +- internal/httpapis/paths.go | 6 - internal/{http => httputil}/http.go | 19 +- internal/{ => httputil}/httpapi.go | 21 +- internal/{ => httputil}/httpapi_test.go | 16 +- internal/httputil/paths.go | 20 ++ internal/{ => httputil}/routing.go | 4 +- internal/{basecomponent => setup}/base.go | 14 +- internal/{basecomponent => setup}/flags.go | 4 +- internal/setup/monolith.go | 14 ++ .../{ => sqlutil}/partition_offset_table.go | 21 +- internal/{ => sqlutil}/postgres.go | 2 +- internal/{ => sqlutil}/postgres_wasm.go | 2 +- internal/{ => sqlutil}/sql.go | 6 +- internal/sqlutil/trace.go | 5 +- internal/sqlutil/uri.go | 14 ++ keyserver/routing/routing.go | 4 +- mediaapi/routing/routing.go | 6 +- mediaapi/storage/postgres/storage.go | 3 +- mediaapi/storage/postgres/thumbnail_table.go | 1 - mediaapi/storage/sqlite3/storage.go | 3 +- mediaapi/storage/sqlite3/thumbnail_table.go | 1 - mediaapi/storage/storage.go | 4 +- mediaapi/storage/storage_wasm.go | 4 +- publicroomsapi/routing/routing.go | 14 +- publicroomsapi/storage/postgres/storage.go | 20 +- publicroomsapi/storage/sqlite3/storage.go | 20 +- publicroomsapi/storage/storage.go | 4 +- roomserver/internal/input_events.go | 4 +- roomserver/internal/input_latest_events.go | 4 +- roomserver/internal/perform_join.go | 6 +- roomserver/internal/perform_leave.go | 6 +- roomserver/inthttp/client.go | 40 ++-- roomserver/inthttp/server.go | 40 ++-- roomserver/roomserver.go | 4 +- .../storage/postgres/event_json_table.go | 1 - .../postgres/event_state_keys_table.go | 5 +- .../storage/postgres/event_types_table.go | 3 +- roomserver/storage/postgres/events_table.go | 9 +- roomserver/storage/postgres/invite_table.go | 5 +- .../storage/postgres/membership_table.go | 7 +- .../storage/postgres/previous_events_table.go | 6 +- roomserver/storage/postgres/rooms_table.go | 12 +- .../storage/postgres/state_block_table.go | 3 +- roomserver/storage/postgres/storage.go | 3 +- roomserver/storage/shared/storage.go | 8 +- .../storage/sqlite3/event_json_table.go | 5 +- .../storage/sqlite3/event_state_keys_table.go | 9 +- .../storage/sqlite3/event_types_table.go | 9 +- roomserver/storage/sqlite3/events_table.go | 27 +-- roomserver/storage/sqlite3/invite_table.go | 7 +- .../storage/sqlite3/membership_table.go | 7 +- .../storage/sqlite3/previous_events_table.go | 6 +- roomserver/storage/sqlite3/rooms_table.go | 14 +- .../storage/sqlite3/state_block_table.go | 9 +- .../storage/sqlite3/state_snapshot_table.go | 3 +- roomserver/storage/sqlite3/storage.go | 4 +- .../storage/sqlite3/transactions_table.go | 4 +- roomserver/storage/storage.go | 4 +- roomserver/storage/storage_wasm.go | 4 +- roomserver/types/types.go | 6 +- serverkeyapi/inthttp/client.go | 6 +- serverkeyapi/inthttp/server.go | 6 +- serverkeyapi/storage/keydb.go | 4 +- serverkeyapi/storage/keydb_wasm.go | 4 +- serverkeyapi/storage/postgres/keydb.go | 3 +- .../storage/postgres/server_key_table.go | 3 +- serverkeyapi/storage/sqlite3/keydb.go | 3 +- .../storage/sqlite3/server_key_table.go | 3 +- syncapi/consumers/clientapi.go | 3 +- syncapi/routing/routing.go | 8 +- .../storage/postgres/account_data_table.go | 3 +- .../postgres/current_room_state_table.go | 11 +- syncapi/storage/postgres/invites_table.go | 5 +- .../postgres/output_room_events_table.go | 17 +- .../output_room_events_topology_table.go | 1 - .../storage/postgres/send_to_device_table.go | 7 +- syncapi/storage/postgres/syncserver.go | 10 +- syncapi/storage/shared/syncserver.go | 20 +- syncapi/storage/sqlite3/account_data_table.go | 1 - .../sqlite3/current_room_state_table.go | 11 +- syncapi/storage/sqlite3/invites_table.go | 5 +- .../sqlite3/output_room_events_table.go | 17 +- .../output_room_events_topology_table.go | 12 +- .../storage/sqlite3/send_to_device_table.go | 11 +- syncapi/storage/sqlite3/stream_id_table.go | 6 +- syncapi/storage/sqlite3/syncserver.go | 10 +- syncapi/storage/storage.go | 4 +- syncapi/storage/storage_wasm.go | 4 +- 159 files changed, 784 insertions(+), 693 deletions(-) rename internal/{ => eventutil}/eventcontent.go (97%) rename internal/{ => eventutil}/events.go (98%) rename internal/{ => eventutil}/types.go (96%) delete mode 100644 internal/httpapis/paths.go rename internal/{http => httputil}/http.go (69%) rename internal/{ => httputil}/httpapi.go (92%) rename internal/{ => httputil}/httpapi_test.go (75%) create mode 100644 internal/httputil/paths.go rename internal/{ => httputil}/routing.go (94%) rename internal/{basecomponent => setup}/base.go (96%) rename internal/{basecomponent => setup}/flags.go (94%) rename internal/{ => sqlutil}/partition_offset_table.go (88%) rename internal/{ => sqlutil}/postgres.go (98%) rename internal/{ => sqlutil}/postgres_wasm.go (97%) rename internal/{ => sqlutil}/sql.go (96%) diff --git a/appservice/api/query.go b/appservice/api/query.go index 6fcb20890..0a5cc9f1d 100644 --- a/appservice/api/query.go +++ b/appservice/api/query.go @@ -23,9 +23,8 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" + "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/gomatrixserverlib" - - "github.com/matrix-org/dendrite/internal" ) // RoomAliasExistsRequest is a request to an application service @@ -110,7 +109,7 @@ func RetrieveUserProfile( // If no user exists, return if !userResp.UserIDExists { - return nil, internal.ErrProfileNoExists + return nil, eventutil.ErrProfileNoExists } // Try to query the user from the local database again diff --git a/appservice/appservice.go b/appservice/appservice.go index b7adf755b..0fbe3f20b 100644 --- a/appservice/appservice.go +++ b/appservice/appservice.go @@ -31,9 +31,9 @@ import ( "github.com/matrix-org/dendrite/appservice/workers" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" - "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/dendrite/internal/basecomponent" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/setup" + "github.com/matrix-org/dendrite/internal/sqlutil" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/sirupsen/logrus" ) @@ -46,7 +46,7 @@ func AddInternalRoutes(router *mux.Router, queryAPI appserviceAPI.AppServiceQuer // NewInternalAPI returns a concerete implementation of the internal API. Callers // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. func NewInternalAPI( - base *basecomponent.BaseDendrite, + base *setup.BaseDendrite, accountsDB accounts.Database, deviceDB devices.Database, rsAPI roomserverAPI.RoomserverInternalAPI, @@ -114,7 +114,7 @@ func generateAppServiceAccount( // Create an account for the application service _, err := accountsDB.CreateAccount(ctx, as.SenderLocalpart, "", as.ID) if err != nil { - if errors.Is(err, internal.ErrUserExists) { // This account already exists + if errors.Is(err, sqlutil.ErrUserExists) { // This account already exists return nil } return err diff --git a/appservice/inthttp/client.go b/appservice/inthttp/client.go index acbd1211b..7e3cb208f 100644 --- a/appservice/inthttp/client.go +++ b/appservice/inthttp/client.go @@ -6,7 +6,7 @@ import ( "net/http" "github.com/matrix-org/dendrite/appservice/api" - internalHTTP "github.com/matrix-org/dendrite/internal/http" + "github.com/matrix-org/dendrite/internal/httputil" "github.com/opentracing/opentracing-go" ) @@ -46,7 +46,7 @@ func (h *httpAppServiceQueryAPI) RoomAliasExists( defer span.Finish() apiURL := h.appserviceURL + AppServiceRoomAliasExistsPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // UserIDExists implements AppServiceQueryAPI @@ -59,5 +59,5 @@ func (h *httpAppServiceQueryAPI) UserIDExists( defer span.Finish() apiURL := h.appserviceURL + AppServiceUserIDExistsPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } diff --git a/appservice/inthttp/server.go b/appservice/inthttp/server.go index 1900635a8..009b7b5db 100644 --- a/appservice/inthttp/server.go +++ b/appservice/inthttp/server.go @@ -6,7 +6,7 @@ import ( "github.com/gorilla/mux" "github.com/matrix-org/dendrite/appservice/api" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/util" ) @@ -14,7 +14,7 @@ import ( func AddRoutes(a api.AppServiceQueryAPI, internalAPIMux *mux.Router) { internalAPIMux.Handle( AppServiceRoomAliasExistsPath, - internal.MakeInternalAPI("appserviceRoomAliasExists", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("appserviceRoomAliasExists", func(req *http.Request) util.JSONResponse { var request api.RoomAliasExistsRequest var response api.RoomAliasExistsResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -28,7 +28,7 @@ func AddRoutes(a api.AppServiceQueryAPI, internalAPIMux *mux.Router) { ) internalAPIMux.Handle( AppServiceUserIDExistsPath, - internal.MakeInternalAPI("appserviceUserIDExists", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("appserviceUserIDExists", func(req *http.Request) util.JSONResponse { var request api.UserIDExistsRequest var response api.UserIDExistsResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { diff --git a/appservice/storage/postgres/storage.go b/appservice/storage/postgres/storage.go index aeead6ed3..3e12f3a0d 100644 --- a/appservice/storage/postgres/storage.go +++ b/appservice/storage/postgres/storage.go @@ -21,7 +21,6 @@ import ( // Import postgres database driver _ "github.com/lib/pq" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) @@ -34,7 +33,7 @@ type Database struct { } // NewDatabase opens a new database -func NewDatabase(dataSourceName string, dbProperties internal.DbProperties) (*Database, error) { +func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties) (*Database, error) { var result Database var err error if result.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { diff --git a/appservice/storage/sqlite3/storage.go b/appservice/storage/sqlite3/storage.go index 2238b3ff9..44dcba4ed 100644 --- a/appservice/storage/sqlite3/storage.go +++ b/appservice/storage/sqlite3/storage.go @@ -20,7 +20,6 @@ import ( "database/sql" // Import SQLite database driver - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" _ "github.com/mattn/go-sqlite3" @@ -41,7 +40,7 @@ func NewDatabase(dataSourceName string) (*Database, error) { if err != nil { return nil, err } - if result.db, err = sqlutil.Open(internal.SQLiteDriverName(), cs, nil); err != nil { + if result.db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil); err != nil { return nil, err } if err = result.prepare(); err != nil { diff --git a/appservice/storage/storage.go b/appservice/storage/storage.go index 51ba1de53..c848d15d7 100644 --- a/appservice/storage/storage.go +++ b/appservice/storage/storage.go @@ -21,12 +21,12 @@ import ( "github.com/matrix-org/dendrite/appservice/storage/postgres" "github.com/matrix-org/dendrite/appservice/storage/sqlite3" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" ) // NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme) // and sets DB connection parameters -func NewDatabase(dataSourceName string, dbProperties internal.DbProperties) (Database, error) { +func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return postgres.NewDatabase(dataSourceName, dbProperties) diff --git a/appservice/storage/storage_wasm.go b/appservice/storage/storage_wasm.go index a6144b435..1d6c4b4a9 100644 --- a/appservice/storage/storage_wasm.go +++ b/appservice/storage/storage_wasm.go @@ -19,12 +19,12 @@ import ( "net/url" "github.com/matrix-org/dendrite/appservice/storage/sqlite3" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" ) func NewDatabase( dataSourceName string, - dbProperties internal.DbProperties, // nolint:unparam + dbProperties sqlutil.DbProperties, // nolint:unparam ) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { diff --git a/clientapi/auth/storage/accounts/postgres/account_data_table.go b/clientapi/auth/storage/accounts/postgres/account_data_table.go index 2e044b367..2f16c5c02 100644 --- a/clientapi/auth/storage/accounts/postgres/account_data_table.go +++ b/clientapi/auth/storage/accounts/postgres/account_data_table.go @@ -19,7 +19,6 @@ import ( "database/sql" "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/gomatrixserverlib" ) diff --git a/clientapi/auth/storage/accounts/postgres/membership_table.go b/clientapi/auth/storage/accounts/postgres/membership_table.go index d006e916a..623530acc 100644 --- a/clientapi/auth/storage/accounts/postgres/membership_table.go +++ b/clientapi/auth/storage/accounts/postgres/membership_table.go @@ -18,10 +18,9 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/internal" - "github.com/lib/pq" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/internal" ) const membershipSchema = ` diff --git a/clientapi/auth/storage/accounts/postgres/storage.go b/clientapi/auth/storage/accounts/postgres/storage.go index 4be5dca94..fcb592aef 100644 --- a/clientapi/auth/storage/accounts/postgres/storage.go +++ b/clientapi/auth/storage/accounts/postgres/storage.go @@ -21,7 +21,6 @@ import ( "strconv" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" "golang.org/x/crypto/bcrypt" @@ -33,7 +32,7 @@ import ( // Database represents an account database type Database struct { db *sql.DB - internal.PartitionOffsetStatements + sqlutil.PartitionOffsetStatements accounts accountsStatements profiles profilesStatements memberships membershipStatements @@ -44,13 +43,13 @@ type Database struct { } // NewDatabase creates a new accounts and profiles database -func NewDatabase(dataSourceName string, dbProperties internal.DbProperties, serverName gomatrixserverlib.ServerName) (*Database, error) { +func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties, serverName gomatrixserverlib.ServerName) (*Database, error) { var db *sql.DB var err error if db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { return nil, err } - partitions := internal.PartitionOffsetStatements{} + partitions := sqlutil.PartitionOffsetStatements{} if err = partitions.Prepare(db, "account"); err != nil { return nil, err } @@ -123,7 +122,7 @@ func (d *Database) SetDisplayName( // CreateGuestAccount makes a new guest account and creates an empty profile // for this account. func (d *Database) CreateGuestAccount(ctx context.Context) (acc *authtypes.Account, err error) { - err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { + err = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { var numLocalpart int64 numLocalpart, err = d.accounts.selectNewNumericLocalpart(ctx, txn) if err != nil { @@ -138,11 +137,11 @@ func (d *Database) CreateGuestAccount(ctx context.Context) (acc *authtypes.Accou // CreateAccount makes a new account with the given login name and password, and creates an empty profile // for this account. If no password is supplied, the account will be a passwordless account. If the -// account already exists, it will return nil, ErrUserExists. +// account already exists, it will return nil, sqlutil.ErrUserExists. func (d *Database) CreateAccount( ctx context.Context, localpart, plaintextPassword, appserviceID string, ) (acc *authtypes.Account, err error) { - err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { + err = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { acc, err = d.createAccount(ctx, txn, localpart, plaintextPassword, appserviceID) return err }) @@ -163,8 +162,8 @@ func (d *Database) createAccount( } } if err := d.profiles.insertProfile(ctx, txn, localpart); err != nil { - if internal.IsUniqueConstraintViolationErr(err) { - return nil, internal.ErrUserExists + if sqlutil.IsUniqueConstraintViolationErr(err) { + return nil, sqlutil.ErrUserExists } return nil, err } @@ -210,7 +209,7 @@ func (d *Database) removeMembershipsByEventIDs( func (d *Database) UpdateMemberships( ctx context.Context, eventsToAdd []gomatrixserverlib.Event, idsToRemove []string, ) error { - return internal.WithTransaction(d.db, func(txn *sql.Tx) error { + return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { if err := d.removeMembershipsByEventIDs(ctx, txn, idsToRemove); err != nil { return err } @@ -297,7 +296,7 @@ func (d *Database) newMembership( func (d *Database) SaveAccountData( ctx context.Context, localpart, roomID, dataType, content string, ) error { - return internal.WithTransaction(d.db, func(txn *sql.Tx) error { + return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { return d.accountDatas.insertAccountData(ctx, txn, localpart, roomID, dataType, content) }) } @@ -348,7 +347,7 @@ var Err3PIDInUse = errors.New("This third-party identifier is already in use") func (d *Database) SaveThreePIDAssociation( ctx context.Context, threepid, localpart, medium string, ) (err error) { - return internal.WithTransaction(d.db, func(txn *sql.Tx) error { + return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { user, err := d.threepids.selectLocalpartForThreePID( ctx, txn, threepid, medium, ) diff --git a/clientapi/auth/storage/accounts/postgres/threepid_table.go b/clientapi/auth/storage/accounts/postgres/threepid_table.go index 0b12b5c5b..7de96350c 100644 --- a/clientapi/auth/storage/accounts/postgres/threepid_table.go +++ b/clientapi/auth/storage/accounts/postgres/threepid_table.go @@ -18,7 +18,7 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" ) @@ -82,7 +82,7 @@ func (s *threepidStatements) prepare(db *sql.DB) (err error) { func (s *threepidStatements) selectLocalpartForThreePID( ctx context.Context, txn *sql.Tx, threepid string, medium string, ) (localpart string, err error) { - stmt := internal.TxStmt(txn, s.selectLocalpartForThreePIDStmt) + stmt := sqlutil.TxStmt(txn, s.selectLocalpartForThreePIDStmt) err = stmt.QueryRowContext(ctx, threepid, medium).Scan(&localpart) if err == sql.ErrNoRows { return "", nil @@ -117,7 +117,7 @@ func (s *threepidStatements) selectThreePIDsForLocalpart( func (s *threepidStatements) insertThreePID( ctx context.Context, txn *sql.Tx, threepid, medium, localpart string, ) (err error) { - stmt := internal.TxStmt(txn, s.insertThreePIDStmt) + stmt := sqlutil.TxStmt(txn, s.insertThreePIDStmt) _, err = stmt.ExecContext(ctx, threepid, medium, localpart) return } diff --git a/clientapi/auth/storage/accounts/sqlite3/membership_table.go b/clientapi/auth/storage/accounts/sqlite3/membership_table.go index 90e16fb01..67958f27d 100644 --- a/clientapi/auth/storage/accounts/sqlite3/membership_table.go +++ b/clientapi/auth/storage/accounts/sqlite3/membership_table.go @@ -21,6 +21,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" ) const membershipSchema = ` @@ -95,7 +96,7 @@ func (s *membershipStatements) insertMembership( func (s *membershipStatements) deleteMembershipsByEventIDs( ctx context.Context, txn *sql.Tx, eventIDs []string, ) (err error) { - sqlStr := strings.Replace(deleteMembershipsByEventIDsSQL, "($1)", internal.QueryVariadic(len(eventIDs)), 1) + sqlStr := strings.Replace(deleteMembershipsByEventIDsSQL, "($1)", sqlutil.QueryVariadic(len(eventIDs)), 1) iEventIDs := make([]interface{}, len(eventIDs)) for i, e := range eventIDs { iEventIDs[i] = e diff --git a/clientapi/auth/storage/accounts/sqlite3/storage.go b/clientapi/auth/storage/accounts/sqlite3/storage.go index 31426e471..44245a99d 100644 --- a/clientapi/auth/storage/accounts/sqlite3/storage.go +++ b/clientapi/auth/storage/accounts/sqlite3/storage.go @@ -22,7 +22,6 @@ import ( "sync" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" "golang.org/x/crypto/bcrypt" @@ -32,7 +31,7 @@ import ( // Database represents an account database type Database struct { db *sql.DB - internal.PartitionOffsetStatements + sqlutil.PartitionOffsetStatements accounts accountsStatements profiles profilesStatements memberships membershipStatements @@ -52,10 +51,10 @@ func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) if err != nil { return nil, err } - if db, err = sqlutil.Open(internal.SQLiteDriverName(), cs, nil); err != nil { + if db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil); err != nil { return nil, err } - partitions := internal.PartitionOffsetStatements{} + partitions := sqlutil.PartitionOffsetStatements{} if err = partitions.Prepare(db, "account"); err != nil { return nil, err } @@ -128,7 +127,7 @@ func (d *Database) SetDisplayName( // CreateGuestAccount makes a new guest account and creates an empty profile // for this account. func (d *Database) CreateGuestAccount(ctx context.Context) (acc *authtypes.Account, err error) { - err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { + err = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { // We need to lock so we sequentially create numeric localparts. If we don't, two calls to // this function will cause the same number to be selected and one will fail with 'database is locked' // when the first txn upgrades to a write txn. @@ -154,7 +153,7 @@ func (d *Database) CreateGuestAccount(ctx context.Context) (acc *authtypes.Accou func (d *Database) CreateAccount( ctx context.Context, localpart, plaintextPassword, appserviceID string, ) (acc *authtypes.Account, err error) { - err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { + err = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { acc, err = d.createAccount(ctx, txn, localpart, plaintextPassword, appserviceID) return err }) @@ -175,7 +174,7 @@ func (d *Database) createAccount( } if err := d.profiles.insertProfile(ctx, txn, localpart); err != nil { if isConstraintError(err) { - return nil, internal.ErrUserExists + return nil, sqlutil.ErrUserExists } return nil, err } @@ -221,7 +220,7 @@ func (d *Database) removeMembershipsByEventIDs( func (d *Database) UpdateMemberships( ctx context.Context, eventsToAdd []gomatrixserverlib.Event, idsToRemove []string, ) error { - return internal.WithTransaction(d.db, func(txn *sql.Tx) error { + return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { if err := d.removeMembershipsByEventIDs(ctx, txn, idsToRemove); err != nil { return err } @@ -308,7 +307,7 @@ func (d *Database) newMembership( func (d *Database) SaveAccountData( ctx context.Context, localpart, roomID, dataType, content string, ) error { - return internal.WithTransaction(d.db, func(txn *sql.Tx) error { + return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { return d.accountDatas.insertAccountData(ctx, txn, localpart, roomID, dataType, content) }) } @@ -359,7 +358,7 @@ var Err3PIDInUse = errors.New("This third-party identifier is already in use") func (d *Database) SaveThreePIDAssociation( ctx context.Context, threepid, localpart, medium string, ) (err error) { - return internal.WithTransaction(d.db, func(txn *sql.Tx) error { + return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { user, err := d.threepids.selectLocalpartForThreePID( ctx, txn, threepid, medium, ) diff --git a/clientapi/auth/storage/accounts/sqlite3/threepid_table.go b/clientapi/auth/storage/accounts/sqlite3/threepid_table.go index f78a5ffcd..0200dee7f 100644 --- a/clientapi/auth/storage/accounts/sqlite3/threepid_table.go +++ b/clientapi/auth/storage/accounts/sqlite3/threepid_table.go @@ -19,6 +19,7 @@ import ( "database/sql" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" ) @@ -82,7 +83,7 @@ func (s *threepidStatements) prepare(db *sql.DB) (err error) { func (s *threepidStatements) selectLocalpartForThreePID( ctx context.Context, txn *sql.Tx, threepid string, medium string, ) (localpart string, err error) { - stmt := internal.TxStmt(txn, s.selectLocalpartForThreePIDStmt) + stmt := sqlutil.TxStmt(txn, s.selectLocalpartForThreePIDStmt) err = stmt.QueryRowContext(ctx, threepid, medium).Scan(&localpart) if err == sql.ErrNoRows { return "", nil @@ -117,7 +118,7 @@ func (s *threepidStatements) selectThreePIDsForLocalpart( func (s *threepidStatements) insertThreePID( ctx context.Context, txn *sql.Tx, threepid, medium, localpart string, ) (err error) { - stmt := internal.TxStmt(txn, s.insertThreePIDStmt) + stmt := sqlutil.TxStmt(txn, s.insertThreePIDStmt) _, err = stmt.ExecContext(ctx, threepid, medium, localpart) return } diff --git a/clientapi/auth/storage/accounts/storage.go b/clientapi/auth/storage/accounts/storage.go index 37126b30b..42ec14fc4 100644 --- a/clientapi/auth/storage/accounts/storage.go +++ b/clientapi/auth/storage/accounts/storage.go @@ -21,13 +21,13 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/postgres" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/sqlite3" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) // NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme) // and sets postgres connection parameters -func NewDatabase(dataSourceName string, dbProperties internal.DbProperties, serverName gomatrixserverlib.ServerName) (Database, error) { +func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties, serverName gomatrixserverlib.ServerName) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return postgres.NewDatabase(dataSourceName, dbProperties, serverName) diff --git a/clientapi/auth/storage/accounts/storage_wasm.go b/clientapi/auth/storage/accounts/storage_wasm.go index 81e77cf79..6c221ccf5 100644 --- a/clientapi/auth/storage/accounts/storage_wasm.go +++ b/clientapi/auth/storage/accounts/storage_wasm.go @@ -19,13 +19,13 @@ import ( "net/url" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/sqlite3" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) func NewDatabase( dataSourceName string, - dbProperties internal.DbProperties, // nolint:unparam + dbProperties sqlutil.DbProperties, // nolint:unparam serverName gomatrixserverlib.ServerName, ) (Database, error) { uri, err := url.Parse(dataSourceName) diff --git a/clientapi/auth/storage/devices/postgres/devices_table.go b/clientapi/auth/storage/devices/postgres/devices_table.go index c84e83d3d..149ca659f 100644 --- a/clientapi/auth/storage/devices/postgres/devices_table.go +++ b/clientapi/auth/storage/devices/postgres/devices_table.go @@ -23,6 +23,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/userutil" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) @@ -137,7 +138,7 @@ func (s *devicesStatements) insertDevice( ) (*authtypes.Device, error) { createdTimeMS := time.Now().UnixNano() / 1000000 var sessionID int64 - stmt := internal.TxStmt(txn, s.insertDeviceStmt) + stmt := sqlutil.TxStmt(txn, s.insertDeviceStmt) if err := stmt.QueryRowContext(ctx, id, localpart, accessToken, createdTimeMS, displayName).Scan(&sessionID); err != nil { return nil, err } @@ -153,7 +154,7 @@ func (s *devicesStatements) insertDevice( func (s *devicesStatements) deleteDevice( ctx context.Context, txn *sql.Tx, id, localpart string, ) error { - stmt := internal.TxStmt(txn, s.deleteDeviceStmt) + stmt := sqlutil.TxStmt(txn, s.deleteDeviceStmt) _, err := stmt.ExecContext(ctx, id, localpart) return err } @@ -163,7 +164,7 @@ func (s *devicesStatements) deleteDevice( func (s *devicesStatements) deleteDevices( ctx context.Context, txn *sql.Tx, localpart string, devices []string, ) error { - stmt := internal.TxStmt(txn, s.deleteDevicesStmt) + stmt := sqlutil.TxStmt(txn, s.deleteDevicesStmt) _, err := stmt.ExecContext(ctx, localpart, pq.Array(devices)) return err } @@ -173,7 +174,7 @@ func (s *devicesStatements) deleteDevices( func (s *devicesStatements) deleteDevicesByLocalpart( ctx context.Context, txn *sql.Tx, localpart string, ) error { - stmt := internal.TxStmt(txn, s.deleteDevicesByLocalpartStmt) + stmt := sqlutil.TxStmt(txn, s.deleteDevicesByLocalpartStmt) _, err := stmt.ExecContext(ctx, localpart) return err } @@ -181,7 +182,7 @@ func (s *devicesStatements) deleteDevicesByLocalpart( func (s *devicesStatements) updateDeviceName( ctx context.Context, txn *sql.Tx, localpart, deviceID string, displayName *string, ) error { - stmt := internal.TxStmt(txn, s.updateDeviceNameStmt) + stmt := sqlutil.TxStmt(txn, s.updateDeviceNameStmt) _, err := stmt.ExecContext(ctx, displayName, localpart, deviceID) return err } diff --git a/clientapi/auth/storage/devices/postgres/storage.go b/clientapi/auth/storage/devices/postgres/storage.go index e54e7c5d7..2b9aede2f 100644 --- a/clientapi/auth/storage/devices/postgres/storage.go +++ b/clientapi/auth/storage/devices/postgres/storage.go @@ -21,7 +21,6 @@ import ( "encoding/base64" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) @@ -36,7 +35,7 @@ type Database struct { } // NewDatabase creates a new device database -func NewDatabase(dataSourceName string, dbProperties internal.DbProperties, serverName gomatrixserverlib.ServerName) (*Database, error) { +func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties, serverName gomatrixserverlib.ServerName) (*Database, error) { var db *sql.DB var err error if db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { @@ -83,7 +82,7 @@ func (d *Database) CreateDevice( displayName *string, ) (dev *authtypes.Device, returnErr error) { if deviceID != nil { - returnErr = internal.WithTransaction(d.db, func(txn *sql.Tx) error { + returnErr = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { var err error // Revoke existing tokens for this device if err = d.devices.deleteDevice(ctx, txn, *deviceID, localpart); err != nil { @@ -103,7 +102,7 @@ func (d *Database) CreateDevice( return } - returnErr = internal.WithTransaction(d.db, func(txn *sql.Tx) error { + returnErr = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { var err error dev, err = d.devices.insertDevice(ctx, txn, newDeviceID, localpart, accessToken, displayName) return err @@ -133,7 +132,7 @@ func generateDeviceID() (string, error) { func (d *Database) UpdateDevice( ctx context.Context, localpart, deviceID string, displayName *string, ) error { - return internal.WithTransaction(d.db, func(txn *sql.Tx) error { + return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { return d.devices.updateDeviceName(ctx, txn, localpart, deviceID, displayName) }) } @@ -145,7 +144,7 @@ func (d *Database) UpdateDevice( func (d *Database) RemoveDevice( ctx context.Context, deviceID, localpart string, ) error { - return internal.WithTransaction(d.db, func(txn *sql.Tx) error { + return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { if err := d.devices.deleteDevice(ctx, txn, deviceID, localpart); err != sql.ErrNoRows { return err } @@ -160,7 +159,7 @@ func (d *Database) RemoveDevice( func (d *Database) RemoveDevices( ctx context.Context, localpart string, devices []string, ) error { - return internal.WithTransaction(d.db, func(txn *sql.Tx) error { + return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { if err := d.devices.deleteDevices(ctx, txn, localpart, devices); err != sql.ErrNoRows { return err } @@ -174,7 +173,7 @@ func (d *Database) RemoveDevices( func (d *Database) RemoveAllDevices( ctx context.Context, localpart string, ) error { - return internal.WithTransaction(d.db, func(txn *sql.Tx) error { + return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { if err := d.devices.deleteDevicesByLocalpart(ctx, txn, localpart); err != sql.ErrNoRows { return err } diff --git a/clientapi/auth/storage/devices/sqlite3/devices_table.go b/clientapi/auth/storage/devices/sqlite3/devices_table.go index 49f3eaed7..4656b0041 100644 --- a/clientapi/auth/storage/devices/sqlite3/devices_table.go +++ b/clientapi/auth/storage/devices/sqlite3/devices_table.go @@ -20,7 +20,7 @@ import ( "strings" "time" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/userutil" @@ -128,8 +128,8 @@ func (s *devicesStatements) insertDevice( ) (*authtypes.Device, error) { createdTimeMS := time.Now().UnixNano() / 1000000 var sessionID int64 - countStmt := internal.TxStmt(txn, s.selectDevicesCountStmt) - insertStmt := internal.TxStmt(txn, s.insertDeviceStmt) + countStmt := sqlutil.TxStmt(txn, s.selectDevicesCountStmt) + insertStmt := sqlutil.TxStmt(txn, s.insertDeviceStmt) if err := countStmt.QueryRowContext(ctx).Scan(&sessionID); err != nil { return nil, err } @@ -148,7 +148,7 @@ func (s *devicesStatements) insertDevice( func (s *devicesStatements) deleteDevice( ctx context.Context, txn *sql.Tx, id, localpart string, ) error { - stmt := internal.TxStmt(txn, s.deleteDeviceStmt) + stmt := sqlutil.TxStmt(txn, s.deleteDeviceStmt) _, err := stmt.ExecContext(ctx, id, localpart) return err } @@ -156,12 +156,12 @@ func (s *devicesStatements) deleteDevice( func (s *devicesStatements) deleteDevices( ctx context.Context, txn *sql.Tx, localpart string, devices []string, ) error { - orig := strings.Replace(deleteDevicesSQL, "($1)", internal.QueryVariadic(len(devices)), 1) + orig := strings.Replace(deleteDevicesSQL, "($1)", sqlutil.QueryVariadic(len(devices)), 1) prep, err := s.db.Prepare(orig) if err != nil { return err } - stmt := internal.TxStmt(txn, prep) + stmt := sqlutil.TxStmt(txn, prep) params := make([]interface{}, len(devices)+1) params[0] = localpart for i, v := range devices { @@ -175,7 +175,7 @@ func (s *devicesStatements) deleteDevices( func (s *devicesStatements) deleteDevicesByLocalpart( ctx context.Context, txn *sql.Tx, localpart string, ) error { - stmt := internal.TxStmt(txn, s.deleteDevicesByLocalpartStmt) + stmt := sqlutil.TxStmt(txn, s.deleteDevicesByLocalpartStmt) _, err := stmt.ExecContext(ctx, localpart) return err } @@ -183,7 +183,7 @@ func (s *devicesStatements) deleteDevicesByLocalpart( func (s *devicesStatements) updateDeviceName( ctx context.Context, txn *sql.Tx, localpart, deviceID string, displayName *string, ) error { - stmt := internal.TxStmt(txn, s.updateDeviceNameStmt) + stmt := sqlutil.TxStmt(txn, s.updateDeviceNameStmt) _, err := stmt.ExecContext(ctx, displayName, localpart, deviceID) return err } diff --git a/clientapi/auth/storage/devices/sqlite3/storage.go b/clientapi/auth/storage/devices/sqlite3/storage.go index e05a53b46..09e0bc81e 100644 --- a/clientapi/auth/storage/devices/sqlite3/storage.go +++ b/clientapi/auth/storage/devices/sqlite3/storage.go @@ -21,7 +21,6 @@ import ( "encoding/base64" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" @@ -45,7 +44,7 @@ func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) if err != nil { return nil, err } - if db, err = sqlutil.Open(internal.SQLiteDriverName(), cs, nil); err != nil { + if db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil); err != nil { return nil, err } d := devicesStatements{} @@ -89,7 +88,7 @@ func (d *Database) CreateDevice( displayName *string, ) (dev *authtypes.Device, returnErr error) { if deviceID != nil { - returnErr = internal.WithTransaction(d.db, func(txn *sql.Tx) error { + returnErr = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { var err error // Revoke existing tokens for this device if err = d.devices.deleteDevice(ctx, txn, *deviceID, localpart); err != nil { @@ -109,7 +108,7 @@ func (d *Database) CreateDevice( return } - returnErr = internal.WithTransaction(d.db, func(txn *sql.Tx) error { + returnErr = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { var err error dev, err = d.devices.insertDevice(ctx, txn, newDeviceID, localpart, accessToken, displayName) return err @@ -139,7 +138,7 @@ func generateDeviceID() (string, error) { func (d *Database) UpdateDevice( ctx context.Context, localpart, deviceID string, displayName *string, ) error { - return internal.WithTransaction(d.db, func(txn *sql.Tx) error { + return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { return d.devices.updateDeviceName(ctx, txn, localpart, deviceID, displayName) }) } @@ -151,7 +150,7 @@ func (d *Database) UpdateDevice( func (d *Database) RemoveDevice( ctx context.Context, deviceID, localpart string, ) error { - return internal.WithTransaction(d.db, func(txn *sql.Tx) error { + return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { if err := d.devices.deleteDevice(ctx, txn, deviceID, localpart); err != sql.ErrNoRows { return err } @@ -166,7 +165,7 @@ func (d *Database) RemoveDevice( func (d *Database) RemoveDevices( ctx context.Context, localpart string, devices []string, ) error { - return internal.WithTransaction(d.db, func(txn *sql.Tx) error { + return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { if err := d.devices.deleteDevices(ctx, txn, localpart, devices); err != sql.ErrNoRows { return err } @@ -180,7 +179,7 @@ func (d *Database) RemoveDevices( func (d *Database) RemoveAllDevices( ctx context.Context, localpart string, ) error { - return internal.WithTransaction(d.db, func(txn *sql.Tx) error { + return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { if err := d.devices.deleteDevicesByLocalpart(ctx, txn, localpart); err != sql.ErrNoRows { return err } diff --git a/clientapi/auth/storage/devices/storage.go b/clientapi/auth/storage/devices/storage.go index c132598bd..d0d203427 100644 --- a/clientapi/auth/storage/devices/storage.go +++ b/clientapi/auth/storage/devices/storage.go @@ -21,13 +21,13 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/storage/devices/postgres" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices/sqlite3" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) // NewDatabase opens a new Postgres or Sqlite database (based on dataSourceName scheme) // and sets postgres connection parameters -func NewDatabase(dataSourceName string, dbProperties internal.DbProperties, serverName gomatrixserverlib.ServerName) (Database, error) { +func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties, serverName gomatrixserverlib.ServerName) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return postgres.NewDatabase(dataSourceName, dbProperties, serverName) diff --git a/clientapi/auth/storage/devices/storage_wasm.go b/clientapi/auth/storage/devices/storage_wasm.go index 14c19e74b..e32471d8c 100644 --- a/clientapi/auth/storage/devices/storage_wasm.go +++ b/clientapi/auth/storage/devices/storage_wasm.go @@ -19,13 +19,13 @@ import ( "net/url" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices/sqlite3" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) func NewDatabase( dataSourceName string, - dbProperties internal.DbProperties, // nolint:unparam + dbProperties sqlutil.DbProperties, // nolint:unparam serverName gomatrixserverlib.ServerName, ) (Database, error) { uri, err := url.Parse(dataSourceName) diff --git a/clientapi/producers/syncapi.go b/clientapi/producers/syncapi.go index 375b1eee4..6ab8eef28 100644 --- a/clientapi/producers/syncapi.go +++ b/clientapi/producers/syncapi.go @@ -18,7 +18,7 @@ import ( "encoding/json" "github.com/Shopify/sarama" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/eventutil" log "github.com/sirupsen/logrus" ) @@ -32,7 +32,7 @@ type SyncAPIProducer struct { func (p *SyncAPIProducer) SendData(userID string, roomID string, dataType string) error { var m sarama.ProducerMessage - data := internal.AccountData{ + data := eventutil.AccountData{ RoomID: roomID, Type: dataType, } diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index 89f49d356..fd91a1060 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -30,8 +30,8 @@ import ( "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/threepid" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" log "github.com/sirupsen/logrus" @@ -98,7 +98,7 @@ func (r createRoomRequest) Validate() *util.JSONResponse { // Validate creation_content fields defined in the spec by marshalling the // creation_content map into bytes and then unmarshalling the bytes into - // internal.CreateContent. + // eventutil.CreateContent. creationContentBytes, err := json.Marshal(r.CreationContent) if err != nil { @@ -279,25 +279,25 @@ func createRoom( eventsToMake := []fledglingEvent{ {"m.room.create", "", r.CreationContent}, {"m.room.member", userID, membershipContent}, - {"m.room.power_levels", "", internal.InitialPowerLevelsContent(userID)}, + {"m.room.power_levels", "", eventutil.InitialPowerLevelsContent(userID)}, {"m.room.join_rules", "", gomatrixserverlib.JoinRuleContent{JoinRule: joinRules}}, - {"m.room.history_visibility", "", internal.HistoryVisibilityContent{HistoryVisibility: historyVisibility}}, + {"m.room.history_visibility", "", eventutil.HistoryVisibilityContent{HistoryVisibility: historyVisibility}}, } if roomAlias != "" { // TODO: bit of a chicken and egg problem here as the alias doesn't exist and cannot until we have made the room. // This means we might fail creating the alias but say the canonical alias is something that doesn't exist. // m.room.aliases is handled when we call roomserver.SetRoomAlias - eventsToMake = append(eventsToMake, fledglingEvent{"m.room.canonical_alias", "", internal.CanonicalAlias{Alias: roomAlias}}) + eventsToMake = append(eventsToMake, fledglingEvent{"m.room.canonical_alias", "", eventutil.CanonicalAlias{Alias: roomAlias}}) } if r.GuestCanJoin { - eventsToMake = append(eventsToMake, fledglingEvent{"m.room.guest_access", "", internal.GuestAccessContent{GuestAccess: "can_join"}}) + eventsToMake = append(eventsToMake, fledglingEvent{"m.room.guest_access", "", eventutil.GuestAccessContent{GuestAccess: "can_join"}}) } eventsToMake = append(eventsToMake, r.InitialState...) if r.Name != "" { - eventsToMake = append(eventsToMake, fledglingEvent{"m.room.name", "", internal.NameContent{Name: r.Name}}) + eventsToMake = append(eventsToMake, fledglingEvent{"m.room.name", "", eventutil.NameContent{Name: r.Name}}) } if r.Topic != "" { - eventsToMake = append(eventsToMake, fledglingEvent{"m.room.topic", "", internal.TopicContent{Topic: r.Topic}}) + eventsToMake = append(eventsToMake, fledglingEvent{"m.room.topic", "", eventutil.TopicContent{Topic: r.Topic}}) } // TODO: invite events // TODO: 3pid invite events diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index 0b2c2bc5e..53484a1a6 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -26,8 +26,8 @@ import ( "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/threepid" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/roomserver/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" @@ -95,7 +95,7 @@ func SendMembership( Code: http.StatusBadRequest, JSON: jsonerror.BadJSON(err.Error()), } - } else if err == internal.ErrRoomNoExists { + } else if err == eventutil.ErrRoomNoExists { return util.JSONResponse{ Code: http.StatusNotFound, JSON: jsonerror.NotFound(err.Error()), @@ -188,7 +188,7 @@ func buildMembershipEvent( return nil, err } - return internal.BuildEvent(ctx, &builder, cfg, evTime, rsAPI, nil) + return eventutil.BuildEvent(ctx, &builder, cfg, evTime, rsAPI, nil) } // loadProfile lookups the profile of a given user from the database and returns @@ -268,7 +268,7 @@ func checkAndProcessThreepid( Code: http.StatusBadRequest, JSON: jsonerror.NotTrusted(body.IDServer), } - } else if err == internal.ErrRoomNoExists { + } else if err == eventutil.ErrRoomNoExists { return inviteStored, &util.JSONResponse{ Code: http.StatusNotFound, JSON: jsonerror.NotFound(err.Error()), diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index 642a72887..a7a82ed58 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -24,8 +24,8 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" @@ -42,7 +42,7 @@ func GetProfile( ) util.JSONResponse { profile, err := getProfile(req.Context(), accountDB, cfg, userID, asAPI, federation) if err != nil { - if err == internal.ErrProfileNoExists { + if err == eventutil.ErrProfileNoExists { return util.JSONResponse{ Code: http.StatusNotFound, JSON: jsonerror.NotFound("The user does not exist or does not have a profile"), @@ -55,7 +55,7 @@ func GetProfile( return util.JSONResponse{ Code: http.StatusOK, - JSON: internal.ProfileResponse{ + JSON: eventutil.ProfileResponse{ AvatarURL: profile.AvatarURL, DisplayName: profile.DisplayName, }, @@ -70,7 +70,7 @@ func GetAvatarURL( ) util.JSONResponse { profile, err := getProfile(req.Context(), accountDB, cfg, userID, asAPI, federation) if err != nil { - if err == internal.ErrProfileNoExists { + if err == eventutil.ErrProfileNoExists { return util.JSONResponse{ Code: http.StatusNotFound, JSON: jsonerror.NotFound("The user does not exist or does not have a profile"), @@ -83,7 +83,7 @@ func GetAvatarURL( return util.JSONResponse{ Code: http.StatusOK, - JSON: internal.AvatarURL{ + JSON: eventutil.AvatarURL{ AvatarURL: profile.AvatarURL, }, } @@ -102,7 +102,7 @@ func SetAvatarURL( } } - var r internal.AvatarURL + var r eventutil.AvatarURL if resErr := httputil.UnmarshalJSONRequest(req, &r); resErr != nil { return *resErr } @@ -184,7 +184,7 @@ func GetDisplayName( ) util.JSONResponse { profile, err := getProfile(req.Context(), accountDB, cfg, userID, asAPI, federation) if err != nil { - if err == internal.ErrProfileNoExists { + if err == eventutil.ErrProfileNoExists { return util.JSONResponse{ Code: http.StatusNotFound, JSON: jsonerror.NotFound("The user does not exist or does not have a profile"), @@ -197,7 +197,7 @@ func GetDisplayName( return util.JSONResponse{ Code: http.StatusOK, - JSON: internal.DisplayName{ + JSON: eventutil.DisplayName{ DisplayName: profile.DisplayName, }, } @@ -216,7 +216,7 @@ func SetDisplayName( } } - var r internal.DisplayName + var r eventutil.DisplayName if resErr := httputil.UnmarshalJSONRequest(req, &r); resErr != nil { return *resErr } @@ -293,7 +293,7 @@ func SetDisplayName( // getProfile gets the full profile of a user by querying the database or a // remote homeserver. // Returns an error when something goes wrong or specifically -// internal.ErrProfileNoExists when the profile doesn't exist. +// eventutil.ErrProfileNoExists when the profile doesn't exist. func getProfile( ctx context.Context, accountDB accounts.Database, cfg *config.Dendrite, userID string, @@ -310,7 +310,7 @@ func getProfile( if fedErr != nil { if x, ok := fedErr.(gomatrix.HTTPError); ok { if x.Code == http.StatusNotFound { - return nil, internal.ErrProfileNoExists + return nil, eventutil.ErrProfileNoExists } } @@ -365,7 +365,7 @@ func buildMembershipEvents( return nil, err } - event, err := internal.BuildEvent(ctx, &builder, cfg, evTime, rsAPI, nil) + event, err := eventutil.BuildEvent(ctx, &builder, cfg, evTime, rsAPI, nil) if err != nil { return nil, err } diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index d356db2cb..8988dbd05 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -33,6 +33,8 @@ import ( "time" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/eventutil" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/clientapi/auth" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" @@ -41,7 +43,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/userutil" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/tokens" "github.com/matrix-org/util" @@ -136,7 +137,7 @@ type registerRequest struct { DeviceID *string `json:"device_id"` // Prevent this user from logging in - InhibitLogin internal.WeakBoolean `json:"inhibit_login"` + InhibitLogin eventutil.WeakBoolean `json:"inhibit_login"` // Application Services place Type in the root of their registration // request, whereas clients place it in the authDict struct. @@ -811,7 +812,7 @@ func completeRegistration( accountDB accounts.Database, deviceDB devices.Database, username, password, appserviceID string, - inhibitLogin internal.WeakBoolean, + inhibitLogin eventutil.WeakBoolean, displayName, deviceID *string, ) util.JSONResponse { if username == "" { @@ -830,7 +831,7 @@ func completeRegistration( acc, err := accountDB.CreateAccount(ctx, username, password, appserviceID) if err != nil { - if errors.Is(err, internal.ErrUserExists) { // user already exists + if errors.Is(err, sqlutil.ErrUserExists) { // user already exists return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.UserInUse("Desired user ID is already taken."), diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 024707759..f6aff0f0b 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -29,8 +29,8 @@ import ( "github.com/matrix-org/dendrite/clientapi/producers" eduServerAPI "github.com/matrix-org/dendrite/eduserver/api" federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/transactions" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" @@ -62,7 +62,7 @@ func Setup( ) { publicAPIMux.Handle("/client/versions", - internal.MakeExternalAPI("versions", func(req *http.Request) util.JSONResponse { + httputil.MakeExternalAPI("versions", func(req *http.Request) util.JSONResponse { return util.JSONResponse{ Code: http.StatusOK, JSON: struct { @@ -88,13 +88,13 @@ func Setup( } r0mux.Handle("/createRoom", - internal.MakeAuthAPI("createRoom", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("createRoom", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return CreateRoom(req, device, cfg, accountDB, rsAPI, asAPI) }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/join/{roomIDOrAlias}", - internal.MakeAuthAPI(gomatrixserverlib.Join, authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI(gomatrixserverlib.Join, authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -104,13 +104,13 @@ func Setup( }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/joined_rooms", - internal.MakeAuthAPI("joined_rooms", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("joined_rooms", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return GetJoinedRooms(req, device, accountDB) }), ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/leave", - internal.MakeAuthAPI("membership", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("membership", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -120,8 +120,8 @@ func Setup( }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/{membership:(?:join|kick|ban|unban|invite)}", - internal.MakeAuthAPI("membership", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("membership", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -129,8 +129,8 @@ func Setup( }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/send/{eventType}", - internal.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -138,8 +138,8 @@ func Setup( }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/send/{eventType}/{txnID}", - internal.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -149,8 +149,8 @@ func Setup( }), ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/event/{eventID}", - internal.MakeAuthAPI("rooms_get_event", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("rooms_get_event", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -158,24 +158,24 @@ func Setup( }), ).Methods(http.MethodGet, http.MethodOptions) - r0mux.Handle("/rooms/{roomID}/state", internal.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + r0mux.Handle("/rooms/{roomID}/state", httputil.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } return OnIncomingStateRequest(req.Context(), rsAPI, vars["roomID"]) })).Methods(http.MethodGet, http.MethodOptions) - r0mux.Handle("/rooms/{roomID}/state/{type}", internal.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + r0mux.Handle("/rooms/{roomID}/state/{type}", httputil.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } return OnIncomingStateTypeRequest(req.Context(), rsAPI, vars["roomID"], vars["type"], "") })).Methods(http.MethodGet, http.MethodOptions) - r0mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", internal.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + r0mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", httputil.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -183,8 +183,8 @@ func Setup( })).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/state/{eventType:[^/]+/?}", - internal.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -199,8 +199,8 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/state/{eventType}/{stateKey}", - internal.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -209,21 +209,21 @@ func Setup( }), ).Methods(http.MethodPut, http.MethodOptions) - r0mux.Handle("/register", internal.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse { + r0mux.Handle("/register", httputil.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse { return Register(req, accountDB, deviceDB, cfg) })).Methods(http.MethodPost, http.MethodOptions) - v1mux.Handle("/register", internal.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse { + v1mux.Handle("/register", httputil.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse { return LegacyRegister(req, accountDB, deviceDB, cfg) })).Methods(http.MethodPost, http.MethodOptions) - r0mux.Handle("/register/available", internal.MakeExternalAPI("registerAvailable", func(req *http.Request) util.JSONResponse { + r0mux.Handle("/register/available", httputil.MakeExternalAPI("registerAvailable", func(req *http.Request) util.JSONResponse { return RegisterAvailable(req, cfg, accountDB) })).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/directory/room/{roomAlias}", - internal.MakeExternalAPI("directory_room", func(req *http.Request) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeExternalAPI("directory_room", func(req *http.Request) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -232,8 +232,8 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/directory/room/{roomAlias}", - internal.MakeAuthAPI("directory_room", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("directory_room", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -242,8 +242,8 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/directory/room/{roomAlias}", - internal.MakeAuthAPI("directory_room", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("directory_room", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -252,20 +252,20 @@ func Setup( ).Methods(http.MethodDelete, http.MethodOptions) r0mux.Handle("/logout", - internal.MakeAuthAPI("logout", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("logout", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return Logout(req, deviceDB, device) }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/logout/all", - internal.MakeAuthAPI("logout", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("logout", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return LogoutAll(req, deviceDB, device) }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/typing/{userID}", - internal.MakeAuthAPI("rooms_typing", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("rooms_typing", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -274,8 +274,8 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/sendToDevice/{eventType}/{txnID}", - internal.MakeAuthAPI("send_to_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("send_to_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -288,8 +288,8 @@ func Setup( // rather than r0. It's an exact duplicate of the above handler. // TODO: Remove this if/when sytest is fixed! unstableMux.Handle("/sendToDevice/{eventType}/{txnID}", - internal.MakeAuthAPI("send_to_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("send_to_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -299,7 +299,7 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/account/whoami", - internal.MakeAuthAPI("whoami", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("whoami", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return Whoami(req, device) }), ).Methods(http.MethodGet, http.MethodOptions) @@ -307,20 +307,20 @@ func Setup( // Stub endpoints required by Riot r0mux.Handle("/login", - internal.MakeExternalAPI("login", func(req *http.Request) util.JSONResponse { + httputil.MakeExternalAPI("login", func(req *http.Request) util.JSONResponse { return Login(req, accountDB, deviceDB, cfg) }), ).Methods(http.MethodGet, http.MethodPost, http.MethodOptions) r0mux.Handle("/auth/{authType}/fallback/web", - internal.MakeHTMLAPI("auth_fallback", func(w http.ResponseWriter, req *http.Request) *util.JSONResponse { + httputil.MakeHTMLAPI("auth_fallback", func(w http.ResponseWriter, req *http.Request) *util.JSONResponse { vars := mux.Vars(req) return AuthFallback(w, req, vars["authType"], cfg) }), ).Methods(http.MethodGet, http.MethodPost, http.MethodOptions) r0mux.Handle("/pushrules/", - internal.MakeExternalAPI("push_rules", func(req *http.Request) util.JSONResponse { + httputil.MakeExternalAPI("push_rules", func(req *http.Request) util.JSONResponse { // TODO: Implement push rules API res := json.RawMessage(`{ "global": { @@ -339,8 +339,8 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/user/{userId}/filter", - internal.MakeAuthAPI("put_filter", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("put_filter", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -349,8 +349,8 @@ func Setup( ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/user/{userId}/filter/{filterId}", - internal.MakeAuthAPI("get_filter", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("get_filter", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -361,8 +361,8 @@ func Setup( // Riot user settings r0mux.Handle("/profile/{userID}", - internal.MakeExternalAPI("profile", func(req *http.Request) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeExternalAPI("profile", func(req *http.Request) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -371,8 +371,8 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/profile/{userID}/avatar_url", - internal.MakeExternalAPI("profile_avatar_url", func(req *http.Request) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeExternalAPI("profile_avatar_url", func(req *http.Request) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -381,8 +381,8 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/profile/{userID}/avatar_url", - internal.MakeAuthAPI("profile_avatar_url", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("profile_avatar_url", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -393,8 +393,8 @@ func Setup( // PUT requests, so we need to allow this method r0mux.Handle("/profile/{userID}/displayname", - internal.MakeExternalAPI("profile_displayname", func(req *http.Request) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeExternalAPI("profile_displayname", func(req *http.Request) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -403,8 +403,8 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/profile/{userID}/displayname", - internal.MakeAuthAPI("profile_displayname", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("profile_displayname", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -415,32 +415,32 @@ func Setup( // PUT requests, so we need to allow this method r0mux.Handle("/account/3pid", - internal.MakeAuthAPI("account_3pid", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("account_3pid", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return GetAssociated3PIDs(req, accountDB, device) }), ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/account/3pid", - internal.MakeAuthAPI("account_3pid", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("account_3pid", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return CheckAndSave3PIDAssociation(req, accountDB, device, cfg) }), ).Methods(http.MethodPost, http.MethodOptions) unstableMux.Handle("/account/3pid/delete", - internal.MakeAuthAPI("account_3pid", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("account_3pid", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return Forget3PID(req, accountDB) }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/{path:(?:account/3pid|register)}/email/requestToken", - internal.MakeExternalAPI("account_3pid_request_token", func(req *http.Request) util.JSONResponse { + httputil.MakeExternalAPI("account_3pid_request_token", func(req *http.Request) util.JSONResponse { return RequestEmailToken(req, accountDB, cfg) }), ).Methods(http.MethodPost, http.MethodOptions) // Riot logs get flooded unless this is handled r0mux.Handle("/presence/{userID}/status", - internal.MakeExternalAPI("presence", func(req *http.Request) util.JSONResponse { + httputil.MakeExternalAPI("presence", func(req *http.Request) util.JSONResponse { // TODO: Set presence (probably the responsibility of a presence server not clientapi) return util.JSONResponse{ Code: http.StatusOK, @@ -450,13 +450,13 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/voip/turnServer", - internal.MakeAuthAPI("turn_server", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("turn_server", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return RequestTurnServer(req, device, cfg) }), ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/thirdparty/protocols", - internal.MakeExternalAPI("thirdparty_protocols", func(req *http.Request) util.JSONResponse { + httputil.MakeExternalAPI("thirdparty_protocols", func(req *http.Request) util.JSONResponse { // TODO: Return the third party protcols return util.JSONResponse{ Code: http.StatusOK, @@ -466,7 +466,7 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/initialSync", - internal.MakeExternalAPI("rooms_initial_sync", func(req *http.Request) util.JSONResponse { + httputil.MakeExternalAPI("rooms_initial_sync", func(req *http.Request) util.JSONResponse { // TODO: Allow people to peek into rooms. return util.JSONResponse{ Code: http.StatusForbidden, @@ -476,8 +476,8 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/user/{userID}/account_data/{type}", - internal.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -486,8 +486,8 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/user/{userID}/rooms/{roomID}/account_data/{type}", - internal.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -496,8 +496,8 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/user/{userID}/account_data/{type}", - internal.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -506,8 +506,8 @@ func Setup( ).Methods(http.MethodGet) r0mux.Handle("/user/{userID}/rooms/{roomID}/account_data/{type}", - internal.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -516,8 +516,8 @@ func Setup( ).Methods(http.MethodGet) r0mux.Handle("/rooms/{roomID}/members", - internal.MakeAuthAPI("rooms_members", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("rooms_members", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -526,8 +526,8 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/joined_members", - internal.MakeAuthAPI("rooms_members", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("rooms_members", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -536,21 +536,21 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/read_markers", - internal.MakeExternalAPI("rooms_read_markers", func(req *http.Request) util.JSONResponse { + httputil.MakeExternalAPI("rooms_read_markers", func(req *http.Request) util.JSONResponse { // TODO: return the read_markers. return util.JSONResponse{Code: http.StatusOK, JSON: struct{}{}} }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/devices", - internal.MakeAuthAPI("get_devices", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("get_devices", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return GetDevicesByLocalpart(req, deviceDB, device) }), ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/devices/{deviceID}", - internal.MakeAuthAPI("get_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("get_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -559,8 +559,8 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/devices/{deviceID}", - internal.MakeAuthAPI("device_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("device_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -569,8 +569,8 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/devices/{deviceID}", - internal.MakeAuthAPI("delete_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("delete_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -579,14 +579,14 @@ func Setup( ).Methods(http.MethodDelete, http.MethodOptions) r0mux.Handle("/delete_devices", - internal.MakeAuthAPI("delete_devices", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("delete_devices", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return DeleteDevices(req, deviceDB, device) }), ).Methods(http.MethodPost, http.MethodOptions) // Stub implementations for sytest r0mux.Handle("/events", - internal.MakeExternalAPI("events", func(req *http.Request) util.JSONResponse { + httputil.MakeExternalAPI("events", func(req *http.Request) util.JSONResponse { return util.JSONResponse{Code: http.StatusOK, JSON: map[string]interface{}{ "chunk": []interface{}{}, "start": "", @@ -596,7 +596,7 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/initialSync", - internal.MakeExternalAPI("initial_sync", func(req *http.Request) util.JSONResponse { + httputil.MakeExternalAPI("initial_sync", func(req *http.Request) util.JSONResponse { return util.JSONResponse{Code: http.StatusOK, JSON: map[string]interface{}{ "end": "", }} @@ -604,8 +604,8 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/user/{userId}/rooms/{roomId}/tags", - internal.MakeAuthAPI("get_tags", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("get_tags", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -614,8 +614,8 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/user/{userId}/rooms/{roomId}/tags/{tag}", - internal.MakeAuthAPI("put_tag", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("put_tag", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -624,8 +624,8 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/user/{userId}/rooms/{roomId}/tags/{tag}", - internal.MakeAuthAPI("delete_tag", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("delete_tag", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -634,7 +634,7 @@ func Setup( ).Methods(http.MethodDelete, http.MethodOptions) r0mux.Handle("/capabilities", - internal.MakeAuthAPI("capabilities", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("capabilities", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return GetCapabilities(req, rsAPI) }), ).Methods(http.MethodGet) diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go index 5d5507e85..77a370778 100644 --- a/clientapi/routing/sendevent.go +++ b/clientapi/routing/sendevent.go @@ -20,8 +20,8 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/internal/transactions" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" @@ -146,8 +146,8 @@ func generateSendEvent( } var queryRes api.QueryLatestEventsAndStateResponse - e, err := internal.BuildEvent(req.Context(), &builder, cfg, evTime, rsAPI, &queryRes) - if err == internal.ErrRoomNoExists { + e, err := eventutil.BuildEvent(req.Context(), &builder, cfg, evTime, rsAPI, &queryRes) + if err == eventutil.ErrRoomNoExists { return nil, &util.JSONResponse{ Code: http.StatusNotFound, JSON: jsonerror.NotFound("Room does not exist"), @@ -158,7 +158,7 @@ func generateSendEvent( JSON: jsonerror.BadJSON(e.Error()), } } else if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("internal.BuildEvent failed") + util.GetLogger(req.Context()).WithError(err).Error("eventutil.BuildEvent failed") resErr := jsonerror.InternalServerError() return nil, &resErr } diff --git a/clientapi/threepid/invites.go b/clientapi/threepid/invites.go index b0df0dd4d..11bf965d4 100644 --- a/clientapi/threepid/invites.go +++ b/clientapi/threepid/invites.go @@ -26,8 +26,8 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" ) @@ -353,7 +353,7 @@ func emit3PIDInviteEvent( } queryRes := api.QueryLatestEventsAndStateResponse{} - event, err := internal.BuildEvent(ctx, builder, cfg, evTime, rsAPI, &queryRes) + event, err := eventutil.BuildEvent(ctx, builder, cfg, evTime, rsAPI, &queryRes) if err != nil { return err } diff --git a/cmd/dendrite-appservice-server/main.go b/cmd/dendrite-appservice-server/main.go index fe00a117d..ec68940af 100644 --- a/cmd/dendrite-appservice-server/main.go +++ b/cmd/dendrite-appservice-server/main.go @@ -16,12 +16,12 @@ package main import ( "github.com/matrix-org/dendrite/appservice" - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" ) func main() { - cfg := basecomponent.ParseFlags(false) - base := basecomponent.NewBaseDendrite(cfg, "AppServiceAPI", true) + cfg := setup.ParseFlags(false) + base := setup.NewBaseDendrite(cfg, "AppServiceAPI", true) defer base.Close() // nolint: errcheck accountDB := base.CreateAccountsDB() diff --git a/cmd/dendrite-client-api-server/main.go b/cmd/dendrite-client-api-server/main.go index 9cf57ef68..8ed18c99b 100644 --- a/cmd/dendrite-client-api-server/main.go +++ b/cmd/dendrite-client-api-server/main.go @@ -16,14 +16,14 @@ package main import ( "github.com/matrix-org/dendrite/clientapi" - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/internal/transactions" ) func main() { - cfg := basecomponent.ParseFlags(false) + cfg := setup.ParseFlags(false) - base := basecomponent.NewBaseDendrite(cfg, "ClientAPI", true) + base := setup.NewBaseDendrite(cfg, "ClientAPI", true) defer base.Close() // nolint: errcheck accountDB := base.CreateAccountsDB() diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index b7cbcdc9c..f215606e8 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -32,8 +32,8 @@ import ( "github.com/matrix-org/dendrite/cmd/dendrite-demo-libp2p/storage" "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/federationsender" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/serverkeyapi" @@ -172,7 +172,7 @@ func main() { } monolith.AddAllPublicRoutes(base.Base.PublicAPIMux) - internal.SetupHTTPAPI( + httputil.SetupHTTPAPI( http.DefaultServeMux, base.Base.PublicAPIMux, base.Base.InternalAPIMux, diff --git a/cmd/dendrite-demo-libp2p/p2pdendrite.go b/cmd/dendrite-demo-libp2p/p2pdendrite.go index b7c5c66b0..4270143f5 100644 --- a/cmd/dendrite-demo-libp2p/p2pdendrite.go +++ b/cmd/dendrite-demo-libp2p/p2pdendrite.go @@ -22,7 +22,7 @@ import ( pstore "github.com/libp2p/go-libp2p-core/peerstore" record "github.com/libp2p/go-libp2p-record" - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" "github.com/libp2p/go-libp2p" circuit "github.com/libp2p/go-libp2p-circuit" @@ -39,7 +39,7 @@ import ( // P2PDendrite is a Peer-to-Peer variant of BaseDendrite. type P2PDendrite struct { - Base basecomponent.BaseDendrite + Base setup.BaseDendrite // Store our libp2p object so that we can make outgoing connections from it // later @@ -54,7 +54,7 @@ type P2PDendrite struct { // The componentName is used for logging purposes, and should be a friendly name // of the component running, e.g. SyncAPI. func NewP2PDendrite(cfg *config.Dendrite, componentName string) *P2PDendrite { - baseDendrite := basecomponent.NewBaseDendrite(cfg, componentName, false) + baseDendrite := setup.NewBaseDendrite(cfg, componentName, false) ctx, cancel := context.WithCancel(context.Background()) diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index 6ef56d325..e4d4fe9e6 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -34,9 +34,8 @@ import ( "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/federationsender" - "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/dendrite/internal/basecomponent" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/roomserver" @@ -61,7 +60,7 @@ func (y *yggroundtripper) RoundTrip(req *http.Request) (*http.Response, error) { } func createFederationClient( - base *basecomponent.BaseDendrite, n *yggconn.Node, + base *setup.BaseDendrite, n *yggconn.Node, ) *gomatrixserverlib.FederationClient { yggdialer := func(_, address string) (net.Conn, error) { tokens := strings.Split(address, ":") @@ -135,7 +134,7 @@ func main() { panic(err) } - base := basecomponent.NewBaseDendrite(cfg, "Monolith", false) + base := setup.NewBaseDendrite(cfg, "Monolith", false) defer base.Close() // nolint: errcheck accountDB := base.CreateAccountsDB() @@ -188,7 +187,7 @@ func main() { } monolith.AddAllPublicRoutes(base.PublicAPIMux) - internal.SetupHTTPAPI( + httputil.SetupHTTPAPI( http.DefaultServeMux, base.PublicAPIMux, base.InternalAPIMux, diff --git a/cmd/dendrite-edu-server/main.go b/cmd/dendrite-edu-server/main.go index a86f66c89..1ecce884d 100644 --- a/cmd/dendrite-edu-server/main.go +++ b/cmd/dendrite-edu-server/main.go @@ -17,13 +17,13 @@ import ( "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/eduserver/cache" - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" "github.com/sirupsen/logrus" ) func main() { - cfg := basecomponent.ParseFlags(false) - base := basecomponent.NewBaseDendrite(cfg, "EDUServerAPI", true) + cfg := setup.ParseFlags(false) + base := setup.NewBaseDendrite(cfg, "EDUServerAPI", true) defer func() { if err := base.Close(); err != nil { logrus.WithError(err).Warn("BaseDendrite close failed") diff --git a/cmd/dendrite-federation-api-server/main.go b/cmd/dendrite-federation-api-server/main.go index fcdc000c4..b8db7927a 100644 --- a/cmd/dendrite-federation-api-server/main.go +++ b/cmd/dendrite-federation-api-server/main.go @@ -16,12 +16,12 @@ package main import ( "github.com/matrix-org/dendrite/federationapi" - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" ) func main() { - cfg := basecomponent.ParseFlags(false) - base := basecomponent.NewBaseDendrite(cfg, "FederationAPI", true) + cfg := setup.ParseFlags(false) + base := setup.NewBaseDendrite(cfg, "FederationAPI", true) defer base.Close() // nolint: errcheck accountDB := base.CreateAccountsDB() diff --git a/cmd/dendrite-federation-sender-server/main.go b/cmd/dendrite-federation-sender-server/main.go index 7b60ad055..20bc1070f 100644 --- a/cmd/dendrite-federation-sender-server/main.go +++ b/cmd/dendrite-federation-sender-server/main.go @@ -16,12 +16,12 @@ package main import ( "github.com/matrix-org/dendrite/federationsender" - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" ) func main() { - cfg := basecomponent.ParseFlags(false) - base := basecomponent.NewBaseDendrite(cfg, "FederationSender", true) + cfg := setup.ParseFlags(false) + base := setup.NewBaseDendrite(cfg, "FederationSender", true) defer base.Close() // nolint: errcheck federation := base.CreateFederationClient() diff --git a/cmd/dendrite-key-server/main.go b/cmd/dendrite-key-server/main.go index 5dccc3004..06629d39a 100644 --- a/cmd/dendrite-key-server/main.go +++ b/cmd/dendrite-key-server/main.go @@ -15,13 +15,13 @@ package main import ( - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/keyserver" ) func main() { - cfg := basecomponent.ParseFlags(false) - base := basecomponent.NewBaseDendrite(cfg, "KeyServer", true) + cfg := setup.ParseFlags(false) + base := setup.NewBaseDendrite(cfg, "KeyServer", true) defer base.Close() // nolint: errcheck accountDB := base.CreateAccountsDB() diff --git a/cmd/dendrite-media-api-server/main.go b/cmd/dendrite-media-api-server/main.go index 5c65fad05..52c760273 100644 --- a/cmd/dendrite-media-api-server/main.go +++ b/cmd/dendrite-media-api-server/main.go @@ -15,13 +15,13 @@ package main import ( - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/mediaapi" ) func main() { - cfg := basecomponent.ParseFlags(false) - base := basecomponent.NewBaseDendrite(cfg, "MediaAPI", true) + cfg := setup.ParseFlags(false) + base := setup.NewBaseDendrite(cfg, "MediaAPI", true) defer base.Close() // nolint: errcheck deviceDB := base.CreateDeviceDB() diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 2d538027e..ea8160b84 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -23,9 +23,8 @@ import ( "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/federationsender" - "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/dendrite/internal/basecomponent" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/roomserver" @@ -45,7 +44,7 @@ var ( ) func main() { - cfg := basecomponent.ParseFlags(true) + cfg := setup.ParseFlags(true) if *enableHTTPAPIs { // If the HTTP APIs are enabled then we need to update the Listen // statements in the configuration so that we know where to find @@ -59,7 +58,7 @@ func main() { cfg.Listen.ServerKeyAPI = addr } - base := basecomponent.NewBaseDendrite(cfg, "Monolith", *enableHTTPAPIs) + base := setup.NewBaseDendrite(cfg, "Monolith", *enableHTTPAPIs) defer base.Close() // nolint: errcheck accountDB := base.CreateAccountsDB() @@ -135,7 +134,7 @@ func main() { } monolith.AddAllPublicRoutes(base.PublicAPIMux) - internal.SetupHTTPAPI( + httputil.SetupHTTPAPI( http.DefaultServeMux, base.PublicAPIMux, base.InternalAPIMux, @@ -147,7 +146,7 @@ func main() { go func() { serv := http.Server{ Addr: *httpBindAddr, - WriteTimeout: basecomponent.HTTPServerTimeout, + WriteTimeout: setup.HTTPServerTimeout, } logrus.Info("Listening on ", serv.Addr) @@ -158,7 +157,7 @@ func main() { go func() { serv := http.Server{ Addr: *httpsBindAddr, - WriteTimeout: basecomponent.HTTPServerTimeout, + WriteTimeout: setup.HTTPServerTimeout, } logrus.Info("Listening on ", serv.Addr) diff --git a/cmd/dendrite-public-rooms-api-server/main.go b/cmd/dendrite-public-rooms-api-server/main.go index ff7d7958f..3ba45dc6d 100644 --- a/cmd/dendrite-public-rooms-api-server/main.go +++ b/cmd/dendrite-public-rooms-api-server/main.go @@ -15,15 +15,15 @@ package main import ( - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/publicroomsapi" "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/sirupsen/logrus" ) func main() { - cfg := basecomponent.ParseFlags(false) - base := basecomponent.NewBaseDendrite(cfg, "PublicRoomsAPI", true) + cfg := setup.ParseFlags(false) + base := setup.NewBaseDendrite(cfg, "PublicRoomsAPI", true) defer base.Close() // nolint: errcheck deviceDB := base.CreateDeviceDB() diff --git a/cmd/dendrite-room-server/main.go b/cmd/dendrite-room-server/main.go index a2f1e9415..627a68677 100644 --- a/cmd/dendrite-room-server/main.go +++ b/cmd/dendrite-room-server/main.go @@ -15,13 +15,13 @@ package main import ( - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/roomserver" ) func main() { - cfg := basecomponent.ParseFlags(false) - base := basecomponent.NewBaseDendrite(cfg, "RoomServerAPI", true) + cfg := setup.ParseFlags(false) + base := setup.NewBaseDendrite(cfg, "RoomServerAPI", true) defer base.Close() // nolint: errcheck federation := base.CreateFederationClient() diff --git a/cmd/dendrite-server-key-api-server/main.go b/cmd/dendrite-server-key-api-server/main.go index b4bfcbffb..9ffaeee31 100644 --- a/cmd/dendrite-server-key-api-server/main.go +++ b/cmd/dendrite-server-key-api-server/main.go @@ -15,13 +15,13 @@ package main import ( - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/serverkeyapi" ) func main() { - cfg := basecomponent.ParseFlags(false) - base := basecomponent.NewBaseDendrite(cfg, "ServerKeyAPI", true) + cfg := setup.ParseFlags(false) + base := setup.NewBaseDendrite(cfg, "ServerKeyAPI", true) defer base.Close() // nolint: errcheck federation := base.CreateFederationClient() diff --git a/cmd/dendrite-sync-api-server/main.go b/cmd/dendrite-sync-api-server/main.go index a5302f74e..41e796802 100644 --- a/cmd/dendrite-sync-api-server/main.go +++ b/cmd/dendrite-sync-api-server/main.go @@ -15,13 +15,13 @@ package main import ( - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/syncapi" ) func main() { - cfg := basecomponent.ParseFlags(false) - base := basecomponent.NewBaseDendrite(cfg, "SyncAPI", true) + cfg := setup.ParseFlags(false) + base := setup.NewBaseDendrite(cfg, "SyncAPI", true) defer base.Close() // nolint: errcheck deviceDB := base.CreateDeviceDB() diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 0512ee5c9..70672f4df 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -26,9 +26,8 @@ import ( "github.com/matrix-org/dendrite/eduserver" "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/federationsender" - "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/dendrite/internal/basecomponent" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/roomserver" @@ -184,7 +183,7 @@ func main() { if err := cfg.Derive(); err != nil { logrus.Fatalf("Failed to derive values from config: %s", err) } - base := basecomponent.NewBaseDendrite(cfg, "Monolith", false) + base := setup.NewBaseDendrite(cfg, "Monolith", false) defer base.Close() // nolint: errcheck accountDB := base.CreateAccountsDB() @@ -233,7 +232,7 @@ func main() { } monolith.AddAllPublicRoutes(base.PublicAPIMux) - internal.SetupHTTPAPI( + httputil.SetupHTTPAPI( http.DefaultServeMux, base.PublicAPIMux, base.InternalAPIMux, diff --git a/eduserver/eduserver.go b/eduserver/eduserver.go index c14be3f67..aa65ff239 100644 --- a/eduserver/eduserver.go +++ b/eduserver/eduserver.go @@ -23,7 +23,7 @@ import ( "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/eduserver/input" "github.com/matrix-org/dendrite/eduserver/inthttp" - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" ) // AddInternalRoutes registers HTTP handlers for the internal API. Invokes functions @@ -35,7 +35,7 @@ func AddInternalRoutes(internalMux *mux.Router, inputAPI api.EDUServerInputAPI) // NewInternalAPI returns a concerete implementation of the internal API. Callers // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. func NewInternalAPI( - base *basecomponent.BaseDendrite, + base *setup.BaseDendrite, eduCache *cache.EDUCache, deviceDB devices.Database, ) api.EDUServerInputAPI { diff --git a/eduserver/inthttp/client.go b/eduserver/inthttp/client.go index 149e4fb04..7d0bc1603 100644 --- a/eduserver/inthttp/client.go +++ b/eduserver/inthttp/client.go @@ -6,7 +6,7 @@ import ( "net/http" "github.com/matrix-org/dendrite/eduserver/api" - internalHTTP "github.com/matrix-org/dendrite/internal/http" + "github.com/matrix-org/dendrite/internal/httputil" "github.com/opentracing/opentracing-go" ) @@ -39,7 +39,7 @@ func (h *httpEDUServerInputAPI) InputTypingEvent( defer span.Finish() apiURL := h.eduServerURL + EDUServerInputTypingEventPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // InputSendToDeviceEvent implements EDUServerInputAPI @@ -52,5 +52,5 @@ func (h *httpEDUServerInputAPI) InputSendToDeviceEvent( defer span.Finish() apiURL := h.eduServerURL + EDUServerInputSendToDeviceEventPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } diff --git a/eduserver/inthttp/server.go b/eduserver/inthttp/server.go index 6c7d21a4e..e374513a3 100644 --- a/eduserver/inthttp/server.go +++ b/eduserver/inthttp/server.go @@ -6,14 +6,14 @@ import ( "github.com/gorilla/mux" "github.com/matrix-org/dendrite/eduserver/api" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/util" ) // AddRoutes adds the EDUServerInputAPI handlers to the http.ServeMux. func AddRoutes(t api.EDUServerInputAPI, internalAPIMux *mux.Router) { internalAPIMux.Handle(EDUServerInputTypingEventPath, - internal.MakeInternalAPI("inputTypingEvents", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("inputTypingEvents", func(req *http.Request) util.JSONResponse { var request api.InputTypingEventRequest var response api.InputTypingEventResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -26,7 +26,7 @@ func AddRoutes(t api.EDUServerInputAPI, internalAPIMux *mux.Router) { }), ) internalAPIMux.Handle(EDUServerInputSendToDeviceEventPath, - internal.MakeInternalAPI("inputSendToDeviceEvents", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("inputSendToDeviceEvents", func(req *http.Request) util.JSONResponse { var request api.InputSendToDeviceEventRequest var response api.InputSendToDeviceEventResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go index 593aa169f..e01f077a3 100644 --- a/federationapi/routing/join.go +++ b/federationapi/routing/join.go @@ -21,8 +21,8 @@ import ( "time" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -95,8 +95,8 @@ func MakeJoin( queryRes := api.QueryLatestEventsAndStateResponse{ RoomVersion: verRes.RoomVersion, } - event, err := internal.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), rsAPI, &queryRes) - if err == internal.ErrRoomNoExists { + event, err := eventutil.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), rsAPI, &queryRes) + if err == eventutil.ErrRoomNoExists { return util.JSONResponse{ Code: http.StatusNotFound, JSON: jsonerror.NotFound("Room does not exist"), @@ -107,7 +107,7 @@ func MakeJoin( JSON: jsonerror.BadJSON(e.Error()), } } else if err != nil { - util.GetLogger(httpReq.Context()).WithError(err).Error("internal.BuildEvent failed") + util.GetLogger(httpReq.Context()).WithError(err).Error("eventutil.BuildEvent failed") return jsonerror.InternalServerError() } diff --git a/federationapi/routing/leave.go b/federationapi/routing/leave.go index f998be45a..de15c32ad 100644 --- a/federationapi/routing/leave.go +++ b/federationapi/routing/leave.go @@ -17,8 +17,8 @@ import ( "time" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -69,8 +69,8 @@ func MakeLeave( } var queryRes api.QueryLatestEventsAndStateResponse - event, err := internal.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), rsAPI, &queryRes) - if err == internal.ErrRoomNoExists { + event, err := eventutil.BuildEvent(httpReq.Context(), &builder, cfg, time.Now(), rsAPI, &queryRes) + if err == eventutil.ErrRoomNoExists { return util.JSONResponse{ Code: http.StatusNotFound, JSON: jsonerror.NotFound("Room does not exist"), @@ -81,7 +81,7 @@ func MakeLeave( JSON: jsonerror.BadJSON(e.Error()), } } else if err != nil { - util.GetLogger(httpReq.Context()).WithError(err).Error("internal.BuildEvent failed") + util.GetLogger(httpReq.Context()).WithError(err).Error("eventutil.BuildEvent failed") return jsonerror.InternalServerError() } diff --git a/federationapi/routing/profile.go b/federationapi/routing/profile.go index 832849845..61d0682bd 100644 --- a/federationapi/routing/profile.go +++ b/federationapi/routing/profile.go @@ -21,8 +21,8 @@ import ( appserviceAPI "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -72,11 +72,11 @@ func GetProfile( if field != "" { switch field { case "displayname": - res = internal.DisplayName{ + res = eventutil.DisplayName{ DisplayName: profile.DisplayName, } case "avatar_url": - res = internal.AvatarURL{ + res = eventutil.AvatarURL{ AvatarURL: profile.AvatarURL, } default: @@ -84,7 +84,7 @@ func GetProfile( res = jsonerror.InvalidArgumentValue("The request body did not contain an allowed value of argument 'field'. Allowed values are either: 'avatar_url', 'displayname'.") } } else { - res = internal.ProfileResponse{ + res = eventutil.ProfileResponse{ AvatarURL: profile.AvatarURL, DisplayName: profile.DisplayName, } diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index 754dcdfb0..70b77a4c3 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -23,8 +23,8 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" eduserverAPI "github.com/matrix-org/dendrite/eduserver/api" federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/httputil" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -60,11 +60,11 @@ func Setup( v1fedmux := publicAPIMux.PathPrefix(pathPrefixV1Federation).Subrouter() v2fedmux := publicAPIMux.PathPrefix(pathPrefixV2Federation).Subrouter() - wakeup := &internal.FederationWakeups{ + wakeup := &httputil.FederationWakeups{ FsAPI: fsAPI, } - localKeys := internal.MakeExternalAPI("localkeys", func(req *http.Request) util.JSONResponse { + localKeys := httputil.MakeExternalAPI("localkeys", func(req *http.Request) util.JSONResponse { return LocalKeys(cfg) }) @@ -76,7 +76,7 @@ func Setup( v2keysmux.Handle("/server/", localKeys).Methods(http.MethodGet) v2keysmux.Handle("/server", localKeys).Methods(http.MethodGet) - v1fedmux.Handle("/send/{txnID}", internal.MakeFedAPI( + v1fedmux.Handle("/send/{txnID}", httputil.MakeFedAPI( "federation_send", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return Send( @@ -86,7 +86,7 @@ func Setup( }, )).Methods(http.MethodPut, http.MethodOptions) - v2fedmux.Handle("/invite/{roomID}/{eventID}", internal.MakeFedAPI( + v2fedmux.Handle("/invite/{roomID}/{eventID}", httputil.MakeFedAPI( "federation_invite", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return Invite( @@ -96,13 +96,13 @@ func Setup( }, )).Methods(http.MethodPut, http.MethodOptions) - v1fedmux.Handle("/3pid/onbind", internal.MakeExternalAPI("3pid_onbind", + v1fedmux.Handle("/3pid/onbind", httputil.MakeExternalAPI("3pid_onbind", func(req *http.Request) util.JSONResponse { return CreateInvitesFrom3PIDInvites(req, rsAPI, asAPI, cfg, federation, accountDB) }, )).Methods(http.MethodPost, http.MethodOptions) - v1fedmux.Handle("/exchange_third_party_invite/{roomID}", internal.MakeFedAPI( + v1fedmux.Handle("/exchange_third_party_invite/{roomID}", httputil.MakeFedAPI( "exchange_third_party_invite", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return ExchangeThirdPartyInvite( @@ -111,7 +111,7 @@ func Setup( }, )).Methods(http.MethodPut, http.MethodOptions) - v1fedmux.Handle("/event/{eventID}", internal.MakeFedAPI( + v1fedmux.Handle("/event/{eventID}", httputil.MakeFedAPI( "federation_get_event", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return GetEvent( @@ -120,7 +120,7 @@ func Setup( }, )).Methods(http.MethodGet) - v1fedmux.Handle("/state/{roomID}", internal.MakeFedAPI( + v1fedmux.Handle("/state/{roomID}", httputil.MakeFedAPI( "federation_get_state", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return GetState( @@ -129,7 +129,7 @@ func Setup( }, )).Methods(http.MethodGet) - v1fedmux.Handle("/state_ids/{roomID}", internal.MakeFedAPI( + v1fedmux.Handle("/state_ids/{roomID}", httputil.MakeFedAPI( "federation_get_state_ids", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return GetStateIDs( @@ -138,7 +138,7 @@ func Setup( }, )).Methods(http.MethodGet) - v1fedmux.Handle("/event_auth/{roomID}/{eventID}", internal.MakeFedAPI( + v1fedmux.Handle("/event_auth/{roomID}/{eventID}", httputil.MakeFedAPI( "federation_get_event_auth", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return GetEventAuth( @@ -147,7 +147,7 @@ func Setup( }, )).Methods(http.MethodGet) - v1fedmux.Handle("/query/directory", internal.MakeFedAPI( + v1fedmux.Handle("/query/directory", httputil.MakeFedAPI( "federation_query_room_alias", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return RoomAliasToID( @@ -156,7 +156,7 @@ func Setup( }, )).Methods(http.MethodGet) - v1fedmux.Handle("/query/profile", internal.MakeFedAPI( + v1fedmux.Handle("/query/profile", httputil.MakeFedAPI( "federation_query_profile", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return GetProfile( @@ -165,7 +165,7 @@ func Setup( }, )).Methods(http.MethodGet) - v1fedmux.Handle("/user/devices/{userID}", internal.MakeFedAPI( + v1fedmux.Handle("/user/devices/{userID}", httputil.MakeFedAPI( "federation_user_devices", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return GetUserDevices( @@ -174,7 +174,7 @@ func Setup( }, )).Methods(http.MethodGet) - v1fedmux.Handle("/make_join/{roomID}/{eventID}", internal.MakeFedAPI( + v1fedmux.Handle("/make_join/{roomID}/{eventID}", httputil.MakeFedAPI( "federation_make_join", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { roomID := vars["roomID"] @@ -199,7 +199,7 @@ func Setup( }, )).Methods(http.MethodGet) - v1fedmux.Handle("/send_join/{roomID}/{eventID}", internal.MakeFedAPI( + v1fedmux.Handle("/send_join/{roomID}/{eventID}", httputil.MakeFedAPI( "federation_send_join", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { roomID := vars["roomID"] @@ -217,7 +217,7 @@ func Setup( }, )).Methods(http.MethodPut) - v2fedmux.Handle("/send_join/{roomID}/{eventID}", internal.MakeFedAPI( + v2fedmux.Handle("/send_join/{roomID}/{eventID}", httputil.MakeFedAPI( "federation_send_join", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { roomID := vars["roomID"] @@ -228,7 +228,7 @@ func Setup( }, )).Methods(http.MethodPut) - v1fedmux.Handle("/make_leave/{roomID}/{eventID}", internal.MakeFedAPI( + v1fedmux.Handle("/make_leave/{roomID}/{eventID}", httputil.MakeFedAPI( "federation_make_leave", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { roomID := vars["roomID"] @@ -239,7 +239,7 @@ func Setup( }, )).Methods(http.MethodGet) - v2fedmux.Handle("/send_leave/{roomID}/{eventID}", internal.MakeFedAPI( + v2fedmux.Handle("/send_leave/{roomID}/{eventID}", httputil.MakeFedAPI( "federation_send_leave", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { roomID := vars["roomID"] @@ -250,21 +250,21 @@ func Setup( }, )).Methods(http.MethodPut) - v1fedmux.Handle("/version", internal.MakeExternalAPI( + v1fedmux.Handle("/version", httputil.MakeExternalAPI( "federation_version", func(httpReq *http.Request) util.JSONResponse { return Version() }, )).Methods(http.MethodGet) - v1fedmux.Handle("/get_missing_events/{roomID}", internal.MakeFedAPI( + v1fedmux.Handle("/get_missing_events/{roomID}", httputil.MakeFedAPI( "federation_get_missing_events", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return GetMissingEvents(httpReq, request, rsAPI, vars["roomID"]) }, )).Methods(http.MethodPost) - v1fedmux.Handle("/backfill/{roomID}", internal.MakeFedAPI( + v1fedmux.Handle("/backfill/{roomID}", httputil.MakeFedAPI( "federation_backfill", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return Backfill(httpReq, request, rsAPI, vars["roomID"], cfg) diff --git a/federationsender/federationsender.go b/federationsender/federationsender.go index 621920ef5..10ac51c8a 100644 --- a/federationsender/federationsender.go +++ b/federationsender/federationsender.go @@ -23,7 +23,7 @@ import ( "github.com/matrix-org/dendrite/federationsender/queue" "github.com/matrix-org/dendrite/federationsender/storage" "github.com/matrix-org/dendrite/federationsender/types" - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" @@ -38,7 +38,7 @@ func AddInternalRoutes(router *mux.Router, intAPI api.FederationSenderInternalAP // NewInternalAPI returns a concerete implementation of the internal API. Callers // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. func NewInternalAPI( - base *basecomponent.BaseDendrite, + base *setup.BaseDendrite, federation *gomatrixserverlib.FederationClient, rsAPI roomserverAPI.RoomserverInternalAPI, keyRing *gomatrixserverlib.KeyRing, diff --git a/federationsender/inthttp/client.go b/federationsender/inthttp/client.go index 1e243c607..5da4b35f9 100644 --- a/federationsender/inthttp/client.go +++ b/federationsender/inthttp/client.go @@ -6,7 +6,7 @@ import ( "net/http" "github.com/matrix-org/dendrite/federationsender/api" - internalHTTP "github.com/matrix-org/dendrite/internal/http" + "github.com/matrix-org/dendrite/internal/httputil" "github.com/opentracing/opentracing-go" ) @@ -44,7 +44,7 @@ func (h *httpFederationSenderInternalAPI) PerformLeave( defer span.Finish() apiURL := h.federationSenderURL + FederationSenderPerformLeaveRequestPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } func (h *httpFederationSenderInternalAPI) PerformServersAlive( @@ -56,7 +56,7 @@ func (h *httpFederationSenderInternalAPI) PerformServersAlive( defer span.Finish() apiURL := h.federationSenderURL + FederationSenderPerformServersAlivePath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // QueryJoinedHostServerNamesInRoom implements FederationSenderInternalAPI @@ -69,7 +69,7 @@ func (h *httpFederationSenderInternalAPI) QueryJoinedHostServerNamesInRoom( defer span.Finish() apiURL := h.federationSenderURL + FederationSenderQueryJoinedHostServerNamesInRoomPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // Handle an instruction to make_join & send_join with a remote server. @@ -82,7 +82,7 @@ func (h *httpFederationSenderInternalAPI) PerformJoin( defer span.Finish() apiURL := h.federationSenderURL + FederationSenderPerformJoinRequestPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // Handle an instruction to make_join & send_join with a remote server. @@ -95,5 +95,5 @@ func (h *httpFederationSenderInternalAPI) PerformDirectoryLookup( defer span.Finish() apiURL := h.federationSenderURL + FederationSenderPerformDirectoryLookupRequestPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } diff --git a/federationsender/inthttp/server.go b/federationsender/inthttp/server.go index a9076661d..babd3ae13 100644 --- a/federationsender/inthttp/server.go +++ b/federationsender/inthttp/server.go @@ -6,7 +6,7 @@ import ( "github.com/gorilla/mux" "github.com/matrix-org/dendrite/federationsender/api" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/util" ) @@ -14,7 +14,7 @@ import ( func AddRoutes(intAPI api.FederationSenderInternalAPI, internalAPIMux *mux.Router) { internalAPIMux.Handle( FederationSenderQueryJoinedHostServerNamesInRoomPath, - internal.MakeInternalAPI("QueryJoinedHostServerNamesInRoom", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("QueryJoinedHostServerNamesInRoom", func(req *http.Request) util.JSONResponse { var request api.QueryJoinedHostServerNamesInRoomRequest var response api.QueryJoinedHostServerNamesInRoomResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -27,7 +27,7 @@ func AddRoutes(intAPI api.FederationSenderInternalAPI, internalAPIMux *mux.Route }), ) internalAPIMux.Handle(FederationSenderPerformJoinRequestPath, - internal.MakeInternalAPI("PerformJoinRequest", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("PerformJoinRequest", func(req *http.Request) util.JSONResponse { var request api.PerformJoinRequest var response api.PerformJoinResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -40,7 +40,7 @@ func AddRoutes(intAPI api.FederationSenderInternalAPI, internalAPIMux *mux.Route }), ) internalAPIMux.Handle(FederationSenderPerformLeaveRequestPath, - internal.MakeInternalAPI("PerformLeaveRequest", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("PerformLeaveRequest", func(req *http.Request) util.JSONResponse { var request api.PerformLeaveRequest var response api.PerformLeaveResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -53,7 +53,7 @@ func AddRoutes(intAPI api.FederationSenderInternalAPI, internalAPIMux *mux.Route }), ) internalAPIMux.Handle(FederationSenderPerformDirectoryLookupRequestPath, - internal.MakeInternalAPI("PerformDirectoryLookupRequest", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("PerformDirectoryLookupRequest", func(req *http.Request) util.JSONResponse { var request api.PerformDirectoryLookupRequest var response api.PerformDirectoryLookupResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -66,7 +66,7 @@ func AddRoutes(intAPI api.FederationSenderInternalAPI, internalAPIMux *mux.Route }), ) internalAPIMux.Handle(FederationSenderPerformServersAlivePath, - internal.MakeInternalAPI("PerformServersAliveRequest", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("PerformServersAliveRequest", func(req *http.Request) util.JSONResponse { var request api.PerformServersAliveRequest var response api.PerformServersAliveResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { diff --git a/federationsender/storage/postgres/joined_hosts_table.go b/federationsender/storage/postgres/joined_hosts_table.go index 1b898f485..c0f9a7d5b 100644 --- a/federationsender/storage/postgres/joined_hosts_table.go +++ b/federationsender/storage/postgres/joined_hosts_table.go @@ -22,6 +22,7 @@ import ( "github.com/lib/pq" "github.com/matrix-org/dendrite/federationsender/types" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) @@ -85,7 +86,7 @@ func (s *joinedHostsStatements) insertJoinedHosts( roomID, eventID string, serverName gomatrixserverlib.ServerName, ) error { - stmt := internal.TxStmt(txn, s.insertJoinedHostsStmt) + stmt := sqlutil.TxStmt(txn, s.insertJoinedHostsStmt) _, err := stmt.ExecContext(ctx, roomID, eventID, serverName) return err } @@ -93,7 +94,7 @@ func (s *joinedHostsStatements) insertJoinedHosts( func (s *joinedHostsStatements) deleteJoinedHosts( ctx context.Context, txn *sql.Tx, eventIDs []string, ) error { - stmt := internal.TxStmt(txn, s.deleteJoinedHostsStmt) + stmt := sqlutil.TxStmt(txn, s.deleteJoinedHostsStmt) _, err := stmt.ExecContext(ctx, pq.StringArray(eventIDs)) return err } @@ -101,7 +102,7 @@ func (s *joinedHostsStatements) deleteJoinedHosts( func (s *joinedHostsStatements) selectJoinedHostsWithTx( ctx context.Context, txn *sql.Tx, roomID string, ) ([]types.JoinedHost, error) { - stmt := internal.TxStmt(txn, s.selectJoinedHostsStmt) + stmt := sqlutil.TxStmt(txn, s.selectJoinedHostsStmt) return joinedHostsFromStmt(ctx, stmt, roomID) } diff --git a/federationsender/storage/postgres/room_table.go b/federationsender/storage/postgres/room_table.go index c945475ba..e5266c635 100644 --- a/federationsender/storage/postgres/room_table.go +++ b/federationsender/storage/postgres/room_table.go @@ -19,7 +19,7 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" ) const roomSchema = ` @@ -71,7 +71,7 @@ func (s *roomStatements) prepare(db *sql.DB) (err error) { func (s *roomStatements) insertRoom( ctx context.Context, txn *sql.Tx, roomID string, ) error { - _, err := internal.TxStmt(txn, s.insertRoomStmt).ExecContext(ctx, roomID) + _, err := sqlutil.TxStmt(txn, s.insertRoomStmt).ExecContext(ctx, roomID) return err } @@ -82,7 +82,7 @@ func (s *roomStatements) selectRoomForUpdate( ctx context.Context, txn *sql.Tx, roomID string, ) (string, error) { var lastEventID string - stmt := internal.TxStmt(txn, s.selectRoomForUpdateStmt) + stmt := sqlutil.TxStmt(txn, s.selectRoomForUpdateStmt) err := stmt.QueryRowContext(ctx, roomID).Scan(&lastEventID) if err != nil { return "", err @@ -95,7 +95,7 @@ func (s *roomStatements) selectRoomForUpdate( func (s *roomStatements) updateRoom( ctx context.Context, txn *sql.Tx, roomID, lastEventID string, ) error { - stmt := internal.TxStmt(txn, s.updateRoomStmt) + stmt := sqlutil.TxStmt(txn, s.updateRoomStmt) _, err := stmt.ExecContext(ctx, roomID, lastEventID) return err } diff --git a/federationsender/storage/postgres/storage.go b/federationsender/storage/postgres/storage.go index a4733a20c..8fd4c11a3 100644 --- a/federationsender/storage/postgres/storage.go +++ b/federationsender/storage/postgres/storage.go @@ -20,7 +20,6 @@ import ( "database/sql" "github.com/matrix-org/dendrite/federationsender/types" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" ) @@ -28,12 +27,12 @@ import ( type Database struct { joinedHostsStatements roomStatements - internal.PartitionOffsetStatements + sqlutil.PartitionOffsetStatements db *sql.DB } // NewDatabase opens a new database -func NewDatabase(dataSourceName string, dbProperties internal.DbProperties) (*Database, error) { +func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties) (*Database, error) { var result Database var err error if result.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { @@ -70,7 +69,7 @@ func (d *Database) UpdateRoom( addHosts []types.JoinedHost, removeHosts []string, ) (joinedHosts []types.JoinedHost, err error) { - err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { + err = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { err = d.insertRoom(ctx, txn, roomID) if err != nil { return err diff --git a/federationsender/storage/sqlite3/joined_hosts_table.go b/federationsender/storage/sqlite3/joined_hosts_table.go index 795d33208..d9824658c 100644 --- a/federationsender/storage/sqlite3/joined_hosts_table.go +++ b/federationsender/storage/sqlite3/joined_hosts_table.go @@ -21,6 +21,7 @@ import ( "github.com/matrix-org/dendrite/federationsender/types" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) @@ -84,7 +85,7 @@ func (s *joinedHostsStatements) insertJoinedHosts( roomID, eventID string, serverName gomatrixserverlib.ServerName, ) error { - stmt := internal.TxStmt(txn, s.insertJoinedHostsStmt) + stmt := sqlutil.TxStmt(txn, s.insertJoinedHostsStmt) _, err := stmt.ExecContext(ctx, roomID, eventID, serverName) return err } @@ -93,7 +94,7 @@ func (s *joinedHostsStatements) deleteJoinedHosts( ctx context.Context, txn *sql.Tx, eventIDs []string, ) error { for _, eventID := range eventIDs { - stmt := internal.TxStmt(txn, s.deleteJoinedHostsStmt) + stmt := sqlutil.TxStmt(txn, s.deleteJoinedHostsStmt) if _, err := stmt.ExecContext(ctx, eventID); err != nil { return err } @@ -104,7 +105,7 @@ func (s *joinedHostsStatements) deleteJoinedHosts( func (s *joinedHostsStatements) selectJoinedHostsWithTx( ctx context.Context, txn *sql.Tx, roomID string, ) ([]types.JoinedHost, error) { - stmt := internal.TxStmt(txn, s.selectJoinedHostsStmt) + stmt := sqlutil.TxStmt(txn, s.selectJoinedHostsStmt) return joinedHostsFromStmt(ctx, stmt, roomID) } diff --git a/federationsender/storage/sqlite3/room_table.go b/federationsender/storage/sqlite3/room_table.go index 4bf9ab81d..ca0c4d0b6 100644 --- a/federationsender/storage/sqlite3/room_table.go +++ b/federationsender/storage/sqlite3/room_table.go @@ -19,7 +19,7 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" ) const roomSchema = ` @@ -71,7 +71,7 @@ func (s *roomStatements) prepare(db *sql.DB) (err error) { func (s *roomStatements) insertRoom( ctx context.Context, txn *sql.Tx, roomID string, ) error { - _, err := internal.TxStmt(txn, s.insertRoomStmt).ExecContext(ctx, roomID) + _, err := sqlutil.TxStmt(txn, s.insertRoomStmt).ExecContext(ctx, roomID) return err } @@ -82,7 +82,7 @@ func (s *roomStatements) selectRoomForUpdate( ctx context.Context, txn *sql.Tx, roomID string, ) (string, error) { var lastEventID string - stmt := internal.TxStmt(txn, s.selectRoomForUpdateStmt) + stmt := sqlutil.TxStmt(txn, s.selectRoomForUpdateStmt) err := stmt.QueryRowContext(ctx, roomID).Scan(&lastEventID) if err != nil { return "", err @@ -95,7 +95,7 @@ func (s *roomStatements) selectRoomForUpdate( func (s *roomStatements) updateRoom( ctx context.Context, txn *sql.Tx, roomID, lastEventID string, ) error { - stmt := internal.TxStmt(txn, s.updateRoomStmt) + stmt := sqlutil.TxStmt(txn, s.updateRoomStmt) _, err := stmt.ExecContext(ctx, roomID, lastEventID) return err } diff --git a/federationsender/storage/sqlite3/storage.go b/federationsender/storage/sqlite3/storage.go index 8699fc19a..ac303f646 100644 --- a/federationsender/storage/sqlite3/storage.go +++ b/federationsender/storage/sqlite3/storage.go @@ -22,7 +22,6 @@ import ( _ "github.com/mattn/go-sqlite3" "github.com/matrix-org/dendrite/federationsender/types" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" ) @@ -30,7 +29,7 @@ import ( type Database struct { joinedHostsStatements roomStatements - internal.PartitionOffsetStatements + sqlutil.PartitionOffsetStatements db *sql.DB } @@ -42,7 +41,7 @@ func NewDatabase(dataSourceName string) (*Database, error) { if err != nil { return nil, err } - if result.db, err = sqlutil.Open(internal.SQLiteDriverName(), cs, nil); err != nil { + if result.db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil); err != nil { return nil, err } if err = result.prepare(); err != nil { @@ -76,7 +75,7 @@ func (d *Database) UpdateRoom( addHosts []types.JoinedHost, removeHosts []string, ) (joinedHosts []types.JoinedHost, err error) { - err = internal.WithTransaction(d.db, func(txn *sql.Tx) error { + err = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { err = d.insertRoom(ctx, txn, roomID) if err != nil { return err diff --git a/federationsender/storage/storage.go b/federationsender/storage/storage.go index df8433eba..d37360056 100644 --- a/federationsender/storage/storage.go +++ b/federationsender/storage/storage.go @@ -21,11 +21,11 @@ import ( "github.com/matrix-org/dendrite/federationsender/storage/postgres" "github.com/matrix-org/dendrite/federationsender/storage/sqlite3" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" ) // NewDatabase opens a new database -func NewDatabase(dataSourceName string, dbProperties internal.DbProperties) (Database, error) { +func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return postgres.NewDatabase(dataSourceName, dbProperties) diff --git a/federationsender/storage/storage_wasm.go b/federationsender/storage/storage_wasm.go index f593fd448..e5c8f293b 100644 --- a/federationsender/storage/storage_wasm.go +++ b/federationsender/storage/storage_wasm.go @@ -19,13 +19,13 @@ import ( "net/url" "github.com/matrix-org/dendrite/federationsender/storage/sqlite3" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" ) // NewDatabase opens a new database func NewDatabase( dataSourceName string, - dbProperties internal.DbProperties, // nolint:unparam + dbProperties sqlutil.DbProperties, // nolint:unparam ) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { diff --git a/go.mod b/go.mod index a05bb71e2..2f7874082 100644 --- a/go.mod +++ b/go.mod @@ -38,6 +38,7 @@ require ( github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20200530233943-aec82d7a391b go.uber.org/atomic v1.4.0 golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d + golang.org/x/tools v0.0.0-20200612022331-742c5eb664c2 // indirect gopkg.in/h2non/bimg.v1 v1.0.18 gopkg.in/yaml.v2 v2.2.8 ) diff --git a/go.sum b/go.sum index 00d25a3f6..301066b90 100644 --- a/go.sum +++ b/go.sum @@ -570,6 +570,7 @@ github.com/yggdrasil-network/yggdrasil-go v0.3.14 h1:vWzYzCQxOruS+J5FkLfXOS0JhCJ github.com/yggdrasil-network/yggdrasil-go v0.3.14/go.mod h1:rkQzLzVHlFdzsEMG+bDdTI+KeWPCZq1HpXRFzwinf6M= github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20200530233943-aec82d7a391b h1:ELOisSxFXCcptRs4LFub+Hz5fYUvV12wZrTps99Eb3E= github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20200530233943-aec82d7a391b/go.mod h1:d+Nz6SPeG6kmeSPFL0cvfWfgwEql75fUnZiAONgvyBE= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -603,6 +604,8 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -670,7 +673,11 @@ golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd h1:/e+gpKk9r3dJobndpTytxS2gOy6m5uvpg+ISQoEcusQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200612022331-742c5eb664c2 h1:DVqHa33CzfnTKwUV6be+I4hp31W6iXn3ZiEcdKGzLyI= +golang.org/x/tools v0.0.0-20200612022331-742c5eb664c2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= diff --git a/internal/consumers.go b/internal/consumers.go index df68cbfaa..d7917f235 100644 --- a/internal/consumers.go +++ b/internal/consumers.go @@ -19,20 +19,13 @@ import ( "fmt" "github.com/Shopify/sarama" + "github.com/matrix-org/dendrite/internal/sqlutil" ) -// A PartitionOffset is the offset into a partition of the input log. -type PartitionOffset struct { - // The ID of the partition. - Partition int32 - // The offset into the partition. - Offset int64 -} - // A PartitionStorer has the storage APIs needed by the consumer. type PartitionStorer interface { // PartitionOffsets returns the offsets the consumer has reached for each partition. - PartitionOffsets(ctx context.Context, topic string) ([]PartitionOffset, error) + PartitionOffsets(ctx context.Context, topic string) ([]sqlutil.PartitionOffset, error) // SetPartitionOffset records where the consumer has reached for a partition. SetPartitionOffset(ctx context.Context, topic string, partition int32, offset int64) error } diff --git a/internal/eventcontent.go b/internal/eventutil/eventcontent.go similarity index 97% rename from internal/eventcontent.go rename to internal/eventutil/eventcontent.go index 645128369..873e20a8e 100644 --- a/internal/eventcontent.go +++ b/internal/eventutil/eventcontent.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package internal +package eventutil import "github.com/matrix-org/gomatrixserverlib" diff --git a/internal/events.go b/internal/eventutil/events.go similarity index 98% rename from internal/events.go rename to internal/eventutil/events.go index 89c82e03b..e6c7a4ff7 100644 --- a/internal/events.go +++ b/internal/eventutil/events.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package internal +package eventutil import ( "context" diff --git a/internal/types.go b/internal/eventutil/types.go similarity index 96% rename from internal/types.go rename to internal/eventutil/types.go index be2717f39..6d119ce6d 100644 --- a/internal/types.go +++ b/internal/eventutil/types.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package internal +package eventutil import ( "errors" diff --git a/internal/httpapis/paths.go b/internal/httpapis/paths.go deleted file mode 100644 index 8adec2dff..000000000 --- a/internal/httpapis/paths.go +++ /dev/null @@ -1,6 +0,0 @@ -package httpapis - -const ( - PublicPathPrefix = "/_matrix/" - InternalPathPrefix = "/api/" -) diff --git a/internal/http/http.go b/internal/httputil/http.go similarity index 69% rename from internal/http/http.go rename to internal/httputil/http.go index 2b1891403..9197371aa 100644 --- a/internal/http/http.go +++ b/internal/httputil/http.go @@ -1,4 +1,18 @@ -package http +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package httputil import ( "bytes" @@ -9,7 +23,6 @@ import ( "net/url" "strings" - "github.com/matrix-org/dendrite/internal/httpapis" opentracing "github.com/opentracing/opentracing-go" "github.com/opentracing/opentracing-go/ext" ) @@ -29,7 +42,7 @@ func PostJSON( return err } - parsedAPIURL.Path = httpapis.InternalPathPrefix + strings.TrimLeft(parsedAPIURL.Path, "/") + parsedAPIURL.Path = InternalPathPrefix + strings.TrimLeft(parsedAPIURL.Path, "/") apiURL = parsedAPIURL.String() req, err := http.NewRequest(http.MethodPost, apiURL, bytes.NewReader(jsonBytes)) diff --git a/internal/httpapi.go b/internal/httputil/httpapi.go similarity index 92% rename from internal/httpapi.go rename to internal/httputil/httpapi.go index 991a98619..0a37f06c2 100644 --- a/internal/httpapi.go +++ b/internal/httputil/httpapi.go @@ -1,4 +1,18 @@ -package internal +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package httputil import ( "context" @@ -16,7 +30,6 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" federationsenderAPI "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/internal/config" - "github.com/matrix-org/dendrite/internal/httpapis" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" opentracing "github.com/opentracing/opentracing-go" @@ -227,9 +240,9 @@ func SetupHTTPAPI(servMux *http.ServeMux, publicApiMux *mux.Router, internalApiM servMux.Handle("/metrics", WrapHandlerInBasicAuth(promhttp.Handler(), cfg.Metrics.BasicAuth)) } if enableHTTPAPIs { - servMux.Handle(httpapis.InternalPathPrefix, internalApiMux) + servMux.Handle(InternalPathPrefix, internalApiMux) } - servMux.Handle(httpapis.PublicPathPrefix, WrapHandlerInCORS(publicApiMux)) + servMux.Handle(PublicPathPrefix, WrapHandlerInCORS(publicApiMux)) } // WrapHandlerInBasicAuth adds basic auth to a handler. Only used for /metrics diff --git a/internal/httpapi_test.go b/internal/httputil/httpapi_test.go similarity index 75% rename from internal/httpapi_test.go rename to internal/httputil/httpapi_test.go index 6f159c8d6..de6ccf9b4 100644 --- a/internal/httpapi_test.go +++ b/internal/httputil/httpapi_test.go @@ -1,4 +1,18 @@ -package internal +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package httputil import ( "net/http" diff --git a/internal/httputil/paths.go b/internal/httputil/paths.go new file mode 100644 index 000000000..728b5a871 --- /dev/null +++ b/internal/httputil/paths.go @@ -0,0 +1,20 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package httputil + +const ( + PublicPathPrefix = "/_matrix/" + InternalPathPrefix = "/api/" +) diff --git a/internal/routing.go b/internal/httputil/routing.go similarity index 94% rename from internal/routing.go rename to internal/httputil/routing.go index 4462c70ca..0bd3655ec 100644 --- a/internal/routing.go +++ b/internal/httputil/routing.go @@ -1,4 +1,4 @@ -// Copyright 2019 The Matrix.org Foundation C.I.C. +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package internal +package httputil import ( "net/url" diff --git a/internal/basecomponent/base.go b/internal/setup/base.go similarity index 96% rename from internal/basecomponent/base.go rename to internal/setup/base.go index 3ad1e4af2..fb304893a 100644 --- a/internal/basecomponent/base.go +++ b/internal/setup/base.go @@ -1,4 +1,4 @@ -// Copyright 2017 New Vector Ltd +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package basecomponent +package setup import ( "database/sql" @@ -23,7 +23,7 @@ import ( "time" "github.com/matrix-org/dendrite/internal/caching" - "github.com/matrix-org/dendrite/internal/httpapis" + "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/naffka" @@ -127,8 +127,8 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo tracerCloser: closer, Cfg: cfg, Caches: cache, - PublicAPIMux: httpmux.PathPrefix(httpapis.PublicPathPrefix).Subrouter().UseEncodedPath(), - InternalAPIMux: httpmux.PathPrefix(httpapis.InternalPathPrefix).Subrouter().UseEncodedPath(), + PublicAPIMux: httpmux.PathPrefix(httputil.PublicPathPrefix).Subrouter().UseEncodedPath(), + InternalAPIMux: httpmux.PathPrefix(httputil.InternalPathPrefix).Subrouter().UseEncodedPath(), httpClient: &client, KafkaConsumer: kafkaConsumer, KafkaProducer: kafkaProducer, @@ -237,7 +237,7 @@ func (b *BaseDendrite) SetupAndServeHTTP(bindaddr string, listenaddr string) { WriteTimeout: HTTPServerTimeout, } - internal.SetupHTTPAPI( + httputil.SetupHTTPAPI( http.DefaultServeMux, b.PublicAPIMux, b.InternalAPIMux, @@ -282,7 +282,7 @@ func setupNaffka(cfg *config.Dendrite) (sarama.Consumer, sarama.SyncProducer) { if err != nil { logrus.WithError(err).Panic("Failed to parse naffka database file URI") } - db, err = sqlutil.Open(internal.SQLiteDriverName(), cs, nil) + db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil) if err != nil { logrus.WithError(err).Panic("Failed to open naffka database") } diff --git a/internal/basecomponent/flags.go b/internal/setup/flags.go similarity index 94% rename from internal/basecomponent/flags.go rename to internal/setup/flags.go index 117df0796..e4fc58d60 100644 --- a/internal/basecomponent/flags.go +++ b/internal/setup/flags.go @@ -1,4 +1,4 @@ -// Copyright 2017 New Vector Ltd +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package basecomponent +package setup import ( "flag" diff --git a/internal/setup/monolith.go b/internal/setup/monolith.go index 35fcd3119..55ceffd6b 100644 --- a/internal/setup/monolith.go +++ b/internal/setup/monolith.go @@ -1,3 +1,17 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package setup import ( diff --git a/internal/partition_offset_table.go b/internal/sqlutil/partition_offset_table.go similarity index 88% rename from internal/partition_offset_table.go rename to internal/sqlutil/partition_offset_table.go index 8b72819b7..348829025 100644 --- a/internal/partition_offset_table.go +++ b/internal/sqlutil/partition_offset_table.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,14 +12,24 @@ // See the License for the specific language governing permissions and // limitations under the License. -package internal +package sqlutil import ( "context" "database/sql" "strings" + + "github.com/matrix-org/util" ) +// A PartitionOffset is the offset into a partition of the input log. +type PartitionOffset struct { + // The ID of the partition. + Partition int32 + // The offset into the partition. + Offset int64 +} + const partitionOffsetsSchema = ` -- The offsets that the server has processed up to. CREATE TABLE IF NOT EXISTS ${prefix}_partition_offsets ( @@ -90,7 +100,12 @@ func (s *PartitionOffsetStatements) selectPartitionOffsets( if err != nil { return nil, err } - defer CloseAndLogIfError(ctx, rows, "selectPartitionOffsets: rows.close() failed") + defer func() { + err2 := rows.Close() + if err2 != nil { + util.GetLogger(ctx).WithError(err2).Error("selectPartitionOffsets: rows.close() failed") + } + }() var results []PartitionOffset for rows.Next() { var offset PartitionOffset diff --git a/internal/postgres.go b/internal/sqlutil/postgres.go similarity index 98% rename from internal/postgres.go rename to internal/sqlutil/postgres.go index 7ae40d8fd..41a5508a1 100644 --- a/internal/postgres.go +++ b/internal/sqlutil/postgres.go @@ -14,7 +14,7 @@ // +build !wasm -package internal +package sqlutil import "github.com/lib/pq" diff --git a/internal/postgres_wasm.go b/internal/sqlutil/postgres_wasm.go similarity index 97% rename from internal/postgres_wasm.go rename to internal/sqlutil/postgres_wasm.go index 64d328291..c45842f0c 100644 --- a/internal/postgres_wasm.go +++ b/internal/sqlutil/postgres_wasm.go @@ -14,7 +14,7 @@ // +build wasm -package internal +package sqlutil // IsUniqueConstraintViolationErr no-ops for this architecture func IsUniqueConstraintViolationErr(err error) bool { diff --git a/internal/sql.go b/internal/sqlutil/sql.go similarity index 96% rename from internal/sql.go rename to internal/sqlutil/sql.go index e3c10afca..a25a4a5b6 100644 --- a/internal/sql.go +++ b/internal/sqlutil/sql.go @@ -1,6 +1,4 @@ -// Copyright 2017 Vector Creations Ltd -// Copyright 2017-2018 New Vector Ltd -// Copyright 2019-2020 The Matrix.org Foundation C.I.C. +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package internal +package sqlutil import ( "database/sql" diff --git a/internal/sqlutil/trace.go b/internal/sqlutil/trace.go index 1b008e1b7..f6644d591 100644 --- a/internal/sqlutil/trace.go +++ b/internal/sqlutil/trace.go @@ -25,7 +25,6 @@ import ( "strings" "time" - "github.com/matrix-org/dendrite/internal" "github.com/ngrok/sqlmw" "github.com/sirupsen/logrus" ) @@ -78,7 +77,7 @@ func (in *traceInterceptor) RowsNext(c context.Context, rows driver.Rows, dest [ // Open opens a database specified by its database driver name and a driver-specific data source name, // usually consisting of at least a database name and connection information. Includes tracing driver // if DENDRITE_TRACE_SQL=1 -func Open(driverName, dsn string, dbProperties internal.DbProperties) (*sql.DB, error) { +func Open(driverName, dsn string, dbProperties DbProperties) (*sql.DB, error) { if tracingEnabled { // install the wrapped driver driverName += "-trace" @@ -87,7 +86,7 @@ func Open(driverName, dsn string, dbProperties internal.DbProperties) (*sql.DB, if err != nil { return nil, err } - if driverName != internal.SQLiteDriverName() && dbProperties != nil { + if driverName != SQLiteDriverName() && dbProperties != nil { logrus.WithFields(logrus.Fields{ "MaxOpenConns": dbProperties.MaxOpenConns(), "MaxIdleConns": dbProperties.MaxIdleConns(), diff --git a/internal/sqlutil/uri.go b/internal/sqlutil/uri.go index f72e02426..703258e6c 100644 --- a/internal/sqlutil/uri.go +++ b/internal/sqlutil/uri.go @@ -1,3 +1,17 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package sqlutil import ( diff --git a/keyserver/routing/routing.go b/keyserver/routing/routing.go index 2d1122c62..bce3c46ba 100644 --- a/keyserver/routing/routing.go +++ b/keyserver/routing/routing.go @@ -22,8 +22,8 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/util" ) @@ -49,7 +49,7 @@ func Setup( } r0mux.Handle("/keys/query", - internal.MakeAuthAPI("queryKeys", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("queryKeys", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return QueryKeys(req) }), ).Methods(http.MethodPost, http.MethodOptions) diff --git a/mediaapi/routing/routing.go b/mediaapi/routing/routing.go index 11e4ecad5..6bcd3f552 100644 --- a/mediaapi/routing/routing.go +++ b/mediaapi/routing/routing.go @@ -22,8 +22,8 @@ import ( "github.com/gorilla/mux" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/mediaapi/storage" "github.com/matrix-org/dendrite/mediaapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -59,7 +59,7 @@ func Setup( } // TODO: Add AS support - r0mux.Handle("/upload", internal.MakeAuthAPI( + r0mux.Handle("/upload", httputil.MakeAuthAPI( "upload", authData, func(req *http.Request, _ *authtypes.Device) util.JSONResponse { return Upload(req, cfg, db, activeThumbnailGeneration) @@ -99,7 +99,7 @@ func makeDownloadAPI( util.SetCORSHeaders(w) // Content-Type will be overridden in case of returning file data, else we respond with JSON-formatted errors w.Header().Set("Content-Type", "application/json") - vars, _ := internal.URLDecodeMapValues(mux.Vars(req)) + vars, _ := httputil.URLDecodeMapValues(mux.Vars(req)) Download( w, req, diff --git a/mediaapi/storage/postgres/storage.go b/mediaapi/storage/postgres/storage.go index 72ee05cd8..e45e08416 100644 --- a/mediaapi/storage/postgres/storage.go +++ b/mediaapi/storage/postgres/storage.go @@ -21,7 +21,6 @@ import ( // Import the postgres database driver. _ "github.com/lib/pq" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/mediaapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -34,7 +33,7 @@ type Database struct { } // Open opens a postgres database. -func Open(dataSourceName string, dbProperties internal.DbProperties) (*Database, error) { +func Open(dataSourceName string, dbProperties sqlutil.DbProperties) (*Database, error) { var d Database var err error if d.db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { diff --git a/mediaapi/storage/postgres/thumbnail_table.go b/mediaapi/storage/postgres/thumbnail_table.go index dbc97cf22..3f28cdbbf 100644 --- a/mediaapi/storage/postgres/thumbnail_table.go +++ b/mediaapi/storage/postgres/thumbnail_table.go @@ -21,7 +21,6 @@ import ( "time" "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/dendrite/mediaapi/types" "github.com/matrix-org/gomatrixserverlib" ) diff --git a/mediaapi/storage/sqlite3/storage.go b/mediaapi/storage/sqlite3/storage.go index ea81f9120..010c0a66e 100644 --- a/mediaapi/storage/sqlite3/storage.go +++ b/mediaapi/storage/sqlite3/storage.go @@ -20,7 +20,6 @@ import ( "database/sql" // Import the postgres database driver. - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/mediaapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -41,7 +40,7 @@ func Open(dataSourceName string) (*Database, error) { if err != nil { return nil, err } - if d.db, err = sqlutil.Open(internal.SQLiteDriverName(), cs, nil); err != nil { + if d.db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil); err != nil { return nil, err } if err = d.statements.prepare(d.db); err != nil { diff --git a/mediaapi/storage/sqlite3/thumbnail_table.go b/mediaapi/storage/sqlite3/thumbnail_table.go index a0956f68c..432a1590c 100644 --- a/mediaapi/storage/sqlite3/thumbnail_table.go +++ b/mediaapi/storage/sqlite3/thumbnail_table.go @@ -21,7 +21,6 @@ import ( "time" "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/dendrite/mediaapi/types" "github.com/matrix-org/gomatrixserverlib" ) diff --git a/mediaapi/storage/storage.go b/mediaapi/storage/storage.go index 2694e197d..5ff114db6 100644 --- a/mediaapi/storage/storage.go +++ b/mediaapi/storage/storage.go @@ -19,13 +19,13 @@ package storage import ( "net/url" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/mediaapi/storage/postgres" "github.com/matrix-org/dendrite/mediaapi/storage/sqlite3" ) // Open opens a postgres database. -func Open(dataSourceName string, dbProperties internal.DbProperties) (Database, error) { +func Open(dataSourceName string, dbProperties sqlutil.DbProperties) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return postgres.Open(dataSourceName, dbProperties) diff --git a/mediaapi/storage/storage_wasm.go b/mediaapi/storage/storage_wasm.go index 78de2cb82..a672271f9 100644 --- a/mediaapi/storage/storage_wasm.go +++ b/mediaapi/storage/storage_wasm.go @@ -18,14 +18,14 @@ import ( "fmt" "net/url" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/mediaapi/storage/sqlite3" ) // Open opens a postgres database. func Open( dataSourceName string, - dbProperties internal.DbProperties, // nolint:unparam + dbProperties sqlutil.DbProperties, // nolint:unparam ) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { diff --git a/publicroomsapi/routing/routing.go b/publicroomsapi/routing/routing.go index ee1434392..2da555f98 100644 --- a/publicroomsapi/routing/routing.go +++ b/publicroomsapi/routing/routing.go @@ -17,13 +17,13 @@ package routing import ( "net/http" + "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/roomserver/api" "github.com/gorilla/mux" "github.com/matrix-org/dendrite/clientapi/auth" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/publicroomsapi/directory" "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/publicroomsapi/types" @@ -51,8 +51,8 @@ func Setup( } r0mux.Handle("/directory/list/room/{roomID}", - internal.MakeExternalAPI("directory_list", func(req *http.Request) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeExternalAPI("directory_list", func(req *http.Request) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -61,8 +61,8 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) // TODO: Add AS support r0mux.Handle("/directory/list/room/{roomID}", - internal.MakeAuthAPI("directory_list", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + httputil.MakeAuthAPI("directory_list", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } @@ -70,7 +70,7 @@ func Setup( }), ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/publicRooms", - internal.MakeExternalAPI("public_rooms", func(req *http.Request) util.JSONResponse { + httputil.MakeExternalAPI("public_rooms", func(req *http.Request) util.JSONResponse { if extRoomsProvider != nil { return directory.GetPostPublicRoomsWithExternal(req, publicRoomsDB, fedClient, extRoomsProvider) } @@ -80,7 +80,7 @@ func Setup( // Federation - TODO: should this live here or in federation API? It's sure easier if it's here so here it is. publicAPIMux.Handle("/federation/v1/publicRooms", - internal.MakeExternalAPI("federation_public_rooms", func(req *http.Request) util.JSONResponse { + httputil.MakeExternalAPI("federation_public_rooms", func(req *http.Request) util.JSONResponse { return directory.GetPostPublicRooms(req, publicRoomsDB) }), ).Methods(http.MethodGet) diff --git a/publicroomsapi/storage/postgres/storage.go b/publicroomsapi/storage/postgres/storage.go index 5f0f551a7..36c6aec64 100644 --- a/publicroomsapi/storage/postgres/storage.go +++ b/publicroomsapi/storage/postgres/storage.go @@ -20,7 +20,7 @@ import ( "database/sql" "encoding/json" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" @@ -29,7 +29,7 @@ import ( // PublicRoomsServerDatabase represents a public rooms server database. type PublicRoomsServerDatabase struct { db *sql.DB - internal.PartitionOffsetStatements + sqlutil.PartitionOffsetStatements statements publicRoomsStatements localServerName gomatrixserverlib.ServerName } @@ -37,7 +37,7 @@ type PublicRoomsServerDatabase struct { type attributeValue interface{} // NewPublicRoomsServerDatabase creates a new public rooms server database. -func NewPublicRoomsServerDatabase(dataSourceName string, dbProperties internal.DbProperties, localServerName gomatrixserverlib.ServerName) (*PublicRoomsServerDatabase, error) { +func NewPublicRoomsServerDatabase(dataSourceName string, dbProperties sqlutil.DbProperties, localServerName gomatrixserverlib.ServerName) (*PublicRoomsServerDatabase, error) { var db *sql.DB var err error if db, err = sqlutil.Open("postgres", dataSourceName, dbProperties); err != nil { @@ -138,33 +138,33 @@ func (d *PublicRoomsServerDatabase) UpdateRoomFromEvent( case "m.room.aliases": return d.updateRoomAliases(ctx, event) case "m.room.canonical_alias": - var content internal.CanonicalAliasContent + var content eventutil.CanonicalAliasContent field := &(content.Alias) attrName := "canonical_alias" return d.updateStringAttribute(ctx, attrName, event, &content, field) case "m.room.name": - var content internal.NameContent + var content eventutil.NameContent field := &(content.Name) attrName := "name" return d.updateStringAttribute(ctx, attrName, event, &content, field) case "m.room.topic": - var content internal.TopicContent + var content eventutil.TopicContent field := &(content.Topic) attrName := "topic" return d.updateStringAttribute(ctx, attrName, event, &content, field) case "m.room.avatar": - var content internal.AvatarContent + var content eventutil.AvatarContent field := &(content.URL) attrName := "avatar_url" return d.updateStringAttribute(ctx, attrName, event, &content, field) case "m.room.history_visibility": - var content internal.HistoryVisibilityContent + var content eventutil.HistoryVisibilityContent field := &(content.HistoryVisibility) attrName := "world_readable" strForTrue := "world_readable" return d.updateBooleanAttribute(ctx, attrName, event, &content, field, strForTrue) case "m.room.guest_access": - var content internal.GuestAccessContent + var content eventutil.GuestAccessContent field := &(content.GuestAccess) attrName := "guest_can_join" strForTrue := "can_join" @@ -248,7 +248,7 @@ func (d *PublicRoomsServerDatabase) updateRoomAliases( if aliasesEvent.StateKey() == nil || *aliasesEvent.StateKey() != string(d.localServerName) { return nil // only store our own aliases } - var content internal.AliasesContent + var content eventutil.AliasesContent if err := json.Unmarshal(aliasesEvent.Content(), &content); err != nil { return err } diff --git a/publicroomsapi/storage/sqlite3/storage.go b/publicroomsapi/storage/sqlite3/storage.go index f51051823..5c685d131 100644 --- a/publicroomsapi/storage/sqlite3/storage.go +++ b/publicroomsapi/storage/sqlite3/storage.go @@ -22,7 +22,7 @@ import ( _ "github.com/mattn/go-sqlite3" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" @@ -31,7 +31,7 @@ import ( // PublicRoomsServerDatabase represents a public rooms server database. type PublicRoomsServerDatabase struct { db *sql.DB - internal.PartitionOffsetStatements + sqlutil.PartitionOffsetStatements statements publicRoomsStatements localServerName gomatrixserverlib.ServerName } @@ -46,7 +46,7 @@ func NewPublicRoomsServerDatabase(dataSourceName string, localServerName gomatri if err != nil { return nil, err } - if db, err = sqlutil.Open(internal.SQLiteDriverName(), cs, nil); err != nil { + if db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil); err != nil { return nil, err } storage := PublicRoomsServerDatabase{ @@ -144,33 +144,33 @@ func (d *PublicRoomsServerDatabase) UpdateRoomFromEvent( case "m.room.aliases": return d.updateRoomAliases(ctx, event) case "m.room.canonical_alias": - var content internal.CanonicalAliasContent + var content eventutil.CanonicalAliasContent field := &(content.Alias) attrName := "canonical_alias" return d.updateStringAttribute(ctx, attrName, event, &content, field) case "m.room.name": - var content internal.NameContent + var content eventutil.NameContent field := &(content.Name) attrName := "name" return d.updateStringAttribute(ctx, attrName, event, &content, field) case "m.room.topic": - var content internal.TopicContent + var content eventutil.TopicContent field := &(content.Topic) attrName := "topic" return d.updateStringAttribute(ctx, attrName, event, &content, field) case "m.room.avatar": - var content internal.AvatarContent + var content eventutil.AvatarContent field := &(content.URL) attrName := "avatar_url" return d.updateStringAttribute(ctx, attrName, event, &content, field) case "m.room.history_visibility": - var content internal.HistoryVisibilityContent + var content eventutil.HistoryVisibilityContent field := &(content.HistoryVisibility) attrName := "world_readable" strForTrue := "world_readable" return d.updateBooleanAttribute(ctx, attrName, event, &content, field, strForTrue) case "m.room.guest_access": - var content internal.GuestAccessContent + var content eventutil.GuestAccessContent field := &(content.GuestAccess) attrName := "guest_can_join" strForTrue := "can_join" @@ -254,7 +254,7 @@ func (d *PublicRoomsServerDatabase) updateRoomAliases( if aliasesEvent.StateKey() == nil || *aliasesEvent.StateKey() != string(d.localServerName) { return nil // only store our own aliases } - var content internal.AliasesContent + var content eventutil.AliasesContent if err := json.Unmarshal(aliasesEvent.Content(), &content); err != nil { return err } diff --git a/publicroomsapi/storage/storage.go b/publicroomsapi/storage/storage.go index 507b9fa6c..f66188040 100644 --- a/publicroomsapi/storage/storage.go +++ b/publicroomsapi/storage/storage.go @@ -19,7 +19,7 @@ package storage import ( "net/url" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/publicroomsapi/storage/postgres" "github.com/matrix-org/dendrite/publicroomsapi/storage/sqlite3" "github.com/matrix-org/gomatrixserverlib" @@ -29,7 +29,7 @@ const schemePostgres = "postgres" const schemeFile = "file" // NewPublicRoomsServerDatabase opens a database connection. -func NewPublicRoomsServerDatabase(dataSourceName string, dbProperties internal.DbProperties, localServerName gomatrixserverlib.ServerName) (Database, error) { +func NewPublicRoomsServerDatabase(dataSourceName string, dbProperties sqlutil.DbProperties, localServerName gomatrixserverlib.ServerName) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return postgres.NewPublicRoomsServerDatabase(dataSourceName, dbProperties, localServerName) diff --git a/roomserver/internal/input_events.go b/roomserver/internal/input_events.go index 3a38636f8..4487aea02 100644 --- a/roomserver/internal/input_events.go +++ b/roomserver/internal/input_events.go @@ -21,7 +21,7 @@ import ( "errors" "fmt" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/state" "github.com/matrix-org/dendrite/roomserver/storage" @@ -177,7 +177,7 @@ func (r *RoomserverInternalAPI) processInviteEvent( } succeeded := false defer func() { - txerr := internal.EndTransaction(updater, &succeeded) + txerr := sqlutil.EndTransaction(updater, &succeeded) if err == nil && txerr != nil { err = txerr } diff --git a/roomserver/internal/input_latest_events.go b/roomserver/internal/input_latest_events.go index e69307ada..66316ac4f 100644 --- a/roomserver/internal/input_latest_events.go +++ b/roomserver/internal/input_latest_events.go @@ -21,7 +21,7 @@ import ( "context" "fmt" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/state" "github.com/matrix-org/dendrite/roomserver/types" @@ -60,7 +60,7 @@ func (r *RoomserverInternalAPI) updateLatestEvents( } succeeded := false defer func() { - txerr := internal.EndTransaction(updater, &succeeded) + txerr := sqlutil.EndTransaction(updater, &succeeded) if err == nil && txerr != nil { err = txerr } diff --git a/roomserver/internal/perform_join.go b/roomserver/internal/perform_join.go index 75dfdd19c..1c951bb15 100644 --- a/roomserver/internal/perform_join.go +++ b/roomserver/internal/perform_join.go @@ -7,7 +7,7 @@ import ( "time" fsAPI "github.com/matrix-org/dendrite/federationsender/api" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" @@ -159,7 +159,7 @@ func (r *RoomserverInternalAPI) performJoinRoomByID( // TODO: Check what happens if the room exists on the server // but everyone has since left. I suspect it does the wrong thing. buildRes := api.QueryLatestEventsAndStateResponse{} - event, err := internal.BuildEvent( + event, err := eventutil.BuildEvent( ctx, // the request context &eb, // the template join event r.Cfg, // the server configuration @@ -202,7 +202,7 @@ func (r *RoomserverInternalAPI) performJoinRoomByID( } } - case internal.ErrRoomNoExists: + case eventutil.ErrRoomNoExists: // The room doesn't exist. First of all check if the room is a local // room. If it is then there's nothing more to do - the room just // hasn't been created yet. diff --git a/roomserver/internal/perform_leave.go b/roomserver/internal/perform_leave.go index b60c2e99f..880c8b203 100644 --- a/roomserver/internal/perform_leave.go +++ b/roomserver/internal/perform_leave.go @@ -7,7 +7,7 @@ import ( "time" fsAPI "github.com/matrix-org/dendrite/federationsender/api" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" ) @@ -97,7 +97,7 @@ func (r *RoomserverInternalAPI) performLeaveRoomByID( // TODO: Check what happens if the room exists on the server // but everyone has since left. I suspect it does the wrong thing. buildRes := api.QueryLatestEventsAndStateResponse{} - event, err := internal.BuildEvent( + event, err := eventutil.BuildEvent( ctx, // the request context &eb, // the template leave event r.Cfg, // the server configuration @@ -106,7 +106,7 @@ func (r *RoomserverInternalAPI) performLeaveRoomByID( &buildRes, // the query response ) if err != nil { - return fmt.Errorf("internal.BuildEvent: %w", err) + return fmt.Errorf("eventutil.BuildEvent: %w", err) } // Give our leave event to the roomserver input stream. The diff --git a/roomserver/inthttp/client.go b/roomserver/inthttp/client.go index 1244300d4..e41adb993 100644 --- a/roomserver/inthttp/client.go +++ b/roomserver/inthttp/client.go @@ -7,7 +7,7 @@ import ( fsInputAPI "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/internal/caching" - internalHTTP "github.com/matrix-org/dendrite/internal/http" + "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/roomserver/api" "github.com/opentracing/opentracing-go" ) @@ -78,7 +78,7 @@ func (h *httpRoomserverInternalAPI) SetRoomAlias( defer span.Finish() apiURL := h.roomserverURL + RoomserverSetRoomAliasPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // GetRoomIDForAlias implements RoomserverAliasAPI @@ -91,7 +91,7 @@ func (h *httpRoomserverInternalAPI) GetRoomIDForAlias( defer span.Finish() apiURL := h.roomserverURL + RoomserverGetRoomIDForAliasPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // GetAliasesForRoomID implements RoomserverAliasAPI @@ -104,7 +104,7 @@ func (h *httpRoomserverInternalAPI) GetAliasesForRoomID( defer span.Finish() apiURL := h.roomserverURL + RoomserverGetAliasesForRoomIDPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // GetCreatorIDForAlias implements RoomserverAliasAPI @@ -117,7 +117,7 @@ func (h *httpRoomserverInternalAPI) GetCreatorIDForAlias( defer span.Finish() apiURL := h.roomserverURL + RoomserverGetCreatorIDForAliasPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // RemoveRoomAlias implements RoomserverAliasAPI @@ -130,7 +130,7 @@ func (h *httpRoomserverInternalAPI) RemoveRoomAlias( defer span.Finish() apiURL := h.roomserverURL + RoomserverRemoveRoomAliasPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // InputRoomEvents implements RoomserverInputAPI @@ -143,7 +143,7 @@ func (h *httpRoomserverInternalAPI) InputRoomEvents( defer span.Finish() apiURL := h.roomserverURL + RoomserverInputRoomEventsPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } func (h *httpRoomserverInternalAPI) PerformJoin( @@ -155,7 +155,7 @@ func (h *httpRoomserverInternalAPI) PerformJoin( defer span.Finish() apiURL := h.roomserverURL + RoomserverPerformJoinPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } func (h *httpRoomserverInternalAPI) PerformLeave( @@ -167,7 +167,7 @@ func (h *httpRoomserverInternalAPI) PerformLeave( defer span.Finish() apiURL := h.roomserverURL + RoomserverPerformLeavePath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // QueryLatestEventsAndState implements RoomserverQueryAPI @@ -180,7 +180,7 @@ func (h *httpRoomserverInternalAPI) QueryLatestEventsAndState( defer span.Finish() apiURL := h.roomserverURL + RoomserverQueryLatestEventsAndStatePath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // QueryStateAfterEvents implements RoomserverQueryAPI @@ -193,7 +193,7 @@ func (h *httpRoomserverInternalAPI) QueryStateAfterEvents( defer span.Finish() apiURL := h.roomserverURL + RoomserverQueryStateAfterEventsPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // QueryEventsByID implements RoomserverQueryAPI @@ -206,7 +206,7 @@ func (h *httpRoomserverInternalAPI) QueryEventsByID( defer span.Finish() apiURL := h.roomserverURL + RoomserverQueryEventsByIDPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // QueryMembershipForUser implements RoomserverQueryAPI @@ -219,7 +219,7 @@ func (h *httpRoomserverInternalAPI) QueryMembershipForUser( defer span.Finish() apiURL := h.roomserverURL + RoomserverQueryMembershipForUserPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // QueryMembershipsForRoom implements RoomserverQueryAPI @@ -232,7 +232,7 @@ func (h *httpRoomserverInternalAPI) QueryMembershipsForRoom( defer span.Finish() apiURL := h.roomserverURL + RoomserverQueryMembershipsForRoomPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // QueryServerAllowedToSeeEvent implements RoomserverQueryAPI @@ -245,7 +245,7 @@ func (h *httpRoomserverInternalAPI) QueryServerAllowedToSeeEvent( defer span.Finish() apiURL := h.roomserverURL + RoomserverQueryServerAllowedToSeeEventPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // QueryMissingEvents implements RoomServerQueryAPI @@ -258,7 +258,7 @@ func (h *httpRoomserverInternalAPI) QueryMissingEvents( defer span.Finish() apiURL := h.roomserverURL + RoomserverQueryMissingEventsPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // QueryStateAndAuthChain implements RoomserverQueryAPI @@ -271,7 +271,7 @@ func (h *httpRoomserverInternalAPI) QueryStateAndAuthChain( defer span.Finish() apiURL := h.roomserverURL + RoomserverQueryStateAndAuthChainPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // PerformBackfill implements RoomServerQueryAPI @@ -284,7 +284,7 @@ func (h *httpRoomserverInternalAPI) PerformBackfill( defer span.Finish() apiURL := h.roomserverURL + RoomserverPerformBackfillPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // QueryRoomVersionCapabilities implements RoomServerQueryAPI @@ -297,7 +297,7 @@ func (h *httpRoomserverInternalAPI) QueryRoomVersionCapabilities( defer span.Finish() apiURL := h.roomserverURL + RoomserverQueryRoomVersionCapabilitiesPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } // QueryRoomVersionForRoom implements RoomServerQueryAPI @@ -315,7 +315,7 @@ func (h *httpRoomserverInternalAPI) QueryRoomVersionForRoom( defer span.Finish() apiURL := h.roomserverURL + RoomserverQueryRoomVersionForRoomPath - err := internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) if err == nil { h.cache.StoreRoomVersion(request.RoomID, response.RoomVersion) } diff --git a/roomserver/inthttp/server.go b/roomserver/inthttp/server.go index 8ac815f3e..822acd15b 100644 --- a/roomserver/inthttp/server.go +++ b/roomserver/inthttp/server.go @@ -5,7 +5,7 @@ import ( "net/http" "github.com/gorilla/mux" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/util" ) @@ -14,7 +14,7 @@ import ( // nolint: gocyclo func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { internalAPIMux.Handle(RoomserverInputRoomEventsPath, - internal.MakeInternalAPI("inputRoomEvents", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("inputRoomEvents", func(req *http.Request) util.JSONResponse { var request api.InputRoomEventsRequest var response api.InputRoomEventsResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -27,7 +27,7 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { }), ) internalAPIMux.Handle(RoomserverPerformJoinPath, - internal.MakeInternalAPI("performJoin", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("performJoin", func(req *http.Request) util.JSONResponse { var request api.PerformJoinRequest var response api.PerformJoinResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -40,7 +40,7 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { }), ) internalAPIMux.Handle(RoomserverPerformLeavePath, - internal.MakeInternalAPI("performLeave", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("performLeave", func(req *http.Request) util.JSONResponse { var request api.PerformLeaveRequest var response api.PerformLeaveResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -54,7 +54,7 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { ) internalAPIMux.Handle( RoomserverQueryLatestEventsAndStatePath, - internal.MakeInternalAPI("queryLatestEventsAndState", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("queryLatestEventsAndState", func(req *http.Request) util.JSONResponse { var request api.QueryLatestEventsAndStateRequest var response api.QueryLatestEventsAndStateResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -68,7 +68,7 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { ) internalAPIMux.Handle( RoomserverQueryStateAfterEventsPath, - internal.MakeInternalAPI("queryStateAfterEvents", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("queryStateAfterEvents", func(req *http.Request) util.JSONResponse { var request api.QueryStateAfterEventsRequest var response api.QueryStateAfterEventsResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -82,7 +82,7 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { ) internalAPIMux.Handle( RoomserverQueryEventsByIDPath, - internal.MakeInternalAPI("queryEventsByID", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("queryEventsByID", func(req *http.Request) util.JSONResponse { var request api.QueryEventsByIDRequest var response api.QueryEventsByIDResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -96,7 +96,7 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { ) internalAPIMux.Handle( RoomserverQueryMembershipForUserPath, - internal.MakeInternalAPI("QueryMembershipForUser", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("QueryMembershipForUser", func(req *http.Request) util.JSONResponse { var request api.QueryMembershipForUserRequest var response api.QueryMembershipForUserResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -110,7 +110,7 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { ) internalAPIMux.Handle( RoomserverQueryMembershipsForRoomPath, - internal.MakeInternalAPI("queryMembershipsForRoom", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("queryMembershipsForRoom", func(req *http.Request) util.JSONResponse { var request api.QueryMembershipsForRoomRequest var response api.QueryMembershipsForRoomResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -124,7 +124,7 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { ) internalAPIMux.Handle( RoomserverQueryServerAllowedToSeeEventPath, - internal.MakeInternalAPI("queryServerAllowedToSeeEvent", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("queryServerAllowedToSeeEvent", func(req *http.Request) util.JSONResponse { var request api.QueryServerAllowedToSeeEventRequest var response api.QueryServerAllowedToSeeEventResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -138,7 +138,7 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { ) internalAPIMux.Handle( RoomserverQueryMissingEventsPath, - internal.MakeInternalAPI("queryMissingEvents", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("queryMissingEvents", func(req *http.Request) util.JSONResponse { var request api.QueryMissingEventsRequest var response api.QueryMissingEventsResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -152,7 +152,7 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { ) internalAPIMux.Handle( RoomserverQueryStateAndAuthChainPath, - internal.MakeInternalAPI("queryStateAndAuthChain", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("queryStateAndAuthChain", func(req *http.Request) util.JSONResponse { var request api.QueryStateAndAuthChainRequest var response api.QueryStateAndAuthChainResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -166,7 +166,7 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { ) internalAPIMux.Handle( RoomserverPerformBackfillPath, - internal.MakeInternalAPI("PerformBackfill", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("PerformBackfill", func(req *http.Request) util.JSONResponse { var request api.PerformBackfillRequest var response api.PerformBackfillResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -180,7 +180,7 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { ) internalAPIMux.Handle( RoomserverQueryRoomVersionCapabilitiesPath, - internal.MakeInternalAPI("QueryRoomVersionCapabilities", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("QueryRoomVersionCapabilities", func(req *http.Request) util.JSONResponse { var request api.QueryRoomVersionCapabilitiesRequest var response api.QueryRoomVersionCapabilitiesResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -194,7 +194,7 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { ) internalAPIMux.Handle( RoomserverQueryRoomVersionForRoomPath, - internal.MakeInternalAPI("QueryRoomVersionForRoom", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("QueryRoomVersionForRoom", func(req *http.Request) util.JSONResponse { var request api.QueryRoomVersionForRoomRequest var response api.QueryRoomVersionForRoomResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -208,7 +208,7 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { ) internalAPIMux.Handle( RoomserverSetRoomAliasPath, - internal.MakeInternalAPI("setRoomAlias", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("setRoomAlias", func(req *http.Request) util.JSONResponse { var request api.SetRoomAliasRequest var response api.SetRoomAliasResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -222,7 +222,7 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { ) internalAPIMux.Handle( RoomserverGetRoomIDForAliasPath, - internal.MakeInternalAPI("GetRoomIDForAlias", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("GetRoomIDForAlias", func(req *http.Request) util.JSONResponse { var request api.GetRoomIDForAliasRequest var response api.GetRoomIDForAliasResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -236,7 +236,7 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { ) internalAPIMux.Handle( RoomserverGetCreatorIDForAliasPath, - internal.MakeInternalAPI("GetCreatorIDForAlias", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("GetCreatorIDForAlias", func(req *http.Request) util.JSONResponse { var request api.GetCreatorIDForAliasRequest var response api.GetCreatorIDForAliasResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -250,7 +250,7 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { ) internalAPIMux.Handle( RoomserverGetAliasesForRoomIDPath, - internal.MakeInternalAPI("getAliasesForRoomID", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("getAliasesForRoomID", func(req *http.Request) util.JSONResponse { var request api.GetAliasesForRoomIDRequest var response api.GetAliasesForRoomIDResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -264,7 +264,7 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { ) internalAPIMux.Handle( RoomserverRemoveRoomAliasPath, - internal.MakeInternalAPI("removeRoomAlias", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("removeRoomAlias", func(req *http.Request) util.JSONResponse { var request api.RemoveRoomAliasRequest var response api.RemoveRoomAliasResponse if err := json.NewDecoder(req.Body).Decode(&request); err != nil { diff --git a/roomserver/roomserver.go b/roomserver/roomserver.go index a9db22d7a..427d5ff36 100644 --- a/roomserver/roomserver.go +++ b/roomserver/roomserver.go @@ -20,7 +20,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/inthttp" "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/dendrite/internal/basecomponent" + "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/roomserver/internal" "github.com/matrix-org/dendrite/roomserver/storage" "github.com/sirupsen/logrus" @@ -35,7 +35,7 @@ func AddInternalRoutes(router *mux.Router, intAPI api.RoomserverInternalAPI) { // NewInternalAPI returns a concerete implementation of the internal API. Callers // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. func NewInternalAPI( - base *basecomponent.BaseDendrite, + base *setup.BaseDendrite, keyRing gomatrixserverlib.JSONVerifier, fedClient *gomatrixserverlib.FederationClient, ) api.RoomserverInternalAPI { diff --git a/roomserver/storage/postgres/event_json_table.go b/roomserver/storage/postgres/event_json_table.go index 3bce9f086..7df175954 100644 --- a/roomserver/storage/postgres/event_json_table.go +++ b/roomserver/storage/postgres/event_json_table.go @@ -20,7 +20,6 @@ import ( "database/sql" "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" diff --git a/roomserver/storage/postgres/event_state_keys_table.go b/roomserver/storage/postgres/event_state_keys_table.go index b7e2cb6b7..500ff20e4 100644 --- a/roomserver/storage/postgres/event_state_keys_table.go +++ b/roomserver/storage/postgres/event_state_keys_table.go @@ -21,6 +21,7 @@ import ( "github.com/lib/pq" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" @@ -94,7 +95,7 @@ func (s *eventStateKeyStatements) InsertEventStateKeyNID( ctx context.Context, txn *sql.Tx, eventStateKey string, ) (types.EventStateKeyNID, error) { var eventStateKeyNID int64 - stmt := internal.TxStmt(txn, s.insertEventStateKeyNIDStmt) + stmt := sqlutil.TxStmt(txn, s.insertEventStateKeyNIDStmt) err := stmt.QueryRowContext(ctx, eventStateKey).Scan(&eventStateKeyNID) return types.EventStateKeyNID(eventStateKeyNID), err } @@ -103,7 +104,7 @@ func (s *eventStateKeyStatements) SelectEventStateKeyNID( ctx context.Context, txn *sql.Tx, eventStateKey string, ) (types.EventStateKeyNID, error) { var eventStateKeyNID int64 - stmt := internal.TxStmt(txn, s.selectEventStateKeyNIDStmt) + stmt := sqlutil.TxStmt(txn, s.selectEventStateKeyNIDStmt) err := stmt.QueryRowContext(ctx, eventStateKey).Scan(&eventStateKeyNID) return types.EventStateKeyNID(eventStateKeyNID), err } diff --git a/roomserver/storage/postgres/event_types_table.go b/roomserver/storage/postgres/event_types_table.go index f8a7bff1b..037d98fe7 100644 --- a/roomserver/storage/postgres/event_types_table.go +++ b/roomserver/storage/postgres/event_types_table.go @@ -19,9 +19,8 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/internal" - "github.com/lib/pq" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" diff --git a/roomserver/storage/postgres/events_table.go b/roomserver/storage/postgres/events_table.go index d39e9511a..bdbf5e7cb 100644 --- a/roomserver/storage/postgres/events_table.go +++ b/roomserver/storage/postgres/events_table.go @@ -22,6 +22,7 @@ import ( "github.com/lib/pq" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" @@ -284,13 +285,13 @@ func (s *eventStatements) UpdateEventState( func (s *eventStatements) SelectEventSentToOutput( ctx context.Context, txn *sql.Tx, eventNID types.EventNID, ) (sentToOutput bool, err error) { - stmt := internal.TxStmt(txn, s.selectEventSentToOutputStmt) + stmt := sqlutil.TxStmt(txn, s.selectEventSentToOutputStmt) err = stmt.QueryRowContext(ctx, int64(eventNID)).Scan(&sentToOutput) return } func (s *eventStatements) UpdateEventSentToOutput(ctx context.Context, txn *sql.Tx, eventNID types.EventNID) error { - stmt := internal.TxStmt(txn, s.updateEventSentToOutputStmt) + stmt := sqlutil.TxStmt(txn, s.updateEventSentToOutputStmt) _, err := stmt.ExecContext(ctx, int64(eventNID)) return err } @@ -298,7 +299,7 @@ func (s *eventStatements) UpdateEventSentToOutput(ctx context.Context, txn *sql. func (s *eventStatements) SelectEventID( ctx context.Context, txn *sql.Tx, eventNID types.EventNID, ) (eventID string, err error) { - stmt := internal.TxStmt(txn, s.selectEventIDStmt) + stmt := sqlutil.TxStmt(txn, s.selectEventIDStmt) err = stmt.QueryRowContext(ctx, int64(eventNID)).Scan(&eventID) return } @@ -306,7 +307,7 @@ func (s *eventStatements) SelectEventID( func (s *eventStatements) BulkSelectStateAtEventAndReference( ctx context.Context, txn *sql.Tx, eventNIDs []types.EventNID, ) ([]types.StateAtEventAndReference, error) { - stmt := internal.TxStmt(txn, s.bulkSelectStateAtEventAndReferenceStmt) + stmt := sqlutil.TxStmt(txn, s.bulkSelectStateAtEventAndReferenceStmt) rows, err := stmt.QueryContext(ctx, eventNIDsAsArray(eventNIDs)) if err != nil { return nil, err diff --git a/roomserver/storage/postgres/invite_table.go b/roomserver/storage/postgres/invite_table.go index 55a5bc7d7..048a094dc 100644 --- a/roomserver/storage/postgres/invite_table.go +++ b/roomserver/storage/postgres/invite_table.go @@ -20,6 +20,7 @@ import ( "database/sql" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" @@ -101,7 +102,7 @@ func (s *inviteStatements) InsertInviteEvent( targetUserNID, senderUserNID types.EventStateKeyNID, inviteEventJSON []byte, ) (bool, error) { - result, err := internal.TxStmt(txn, s.insertInviteEventStmt).ExecContext( + result, err := sqlutil.TxStmt(txn, s.insertInviteEventStmt).ExecContext( ctx, inviteEventID, roomNID, targetUserNID, senderUserNID, inviteEventJSON, ) if err != nil { @@ -118,7 +119,7 @@ func (s *inviteStatements) UpdateInviteRetired( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, ) ([]string, error) { - stmt := internal.TxStmt(txn, s.updateInviteRetiredStmt) + stmt := sqlutil.TxStmt(txn, s.updateInviteRetiredStmt) rows, err := stmt.QueryContext(ctx, roomNID, targetUserNID) if err != nil { return nil, err diff --git a/roomserver/storage/postgres/membership_table.go b/roomserver/storage/postgres/membership_table.go index c635ce28a..13cef638f 100644 --- a/roomserver/storage/postgres/membership_table.go +++ b/roomserver/storage/postgres/membership_table.go @@ -20,6 +20,7 @@ import ( "database/sql" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" @@ -133,7 +134,7 @@ func (s *membershipStatements) InsertMembership( txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, localTarget bool, ) error { - stmt := internal.TxStmt(txn, s.insertMembershipStmt) + stmt := sqlutil.TxStmt(txn, s.insertMembershipStmt) _, err := stmt.ExecContext(ctx, roomNID, targetUserNID, localTarget) return err } @@ -142,7 +143,7 @@ func (s *membershipStatements) SelectMembershipForUpdate( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, ) (membership tables.MembershipState, err error) { - err = internal.TxStmt(txn, s.selectMembershipForUpdateStmt).QueryRowContext( + err = sqlutil.TxStmt(txn, s.selectMembershipForUpdateStmt).QueryRowContext( ctx, roomNID, targetUserNID, ).Scan(&membership) return @@ -216,7 +217,7 @@ func (s *membershipStatements) UpdateMembership( senderUserNID types.EventStateKeyNID, membership tables.MembershipState, eventNID types.EventNID, ) error { - _, err := internal.TxStmt(txn, s.updateMembershipStmt).ExecContext( + _, err := sqlutil.TxStmt(txn, s.updateMembershipStmt).ExecContext( ctx, roomNID, targetUserNID, senderUserNID, membership, eventNID, ) return err diff --git a/roomserver/storage/postgres/previous_events_table.go b/roomserver/storage/postgres/previous_events_table.go index ff0306967..1a4ba6732 100644 --- a/roomserver/storage/postgres/previous_events_table.go +++ b/roomserver/storage/postgres/previous_events_table.go @@ -19,7 +19,7 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" @@ -85,7 +85,7 @@ func (s *previousEventStatements) InsertPreviousEvent( previousEventReferenceSHA256 []byte, eventNID types.EventNID, ) error { - stmt := internal.TxStmt(txn, s.insertPreviousEventStmt) + stmt := sqlutil.TxStmt(txn, s.insertPreviousEventStmt) _, err := stmt.ExecContext( ctx, previousEventID, previousEventReferenceSHA256, int64(eventNID), ) @@ -98,6 +98,6 @@ func (s *previousEventStatements) SelectPreviousEventExists( ctx context.Context, txn *sql.Tx, eventID string, eventReferenceSHA256 []byte, ) error { var ok int64 - stmt := internal.TxStmt(txn, s.selectPreviousEventExistsStmt) + stmt := sqlutil.TxStmt(txn, s.selectPreviousEventExistsStmt) return stmt.QueryRowContext(ctx, eventID, eventReferenceSHA256).Scan(&ok) } diff --git a/roomserver/storage/postgres/rooms_table.go b/roomserver/storage/postgres/rooms_table.go index 5a353ed1c..8e00cfdb8 100644 --- a/roomserver/storage/postgres/rooms_table.go +++ b/roomserver/storage/postgres/rooms_table.go @@ -21,7 +21,7 @@ import ( "errors" "github.com/lib/pq" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" @@ -106,7 +106,7 @@ func (s *roomStatements) InsertRoomNID( roomID string, roomVersion gomatrixserverlib.RoomVersion, ) (types.RoomNID, error) { var roomNID int64 - stmt := internal.TxStmt(txn, s.insertRoomNIDStmt) + stmt := sqlutil.TxStmt(txn, s.insertRoomNIDStmt) err := stmt.QueryRowContext(ctx, roomID, roomVersion).Scan(&roomNID) return types.RoomNID(roomNID), err } @@ -115,7 +115,7 @@ func (s *roomStatements) SelectRoomNID( ctx context.Context, txn *sql.Tx, roomID string, ) (types.RoomNID, error) { var roomNID int64 - stmt := internal.TxStmt(txn, s.selectRoomNIDStmt) + stmt := sqlutil.TxStmt(txn, s.selectRoomNIDStmt) err := stmt.QueryRowContext(ctx, roomID).Scan(&roomNID) return types.RoomNID(roomNID), err } @@ -143,7 +143,7 @@ func (s *roomStatements) SelectLatestEventsNIDsForUpdate( var nids pq.Int64Array var lastEventSentNID int64 var stateSnapshotNID int64 - stmt := internal.TxStmt(txn, s.selectLatestEventNIDsForUpdateStmt) + stmt := sqlutil.TxStmt(txn, s.selectLatestEventNIDsForUpdateStmt) err := stmt.QueryRowContext(ctx, int64(roomNID)).Scan(&nids, &lastEventSentNID, &stateSnapshotNID) if err != nil { return nil, 0, 0, err @@ -163,7 +163,7 @@ func (s *roomStatements) UpdateLatestEventNIDs( lastEventSentNID types.EventNID, stateSnapshotNID types.StateSnapshotNID, ) error { - stmt := internal.TxStmt(txn, s.updateLatestEventNIDsStmt) + stmt := sqlutil.TxStmt(txn, s.updateLatestEventNIDsStmt) _, err := stmt.ExecContext( ctx, roomNID, @@ -178,7 +178,7 @@ func (s *roomStatements) SelectRoomVersionForRoomID( ctx context.Context, txn *sql.Tx, roomID string, ) (gomatrixserverlib.RoomVersion, error) { var roomVersion gomatrixserverlib.RoomVersion - stmt := internal.TxStmt(txn, s.selectRoomVersionForRoomIDStmt) + stmt := sqlutil.TxStmt(txn, s.selectRoomVersionForRoomIDStmt) err := stmt.QueryRowContext(ctx, roomID).Scan(&roomVersion) if err == sql.ErrNoRows { return roomVersion, errors.New("room not found") diff --git a/roomserver/storage/postgres/state_block_table.go b/roomserver/storage/postgres/state_block_table.go index dbe21d989..d618686f7 100644 --- a/roomserver/storage/postgres/state_block_table.go +++ b/roomserver/storage/postgres/state_block_table.go @@ -21,9 +21,8 @@ import ( "fmt" "sort" - "github.com/matrix-org/dendrite/internal" - "github.com/lib/pq" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" diff --git a/roomserver/storage/postgres/storage.go b/roomserver/storage/postgres/storage.go index 971f2b9e6..d76ee0a92 100644 --- a/roomserver/storage/postgres/storage.go +++ b/roomserver/storage/postgres/storage.go @@ -18,7 +18,6 @@ package postgres import ( "database/sql" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" // Import the postgres database driver. @@ -33,7 +32,7 @@ type Database struct { // Open a postgres database. // nolint: gocyclo -func Open(dataSourceName string, dbProperties internal.DbProperties) (*Database, error) { +func Open(dataSourceName string, dbProperties sqlutil.DbProperties) (*Database, error) { var d Database var db *sql.DB var err error diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index bb5b51562..2751cc557 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -5,7 +5,7 @@ import ( "database/sql" "encoding/json" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" @@ -68,7 +68,7 @@ func (d *Database) AddState( stateBlockNIDs []types.StateBlockNID, state []types.StateEntry, ) (stateNID types.StateSnapshotNID, err error) { - err = internal.WithTransaction(d.DB, func(txn *sql.Tx) error { + err = sqlutil.WithTransaction(d.DB, func(txn *sql.Tx) error { if len(state) > 0 { var stateBlockNID types.StateBlockNID stateBlockNID, err = d.StateBlockTable.BulkInsertStateData(ctx, txn, state) @@ -158,7 +158,7 @@ func (d *Database) RoomNIDExcludingStubs(ctx context.Context, roomID string) (ro func (d *Database) LatestEventIDs( ctx context.Context, roomNID types.RoomNID, ) (references []gomatrixserverlib.EventReference, currentStateSnapshotNID types.StateSnapshotNID, depth int64, err error) { - err = internal.WithTransaction(d.DB, func(txn *sql.Tx) error { + err = sqlutil.WithTransaction(d.DB, func(txn *sql.Tx) error { var eventNIDs []types.EventNID eventNIDs, currentStateSnapshotNID, err = d.RoomsTable.SelectLatestEventNIDs(ctx, txn, roomNID) if err != nil { @@ -337,7 +337,7 @@ func (d *Database) StoreEvent( err error ) - err = internal.WithTransaction(d.DB, func(txn *sql.Tx) error { + err = sqlutil.WithTransaction(d.DB, func(txn *sql.Tx) error { if txnAndSessionID != nil { if err = d.TransactionsTable.InsertTransaction( ctx, txn, txnAndSessionID.TransactionID, diff --git a/roomserver/storage/sqlite3/event_json_table.go b/roomserver/storage/sqlite3/event_json_table.go index 7ff4c1d4d..da0c448dc 100644 --- a/roomserver/storage/sqlite3/event_json_table.go +++ b/roomserver/storage/sqlite3/event_json_table.go @@ -21,6 +21,7 @@ import ( "strings" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" @@ -69,7 +70,7 @@ func NewSqliteEventJSONTable(db *sql.DB) (tables.EventJSON, error) { func (s *eventJSONStatements) InsertEventJSON( ctx context.Context, txn *sql.Tx, eventNID types.EventNID, eventJSON []byte, ) error { - _, err := internal.TxStmt(txn, s.insertEventJSONStmt).ExecContext(ctx, int64(eventNID), eventJSON) + _, err := sqlutil.TxStmt(txn, s.insertEventJSONStmt).ExecContext(ctx, int64(eventNID), eventJSON) return err } @@ -80,7 +81,7 @@ func (s *eventJSONStatements) BulkSelectEventJSON( for k, v := range eventNIDs { iEventNIDs[k] = v } - selectOrig := strings.Replace(bulkSelectEventJSONSQL, "($1)", internal.QueryVariadic(len(iEventNIDs)), 1) + selectOrig := strings.Replace(bulkSelectEventJSONSQL, "($1)", sqlutil.QueryVariadic(len(iEventNIDs)), 1) rows, err := s.db.QueryContext(ctx, selectOrig, iEventNIDs...) if err != nil { diff --git a/roomserver/storage/sqlite3/event_state_keys_table.go b/roomserver/storage/sqlite3/event_state_keys_table.go index 2d2c04d26..cbea8428c 100644 --- a/roomserver/storage/sqlite3/event_state_keys_table.go +++ b/roomserver/storage/sqlite3/event_state_keys_table.go @@ -21,6 +21,7 @@ import ( "strings" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" @@ -90,7 +91,7 @@ func (s *eventStateKeyStatements) InsertEventStateKeyNID( var eventStateKeyNID int64 var err error var res sql.Result - insertStmt := internal.TxStmt(txn, s.insertEventStateKeyNIDStmt) + insertStmt := sqlutil.TxStmt(txn, s.insertEventStateKeyNIDStmt) if res, err = insertStmt.ExecContext(ctx, eventStateKey); err == nil { eventStateKeyNID, err = res.LastInsertId() } @@ -101,7 +102,7 @@ func (s *eventStateKeyStatements) SelectEventStateKeyNID( ctx context.Context, txn *sql.Tx, eventStateKey string, ) (types.EventStateKeyNID, error) { var eventStateKeyNID int64 - stmt := internal.TxStmt(txn, s.selectEventStateKeyNIDStmt) + stmt := sqlutil.TxStmt(txn, s.selectEventStateKeyNIDStmt) err := stmt.QueryRowContext(ctx, eventStateKey).Scan(&eventStateKeyNID) return types.EventStateKeyNID(eventStateKeyNID), err } @@ -113,7 +114,7 @@ func (s *eventStateKeyStatements) BulkSelectEventStateKeyNID( for k, v := range eventStateKeys { iEventStateKeys[k] = v } - selectOrig := strings.Replace(bulkSelectEventStateKeySQL, "($1)", internal.QueryVariadic(len(eventStateKeys)), 1) + selectOrig := strings.Replace(bulkSelectEventStateKeySQL, "($1)", sqlutil.QueryVariadic(len(eventStateKeys)), 1) rows, err := s.db.QueryContext(ctx, selectOrig, iEventStateKeys...) if err != nil { @@ -139,7 +140,7 @@ func (s *eventStateKeyStatements) BulkSelectEventStateKey( for k, v := range eventStateKeyNIDs { iEventStateKeyNIDs[k] = v } - selectOrig := strings.Replace(bulkSelectEventStateKeyNIDSQL, "($1)", internal.QueryVariadic(len(eventStateKeyNIDs)), 1) + selectOrig := strings.Replace(bulkSelectEventStateKeyNIDSQL, "($1)", sqlutil.QueryVariadic(len(eventStateKeyNIDs)), 1) rows, err := s.db.QueryContext(ctx, selectOrig, iEventStateKeyNIDs...) if err != nil { diff --git a/roomserver/storage/sqlite3/event_types_table.go b/roomserver/storage/sqlite3/event_types_table.go index 6d229bbd7..c9a461f99 100644 --- a/roomserver/storage/sqlite3/event_types_table.go +++ b/roomserver/storage/sqlite3/event_types_table.go @@ -21,6 +21,7 @@ import ( "strings" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" @@ -104,8 +105,8 @@ func (s *eventTypeStatements) InsertEventTypeNID( ) (types.EventTypeNID, error) { var eventTypeNID int64 var err error - insertStmt := internal.TxStmt(tx, s.insertEventTypeNIDStmt) - resultStmt := internal.TxStmt(tx, s.insertEventTypeNIDResultStmt) + insertStmt := sqlutil.TxStmt(tx, s.insertEventTypeNIDStmt) + resultStmt := sqlutil.TxStmt(tx, s.insertEventTypeNIDResultStmt) if _, err = insertStmt.ExecContext(ctx, eventType); err == nil { err = resultStmt.QueryRowContext(ctx).Scan(&eventTypeNID) } @@ -116,7 +117,7 @@ func (s *eventTypeStatements) SelectEventTypeNID( ctx context.Context, tx *sql.Tx, eventType string, ) (types.EventTypeNID, error) { var eventTypeNID int64 - selectStmt := internal.TxStmt(tx, s.selectEventTypeNIDStmt) + selectStmt := sqlutil.TxStmt(tx, s.selectEventTypeNIDStmt) err := selectStmt.QueryRowContext(ctx, eventType).Scan(&eventTypeNID) return types.EventTypeNID(eventTypeNID), err } @@ -129,7 +130,7 @@ func (s *eventTypeStatements) BulkSelectEventTypeNID( for k, v := range eventTypes { iEventTypes[k] = v } - selectOrig := strings.Replace(bulkSelectEventTypeNIDSQL, "($1)", internal.QueryVariadic(len(iEventTypes)), 1) + selectOrig := strings.Replace(bulkSelectEventTypeNIDSQL, "($1)", sqlutil.QueryVariadic(len(iEventTypes)), 1) selectPrep, err := s.db.Prepare(selectOrig) if err != nil { return nil, err diff --git a/roomserver/storage/sqlite3/events_table.go b/roomserver/storage/sqlite3/events_table.go index 491399ce7..d66db4694 100644 --- a/roomserver/storage/sqlite3/events_table.go +++ b/roomserver/storage/sqlite3/events_table.go @@ -23,6 +23,7 @@ import ( "strings" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" @@ -150,7 +151,7 @@ func (s *eventStatements) InsertEvent( depth int64, ) (types.EventNID, types.StateSnapshotNID, error) { // attempt to insert: the last_row_id is the event NID - insertStmt := internal.TxStmt(txn, s.insertEventStmt) + insertStmt := sqlutil.TxStmt(txn, s.insertEventStmt) result, err := insertStmt.ExecContext( ctx, int64(roomNID), int64(eventTypeNID), int64(eventStateKeyNID), eventID, referenceSHA256, eventNIDsAsArray(authEventNIDs), depth, @@ -171,7 +172,7 @@ func (s *eventStatements) SelectEvent( ) (types.EventNID, types.StateSnapshotNID, error) { var eventNID int64 var stateNID int64 - selectStmt := internal.TxStmt(txn, s.selectEventStmt) + selectStmt := sqlutil.TxStmt(txn, s.selectEventStmt) err := selectStmt.QueryRowContext(ctx, eventID).Scan(&eventNID, &stateNID) return types.EventNID(eventNID), types.StateSnapshotNID(stateNID), err } @@ -186,7 +187,7 @@ func (s *eventStatements) BulkSelectStateEventByID( for k, v := range eventIDs { iEventIDs[k] = v } - selectOrig := strings.Replace(bulkSelectStateEventByIDSQL, "($1)", internal.QueryVariadic(len(iEventIDs)), 1) + selectOrig := strings.Replace(bulkSelectStateEventByIDSQL, "($1)", sqlutil.QueryVariadic(len(iEventIDs)), 1) selectStmt, err := s.db.Prepare(selectOrig) if err != nil { return nil, err @@ -238,7 +239,7 @@ func (s *eventStatements) BulkSelectStateAtEventByID( for k, v := range eventIDs { iEventIDs[k] = v } - selectOrig := strings.Replace(bulkSelectStateAtEventByIDSQL, "($1)", internal.QueryVariadic(len(iEventIDs)), 1) + selectOrig := strings.Replace(bulkSelectStateAtEventByIDSQL, "($1)", sqlutil.QueryVariadic(len(iEventIDs)), 1) selectStmt, err := s.db.Prepare(selectOrig) if err != nil { return nil, err @@ -285,7 +286,7 @@ func (s *eventStatements) UpdateEventState( func (s *eventStatements) SelectEventSentToOutput( ctx context.Context, txn *sql.Tx, eventNID types.EventNID, ) (sentToOutput bool, err error) { - selectStmt := internal.TxStmt(txn, s.selectEventSentToOutputStmt) + selectStmt := sqlutil.TxStmt(txn, s.selectEventSentToOutputStmt) err = selectStmt.QueryRowContext(ctx, int64(eventNID)).Scan(&sentToOutput) //err = s.selectEventSentToOutputStmt.QueryRowContext(ctx, int64(eventNID)).Scan(&sentToOutput) if err != nil { @@ -294,7 +295,7 @@ func (s *eventStatements) SelectEventSentToOutput( } func (s *eventStatements) UpdateEventSentToOutput(ctx context.Context, txn *sql.Tx, eventNID types.EventNID) error { - updateStmt := internal.TxStmt(txn, s.updateEventSentToOutputStmt) + updateStmt := sqlutil.TxStmt(txn, s.updateEventSentToOutputStmt) _, err := updateStmt.ExecContext(ctx, int64(eventNID)) //_, err := s.updateEventSentToOutputStmt.ExecContext(ctx, int64(eventNID)) return err @@ -303,7 +304,7 @@ func (s *eventStatements) UpdateEventSentToOutput(ctx context.Context, txn *sql. func (s *eventStatements) SelectEventID( ctx context.Context, txn *sql.Tx, eventNID types.EventNID, ) (eventID string, err error) { - selectStmt := internal.TxStmt(txn, s.selectEventIDStmt) + selectStmt := sqlutil.TxStmt(txn, s.selectEventIDStmt) err = selectStmt.QueryRowContext(ctx, int64(eventNID)).Scan(&eventID) return } @@ -316,7 +317,7 @@ func (s *eventStatements) BulkSelectStateAtEventAndReference( for k, v := range eventNIDs { iEventNIDs[k] = v } - selectOrig := strings.Replace(bulkSelectStateAtEventAndReferenceSQL, "($1)", internal.QueryVariadic(len(iEventNIDs)), 1) + selectOrig := strings.Replace(bulkSelectStateAtEventAndReferenceSQL, "($1)", sqlutil.QueryVariadic(len(iEventNIDs)), 1) ////////////// rows, err := txn.QueryContext(ctx, selectOrig, iEventNIDs...) @@ -362,14 +363,14 @@ func (s *eventStatements) BulkSelectEventReference( for k, v := range eventNIDs { iEventNIDs[k] = v } - selectOrig := strings.Replace(bulkSelectEventReferenceSQL, "($1)", internal.QueryVariadic(len(iEventNIDs)), 1) + selectOrig := strings.Replace(bulkSelectEventReferenceSQL, "($1)", sqlutil.QueryVariadic(len(iEventNIDs)), 1) selectPrep, err := txn.Prepare(selectOrig) if err != nil { return nil, err } /////////////// - selectStmt := internal.TxStmt(txn, selectPrep) + selectStmt := sqlutil.TxStmt(txn, selectPrep) rows, err := selectStmt.QueryContext(ctx, iEventNIDs...) if err != nil { return nil, err @@ -396,7 +397,7 @@ func (s *eventStatements) BulkSelectEventID(ctx context.Context, eventNIDs []typ for k, v := range eventNIDs { iEventNIDs[k] = v } - selectOrig := strings.Replace(bulkSelectEventIDSQL, "($1)", internal.QueryVariadic(len(iEventNIDs)), 1) + selectOrig := strings.Replace(bulkSelectEventIDSQL, "($1)", sqlutil.QueryVariadic(len(iEventNIDs)), 1) selectStmt, err := s.db.Prepare(selectOrig) if err != nil { return nil, err @@ -432,7 +433,7 @@ func (s *eventStatements) BulkSelectEventNID(ctx context.Context, eventIDs []str for k, v := range eventIDs { iEventIDs[k] = v } - selectOrig := strings.Replace(bulkSelectEventNIDSQL, "($1)", internal.QueryVariadic(len(iEventIDs)), 1) + selectOrig := strings.Replace(bulkSelectEventNIDSQL, "($1)", sqlutil.QueryVariadic(len(iEventIDs)), 1) selectStmt, err := s.db.Prepare(selectOrig) if err != nil { return nil, err @@ -461,7 +462,7 @@ func (s *eventStatements) SelectMaxEventDepth(ctx context.Context, txn *sql.Tx, for i, v := range eventNIDs { iEventIDs[i] = v } - sqlStr := strings.Replace(selectMaxEventDepthSQL, "($1)", internal.QueryVariadic(len(iEventIDs)), 1) + sqlStr := strings.Replace(selectMaxEventDepthSQL, "($1)", sqlutil.QueryVariadic(len(iEventIDs)), 1) err := txn.QueryRowContext(ctx, sqlStr, iEventIDs...).Scan(&result) if err != nil { return 0, err diff --git a/roomserver/storage/sqlite3/invite_table.go b/roomserver/storage/sqlite3/invite_table.go index 7dcc2dc0b..21745d1b0 100644 --- a/roomserver/storage/sqlite3/invite_table.go +++ b/roomserver/storage/sqlite3/invite_table.go @@ -20,6 +20,7 @@ import ( "database/sql" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" @@ -89,7 +90,7 @@ func (s *inviteStatements) InsertInviteEvent( targetUserNID, senderUserNID types.EventStateKeyNID, inviteEventJSON []byte, ) (bool, error) { - stmt := internal.TxStmt(txn, s.insertInviteEventStmt) + stmt := sqlutil.TxStmt(txn, s.insertInviteEventStmt) result, err := stmt.ExecContext( ctx, inviteEventID, roomNID, targetUserNID, senderUserNID, inviteEventJSON, ) @@ -108,7 +109,7 @@ func (s *inviteStatements) UpdateInviteRetired( txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, ) (eventIDs []string, err error) { // gather all the event IDs we will retire - stmt := internal.TxStmt(txn, s.selectInvitesAboutToRetireStmt) + stmt := sqlutil.TxStmt(txn, s.selectInvitesAboutToRetireStmt) rows, err := stmt.QueryContext(ctx, roomNID, targetUserNID) if err != nil { return nil, err @@ -123,7 +124,7 @@ func (s *inviteStatements) UpdateInviteRetired( } // now retire the invites - stmt = internal.TxStmt(txn, s.updateInviteRetiredStmt) + stmt = sqlutil.TxStmt(txn, s.updateInviteRetiredStmt) _, err = stmt.ExecContext(ctx, roomNID, targetUserNID) return } diff --git a/roomserver/storage/sqlite3/membership_table.go b/roomserver/storage/sqlite3/membership_table.go index 5f3076c5a..6f0d763e7 100644 --- a/roomserver/storage/sqlite3/membership_table.go +++ b/roomserver/storage/sqlite3/membership_table.go @@ -20,6 +20,7 @@ import ( "database/sql" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" @@ -109,7 +110,7 @@ func (s *membershipStatements) InsertMembership( roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, localTarget bool, ) error { - stmt := internal.TxStmt(txn, s.insertMembershipStmt) + stmt := sqlutil.TxStmt(txn, s.insertMembershipStmt) _, err := stmt.ExecContext(ctx, roomNID, targetUserNID, localTarget) return err } @@ -118,7 +119,7 @@ func (s *membershipStatements) SelectMembershipForUpdate( ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, ) (membership tables.MembershipState, err error) { - stmt := internal.TxStmt(txn, s.selectMembershipForUpdateStmt) + stmt := sqlutil.TxStmt(txn, s.selectMembershipForUpdateStmt) err = stmt.QueryRowContext( ctx, roomNID, targetUserNID, ).Scan(&membership) @@ -193,7 +194,7 @@ func (s *membershipStatements) UpdateMembership( senderUserNID types.EventStateKeyNID, membership tables.MembershipState, eventNID types.EventNID, ) error { - stmt := internal.TxStmt(txn, s.updateMembershipStmt) + stmt := sqlutil.TxStmt(txn, s.updateMembershipStmt) _, err := stmt.ExecContext( ctx, senderUserNID, membership, eventNID, roomNID, targetUserNID, ) diff --git a/roomserver/storage/sqlite3/previous_events_table.go b/roomserver/storage/sqlite3/previous_events_table.go index 2b187bba3..549aecfb7 100644 --- a/roomserver/storage/sqlite3/previous_events_table.go +++ b/roomserver/storage/sqlite3/previous_events_table.go @@ -19,7 +19,7 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" @@ -77,7 +77,7 @@ func (s *previousEventStatements) InsertPreviousEvent( previousEventReferenceSHA256 []byte, eventNID types.EventNID, ) error { - stmt := internal.TxStmt(txn, s.insertPreviousEventStmt) + stmt := sqlutil.TxStmt(txn, s.insertPreviousEventStmt) _, err := stmt.ExecContext( ctx, previousEventID, previousEventReferenceSHA256, int64(eventNID), ) @@ -90,6 +90,6 @@ func (s *previousEventStatements) SelectPreviousEventExists( ctx context.Context, txn *sql.Tx, eventID string, eventReferenceSHA256 []byte, ) error { var ok int64 - stmt := internal.TxStmt(txn, s.selectPreviousEventExistsStmt) + stmt := sqlutil.TxStmt(txn, s.selectPreviousEventExistsStmt) return stmt.QueryRowContext(ctx, eventID, eventReferenceSHA256).Scan(&ok) } diff --git a/roomserver/storage/sqlite3/rooms_table.go b/roomserver/storage/sqlite3/rooms_table.go index d22fceb3b..ab695c5d2 100644 --- a/roomserver/storage/sqlite3/rooms_table.go +++ b/roomserver/storage/sqlite3/rooms_table.go @@ -21,7 +21,7 @@ import ( "encoding/json" "errors" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" @@ -95,7 +95,7 @@ func (s *roomStatements) InsertRoomNID( roomID string, roomVersion gomatrixserverlib.RoomVersion, ) (types.RoomNID, error) { var err error - insertStmt := internal.TxStmt(txn, s.insertRoomNIDStmt) + insertStmt := sqlutil.TxStmt(txn, s.insertRoomNIDStmt) if _, err = insertStmt.ExecContext(ctx, roomID, roomVersion); err == nil { return s.SelectRoomNID(ctx, txn, roomID) } else { @@ -107,7 +107,7 @@ func (s *roomStatements) SelectRoomNID( ctx context.Context, txn *sql.Tx, roomID string, ) (types.RoomNID, error) { var roomNID int64 - stmt := internal.TxStmt(txn, s.selectRoomNIDStmt) + stmt := sqlutil.TxStmt(txn, s.selectRoomNIDStmt) err := stmt.QueryRowContext(ctx, roomID).Scan(&roomNID) return types.RoomNID(roomNID), err } @@ -118,7 +118,7 @@ func (s *roomStatements) SelectLatestEventNIDs( var eventNIDs []types.EventNID var nidsJSON string var stateSnapshotNID int64 - stmt := internal.TxStmt(txn, s.selectLatestEventNIDsStmt) + stmt := sqlutil.TxStmt(txn, s.selectLatestEventNIDsStmt) err := stmt.QueryRowContext(ctx, int64(roomNID)).Scan(&nidsJSON, &stateSnapshotNID) if err != nil { return nil, 0, err @@ -136,7 +136,7 @@ func (s *roomStatements) SelectLatestEventsNIDsForUpdate( var nidsJSON string var lastEventSentNID int64 var stateSnapshotNID int64 - stmt := internal.TxStmt(txn, s.selectLatestEventNIDsForUpdateStmt) + stmt := sqlutil.TxStmt(txn, s.selectLatestEventNIDsForUpdateStmt) err := stmt.QueryRowContext(ctx, int64(roomNID)).Scan(&nidsJSON, &lastEventSentNID, &stateSnapshotNID) if err != nil { return nil, 0, 0, err @@ -155,7 +155,7 @@ func (s *roomStatements) UpdateLatestEventNIDs( lastEventSentNID types.EventNID, stateSnapshotNID types.StateSnapshotNID, ) error { - stmt := internal.TxStmt(txn, s.updateLatestEventNIDsStmt) + stmt := sqlutil.TxStmt(txn, s.updateLatestEventNIDsStmt) _, err := stmt.ExecContext( ctx, eventNIDsAsArray(eventNIDs), @@ -170,7 +170,7 @@ func (s *roomStatements) SelectRoomVersionForRoomID( ctx context.Context, txn *sql.Tx, roomID string, ) (gomatrixserverlib.RoomVersion, error) { var roomVersion gomatrixserverlib.RoomVersion - stmt := internal.TxStmt(txn, s.selectRoomVersionForRoomIDStmt) + stmt := sqlutil.TxStmt(txn, s.selectRoomVersionForRoomIDStmt) err := stmt.QueryRowContext(ctx, roomID).Scan(&roomVersion) if err == sql.ErrNoRows { return roomVersion, errors.New("room not found") diff --git a/roomserver/storage/sqlite3/state_block_table.go b/roomserver/storage/sqlite3/state_block_table.go index 399fdbf63..c058c783a 100644 --- a/roomserver/storage/sqlite3/state_block_table.go +++ b/roomserver/storage/sqlite3/state_block_table.go @@ -23,6 +23,7 @@ import ( "strings" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" @@ -130,7 +131,7 @@ func (s *stateBlockStatements) BulkSelectStateBlockEntries( for k, v := range stateBlockNIDs { nids[k] = v } - selectOrig := strings.Replace(bulkSelectStateBlockEntriesSQL, "($1)", internal.QueryVariadic(len(nids)), 1) + selectOrig := strings.Replace(bulkSelectStateBlockEntriesSQL, "($1)", sqlutil.QueryVariadic(len(nids)), 1) selectStmt, err := s.db.Prepare(selectOrig) if err != nil { return nil, err @@ -186,9 +187,9 @@ func (s *stateBlockStatements) BulkSelectFilteredStateBlockEntries( sort.Sort(tuples) eventTypeNIDArray, eventStateKeyNIDArray := tuples.typesAndStateKeysAsArrays() - sqlStatement := strings.Replace(bulkSelectFilteredStateBlockEntriesSQL, "($1)", internal.QueryVariadic(len(stateBlockNIDs)), 1) - sqlStatement = strings.Replace(sqlStatement, "($2)", internal.QueryVariadicOffset(len(eventTypeNIDArray), len(stateBlockNIDs)), 1) - sqlStatement = strings.Replace(sqlStatement, "($3)", internal.QueryVariadicOffset(len(eventStateKeyNIDArray), len(stateBlockNIDs)+len(eventTypeNIDArray)), 1) + sqlStatement := strings.Replace(bulkSelectFilteredStateBlockEntriesSQL, "($1)", sqlutil.QueryVariadic(len(stateBlockNIDs)), 1) + sqlStatement = strings.Replace(sqlStatement, "($2)", sqlutil.QueryVariadicOffset(len(eventTypeNIDArray), len(stateBlockNIDs)), 1) + sqlStatement = strings.Replace(sqlStatement, "($3)", sqlutil.QueryVariadicOffset(len(eventStateKeyNIDArray), len(stateBlockNIDs)+len(eventTypeNIDArray)), 1) var params []interface{} for _, val := range stateBlockNIDs { diff --git a/roomserver/storage/sqlite3/state_snapshot_table.go b/roomserver/storage/sqlite3/state_snapshot_table.go index 5de3f639a..d077b6171 100644 --- a/roomserver/storage/sqlite3/state_snapshot_table.go +++ b/roomserver/storage/sqlite3/state_snapshot_table.go @@ -23,6 +23,7 @@ import ( "strings" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" @@ -92,7 +93,7 @@ func (s *stateSnapshotStatements) BulkSelectStateBlockNIDs( for k, v := range stateNIDs { nids[k] = v } - selectOrig := strings.Replace(bulkSelectStateBlockNIDsSQL, "($1)", internal.QueryVariadic(len(nids)), 1) + selectOrig := strings.Replace(bulkSelectStateBlockNIDsSQL, "($1)", sqlutil.QueryVariadic(len(nids)), 1) selectStmt, err := s.db.Prepare(selectOrig) if err != nil { return nil, err diff --git a/roomserver/storage/sqlite3/storage.go b/roomserver/storage/sqlite3/storage.go index 5596b5e3c..8e9352192 100644 --- a/roomserver/storage/sqlite3/storage.go +++ b/roomserver/storage/sqlite3/storage.go @@ -20,8 +20,6 @@ import ( "database/sql" "github.com/matrix-org/dendrite/internal/sqlutil" - - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" @@ -52,7 +50,7 @@ func Open(dataSourceName string) (*Database, error) { if err != nil { return nil, err } - if d.db, err = sqlutil.Open(internal.SQLiteDriverName(), cs, nil); err != nil { + if d.db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil); err != nil { return nil, err } //d.db.Exec("PRAGMA journal_mode=WAL;") diff --git a/roomserver/storage/sqlite3/transactions_table.go b/roomserver/storage/sqlite3/transactions_table.go index 7b489ac6e..1e8de1ca8 100644 --- a/roomserver/storage/sqlite3/transactions_table.go +++ b/roomserver/storage/sqlite3/transactions_table.go @@ -19,7 +19,7 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" ) @@ -68,7 +68,7 @@ func (s *transactionStatements) InsertTransaction( userID string, eventID string, ) (err error) { - stmt := internal.TxStmt(txn, s.insertTransactionStmt) + stmt := sqlutil.TxStmt(txn, s.insertTransactionStmt) _, err = stmt.ExecContext( ctx, transactionID, sessionID, userID, eventID, ) diff --git a/roomserver/storage/storage.go b/roomserver/storage/storage.go index 071e2cf1d..d7367e4c7 100644 --- a/roomserver/storage/storage.go +++ b/roomserver/storage/storage.go @@ -19,13 +19,13 @@ package storage import ( "net/url" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/postgres" "github.com/matrix-org/dendrite/roomserver/storage/sqlite3" ) // Open opens a database connection. -func Open(dataSourceName string, dbProperties internal.DbProperties) (Database, error) { +func Open(dataSourceName string, dbProperties sqlutil.DbProperties) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return postgres.Open(dataSourceName, dbProperties) diff --git a/roomserver/storage/storage_wasm.go b/roomserver/storage/storage_wasm.go index ef30ca593..78405b20e 100644 --- a/roomserver/storage/storage_wasm.go +++ b/roomserver/storage/storage_wasm.go @@ -18,14 +18,14 @@ import ( "fmt" "net/url" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/sqlite3" ) // NewPublicRoomsServerDatabase opens a database connection. func Open( dataSourceName string, - dbProperties internal.DbProperties, // nolint:unparam + dbProperties sqlutil.DbProperties, // nolint:unparam ) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { diff --git a/roomserver/types/types.go b/roomserver/types/types.go index 87a4f935c..241e1e15d 100644 --- a/roomserver/types/types.go +++ b/roomserver/types/types.go @@ -16,7 +16,7 @@ package types import ( - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) @@ -174,7 +174,7 @@ type RoomRecentEventsUpdater interface { // It will share the same transaction as this updater. MembershipUpdater(targetUserNID EventStateKeyNID, isTargetLocalUser bool) (MembershipUpdater, error) // Implements Transaction so it can be committed or rolledback - internal.Transaction + sqlutil.Transaction } // A MembershipUpdater is used to update the membership of a user in a room. @@ -199,7 +199,7 @@ type MembershipUpdater interface { // Returns a list of invite event IDs that this state change retired. SetToLeave(senderUserID string, eventID string) (inviteEventIDs []string, err error) // Implements Transaction so it can be committed or rolledback. - internal.Transaction + sqlutil.Transaction } // A MissingEventError is an error that happened because the roomserver was diff --git a/serverkeyapi/inthttp/client.go b/serverkeyapi/inthttp/client.go index 2587160d4..e84cf47f6 100644 --- a/serverkeyapi/inthttp/client.go +++ b/serverkeyapi/inthttp/client.go @@ -6,7 +6,7 @@ import ( "net/http" "github.com/matrix-org/dendrite/internal/caching" - internalHTTP "github.com/matrix-org/dendrite/internal/http" + "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/serverkeyapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/opentracing/opentracing-go" @@ -116,7 +116,7 @@ func (h *httpServerKeyInternalAPI) InputPublicKeys( defer span.Finish() apiURL := h.serverKeyAPIURL + ServerKeyInputPublicKeyPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } func (h *httpServerKeyInternalAPI) QueryPublicKeys( @@ -128,5 +128,5 @@ func (h *httpServerKeyInternalAPI) QueryPublicKeys( defer span.Finish() apiURL := h.serverKeyAPIURL + ServerKeyQueryPublicKeyPath - return internalHTTP.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } diff --git a/serverkeyapi/inthttp/server.go b/serverkeyapi/inthttp/server.go index fd4b72c7e..cd4748392 100644 --- a/serverkeyapi/inthttp/server.go +++ b/serverkeyapi/inthttp/server.go @@ -5,15 +5,15 @@ import ( "net/http" "github.com/gorilla/mux" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/serverkeyapi/api" "github.com/matrix-org/util" ) func AddRoutes(s api.ServerKeyInternalAPI, internalAPIMux *mux.Router, cache caching.ServerKeyCache) { internalAPIMux.Handle(ServerKeyQueryPublicKeyPath, - internal.MakeInternalAPI("queryPublicKeys", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("queryPublicKeys", func(req *http.Request) util.JSONResponse { request := api.QueryPublicKeysRequest{} response := api.QueryPublicKeysResponse{} if err := json.NewDecoder(req.Body).Decode(&request); err != nil { @@ -28,7 +28,7 @@ func AddRoutes(s api.ServerKeyInternalAPI, internalAPIMux *mux.Router, cache cac }), ) internalAPIMux.Handle(ServerKeyInputPublicKeyPath, - internal.MakeInternalAPI("inputPublicKeys", func(req *http.Request) util.JSONResponse { + httputil.MakeInternalAPI("inputPublicKeys", func(req *http.Request) util.JSONResponse { request := api.InputPublicKeysRequest{} response := api.InputPublicKeysResponse{} if err := json.NewDecoder(req.Body).Decode(&request); err != nil { diff --git a/serverkeyapi/storage/keydb.go b/serverkeyapi/storage/keydb.go index b9389bd63..c28c4de1e 100644 --- a/serverkeyapi/storage/keydb.go +++ b/serverkeyapi/storage/keydb.go @@ -21,7 +21,7 @@ import ( "golang.org/x/crypto/ed25519" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/serverkeyapi/storage/postgres" "github.com/matrix-org/dendrite/serverkeyapi/storage/sqlite3" "github.com/matrix-org/gomatrixserverlib" @@ -30,7 +30,7 @@ import ( // NewDatabase opens a database connection. func NewDatabase( dataSourceName string, - dbProperties internal.DbProperties, + dbProperties sqlutil.DbProperties, serverName gomatrixserverlib.ServerName, serverKey ed25519.PublicKey, serverKeyID gomatrixserverlib.KeyID, diff --git a/serverkeyapi/storage/keydb_wasm.go b/serverkeyapi/storage/keydb_wasm.go index 3d2ca5505..de66a1d63 100644 --- a/serverkeyapi/storage/keydb_wasm.go +++ b/serverkeyapi/storage/keydb_wasm.go @@ -22,7 +22,7 @@ import ( "golang.org/x/crypto/ed25519" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/serverkeyapi/storage/sqlite3" "github.com/matrix-org/gomatrixserverlib" ) @@ -30,7 +30,7 @@ import ( // NewDatabase opens a database connection. func NewDatabase( dataSourceName string, - dbProperties internal.DbProperties, // nolint:unparam + dbProperties sqlutil.DbProperties, // nolint:unparam serverName gomatrixserverlib.ServerName, serverKey ed25519.PublicKey, serverKeyID gomatrixserverlib.KeyID, diff --git a/serverkeyapi/storage/postgres/keydb.go b/serverkeyapi/storage/postgres/keydb.go index 6282e13bb..32cdf951b 100644 --- a/serverkeyapi/storage/postgres/keydb.go +++ b/serverkeyapi/storage/postgres/keydb.go @@ -21,7 +21,6 @@ import ( "golang.org/x/crypto/ed25519" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) @@ -38,7 +37,7 @@ type Database struct { // Returns an error if there was a problem talking to the database. func NewDatabase( dataSourceName string, - dbProperties internal.DbProperties, + dbProperties sqlutil.DbProperties, serverName gomatrixserverlib.ServerName, serverKey ed25519.PublicKey, serverKeyID gomatrixserverlib.KeyID, diff --git a/serverkeyapi/storage/postgres/server_key_table.go b/serverkeyapi/storage/postgres/server_key_table.go index b3c26a480..87f1c211c 100644 --- a/serverkeyapi/storage/postgres/server_key_table.go +++ b/serverkeyapi/storage/postgres/server_key_table.go @@ -19,9 +19,8 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/internal" - "github.com/lib/pq" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/gomatrixserverlib" ) diff --git a/serverkeyapi/storage/sqlite3/keydb.go b/serverkeyapi/storage/sqlite3/keydb.go index ab423cff6..268c75420 100644 --- a/serverkeyapi/storage/sqlite3/keydb.go +++ b/serverkeyapi/storage/sqlite3/keydb.go @@ -21,7 +21,6 @@ import ( "golang.org/x/crypto/ed25519" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" @@ -48,7 +47,7 @@ func NewDatabase( if err != nil { return nil, err } - db, err := sqlutil.Open(internal.SQLiteDriverName(), cs, nil) + db, err := sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil) if err != nil { return nil, err } diff --git a/serverkeyapi/storage/sqlite3/server_key_table.go b/serverkeyapi/storage/sqlite3/server_key_table.go index ae24a14d4..4f03dccbb 100644 --- a/serverkeyapi/storage/sqlite3/server_key_table.go +++ b/serverkeyapi/storage/sqlite3/server_key_table.go @@ -21,6 +21,7 @@ import ( "strings" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" ) @@ -90,7 +91,7 @@ func (s *serverKeyStatements) bulkSelectServerKeys( nameAndKeyIDs = append(nameAndKeyIDs, nameAndKeyID(request)) } - query := strings.Replace(bulkSelectServerKeysSQL, "($1)", internal.QueryVariadic(len(nameAndKeyIDs)), 1) + query := strings.Replace(bulkSelectServerKeysSQL, "($1)", sqlutil.QueryVariadic(len(nameAndKeyIDs)), 1) iKeyIDs := make([]interface{}, len(nameAndKeyIDs)) for i, v := range nameAndKeyIDs { diff --git a/syncapi/consumers/clientapi.go b/syncapi/consumers/clientapi.go index 4d43e8111..ad6290e3f 100644 --- a/syncapi/consumers/clientapi.go +++ b/syncapi/consumers/clientapi.go @@ -21,6 +21,7 @@ import ( "github.com/Shopify/sarama" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/sync" "github.com/matrix-org/dendrite/syncapi/types" @@ -67,7 +68,7 @@ func (s *OutputClientDataConsumer) Start() error { // sync stream position may race and be incorrectly calculated. func (s *OutputClientDataConsumer) onMessage(msg *sarama.ConsumerMessage) error { // Parse out the event JSON - var output internal.AccountData + var output eventutil.AccountData if err := json.Unmarshal(msg.Value, &output); err != nil { // If the message was invalid, log it and move on to the next message in the stream log.WithError(err).Errorf("client API server output log: message parse failure") diff --git a/syncapi/routing/routing.go b/syncapi/routing/routing.go index c876164d7..50b469177 100644 --- a/syncapi/routing/routing.go +++ b/syncapi/routing/routing.go @@ -21,8 +21,8 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" - "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/sync" @@ -52,12 +52,12 @@ func Setup( } // TODO: Add AS support for all handlers below. - r0mux.Handle("/sync", internal.MakeAuthAPI("sync", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + r0mux.Handle("/sync", httputil.MakeAuthAPI("sync", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { return srp.OnIncomingSyncRequest(req, device) })).Methods(http.MethodGet, http.MethodOptions) - r0mux.Handle("/rooms/{roomID}/messages", internal.MakeAuthAPI("room_messages", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { - vars, err := internal.URLDecodeMapValues(mux.Vars(req)) + r0mux.Handle("/rooms/{roomID}/messages", httputil.MakeAuthAPI("room_messages", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } diff --git a/syncapi/storage/postgres/account_data_table.go b/syncapi/storage/postgres/account_data_table.go index e0b87bd6d..67eb1e863 100644 --- a/syncapi/storage/postgres/account_data_table.go +++ b/syncapi/storage/postgres/account_data_table.go @@ -21,6 +21,7 @@ import ( "github.com/lib/pq" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -136,7 +137,7 @@ func (s *accountDataStatements) SelectMaxAccountDataID( ctx context.Context, txn *sql.Tx, ) (id int64, err error) { var nullableID sql.NullInt64 - stmt := internal.TxStmt(txn, s.selectMaxAccountDataIDStmt) + stmt := sqlutil.TxStmt(txn, s.selectMaxAccountDataIDStmt) err = stmt.QueryRowContext(ctx).Scan(&nullableID) if nullableID.Valid { id = nullableID.Int64 diff --git a/syncapi/storage/postgres/current_room_state_table.go b/syncapi/storage/postgres/current_room_state_table.go index cc0fb6b1e..25906edb4 100644 --- a/syncapi/storage/postgres/current_room_state_table.go +++ b/syncapi/storage/postgres/current_room_state_table.go @@ -22,6 +22,7 @@ import ( "github.com/lib/pq" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -165,7 +166,7 @@ func (s *currentRoomStateStatements) SelectRoomIDsWithMembership( userID string, membership string, // nolint: unparam ) ([]string, error) { - stmt := internal.TxStmt(txn, s.selectRoomIDsWithMembershipStmt) + stmt := sqlutil.TxStmt(txn, s.selectRoomIDsWithMembershipStmt) rows, err := stmt.QueryContext(ctx, userID, membership) if err != nil { return nil, err @@ -188,7 +189,7 @@ func (s *currentRoomStateStatements) SelectCurrentState( ctx context.Context, txn *sql.Tx, roomID string, stateFilter *gomatrixserverlib.StateFilter, ) ([]gomatrixserverlib.HeaderedEvent, error) { - stmt := internal.TxStmt(txn, s.selectCurrentStateStmt) + stmt := sqlutil.TxStmt(txn, s.selectCurrentStateStmt) rows, err := stmt.QueryContext(ctx, roomID, pq.StringArray(stateFilter.Senders), pq.StringArray(stateFilter.NotSenders), @@ -208,7 +209,7 @@ func (s *currentRoomStateStatements) SelectCurrentState( func (s *currentRoomStateStatements) DeleteRoomStateByEventID( ctx context.Context, txn *sql.Tx, eventID string, ) error { - stmt := internal.TxStmt(txn, s.deleteRoomStateByEventIDStmt) + stmt := sqlutil.TxStmt(txn, s.deleteRoomStateByEventIDStmt) _, err := stmt.ExecContext(ctx, eventID) return err } @@ -231,7 +232,7 @@ func (s *currentRoomStateStatements) UpsertRoomState( } // upsert state event - stmt := internal.TxStmt(txn, s.upsertRoomStateStmt) + stmt := sqlutil.TxStmt(txn, s.upsertRoomStateStmt) _, err = stmt.ExecContext( ctx, event.RoomID(), @@ -250,7 +251,7 @@ func (s *currentRoomStateStatements) UpsertRoomState( func (s *currentRoomStateStatements) SelectEventsWithEventIDs( ctx context.Context, txn *sql.Tx, eventIDs []string, ) ([]types.StreamEvent, error) { - stmt := internal.TxStmt(txn, s.selectEventsWithEventIDsStmt) + stmt := sqlutil.TxStmt(txn, s.selectEventsWithEventIDsStmt) rows, err := stmt.QueryContext(ctx, pq.StringArray(eventIDs)) if err != nil { return nil, err diff --git a/syncapi/storage/postgres/invites_table.go b/syncapi/storage/postgres/invites_table.go index caa12d2f8..5031d64e5 100644 --- a/syncapi/storage/postgres/invites_table.go +++ b/syncapi/storage/postgres/invites_table.go @@ -21,6 +21,7 @@ import ( "encoding/json" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -119,7 +120,7 @@ func (s *inviteEventsStatements) DeleteInviteEvent( func (s *inviteEventsStatements) SelectInviteEventsInRange( ctx context.Context, txn *sql.Tx, targetUserID string, r types.Range, ) (map[string]gomatrixserverlib.HeaderedEvent, error) { - stmt := internal.TxStmt(txn, s.selectInviteEventsInRangeStmt) + stmt := sqlutil.TxStmt(txn, s.selectInviteEventsInRangeStmt) rows, err := stmt.QueryContext(ctx, targetUserID, r.Low(), r.High()) if err != nil { return nil, err @@ -149,7 +150,7 @@ func (s *inviteEventsStatements) SelectMaxInviteID( ctx context.Context, txn *sql.Tx, ) (id int64, err error) { var nullableID sql.NullInt64 - stmt := internal.TxStmt(txn, s.selectMaxInviteIDStmt) + stmt := sqlutil.TxStmt(txn, s.selectMaxInviteIDStmt) err = stmt.QueryRowContext(ctx).Scan(&nullableID) if nullableID.Valid { id = nullableID.Int64 diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go index ae6dfb381..f01b2eabd 100644 --- a/syncapi/storage/postgres/output_room_events_table.go +++ b/syncapi/storage/postgres/output_room_events_table.go @@ -21,12 +21,13 @@ import ( "encoding/json" "sort" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/lib/pq" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" log "github.com/sirupsen/logrus" ) @@ -158,7 +159,7 @@ func (s *outputRoomEventsStatements) SelectStateInRange( ctx context.Context, txn *sql.Tx, r types.Range, stateFilter *gomatrixserverlib.StateFilter, ) (map[string]map[string]bool, map[string]types.StreamEvent, error) { - stmt := internal.TxStmt(txn, s.selectStateInRangeStmt) + stmt := sqlutil.TxStmt(txn, s.selectStateInRangeStmt) rows, err := stmt.QueryContext( ctx, r.Low(), r.High(), @@ -239,7 +240,7 @@ func (s *outputRoomEventsStatements) SelectMaxEventID( ctx context.Context, txn *sql.Tx, ) (id int64, err error) { var nullableID sql.NullInt64 - stmt := internal.TxStmt(txn, s.selectMaxEventIDStmt) + stmt := sqlutil.TxStmt(txn, s.selectMaxEventIDStmt) err = stmt.QueryRowContext(ctx).Scan(&nullableID) if nullableID.Valid { id = nullableID.Int64 @@ -275,7 +276,7 @@ func (s *outputRoomEventsStatements) InsertEvent( return } - stmt := internal.TxStmt(txn, s.insertEventStmt) + stmt := sqlutil.TxStmt(txn, s.insertEventStmt) err = stmt.QueryRowContext( ctx, event.RoomID(), @@ -303,9 +304,9 @@ func (s *outputRoomEventsStatements) SelectRecentEvents( ) ([]types.StreamEvent, error) { var stmt *sql.Stmt if onlySyncEvents { - stmt = internal.TxStmt(txn, s.selectRecentEventsForSyncStmt) + stmt = sqlutil.TxStmt(txn, s.selectRecentEventsForSyncStmt) } else { - stmt = internal.TxStmt(txn, s.selectRecentEventsStmt) + stmt = sqlutil.TxStmt(txn, s.selectRecentEventsStmt) } rows, err := stmt.QueryContext(ctx, roomID, r.Low(), r.High(), limit) if err != nil { @@ -333,7 +334,7 @@ func (s *outputRoomEventsStatements) SelectEarlyEvents( ctx context.Context, txn *sql.Tx, roomID string, r types.Range, limit int, ) ([]types.StreamEvent, error) { - stmt := internal.TxStmt(txn, s.selectEarlyEventsStmt) + stmt := sqlutil.TxStmt(txn, s.selectEarlyEventsStmt) rows, err := stmt.QueryContext(ctx, roomID, r.Low(), r.High(), limit) if err != nil { return nil, err @@ -357,7 +358,7 @@ func (s *outputRoomEventsStatements) SelectEarlyEvents( func (s *outputRoomEventsStatements) SelectEvents( ctx context.Context, txn *sql.Tx, eventIDs []string, ) ([]types.StreamEvent, error) { - stmt := internal.TxStmt(txn, s.selectEventsStmt) + stmt := sqlutil.TxStmt(txn, s.selectEventsStmt) rows, err := stmt.QueryContext(ctx, pq.StringArray(eventIDs)) if err != nil { return nil, err diff --git a/syncapi/storage/postgres/output_room_events_topology_table.go b/syncapi/storage/postgres/output_room_events_topology_table.go index ffbeece30..1ab3a1dc2 100644 --- a/syncapi/storage/postgres/output_room_events_topology_table.go +++ b/syncapi/storage/postgres/output_room_events_topology_table.go @@ -19,7 +19,6 @@ import ( "database/sql" "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" diff --git a/syncapi/storage/postgres/send_to_device_table.go b/syncapi/storage/postgres/send_to_device_table.go index 335a05ef1..07af9ad6b 100644 --- a/syncapi/storage/postgres/send_to_device_table.go +++ b/syncapi/storage/postgres/send_to_device_table.go @@ -21,6 +21,7 @@ import ( "github.com/lib/pq" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" ) @@ -107,14 +108,14 @@ func NewPostgresSendToDeviceTable(db *sql.DB) (tables.SendToDevice, error) { func (s *sendToDeviceStatements) InsertSendToDeviceMessage( ctx context.Context, txn *sql.Tx, userID, deviceID, content string, ) (err error) { - _, err = internal.TxStmt(txn, s.insertSendToDeviceMessageStmt).ExecContext(ctx, userID, deviceID, content) + _, err = sqlutil.TxStmt(txn, s.insertSendToDeviceMessageStmt).ExecContext(ctx, userID, deviceID, content) return } func (s *sendToDeviceStatements) CountSendToDeviceMessages( ctx context.Context, txn *sql.Tx, userID, deviceID string, ) (count int, err error) { - row := internal.TxStmt(txn, s.countSendToDeviceMessagesStmt).QueryRowContext(ctx, userID, deviceID) + row := sqlutil.TxStmt(txn, s.countSendToDeviceMessagesStmt).QueryRowContext(ctx, userID, deviceID) if err = row.Scan(&count); err != nil { return } @@ -124,7 +125,7 @@ func (s *sendToDeviceStatements) CountSendToDeviceMessages( func (s *sendToDeviceStatements) SelectSendToDeviceMessages( ctx context.Context, txn *sql.Tx, userID, deviceID string, ) (events []types.SendToDeviceEvent, err error) { - rows, err := internal.TxStmt(txn, s.selectSendToDeviceMessagesStmt).QueryContext(ctx, userID, deviceID) + rows, err := sqlutil.TxStmt(txn, s.selectSendToDeviceMessagesStmt).QueryContext(ctx, userID, deviceID) if err != nil { return } diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go index 8a8f964a5..573586cc7 100644 --- a/syncapi/storage/postgres/syncserver.go +++ b/syncapi/storage/postgres/syncserver.go @@ -18,12 +18,10 @@ package postgres import ( "database/sql" - "github.com/matrix-org/dendrite/internal/sqlutil" - // Import the postgres database driver. _ "github.com/lib/pq" "github.com/matrix-org/dendrite/eduserver/cache" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage/shared" ) @@ -32,11 +30,11 @@ import ( type SyncServerDatasource struct { shared.Database db *sql.DB - internal.PartitionOffsetStatements + sqlutil.PartitionOffsetStatements } // NewDatabase creates a new sync server database -func NewDatabase(dbDataSourceName string, dbProperties internal.DbProperties) (*SyncServerDatasource, error) { +func NewDatabase(dbDataSourceName string, dbProperties sqlutil.DbProperties) (*SyncServerDatasource, error) { var d SyncServerDatasource var err error if d.db, err = sqlutil.Open("postgres", dbDataSourceName, dbProperties); err != nil { @@ -82,7 +80,7 @@ func NewDatabase(dbDataSourceName string, dbProperties internal.DbProperties) (* CurrentRoomState: currState, BackwardExtremities: backwardExtremities, SendToDevice: sendToDevice, - SendToDeviceWriter: internal.NewTransactionWriter(), + SendToDeviceWriter: sqlutil.NewTransactionWriter(), EDUCache: cache.New(), } return &d, nil diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go index 497c043a6..21d8df375 100644 --- a/syncapi/storage/shared/syncserver.go +++ b/syncapi/storage/shared/syncserver.go @@ -23,7 +23,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/eduserver/cache" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" @@ -42,7 +42,7 @@ type Database struct { CurrentRoomState tables.CurrentRoomState BackwardExtremities tables.BackwardsExtremities SendToDevice tables.SendToDevice - SendToDeviceWriter *internal.TransactionWriter + SendToDeviceWriter *sqlutil.TransactionWriter EDUCache *cache.EDUCache } @@ -126,7 +126,7 @@ func (d *Database) GetStateEvent( func (d *Database) GetStateEventsForRoom( ctx context.Context, roomID string, stateFilter *gomatrixserverlib.StateFilter, ) (stateEvents []gomatrixserverlib.HeaderedEvent, err error) { - err = internal.WithTransaction(d.DB, func(txn *sql.Tx) error { + err = sqlutil.WithTransaction(d.DB, func(txn *sql.Tx) error { stateEvents, err = d.CurrentRoomState.SelectCurrentState(ctx, txn, roomID, stateFilter) return err }) @@ -136,7 +136,7 @@ func (d *Database) GetStateEventsForRoom( func (d *Database) SyncStreamPosition(ctx context.Context) (types.StreamPosition, error) { var maxID int64 var err error - err = internal.WithTransaction(d.DB, func(txn *sql.Tx) error { + err = sqlutil.WithTransaction(d.DB, func(txn *sql.Tx) error { maxID, err = d.OutputEvents.SelectMaxEventID(ctx, txn) if err != nil { return err @@ -168,7 +168,7 @@ func (d *Database) SyncStreamPosition(ctx context.Context) (types.StreamPosition func (d *Database) AddInviteEvent( ctx context.Context, inviteEvent gomatrixserverlib.HeaderedEvent, ) (sp types.StreamPosition, err error) { - err = internal.WithTransaction(d.DB, func(txn *sql.Tx) error { + err = sqlutil.WithTransaction(d.DB, func(txn *sql.Tx) error { sp, err = d.Invites.InsertInviteEvent(ctx, txn, inviteEvent) return err }) @@ -207,7 +207,7 @@ func (d *Database) GetAccountDataInRange( func (d *Database) UpsertAccountData( ctx context.Context, userID, roomID, dataType string, ) (sp types.StreamPosition, err error) { - err = internal.WithTransaction(d.DB, func(txn *sql.Tx) error { + err = sqlutil.WithTransaction(d.DB, func(txn *sql.Tx) error { sp, err = d.AccountData.InsertAccountData(ctx, txn, userID, roomID, dataType) return err }) @@ -275,7 +275,7 @@ func (d *Database) WriteEvent( addStateEventIDs, removeStateEventIDs []string, transactionID *api.TransactionID, excludeFromSync bool, ) (pduPosition types.StreamPosition, returnErr error) { - returnErr = internal.WithTransaction(d.DB, func(txn *sql.Tx) error { + returnErr = sqlutil.WithTransaction(d.DB, func(txn *sql.Tx) error { var err error pos, err := d.OutputEvents.InsertEvent( ctx, txn, ev, addStateEventIDs, removeStateEventIDs, transactionID, excludeFromSync, @@ -375,7 +375,7 @@ func (d *Database) GetEventsInTopologicalRange( } func (d *Database) SyncPosition(ctx context.Context) (tok types.StreamingToken, err error) { - err = internal.WithTransaction(d.DB, func(txn *sql.Tx) error { + err = sqlutil.WithTransaction(d.DB, func(txn *sql.Tx) error { pos, err := d.syncPositionTx(ctx, txn) if err != nil { return err @@ -454,7 +454,7 @@ func (d *Database) addPDUDeltaToResponse( } var succeeded bool defer func() { - txerr := internal.EndTransaction(txn, &succeeded) + txerr := sqlutil.EndTransaction(txn, &succeeded) if err == nil && txerr != nil { err = txerr } @@ -608,7 +608,7 @@ func (d *Database) getResponseWithPDUsForCompleteSync( } var succeeded bool defer func() { - txerr := internal.EndTransaction(txn, &succeeded) + txerr := sqlutil.EndTransaction(txn, &succeeded) if err == nil && txerr != nil { err = txerr } diff --git a/syncapi/storage/sqlite3/account_data_table.go b/syncapi/storage/sqlite3/account_data_table.go index a3c797342..ae5caa4e5 100644 --- a/syncapi/storage/sqlite3/account_data_table.go +++ b/syncapi/storage/sqlite3/account_data_table.go @@ -20,7 +20,6 @@ import ( "database/sql" "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go index b0cf1297c..85f212ad8 100644 --- a/syncapi/storage/sqlite3/current_room_state_table.go +++ b/syncapi/storage/sqlite3/current_room_state_table.go @@ -22,6 +22,7 @@ import ( "strings" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -152,7 +153,7 @@ func (s *currentRoomStateStatements) SelectRoomIDsWithMembership( userID string, membership string, // nolint: unparam ) ([]string, error) { - stmt := internal.TxStmt(txn, s.selectRoomIDsWithMembershipStmt) + stmt := sqlutil.TxStmt(txn, s.selectRoomIDsWithMembershipStmt) rows, err := stmt.QueryContext(ctx, userID, membership) if err != nil { return nil, err @@ -175,7 +176,7 @@ func (s *currentRoomStateStatements) SelectCurrentState( ctx context.Context, txn *sql.Tx, roomID string, stateFilterPart *gomatrixserverlib.StateFilter, ) ([]gomatrixserverlib.HeaderedEvent, error) { - stmt := internal.TxStmt(txn, s.selectCurrentStateStmt) + stmt := sqlutil.TxStmt(txn, s.selectCurrentStateStmt) rows, err := stmt.QueryContext(ctx, roomID, nil, // FIXME: pq.StringArray(stateFilterPart.Senders), nil, // FIXME: pq.StringArray(stateFilterPart.NotSenders), @@ -195,7 +196,7 @@ func (s *currentRoomStateStatements) SelectCurrentState( func (s *currentRoomStateStatements) DeleteRoomStateByEventID( ctx context.Context, txn *sql.Tx, eventID string, ) error { - stmt := internal.TxStmt(txn, s.deleteRoomStateByEventIDStmt) + stmt := sqlutil.TxStmt(txn, s.deleteRoomStateByEventIDStmt) _, err := stmt.ExecContext(ctx, eventID) return err } @@ -218,7 +219,7 @@ func (s *currentRoomStateStatements) UpsertRoomState( } // upsert state event - stmt := internal.TxStmt(txn, s.upsertRoomStateStmt) + stmt := sqlutil.TxStmt(txn, s.upsertRoomStateStmt) _, err = stmt.ExecContext( ctx, event.RoomID(), @@ -241,7 +242,7 @@ func (s *currentRoomStateStatements) SelectEventsWithEventIDs( for k, v := range eventIDs { iEventIDs[k] = v } - query := strings.Replace(selectEventsWithEventIDsSQL, "($1)", internal.QueryVariadic(len(iEventIDs)), 1) + query := strings.Replace(selectEventsWithEventIDsSQL, "($1)", sqlutil.QueryVariadic(len(iEventIDs)), 1) rows, err := txn.QueryContext(ctx, query, iEventIDs...) if err != nil { return nil, err diff --git a/syncapi/storage/sqlite3/invites_table.go b/syncapi/storage/sqlite3/invites_table.go index 64b523394..bb58e3456 100644 --- a/syncapi/storage/sqlite3/invites_table.go +++ b/syncapi/storage/sqlite3/invites_table.go @@ -21,6 +21,7 @@ import ( "encoding/json" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -123,7 +124,7 @@ func (s *inviteEventsStatements) DeleteInviteEvent( func (s *inviteEventsStatements) SelectInviteEventsInRange( ctx context.Context, txn *sql.Tx, targetUserID string, r types.Range, ) (map[string]gomatrixserverlib.HeaderedEvent, error) { - stmt := internal.TxStmt(txn, s.selectInviteEventsInRangeStmt) + stmt := sqlutil.TxStmt(txn, s.selectInviteEventsInRangeStmt) rows, err := stmt.QueryContext(ctx, targetUserID, r.Low(), r.High()) if err != nil { return nil, err @@ -153,7 +154,7 @@ func (s *inviteEventsStatements) SelectMaxInviteID( ctx context.Context, txn *sql.Tx, ) (id int64, err error) { var nullableID sql.NullInt64 - stmt := internal.TxStmt(txn, s.selectMaxInviteIDStmt) + stmt := sqlutil.TxStmt(txn, s.selectMaxInviteIDStmt) err = stmt.QueryRowContext(ctx).Scan(&nullableID) if nullableID.Valid { id = nullableID.Int64 diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index 4a84ecc6c..367ab3c9a 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -21,11 +21,12 @@ import ( "encoding/json" "sort" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" log "github.com/sirupsen/logrus" ) @@ -149,7 +150,7 @@ func (s *outputRoomEventsStatements) SelectStateInRange( ctx context.Context, txn *sql.Tx, r types.Range, stateFilterPart *gomatrixserverlib.StateFilter, ) (map[string]map[string]bool, map[string]types.StreamEvent, error) { - stmt := internal.TxStmt(txn, s.selectStateInRangeStmt) + stmt := sqlutil.TxStmt(txn, s.selectStateInRangeStmt) rows, err := stmt.QueryContext( ctx, r.Low(), r.High(), @@ -236,7 +237,7 @@ func (s *outputRoomEventsStatements) SelectMaxEventID( ctx context.Context, txn *sql.Tx, ) (id int64, err error) { var nullableID sql.NullInt64 - stmt := internal.TxStmt(txn, s.selectMaxEventIDStmt) + stmt := sqlutil.TxStmt(txn, s.selectMaxEventIDStmt) err = stmt.QueryRowContext(ctx).Scan(&nullableID) if nullableID.Valid { id = nullableID.Int64 @@ -286,7 +287,7 @@ func (s *outputRoomEventsStatements) InsertEvent( return } - insertStmt := internal.TxStmt(txn, s.insertEventStmt) + insertStmt := sqlutil.TxStmt(txn, s.insertEventStmt) _, err = insertStmt.ExecContext( ctx, streamPos, @@ -313,9 +314,9 @@ func (s *outputRoomEventsStatements) SelectRecentEvents( ) ([]types.StreamEvent, error) { var stmt *sql.Stmt if onlySyncEvents { - stmt = internal.TxStmt(txn, s.selectRecentEventsForSyncStmt) + stmt = sqlutil.TxStmt(txn, s.selectRecentEventsForSyncStmt) } else { - stmt = internal.TxStmt(txn, s.selectRecentEventsStmt) + stmt = sqlutil.TxStmt(txn, s.selectRecentEventsStmt) } rows, err := stmt.QueryContext(ctx, roomID, r.Low(), r.High(), limit) @@ -342,7 +343,7 @@ func (s *outputRoomEventsStatements) SelectEarlyEvents( ctx context.Context, txn *sql.Tx, roomID string, r types.Range, limit int, ) ([]types.StreamEvent, error) { - stmt := internal.TxStmt(txn, s.selectEarlyEventsStmt) + stmt := sqlutil.TxStmt(txn, s.selectEarlyEventsStmt) rows, err := stmt.QueryContext(ctx, roomID, r.Low(), r.High(), limit) if err != nil { return nil, err @@ -367,7 +368,7 @@ func (s *outputRoomEventsStatements) SelectEvents( ctx context.Context, txn *sql.Tx, eventIDs []string, ) ([]types.StreamEvent, error) { var returnEvents []types.StreamEvent - stmt := internal.TxStmt(txn, s.selectEventsStmt) + stmt := sqlutil.TxStmt(txn, s.selectEventsStmt) for _, eventID := range eventIDs { rows, err := stmt.QueryContext(ctx, eventID) if err != nil { diff --git a/syncapi/storage/sqlite3/output_room_events_topology_table.go b/syncapi/storage/sqlite3/output_room_events_topology_table.go index 0d727cb18..811dfa4f3 100644 --- a/syncapi/storage/sqlite3/output_room_events_topology_table.go +++ b/syncapi/storage/sqlite3/output_room_events_topology_table.go @@ -18,7 +18,7 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -102,7 +102,7 @@ func NewSqliteTopologyTable(db *sql.DB) (tables.Topology, error) { func (s *outputRoomEventsTopologyStatements) InsertEventInTopology( ctx context.Context, txn *sql.Tx, event *gomatrixserverlib.HeaderedEvent, pos types.StreamPosition, ) (err error) { - stmt := internal.TxStmt(txn, s.insertEventInTopologyStmt) + stmt := sqlutil.TxStmt(txn, s.insertEventInTopologyStmt) _, err = stmt.ExecContext( ctx, event.EventID(), event.Depth(), event.RoomID(), pos, ) @@ -118,9 +118,9 @@ func (s *outputRoomEventsTopologyStatements) SelectEventIDsInRange( // is requested or not. var stmt *sql.Stmt if chronologicalOrder { - stmt = internal.TxStmt(txn, s.selectEventIDsInRangeASCStmt) + stmt = sqlutil.TxStmt(txn, s.selectEventIDsInRangeASCStmt) } else { - stmt = internal.TxStmt(txn, s.selectEventIDsInRangeDESCStmt) + stmt = sqlutil.TxStmt(txn, s.selectEventIDsInRangeDESCStmt) } // Query the event IDs. @@ -149,7 +149,7 @@ func (s *outputRoomEventsTopologyStatements) SelectEventIDsInRange( func (s *outputRoomEventsTopologyStatements) SelectPositionInTopology( ctx context.Context, txn *sql.Tx, eventID string, ) (pos types.StreamPosition, spos types.StreamPosition, err error) { - stmt := internal.TxStmt(txn, s.selectPositionInTopologyStmt) + stmt := sqlutil.TxStmt(txn, s.selectPositionInTopologyStmt) err = stmt.QueryRowContext(ctx, eventID).Scan(&pos, &spos) return } @@ -157,7 +157,7 @@ func (s *outputRoomEventsTopologyStatements) SelectPositionInTopology( func (s *outputRoomEventsTopologyStatements) SelectMaxPositionInTopology( ctx context.Context, txn *sql.Tx, roomID string, ) (pos types.StreamPosition, spos types.StreamPosition, err error) { - stmt := internal.TxStmt(txn, s.selectMaxPositionInTopologyStmt) + stmt := sqlutil.TxStmt(txn, s.selectMaxPositionInTopologyStmt) err = stmt.QueryRowContext(ctx, roomID).Scan(&pos, &spos) return } diff --git a/syncapi/storage/sqlite3/send_to_device_table.go b/syncapi/storage/sqlite3/send_to_device_table.go index 0d03f23ef..42bd3c19a 100644 --- a/syncapi/storage/sqlite3/send_to_device_table.go +++ b/syncapi/storage/sqlite3/send_to_device_table.go @@ -21,6 +21,7 @@ import ( "strings" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" ) @@ -97,14 +98,14 @@ func NewSqliteSendToDeviceTable(db *sql.DB) (tables.SendToDevice, error) { func (s *sendToDeviceStatements) InsertSendToDeviceMessage( ctx context.Context, txn *sql.Tx, userID, deviceID, content string, ) (err error) { - _, err = internal.TxStmt(txn, s.insertSendToDeviceMessageStmt).ExecContext(ctx, userID, deviceID, content) + _, err = sqlutil.TxStmt(txn, s.insertSendToDeviceMessageStmt).ExecContext(ctx, userID, deviceID, content) return } func (s *sendToDeviceStatements) CountSendToDeviceMessages( ctx context.Context, txn *sql.Tx, userID, deviceID string, ) (count int, err error) { - row := internal.TxStmt(txn, s.countSendToDeviceMessagesStmt).QueryRowContext(ctx, userID, deviceID) + row := sqlutil.TxStmt(txn, s.countSendToDeviceMessagesStmt).QueryRowContext(ctx, userID, deviceID) if err = row.Scan(&count); err != nil { return } @@ -114,7 +115,7 @@ func (s *sendToDeviceStatements) CountSendToDeviceMessages( func (s *sendToDeviceStatements) SelectSendToDeviceMessages( ctx context.Context, txn *sql.Tx, userID, deviceID string, ) (events []types.SendToDeviceEvent, err error) { - rows, err := internal.TxStmt(txn, s.selectSendToDeviceMessagesStmt).QueryContext(ctx, userID, deviceID) + rows, err := sqlutil.TxStmt(txn, s.selectSendToDeviceMessagesStmt).QueryContext(ctx, userID, deviceID) if err != nil { return } @@ -149,7 +150,7 @@ func (s *sendToDeviceStatements) SelectSendToDeviceMessages( func (s *sendToDeviceStatements) UpdateSentSendToDeviceMessages( ctx context.Context, txn *sql.Tx, token string, nids []types.SendToDeviceNID, ) (err error) { - query := strings.Replace(updateSentSendToDeviceMessagesSQL, "($2)", internal.QueryVariadic(1+len(nids)), 1) + query := strings.Replace(updateSentSendToDeviceMessagesSQL, "($2)", sqlutil.QueryVariadic(1+len(nids)), 1) params := make([]interface{}, 1+len(nids)) params[0] = token for k, v := range nids { @@ -162,7 +163,7 @@ func (s *sendToDeviceStatements) UpdateSentSendToDeviceMessages( func (s *sendToDeviceStatements) DeleteSendToDeviceMessages( ctx context.Context, txn *sql.Tx, nids []types.SendToDeviceNID, ) (err error) { - query := strings.Replace(deleteSendToDeviceMessagesSQL, "($1)", internal.QueryVariadic(len(nids)), 1) + query := strings.Replace(deleteSendToDeviceMessagesSQL, "($1)", sqlutil.QueryVariadic(len(nids)), 1) params := make([]interface{}, 1+len(nids)) for k, v := range nids { params[k] = v diff --git a/syncapi/storage/sqlite3/stream_id_table.go b/syncapi/storage/sqlite3/stream_id_table.go index 3bfe22dd0..57abd9c44 100644 --- a/syncapi/storage/sqlite3/stream_id_table.go +++ b/syncapi/storage/sqlite3/stream_id_table.go @@ -4,7 +4,7 @@ import ( "context" "database/sql" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/types" ) @@ -46,8 +46,8 @@ func (s *streamIDStatements) prepare(db *sql.DB) (err error) { } func (s *streamIDStatements) nextStreamID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error) { - increaseStmt := internal.TxStmt(txn, s.increaseStreamIDStmt) - selectStmt := internal.TxStmt(txn, s.selectStreamIDStmt) + increaseStmt := sqlutil.TxStmt(txn, s.increaseStreamIDStmt) + selectStmt := sqlutil.TxStmt(txn, s.selectStreamIDStmt) if _, err = increaseStmt.ExecContext(ctx, "global"); err != nil { return } diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index 38ce5bcf0..51cdbe325 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -18,13 +18,11 @@ package sqlite3 import ( "database/sql" - "github.com/matrix-org/dendrite/internal/sqlutil" - // Import the sqlite3 package _ "github.com/mattn/go-sqlite3" "github.com/matrix-org/dendrite/eduserver/cache" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage/shared" ) @@ -33,7 +31,7 @@ import ( type SyncServerDatasource struct { shared.Database db *sql.DB - internal.PartitionOffsetStatements + sqlutil.PartitionOffsetStatements streamID streamIDStatements } @@ -45,7 +43,7 @@ func NewDatabase(dataSourceName string) (*SyncServerDatasource, error) { if err != nil { return nil, err } - if d.db, err = sqlutil.Open(internal.SQLiteDriverName(), cs, nil); err != nil { + if d.db, err = sqlutil.Open(sqlutil.SQLiteDriverName(), cs, nil); err != nil { return nil, err } if err = d.prepare(); err != nil { @@ -98,7 +96,7 @@ func (d *SyncServerDatasource) prepare() (err error) { CurrentRoomState: roomState, Topology: topology, SendToDevice: sendToDevice, - SendToDeviceWriter: internal.NewTransactionWriter(), + SendToDeviceWriter: sqlutil.NewTransactionWriter(), EDUCache: cache.New(), } return nil diff --git a/syncapi/storage/storage.go b/syncapi/storage/storage.go index a4275da0c..ea69da3bc 100644 --- a/syncapi/storage/storage.go +++ b/syncapi/storage/storage.go @@ -19,13 +19,13 @@ package storage import ( "net/url" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage/postgres" "github.com/matrix-org/dendrite/syncapi/storage/sqlite3" ) // NewSyncServerDatasource opens a database connection. -func NewSyncServerDatasource(dataSourceName string, dbProperties internal.DbProperties) (Database, error) { +func NewSyncServerDatasource(dataSourceName string, dbProperties sqlutil.DbProperties) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { return postgres.NewDatabase(dataSourceName, dbProperties) diff --git a/syncapi/storage/storage_wasm.go b/syncapi/storage/storage_wasm.go index a2b269a14..0886b8c21 100644 --- a/syncapi/storage/storage_wasm.go +++ b/syncapi/storage/storage_wasm.go @@ -18,14 +18,14 @@ import ( "fmt" "net/url" - "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage/sqlite3" ) // NewPublicRoomsServerDatabase opens a database connection. func NewSyncServerDatasource( dataSourceName string, - dbProperties internal.DbProperties, // nolint:unparam + dbProperties sqlutil.DbProperties, // nolint:unparam ) (Database, error) { uri, err := url.Parse(dataSourceName) if err != nil { From 0dc4ceaa2d8e46aa0134c1aabe96389ba4c1591d Mon Sep 17 00:00:00 2001 From: Kegsay Date: Fri, 12 Jun 2020 15:11:33 +0100 Subject: [PATCH 159/200] Minor perf/debugging improvements (#1121) * Minor perf/debugging improvements - publicroomsapi: Don't call QueryEventsByID with no event IDs - appservice: Consume only if there are 1 or more ASes - roomserver: don't keep a copy of the request "for debugging" - we trace now * fedsender: return early if we have no destinations * Unbreak tests --- appservice/appservice.go | 16 ++++++++++------ federationapi/routing/send_test.go | 3 --- federationsender/queue/queue.go | 3 +++ publicroomsapi/consumers/roomserver.go | 11 ++++++----- roomserver/api/query.go | 8 -------- roomserver/internal/query.go | 5 ----- 6 files changed, 19 insertions(+), 27 deletions(-) diff --git a/appservice/appservice.go b/appservice/appservice.go index 0fbe3f20b..bd261ff9b 100644 --- a/appservice/appservice.go +++ b/appservice/appservice.go @@ -86,12 +86,16 @@ func NewInternalAPI( Cfg: base.Cfg, } - consumer := consumers.NewOutputRoomEventConsumer( - base.Cfg, base.KafkaConsumer, accountsDB, appserviceDB, - rsAPI, workerStates, - ) - if err := consumer.Start(); err != nil { - logrus.WithError(err).Panicf("failed to start appservice roomserver consumer") + // Only consume if we actually have ASes to track, else we'll just chew cycles needlessly. + // We can't add ASes at runtime so this is safe to do. + if len(workerStates) > 0 { + consumer := consumers.NewOutputRoomEventConsumer( + base.Cfg, base.KafkaConsumer, accountsDB, appserviceDB, + rsAPI, workerStates, + ) + if err := consumer.Start(); err != nil { + logrus.WithError(err).Panicf("failed to start appservice roomserver consumer") + } } // Create application service transaction workers diff --git a/federationapi/routing/send_test.go b/federationapi/routing/send_test.go index adae7c221..3123b55c9 100644 --- a/federationapi/routing/send_test.go +++ b/federationapi/routing/send_test.go @@ -128,7 +128,6 @@ func (t *testRoomserverAPI) QueryLatestEventsAndState( response *api.QueryLatestEventsAndStateResponse, ) error { r := t.queryLatestEventsAndState(request) - response.QueryLatestEventsAndStateRequest = *request response.RoomExists = r.RoomExists response.RoomVersion = testRoomVersion response.LatestEvents = r.LatestEvents @@ -144,7 +143,6 @@ func (t *testRoomserverAPI) QueryStateAfterEvents( response *api.QueryStateAfterEventsResponse, ) error { response.RoomVersion = testRoomVersion - response.QueryStateAfterEventsRequest = *request res := t.queryStateAfterEvents(request) response.PrevEventsExist = res.PrevEventsExist response.RoomExists = res.RoomExists @@ -612,7 +610,6 @@ func TestTransactionFetchMissingStateByStateIDs(t *testing.T) { } } } - res.QueryEventsByIDRequest = *req return res }, } diff --git a/federationsender/queue/queue.go b/federationsender/queue/queue.go index 5b8fc3c56..240343559 100644 --- a/federationsender/queue/queue.go +++ b/federationsender/queue/queue.go @@ -107,6 +107,9 @@ func (oqs *OutgoingQueues) SendEvent( // Remove our own server from the list of destinations. destinations = filterAndDedupeDests(oqs.origin, destinations) + if len(destinations) == 0 { + return nil + } log.WithFields(log.Fields{ "destinations": destinations, "event": ev.EventID(), diff --git a/publicroomsapi/consumers/roomserver.go b/publicroomsapi/consumers/roomserver.go index ba187cb11..b9686d56d 100644 --- a/publicroomsapi/consumers/roomserver.go +++ b/publicroomsapi/consumers/roomserver.go @@ -78,18 +78,19 @@ func (s *OutputRoomEventConsumer) onMessage(msg *sarama.ConsumerMessage) error { return nil } - remQueryReq := api.QueryEventsByIDRequest{EventIDs: output.NewRoomEvent.RemovesStateEventIDs} var remQueryRes api.QueryEventsByIDResponse - if err := s.rsAPI.QueryEventsByID(context.TODO(), &remQueryReq, &remQueryRes); err != nil { - log.Warn(err) - return err + if len(output.NewRoomEvent.RemovesStateEventIDs) > 0 { + remQueryReq := api.QueryEventsByIDRequest{EventIDs: output.NewRoomEvent.RemovesStateEventIDs} + if err := s.rsAPI.QueryEventsByID(context.TODO(), &remQueryReq, &remQueryRes); err != nil { + log.Warn(err) + return err + } } var addQueryEvents, remQueryEvents []gomatrixserverlib.Event for _, headeredEvent := range output.NewRoomEvent.AddsState() { addQueryEvents = append(addQueryEvents, headeredEvent.Event) } - addQueryEvents = append(addQueryEvents, output.NewRoomEvent.Event.Unwrap()) for _, headeredEvent := range remQueryRes.Events { remQueryEvents = append(remQueryEvents, headeredEvent.Event) } diff --git a/roomserver/api/query.go b/roomserver/api/query.go index b1525342e..6586b1af3 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -33,8 +33,6 @@ type QueryLatestEventsAndStateRequest struct { // This is used when sending events to set the prev_events, auth_events and depth. // It is also used to tell whether the event is allowed by the event auth rules. type QueryLatestEventsAndStateResponse struct { - // Copy of the request for debugging. - QueryLatestEventsAndStateRequest // Does the room exist? // If the room doesn't exist this will be false and LatestEvents will be empty. RoomExists bool `json:"room_exists"` @@ -66,8 +64,6 @@ type QueryStateAfterEventsRequest struct { // QueryStateAfterEventsResponse is a response to QueryStateAfterEvents type QueryStateAfterEventsResponse struct { - // Copy of the request for debugging. - QueryStateAfterEventsRequest // Does the room exist on this roomserver? // If the room doesn't exist this will be false and StateEvents will be empty. RoomExists bool `json:"room_exists"` @@ -89,8 +85,6 @@ type QueryEventsByIDRequest struct { // QueryEventsByIDResponse is a response to QueryEventsByID type QueryEventsByIDResponse struct { - // Copy of the request for debugging. - QueryEventsByIDRequest // A list of events with the requested IDs. // If the roomserver does not have a copy of a requested event // then it will omit that event from the list. @@ -187,8 +181,6 @@ type QueryStateAndAuthChainRequest struct { // QueryStateAndAuthChainResponse is a response to QueryStateAndAuthChain type QueryStateAndAuthChainResponse struct { - // Copy of the request for debugging. - QueryStateAndAuthChainRequest // Does the room exist on this roomserver? // If the room doesn't exist this will be false and StateEvents will be empty. RoomExists bool `json:"room_exists"` diff --git a/roomserver/internal/query.go b/roomserver/internal/query.go index 375ddc23c..4fc8e4c25 100644 --- a/roomserver/internal/query.go +++ b/roomserver/internal/query.go @@ -45,7 +45,6 @@ func (r *RoomserverInternalAPI) QueryLatestEventsAndState( roomState := state.NewStateResolution(r.DB) - response.QueryLatestEventsAndStateRequest = *request roomNID, err := r.DB.RoomNIDExcludingStubs(ctx, request.RoomID) if err != nil { return err @@ -105,7 +104,6 @@ func (r *RoomserverInternalAPI) QueryStateAfterEvents( roomState := state.NewStateResolution(r.DB) - response.QueryStateAfterEventsRequest = *request roomNID, err := r.DB.RoomNIDExcludingStubs(ctx, request.RoomID) if err != nil { return err @@ -153,8 +151,6 @@ func (r *RoomserverInternalAPI) QueryEventsByID( request *api.QueryEventsByIDRequest, response *api.QueryEventsByIDResponse, ) error { - response.QueryEventsByIDRequest = *request - eventNIDMap, err := r.DB.EventNIDs(ctx, request.EventIDs) if err != nil { return err @@ -734,7 +730,6 @@ func (r *RoomserverInternalAPI) QueryStateAndAuthChain( request *api.QueryStateAndAuthChainRequest, response *api.QueryStateAndAuthChainResponse, ) error { - response.QueryStateAndAuthChainRequest = *request roomNID, err := r.DB.RoomNIDExcludingStubs(ctx, request.RoomID) if err != nil { return err From 6b5996db1736ee962bd081b67b7f38c1591737f8 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Mon, 15 Jun 2020 09:54:11 +0100 Subject: [PATCH 160/200] Add bare bones user API (#1127) * Add bare bones user API with tests! * linting --- userapi/api/api.go | 38 +++++++++++ userapi/internal/api.go | 53 +++++++++++++++ userapi/inthttp/client.go | 62 +++++++++++++++++ userapi/inthttp/server.go | 41 +++++++++++ userapi/userapi.go | 41 +++++++++++ userapi/userapi_test.go | 138 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 373 insertions(+) create mode 100644 userapi/api/api.go create mode 100644 userapi/internal/api.go create mode 100644 userapi/inthttp/client.go create mode 100644 userapi/inthttp/server.go create mode 100644 userapi/userapi.go create mode 100644 userapi/userapi_test.go diff --git a/userapi/api/api.go b/userapi/api/api.go new file mode 100644 index 000000000..8534fb17e --- /dev/null +++ b/userapi/api/api.go @@ -0,0 +1,38 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package api + +import "context" + +// UserInternalAPI is the internal API for information about users and devices. +type UserInternalAPI interface { + QueryProfile(ctx context.Context, req *QueryProfileRequest, res *QueryProfileResponse) error +} + +// QueryProfileRequest is the request for QueryProfile +type QueryProfileRequest struct { + // The user ID to query + UserID string +} + +// QueryProfileResponse is the response for QueryProfile +type QueryProfileResponse struct { + // True if the user has been created. Querying for a profile does not create them. + UserExists bool + // The current display name if set. + DisplayName string + // The current avatar URL if set. + AvatarURL string +} diff --git a/userapi/internal/api.go b/userapi/internal/api.go new file mode 100644 index 000000000..0144526c7 --- /dev/null +++ b/userapi/internal/api.go @@ -0,0 +1,53 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package internal + +import ( + "context" + "database/sql" + "fmt" + + "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" + "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" + "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib" +) + +type UserInternalAPI struct { + AccountDB accounts.Database + DeviceDB devices.Database + ServerName gomatrixserverlib.ServerName +} + +func (a *UserInternalAPI) QueryProfile(ctx context.Context, req *api.QueryProfileRequest, res *api.QueryProfileResponse) error { + local, domain, err := gomatrixserverlib.SplitID('@', req.UserID) + if err != nil { + return err + } + if domain != a.ServerName { + return fmt.Errorf("cannot query profile of remote users: got %s want %s", domain, a.ServerName) + } + prof, err := a.AccountDB.GetProfileByLocalpart(ctx, local) + if err != nil { + if err == sql.ErrNoRows { + return nil + } + return err + } + res.UserExists = true + res.AvatarURL = prof.AvatarURL + res.DisplayName = prof.DisplayName + return nil +} diff --git a/userapi/inthttp/client.go b/userapi/inthttp/client.go new file mode 100644 index 000000000..90cc54a48 --- /dev/null +++ b/userapi/inthttp/client.go @@ -0,0 +1,62 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package inthttp + +import ( + "context" + "errors" + "net/http" + + "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/userapi/api" + "github.com/opentracing/opentracing-go" +) + +// HTTP paths for the internal HTTP APIs +const ( + QueryProfilePath = "/userapi/queryProfile" +) + +// NewUserAPIClient creates a UserInternalAPI implemented by talking to a HTTP POST API. +// If httpClient is nil an error is returned +func NewUserAPIClient( + apiURL string, + httpClient *http.Client, +) (api.UserInternalAPI, error) { + if httpClient == nil { + return nil, errors.New("NewUserAPIClient: httpClient is ") + } + return &httpUserInternalAPI{ + apiURL: apiURL, + httpClient: httpClient, + }, nil +} + +type httpUserInternalAPI struct { + apiURL string + httpClient *http.Client +} + +func (h *httpUserInternalAPI) QueryProfile( + ctx context.Context, + request *api.QueryProfileRequest, + response *api.QueryProfileResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "QueryProfile") + defer span.Finish() + + apiURL := h.apiURL + QueryProfilePath + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} diff --git a/userapi/inthttp/server.go b/userapi/inthttp/server.go new file mode 100644 index 000000000..f3c17ccd4 --- /dev/null +++ b/userapi/inthttp/server.go @@ -0,0 +1,41 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package inthttp + +import ( + "encoding/json" + "net/http" + + "github.com/gorilla/mux" + "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/util" +) + +func AddRoutes(internalAPIMux *mux.Router, s api.UserInternalAPI) { + internalAPIMux.Handle(QueryProfilePath, + httputil.MakeInternalAPI("queryProfile", func(req *http.Request) util.JSONResponse { + request := api.QueryProfileRequest{} + response := api.QueryProfileResponse{} + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := s.QueryProfile(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) +} diff --git a/userapi/userapi.go b/userapi/userapi.go new file mode 100644 index 000000000..32f851cc7 --- /dev/null +++ b/userapi/userapi.go @@ -0,0 +1,41 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package userapi + +import ( + "github.com/gorilla/mux" + "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" + "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" + "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/internal" + "github.com/matrix-org/dendrite/userapi/inthttp" + "github.com/matrix-org/gomatrixserverlib" +) + +// AddInternalRoutes registers HTTP handlers for the internal API. Invokes functions +// on the given input API. +func AddInternalRoutes(router *mux.Router, intAPI api.UserInternalAPI) { + inthttp.AddRoutes(router, intAPI) +} + +// NewInternalAPI returns a concerete implementation of the internal API. Callers +// can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. +func NewInternalAPI(accountDB accounts.Database, deviceDB devices.Database, serverName gomatrixserverlib.ServerName) api.UserInternalAPI { + return &internal.UserInternalAPI{ + AccountDB: accountDB, + DeviceDB: deviceDB, + ServerName: serverName, + } +} diff --git a/userapi/userapi_test.go b/userapi/userapi_test.go new file mode 100644 index 000000000..423a86125 --- /dev/null +++ b/userapi/userapi_test.go @@ -0,0 +1,138 @@ +package userapi_test + +import ( + "context" + "fmt" + "net" + "net/http" + "reflect" + "sync" + "testing" + + "github.com/gorilla/mux" + "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" + "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" + "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/userapi" + "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/inthttp" + "github.com/matrix-org/gomatrixserverlib" +) + +const ( + serverName = gomatrixserverlib.ServerName("example.com") +) + +func MustMakeInternalAPI(t *testing.T) (api.UserInternalAPI, accounts.Database, devices.Database) { + accountDB, err := accounts.NewDatabase("file::memory:", nil, serverName) + if err != nil { + t.Fatalf("failed to create account DB: %s", err) + } + deviceDB, err := devices.NewDatabase("file::memory:", nil, serverName) + if err != nil { + t.Fatalf("failed to create device DB: %s", err) + } + + return userapi.NewInternalAPI(accountDB, deviceDB, serverName), accountDB, deviceDB +} + +func TestQueryProfile(t *testing.T) { + aliceAvatarURL := "mxc://example.com/alice" + aliceDisplayName := "Alice" + userAPI, accountDB, _ := MustMakeInternalAPI(t) + _, err := accountDB.CreateAccount(context.TODO(), "alice", "foobar", "") + if err != nil { + t.Fatalf("failed to make account: %s", err) + } + if err := accountDB.SetAvatarURL(context.TODO(), "alice", aliceAvatarURL); err != nil { + t.Fatalf("failed to set avatar url: %s", err) + } + if err := accountDB.SetDisplayName(context.TODO(), "alice", aliceDisplayName); err != nil { + t.Fatalf("failed to set display name: %s", err) + } + + testCases := []struct { + req api.QueryProfileRequest + wantRes api.QueryProfileResponse + wantErr error + }{ + { + req: api.QueryProfileRequest{ + UserID: fmt.Sprintf("@alice:%s", serverName), + }, + wantRes: api.QueryProfileResponse{ + UserExists: true, + AvatarURL: aliceAvatarURL, + DisplayName: aliceDisplayName, + }, + }, + { + req: api.QueryProfileRequest{ + UserID: fmt.Sprintf("@bob:%s", serverName), + }, + wantRes: api.QueryProfileResponse{ + UserExists: false, + }, + }, + { + req: api.QueryProfileRequest{ + UserID: "@alice:wrongdomain.com", + }, + wantErr: fmt.Errorf("wrong domain"), + }, + } + + runCases := func(testAPI api.UserInternalAPI) { + for _, tc := range testCases { + var gotRes api.QueryProfileResponse + gotErr := testAPI.QueryProfile(context.TODO(), &tc.req, &gotRes) + if tc.wantErr == nil && gotErr != nil || tc.wantErr != nil && gotErr == nil { + t.Errorf("QueryProfile error, got %s want %s", gotErr, tc.wantErr) + continue + } + if !reflect.DeepEqual(tc.wantRes, gotRes) { + t.Errorf("QueryProfile response got %+v want %+v", gotRes, tc.wantRes) + } + } + } + + t.Run("HTTP API", func(t *testing.T) { + router := mux.NewRouter().PathPrefix(httputil.InternalPathPrefix).Subrouter() + userapi.AddInternalRoutes(router, userAPI) + apiURL, cancel := listenAndServe(t, router) + defer cancel() + httpAPI, err := inthttp.NewUserAPIClient(apiURL, &http.Client{}) + if err != nil { + t.Fatalf("failed to create HTTP client") + } + runCases(httpAPI) + }) + t.Run("Monolith", func(t *testing.T) { + runCases(userAPI) + }) +} + +func listenAndServe(t *testing.T, router *mux.Router) (apiURL string, cancel func()) { + listener, err := net.Listen("tcp", ":0") + if err != nil { + t.Fatalf("failed to listen: %s", err) + } + port := listener.Addr().(*net.TCPAddr).Port + srv := http.Server{} + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + srv.Handler = router + err := srv.Serve(listener) + if err != nil && err != http.ErrServerClosed { + t.Logf("Listen failed: %s", err) + } + }() + + return fmt.Sprintf("http://localhost:%d", port), func() { + srv.Shutdown(context.Background()) + wg.Wait() + } +} From 0ba1245a46f2497933a9331c4550eda71376ef7e Mon Sep 17 00:00:00 2001 From: Kegsay Date: Mon, 15 Jun 2020 10:13:57 +0100 Subject: [PATCH 161/200] Current wiring (#1125) * Current wiring * Add ServerKeyAPI lines --- clientapi/clientapi.go | 3 +- clientapi/routing/getevent.go | 3 -- clientapi/routing/routing.go | 3 +- docs/WIRING-Current.md | 71 +++++++++++++++++++++++++++++++++++ internal/setup/monolith.go | 2 +- 5 files changed, 74 insertions(+), 8 deletions(-) create mode 100644 docs/WIRING-Current.md diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go index 2780f367c..545b95b0e 100644 --- a/clientapi/clientapi.go +++ b/clientapi/clientapi.go @@ -41,7 +41,6 @@ func AddPublicRoutes( deviceDB devices.Database, accountsDB accounts.Database, federation *gomatrixserverlib.FederationClient, - keyRing *gomatrixserverlib.KeyRing, rsAPI roomserverAPI.RoomserverInternalAPI, eduInputAPI eduServerAPI.EDUServerInputAPI, asAPI appserviceAPI.AppServiceQueryAPI, @@ -62,7 +61,7 @@ func AddPublicRoutes( routing.Setup( router, cfg, eduInputAPI, rsAPI, asAPI, - accountsDB, deviceDB, federation, *keyRing, + accountsDB, deviceDB, federation, syncProducer, transactionsCache, fsAPI, ) } diff --git a/clientapi/routing/getevent.go b/clientapi/routing/getevent.go index 3ca9d1b62..16f36d661 100644 --- a/clientapi/routing/getevent.go +++ b/clientapi/routing/getevent.go @@ -32,7 +32,6 @@ type getEventRequest struct { eventID string cfg *config.Dendrite federation *gomatrixserverlib.FederationClient - keyRing gomatrixserverlib.KeyRing requestedEvent gomatrixserverlib.Event } @@ -46,7 +45,6 @@ func GetEvent( cfg *config.Dendrite, rsAPI api.RoomserverInternalAPI, federation *gomatrixserverlib.FederationClient, - keyRing gomatrixserverlib.KeyRing, ) util.JSONResponse { eventsReq := api.QueryEventsByIDRequest{ EventIDs: []string{eventID}, @@ -75,7 +73,6 @@ func GetEvent( eventID: eventID, cfg: cfg, federation: federation, - keyRing: keyRing, requestedEvent: requestedEvent, } diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index f6aff0f0b..82a80fff9 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -55,7 +55,6 @@ func Setup( accountDB accounts.Database, deviceDB devices.Database, federation *gomatrixserverlib.FederationClient, - keyRing gomatrixserverlib.KeyRing, syncProducer *producers.SyncAPIProducer, transactionsCache *transactions.Cache, federationSender federationSenderAPI.FederationSenderInternalAPI, @@ -154,7 +153,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return GetEvent(req, device, vars["roomID"], vars["eventID"], cfg, rsAPI, federation, keyRing) + return GetEvent(req, device, vars["roomID"], vars["eventID"], cfg, rsAPI, federation) }), ).Methods(http.MethodGet, http.MethodOptions) diff --git a/docs/WIRING-Current.md b/docs/WIRING-Current.md new file mode 100644 index 000000000..62450f2f8 --- /dev/null +++ b/docs/WIRING-Current.md @@ -0,0 +1,71 @@ +This document details how various components communicate with each other. There are two kinds of components: + - Public-facing: exposes CS/SS API endpoints and need to be routed to via client-api-proxy or equivalent. + - Internal-only: exposes internal APIs and produces Kafka events. + +## Internal HTTP APIs + +Not everything can be done using Kafka logs. For example, requesting the latest events in a room is much better suited to +a request/response model like HTTP or RPC. Therefore, components can expose "internal APIs" which sit outside of Kafka logs. +Note in Monolith mode these are actually direct function calls and are not serialised HTTP requests. + +``` + Tier 1 Sync PublicRooms FederationAPI ClientAPI MediaAPI +Public Facing | .-----1------` | | | | | | | | | + 2 | .-------3-----------------` | | | `--------|-|-|-|--11--------------------. + | | | .--------4----------------------------------` | | | | + | | | | .---5-----------` | | | | | | + | | | | | .---6----------------------------` | | | + | | | | | | | .-----7----------` | | + | | | | | | 8 | | 10 | + | | | | | | | | `---9----. | | + V V V V V V V V V V V + Tier 2 Roomserver EDUServer FedSender AppService KeyServer ServerKeyAPI +Internal only | `------------------------12----------^ ^ + `------------------------------------------------------------13----------` + + Client ---> Server +``` +- 1 (PublicRooms -> Roomserver): Calculating current auth for changing visibility +- 2 (Sync -> Roomserver): When making backfill requests +- 3 (FedAPI -> Roomserver): Calculating (prev/auth events) and sending new events, processing backfill/state/state_ids requests +- 4 (ClientAPI -> Roomserver): Calculating (prev/auth events) and sending new events, processing /state requests +- 5 (FedAPI -> EDUServer): Sending typing/send-to-device events +- 6 (ClientAPI -> EDUServer): Sending typing/send-to-device events +- 7 (ClientAPI -> FedSender): Handling directory lookups +- 8 (FedAPI -> FedSender): Resetting backoffs when receiving traffic from a server. Querying joined hosts when handling alias lookup requests +- 9 (FedAPI -> AppService): Working out if the client is an appservice user +- 10 (ClientAPI -> AppService): Working out if the client is an appservice user +- 11 (FedAPI -> ServerKeyAPI): Verifying incoming event signatures +- 12 (FedSender -> ServerKeyAPI): Verifying event signatures of responses (e.g from send_join) +- 13 (Roomserver -> ServerKeyAPI): Verifying event signatures of backfilled events + +## Kafka logs + +``` + .----1--------------------------------------------. + V | + Tier 1 Sync PublicRooms FederationAPI ClientAPI MediaAPI +Public Facing ^ ^ ^ ^ + | | | | + 2 | | | + | `-3------------. | + | | | | + | | | | + | .------4------` | | + | | .--------5-----|------------------------------` + | | | | + Tier 2 Roomserver EDUServer FedSender AppService KeyServer ServerKeyAPI +Internal only | | ^ ^ + | `-----6----------` | + `--------------------7--------` + + +Producer ----> Consumer +``` +- 1 (ClientAPI -> Sync): For tracking account data +- 2 (Roomserver -> Sync): For all data to send to clients +- 3 (EDUServer -> Sync): For typing/send-to-device data to send to clients +- 4 (Roomserver -> PublicRooms): For tracking the current room name/topic/joined count/etc. +- 5 (Roomserver -> ClientAPI): For tracking memberships for profile updates. +- 6 (EDUServer -> FedSender): For sending EDUs over federation +- 7 (Roomserver -> FedSender): For sending PDUs over federation, for tracking joined hosts. diff --git a/internal/setup/monolith.go b/internal/setup/monolith.go index 55ceffd6b..4dfbf7115 100644 --- a/internal/setup/monolith.go +++ b/internal/setup/monolith.go @@ -67,7 +67,7 @@ type Monolith struct { func (m *Monolith) AddAllPublicRoutes(publicMux *mux.Router) { clientapi.AddPublicRoutes( publicMux, m.Config, m.KafkaConsumer, m.KafkaProducer, m.DeviceDB, m.AccountDB, - m.FedClient, m.KeyRing, m.RoomserverAPI, + m.FedClient, m.RoomserverAPI, m.EDUInternalAPI, m.AppserviceAPI, transactions.New(), m.FederationSenderAPI, ) From 1aac3173410dbe5581f27b2f9104ef850fefa546 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 15 Jun 2020 10:48:43 +0100 Subject: [PATCH 162/200] Unbreak build --- cmd/dendrite-client-api-server/main.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/cmd/dendrite-client-api-server/main.go b/cmd/dendrite-client-api-server/main.go index 8ed18c99b..a28eb8b3b 100644 --- a/cmd/dendrite-client-api-server/main.go +++ b/cmd/dendrite-client-api-server/main.go @@ -30,16 +30,13 @@ func main() { deviceDB := base.CreateDeviceDB() federation := base.CreateFederationClient() - serverKeyAPI := base.ServerKeyAPIClient() - keyRing := serverKeyAPI.KeyRing() - asQuery := base.AppserviceHTTPClient() rsAPI := base.RoomserverHTTPClient() fsAPI := base.FederationSenderHTTPClient() eduInputAPI := base.EDUServerClient() clientapi.AddPublicRoutes( - base.PublicAPIMux, base.Cfg, base.KafkaConsumer, base.KafkaProducer, deviceDB, accountDB, federation, keyRing, + base.PublicAPIMux, base.Cfg, base.KafkaConsumer, base.KafkaProducer, deviceDB, accountDB, federation, rsAPI, eduInputAPI, asQuery, transactions.New(), fsAPI, ) From 7c36fb78a729dcce174a5d1e577edeeeb9ca806d Mon Sep 17 00:00:00 2001 From: Kegsay Date: Mon, 15 Jun 2020 16:57:59 +0100 Subject: [PATCH 163/200] Fix rooms v3 url paths for good - with tests (#1130) * Fix rooms v3 url paths for good - with tests - Add a test rig around `federationapi` to test routing. - Use `JSONVerifier` over `KeyRing` so we can stub things out more easily. - Add `test.NopJSONVerifier` which verifies nothing. - Add `base.BaseMux` which is the original `mux.Router` used to spawn public/internal routers. - Listen on `base.BaseMux` and not the default serve mux as it cleans paths which we don't want. - Factor out `ListenAndServe` to `test.ListenAndServe` and add flag for listening on TLS. * Fix comments * Linting --- cmd/create-room-events/main.go | 5 +- cmd/dendrite-demo-libp2p/main.go | 6 +- cmd/dendrite-demo-yggdrasil/main.go | 29 ++++---- cmd/dendrite-monolith-server/main.go | 4 +- cmd/dendritejs/jsServer.go | 2 +- cmd/dendritejs/main.go | 7 +- federationapi/federationapi.go | 4 +- federationapi/federationapi_test.go | 102 +++++++++++++++++++++++++++ federationapi/routing/invite.go | 2 +- federationapi/routing/join.go | 2 +- federationapi/routing/leave.go | 2 +- federationapi/routing/routing.go | 2 +- federationapi/routing/send.go | 2 +- federationapi/routing/send_test.go | 12 +--- internal/httputil/httpapi.go | 7 +- internal/setup/base.go | 5 +- internal/test/keyring.go | 31 ++++++++ internal/test/server.go | 50 ++++++++++++- userapi/userapi_test.go | 30 +------- 19 files changed, 228 insertions(+), 76 deletions(-) create mode 100644 federationapi/federationapi_test.go create mode 100644 internal/test/keyring.go diff --git a/cmd/create-room-events/main.go b/cmd/create-room-events/main.go index ebce953ce..afe974643 100644 --- a/cmd/create-room-events/main.go +++ b/cmd/create-room-events/main.go @@ -47,6 +47,7 @@ var ( userID = flag.String("user-id", "@userid:$SERVER_NAME", "The user ID to use as the event sender") messageCount = flag.Int("message-count", 10, "The number of m.room.messsage events to generate") format = flag.String("Format", "InputRoomEvent", "The output format to use for the messages: InputRoomEvent or Event") + ver = flag.String("version", string(gomatrixserverlib.RoomVersionV1), "Room version to generate events as") ) // By default we use a private key of 0. @@ -109,7 +110,7 @@ func buildAndOutput() gomatrixserverlib.EventReference { event, err := b.Build( now, name, key, privateKey, - gomatrixserverlib.RoomVersionV1, + gomatrixserverlib.RoomVersion(*ver), ) if err != nil { panic(err) @@ -127,7 +128,7 @@ func writeEvent(event gomatrixserverlib.Event) { if *format == "InputRoomEvent" { var ire api.InputRoomEvent ire.Kind = api.KindNew - ire.Event = event.Headered(gomatrixserverlib.RoomVersionV1) + ire.Event = event.Headered(gomatrixserverlib.RoomVersion(*ver)) authEventIDs := []string{} for _, ref := range b.AuthEvents.([]gomatrixserverlib.EventReference) { authEventIDs = append(authEventIDs, ref.EventID) diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index f215606e8..0e757de97 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -173,7 +173,7 @@ func main() { monolith.AddAllPublicRoutes(base.Base.PublicAPIMux) httputil.SetupHTTPAPI( - http.DefaultServeMux, + base.Base.BaseMux, base.Base.PublicAPIMux, base.Base.InternalAPIMux, &cfg, @@ -184,7 +184,7 @@ func main() { go func() { httpBindAddr := fmt.Sprintf(":%d", *instancePort) logrus.Info("Listening on ", httpBindAddr) - logrus.Fatal(http.ListenAndServe(httpBindAddr, nil)) + logrus.Fatal(http.ListenAndServe(httpBindAddr, base.Base.BaseMux)) }() // Expose the matrix APIs also via libp2p if base.LibP2P != nil { @@ -197,7 +197,7 @@ func main() { defer func() { logrus.Fatal(listener.Close()) }() - logrus.Fatal(http.Serve(listener, nil)) + logrus.Fatal(http.Serve(listener, base.Base.BaseMux)) }() } diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index e4d4fe9e6..6923b68b1 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -94,18 +94,6 @@ func createFederationClient( func main() { flag.Parse() - // Build both ends of a HTTP multiplex. - httpServer := &http.Server{ - Addr: ":0", - TLSNextProto: map[string]func(*http.Server, *tls.Conn, http.Handler){}, - ReadTimeout: 15 * time.Second, - WriteTimeout: 45 * time.Second, - IdleTimeout: 60 * time.Second, - BaseContext: func(_ net.Listener) context.Context { - return context.Background() - }, - } - ygg, err := yggconn.Setup(*instanceName, *instancePeer) if err != nil { panic(err) @@ -188,13 +176,26 @@ func main() { monolith.AddAllPublicRoutes(base.PublicAPIMux) httputil.SetupHTTPAPI( - http.DefaultServeMux, + base.BaseMux, base.PublicAPIMux, base.InternalAPIMux, cfg, base.UseHTTPAPIs, ) + // Build both ends of a HTTP multiplex. + httpServer := &http.Server{ + Addr: ":0", + TLSNextProto: map[string]func(*http.Server, *tls.Conn, http.Handler){}, + ReadTimeout: 15 * time.Second, + WriteTimeout: 45 * time.Second, + IdleTimeout: 60 * time.Second, + BaseContext: func(_ net.Listener) context.Context { + return context.Background() + }, + Handler: base.BaseMux, + } + go func() { logrus.Info("Listening on ", ygg.DerivedServerName()) logrus.Fatal(httpServer.Serve(ygg)) @@ -202,7 +203,7 @@ func main() { go func() { httpBindAddr := fmt.Sprintf("localhost:%d", *instancePort) logrus.Info("Listening on ", httpBindAddr) - logrus.Fatal(http.ListenAndServe(httpBindAddr, nil)) + logrus.Fatal(http.ListenAndServe(httpBindAddr, base.BaseMux)) }() select {} diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index ea8160b84..d7b0bf480 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -135,7 +135,7 @@ func main() { monolith.AddAllPublicRoutes(base.PublicAPIMux) httputil.SetupHTTPAPI( - http.DefaultServeMux, + base.BaseMux, base.PublicAPIMux, base.InternalAPIMux, cfg, @@ -147,6 +147,7 @@ func main() { serv := http.Server{ Addr: *httpBindAddr, WriteTimeout: setup.HTTPServerTimeout, + Handler: base.BaseMux, } logrus.Info("Listening on ", serv.Addr) @@ -158,6 +159,7 @@ func main() { serv := http.Server{ Addr: *httpsBindAddr, WriteTimeout: setup.HTTPServerTimeout, + Handler: base.BaseMux, } logrus.Info("Listening on ", serv.Addr) diff --git a/cmd/dendritejs/jsServer.go b/cmd/dendritejs/jsServer.go index 31f122645..074d20cba 100644 --- a/cmd/dendritejs/jsServer.go +++ b/cmd/dendritejs/jsServer.go @@ -28,7 +28,7 @@ import ( // JSServer exposes an HTTP-like server interface which allows JS to 'send' requests to it. type JSServer struct { // The router which will service requests - Mux *http.ServeMux + Mux http.Handler } // OnRequestFromJS is the function that JS will invoke when there is a new request. diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 70672f4df..8c19eb6d5 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -19,7 +19,6 @@ package main import ( "crypto/ed25519" "fmt" - "net/http" "syscall/js" "github.com/matrix-org/dendrite/appservice" @@ -233,7 +232,7 @@ func main() { monolith.AddAllPublicRoutes(base.PublicAPIMux) httputil.SetupHTTPAPI( - http.DefaultServeMux, + base.BaseMux, base.PublicAPIMux, base.InternalAPIMux, cfg, @@ -245,7 +244,7 @@ func main() { go func() { logrus.Info("Listening on libp2p-js host ID ", node.Id) s := JSServer{ - Mux: http.DefaultServeMux, + Mux: base.BaseMux, } s.ListenAndServe("p2p") }() @@ -255,7 +254,7 @@ func main() { go func() { logrus.Info("Listening for service-worker fetch traffic") s := JSServer{ - Mux: http.DefaultServeMux, + Mux: base.BaseMux, } s.ListenAndServe("fetch") }() diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go index 9299b5016..db272f1c8 100644 --- a/federationapi/federationapi.go +++ b/federationapi/federationapi.go @@ -35,7 +35,7 @@ func AddPublicRoutes( accountsDB accounts.Database, deviceDB devices.Database, federation *gomatrixserverlib.FederationClient, - keyRing *gomatrixserverlib.KeyRing, + keyRing gomatrixserverlib.JSONVerifier, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, federationSenderAPI federationSenderAPI.FederationSenderInternalAPI, @@ -44,7 +44,7 @@ func AddPublicRoutes( routing.Setup( router, cfg, rsAPI, asAPI, - eduAPI, federationSenderAPI, *keyRing, + eduAPI, federationSenderAPI, keyRing, federation, accountsDB, deviceDB, ) } diff --git a/federationapi/federationapi_test.go b/federationapi/federationapi_test.go new file mode 100644 index 000000000..cf7d732bf --- /dev/null +++ b/federationapi/federationapi_test.go @@ -0,0 +1,102 @@ +package federationapi_test + +import ( + "context" + "crypto/ed25519" + "strings" + "testing" + + "github.com/matrix-org/dendrite/federationapi" + "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/internal/setup" + "github.com/matrix-org/dendrite/internal/test" + "github.com/matrix-org/gomatrix" + "github.com/matrix-org/gomatrixserverlib" +) + +// Tests that event IDs with '/' in them (escaped as %2F) are correctly passed to the right handler and don't 404. +// Relevant for v3 rooms and a cause of flakey sytests as the IDs are randomly generated. +func TestRoomsV3URLEscapeDoNot404(t *testing.T) { + _, privKey, _ := ed25519.GenerateKey(nil) + cfg := &config.Dendrite{} + cfg.Matrix.KeyID = gomatrixserverlib.KeyID("ed25519:auto") + cfg.Matrix.ServerName = gomatrixserverlib.ServerName("localhost") + cfg.Matrix.PrivateKey = privKey + cfg.Kafka.UseNaffka = true + cfg.Database.Naffka = "file::memory:" + cfg.SetDefaults() + base := setup.NewBaseDendrite(cfg, "Test", false) + keyRing := &test.NopJSONVerifier{} + fsAPI := base.FederationSenderHTTPClient() + // TODO: This is pretty fragile, as if anything calls anything on these nils this test will break. + // Unfortunately, it makes little sense to instantiate these dependencies when we just want to test routing. + federationapi.AddPublicRoutes(base.PublicAPIMux, cfg, nil, nil, nil, keyRing, nil, nil, fsAPI, nil) + httputil.SetupHTTPAPI( + base.BaseMux, + base.PublicAPIMux, + base.InternalAPIMux, + cfg, + base.UseHTTPAPIs, + ) + baseURL, cancel := test.ListenAndServe(t, base.BaseMux, true) + defer cancel() + serverName := gomatrixserverlib.ServerName(strings.TrimPrefix(baseURL, "https://")) + + fedCli := gomatrixserverlib.NewFederationClient(serverName, cfg.Matrix.KeyID, cfg.Matrix.PrivateKey) + + testCases := []struct { + roomVer gomatrixserverlib.RoomVersion + eventJSON string + }{ + { + eventJSON: `{"auth_events":[["$Nzfbrhc3oaYVKzGM:localhost",{"sha256":"BCBHOgB4qxLPQkBd6th8ydFSyqjth/LF99VNjYffOQ0"}],["$EZzkD2BH1Gtm5v1D:localhost",{"sha256":"3dLUnDBs8/iC5DMw/ydKtmAqVZtzqqtHpsjsQPk7GJA"}]],"content":{"body":"Test Message"},"depth":11,"event_id":"$mGiPO3oGjQfCkIUw:localhost","hashes":{"sha256":"h+t+4DwIBC9UNyJ3jzyAQAAl4H3yQHVuHrm2S1JZizU"},"origin":"localhost","origin_server_ts":0,"prev_events":[["$tFr64vpiSHdLU0Qr:localhost",{"sha256":"+R07ZrIs4c4tjPFE+tmcYIGUfeLGFI/4e0OITb9uEcM"}]],"room_id":"!roomid:localhost","sender":"@userid:localhost","signatures":{"localhost":{"ed25519:auto":"LYFr/rW9m5/7UKBQMF5qWnG82He4VGsRESUgDmvkn5DrJRyS4TLL/7zl0Lymn3pa3q2yaTO74LQX/CRotqG1BA"}},"type":"m.room.message"}`, + roomVer: gomatrixserverlib.RoomVersionV1, + }, + // single / (handlers which do not UseEncodedPath will fail this test) + // EventID: $0SFh2WJbjBs3OT+E0yl95giDKo/3Zp52HsHUUk4uPyg + { + eventJSON: `{"auth_events":["$x4MKEPRSF6OGlo0qpnsP3BfSmYX5HhVlykOsQH3ECyg","$BcEcbZnlFLB5rxSNSZNBn6fO3jU/TKAJ79wfKyCQLiU"],"content":{"body":"Test Message"},"depth":8,"hashes":{"sha256":"dfK0MBn1RZZqCVJqWsn/MGY7QJHjQcwqF0unOonLCTU"},"origin":"localhost","origin_server_ts":0,"prev_events":["$1SwcZ1XY/Y8yKLjP4DzAOHN5WFBcDAZxb5vFDnW2ubA"],"room_id":"!roomid:localhost","sender":"@userid:localhost","signatures":{"localhost":{"ed25519:auto":"INOjuWMg+GmFkUpmzhMB0bqLNs73mSvwldY1ftYIQ/B3lD9soD2OMG3AF+wgZW/I8xqzY4DOHfbnbUeYPf67BA"}},"type":"m.room.message"}`, + roomVer: gomatrixserverlib.RoomVersionV3, + }, + // multiple / + // EventID: $OzENBCuVv/fnRAYCeQudIon/84/V5pxtEjQMTgi3emk + { + eventJSON: `{"auth_events":["$x4MKEPRSF6OGlo0qpnsP3BfSmYX5HhVlykOsQH3ECyg","$BcEcbZnlFLB5rxSNSZNBn6fO3jU/TKAJ79wfKyCQLiU"],"content":{"body":"Test Message"},"depth":2,"hashes":{"sha256":"U5+WsiJAhiEM88J8HTjuUjPImVGVzDFD3v/WS+jb2f0"},"origin":"localhost","origin_server_ts":0,"prev_events":["$BcEcbZnlFLB5rxSNSZNBn6fO3jU/TKAJ79wfKyCQLiU"],"room_id":"!roomid:localhost","sender":"@userid:localhost","signatures":{"localhost":{"ed25519:auto":"tKS469e9+wdWPEKB/LbBJWQ8vfOOdKgTWER5IwbSAH1CxmLvkCziUsgVu85zfzDSLoUi5mU5FHLiMTC6P/qICw"}},"type":"m.room.message"}`, + roomVer: gomatrixserverlib.RoomVersionV3, + }, + // two slashes (handlers which clean paths before UseEncodedPath will fail this test) + // EventID: $EmwNBlHoSOVmCZ1cM//yv/OvxB6r4OFEIGSJea7+Amk + { + eventJSON: `{"auth_events":["$x4MKEPRSF6OGlo0qpnsP3BfSmYX5HhVlykOsQH3ECyg","$BcEcbZnlFLB5rxSNSZNBn6fO3jU/TKAJ79wfKyCQLiU"],"content":{"body":"Test Message"},"depth":3917,"hashes":{"sha256":"cNAWtlHIegrji0mMA6x1rhpYCccY8W1NsWZqSpJFhjs"},"origin":"localhost","origin_server_ts":0,"prev_events":["$4GDB0bVjkWwS3G4noUZCq5oLWzpBYpwzdMcf7gj24CI"],"room_id":"!roomid:localhost","sender":"@userid:localhost","signatures":{"localhost":{"ed25519:auto":"NKym6Kcy3u9mGUr21Hjfe3h7DfDilDhN5PqztT0QZ4NTZ+8Y7owseLolQVXp+TvNjecvzdDywsXXVvGiuQiWAQ"}},"type":"m.room.message"}`, + roomVer: gomatrixserverlib.RoomVersionV3, + }, + } + + for _, tc := range testCases { + ev, err := gomatrixserverlib.NewEventFromTrustedJSON([]byte(tc.eventJSON), false, tc.roomVer) + if err != nil { + t.Errorf("failed to parse event: %s", err) + } + he := ev.Headered(tc.roomVer) + invReq, err := gomatrixserverlib.NewInviteV2Request(&he, nil) + if err != nil { + t.Errorf("failed to create invite v2 request: %s", err) + continue + } + _, err = fedCli.SendInviteV2(context.Background(), serverName, invReq) + if err == nil { + t.Errorf("expected an error, got none") + continue + } + gerr, ok := err.(gomatrix.HTTPError) + if !ok { + t.Errorf("failed to cast response error as gomatrix.HTTPError") + continue + } + t.Logf("Error: %+v", gerr) + if gerr.Code == 404 { + t.Errorf("invite event resulted in a 404") + } + } +} diff --git a/federationapi/routing/invite.go b/federationapi/routing/invite.go index 908a04fcb..7d02bc1d3 100644 --- a/federationapi/routing/invite.go +++ b/federationapi/routing/invite.go @@ -35,7 +35,7 @@ func Invite( eventID string, cfg *config.Dendrite, rsAPI api.RoomserverInternalAPI, - keys gomatrixserverlib.KeyRing, + keys gomatrixserverlib.JSONVerifier, ) util.JSONResponse { inviteReq := gomatrixserverlib.InviteV2Request{} if err := json.Unmarshal(request.Content(), &inviteReq); err != nil { diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go index e01f077a3..8dcd15333 100644 --- a/federationapi/routing/join.go +++ b/federationapi/routing/join.go @@ -143,7 +143,7 @@ func SendJoin( request *gomatrixserverlib.FederationRequest, cfg *config.Dendrite, rsAPI api.RoomserverInternalAPI, - keys gomatrixserverlib.KeyRing, + keys gomatrixserverlib.JSONVerifier, roomID, eventID string, ) util.JSONResponse { verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} diff --git a/federationapi/routing/leave.go b/federationapi/routing/leave.go index de15c32ad..108fc50ae 100644 --- a/federationapi/routing/leave.go +++ b/federationapi/routing/leave.go @@ -113,7 +113,7 @@ func SendLeave( request *gomatrixserverlib.FederationRequest, cfg *config.Dendrite, rsAPI api.RoomserverInternalAPI, - keys gomatrixserverlib.KeyRing, + keys gomatrixserverlib.JSONVerifier, roomID, eventID string, ) util.JSONResponse { verReq := api.QueryRoomVersionForRoomRequest{RoomID: roomID} diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index 70b77a4c3..350febbc1 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -51,7 +51,7 @@ func Setup( asAPI appserviceAPI.AppServiceQueryAPI, eduAPI eduserverAPI.EDUServerInputAPI, fsAPI federationSenderAPI.FederationSenderInternalAPI, - keys gomatrixserverlib.KeyRing, + keys gomatrixserverlib.JSONVerifier, federation *gomatrixserverlib.FederationClient, accountDB accounts.Database, deviceDB devices.Database, diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index a057750f6..cf71b8ba0 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -37,7 +37,7 @@ func Send( cfg *config.Dendrite, rsAPI api.RoomserverInternalAPI, eduAPI eduserverAPI.EDUServerInputAPI, - keys gomatrixserverlib.KeyRing, + keys gomatrixserverlib.JSONVerifier, federation *gomatrixserverlib.FederationClient, ) util.JSONResponse { t := txnReq{ diff --git a/federationapi/routing/send_test.go b/federationapi/routing/send_test.go index 3123b55c9..6e6606c86 100644 --- a/federationapi/routing/send_test.go +++ b/federationapi/routing/send_test.go @@ -10,6 +10,7 @@ import ( eduAPI "github.com/matrix-org/dendrite/eduserver/api" fsAPI "github.com/matrix-org/dendrite/federationsender/api" + "github.com/matrix-org/dendrite/internal/test" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" ) @@ -53,15 +54,6 @@ func init() { } } -type testNopJSONVerifier struct { - // this verifier verifies nothing -} - -func (t *testNopJSONVerifier) VerifyJSONs(ctx context.Context, requests []gomatrixserverlib.VerifyJSONRequest) ([]gomatrixserverlib.VerifyJSONResult, error) { - result := make([]gomatrixserverlib.VerifyJSONResult, len(requests)) - return result, nil -} - type testEDUProducer struct { // this producer keeps track of calls to InputTypingEvent invocations []eduAPI.InputTypingEventRequest @@ -330,7 +322,7 @@ func mustCreateTransaction(rsAPI api.RoomserverInternalAPI, fedClient txnFederat context: context.Background(), rsAPI: rsAPI, eduAPI: &testEDUProducer{}, - keys: &testNopJSONVerifier{}, + keys: &test.NopJSONVerifier{}, federation: fedClient, haveEvents: make(map[string]*gomatrixserverlib.HeaderedEvent), newEvents: make(map[string]bool), diff --git a/internal/httputil/httpapi.go b/internal/httputil/httpapi.go index 0a37f06c2..a35a10d68 100644 --- a/internal/httputil/httpapi.go +++ b/internal/httputil/httpapi.go @@ -185,7 +185,7 @@ func MakeInternalAPI(metricsName string, f func(*http.Request) util.JSONResponse func MakeFedAPI( metricsName string, serverName gomatrixserverlib.ServerName, - keyRing gomatrixserverlib.KeyRing, + keyRing gomatrixserverlib.JSONVerifier, wakeup *FederationWakeups, f func(*http.Request, *gomatrixserverlib.FederationRequest, map[string]string) util.JSONResponse, ) http.Handler { @@ -233,9 +233,8 @@ func (f *FederationWakeups) Wakeup(ctx context.Context, origin gomatrixserverlib } } -// SetupHTTPAPI registers an HTTP API mux under /api and sets up a metrics -// listener. -func SetupHTTPAPI(servMux *http.ServeMux, publicApiMux *mux.Router, internalApiMux *mux.Router, cfg *config.Dendrite, enableHTTPAPIs bool) { +// SetupHTTPAPI registers an HTTP API mux under /api and sets up a metrics listener +func SetupHTTPAPI(servMux, publicApiMux, internalApiMux *mux.Router, cfg *config.Dendrite, enableHTTPAPIs bool) { if cfg.Metrics.Enabled { servMux.Handle("/metrics", WrapHandlerInBasicAuth(promhttp.Handler(), cfg.Metrics.BasicAuth)) } diff --git a/internal/setup/base.go b/internal/setup/base.go index fb304893a..414b7964a 100644 --- a/internal/setup/base.go +++ b/internal/setup/base.go @@ -63,6 +63,7 @@ type BaseDendrite struct { // PublicAPIMux should be used to register new public matrix api endpoints PublicAPIMux *mux.Router InternalAPIMux *mux.Router + BaseMux *mux.Router // base router which created public/internal subrouters UseHTTPAPIs bool httpClient *http.Client Cfg *config.Dendrite @@ -127,6 +128,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo tracerCloser: closer, Cfg: cfg, Caches: cache, + BaseMux: httpmux, PublicAPIMux: httpmux.PathPrefix(httputil.PublicPathPrefix).Subrouter().UseEncodedPath(), InternalAPIMux: httpmux.PathPrefix(httputil.InternalPathPrefix).Subrouter().UseEncodedPath(), httpClient: &client, @@ -238,12 +240,13 @@ func (b *BaseDendrite) SetupAndServeHTTP(bindaddr string, listenaddr string) { } httputil.SetupHTTPAPI( - http.DefaultServeMux, + b.BaseMux, b.PublicAPIMux, b.InternalAPIMux, b.Cfg, b.UseHTTPAPIs, ) + serv.Handler = b.BaseMux logrus.Infof("Starting %s server on %s", b.componentName, serv.Addr) err := serv.ListenAndServe() diff --git a/internal/test/keyring.go b/internal/test/keyring.go new file mode 100644 index 000000000..ed9c34849 --- /dev/null +++ b/internal/test/keyring.go @@ -0,0 +1,31 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "context" + + "github.com/matrix-org/gomatrixserverlib" +) + +// NopJSONVerifier is a JSONVerifier that verifies nothing and returns no errors. +type NopJSONVerifier struct { + // this verifier verifies nothing +} + +func (t *NopJSONVerifier) VerifyJSONs(ctx context.Context, requests []gomatrixserverlib.VerifyJSONRequest) ([]gomatrixserverlib.VerifyJSONResult, error) { + result := make([]gomatrixserverlib.VerifyJSONResult, len(requests)) + return result, nil +} diff --git a/internal/test/server.go b/internal/test/server.go index 1493dac6f..c3348d533 100644 --- a/internal/test/server.go +++ b/internal/test/server.go @@ -1,4 +1,4 @@ -// Copyright 2017 Vector Creations Ltd +// Copyright 2020 The Matrix.org Foundation C.I.C. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,11 +15,16 @@ package test import ( + "context" "fmt" + "net" + "net/http" "os" "os/exec" "path/filepath" "strings" + "sync" + "testing" "github.com/matrix-org/dendrite/internal/config" ) @@ -103,3 +108,46 @@ func StartProxy(bindAddr string, cfg *config.Dendrite) (*exec.Cmd, chan error) { proxyArgs, ) } + +// ListenAndServe will listen on a random high-numbered port and attach the given router. +// Returns the base URL to send requests to. Call `cancel` to shutdown the server, which will block until it has closed. +func ListenAndServe(t *testing.T, router http.Handler, useTLS bool) (apiURL string, cancel func()) { + listener, err := net.Listen("tcp", ":0") + if err != nil { + t.Fatalf("failed to listen: %s", err) + } + port := listener.Addr().(*net.TCPAddr).Port + srv := http.Server{} + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + srv.Handler = router + var err error + if useTLS { + certFile := filepath.Join(os.TempDir(), "dendrite.cert") + keyFile := filepath.Join(os.TempDir(), "dendrite.key") + err = NewTLSKey(keyFile, certFile) + if err != nil { + t.Logf("failed to generate tls key/cert: %s", err) + return + } + err = srv.ServeTLS(listener, certFile, keyFile) + } else { + err = srv.Serve(listener) + } + if err != nil && err != http.ErrServerClosed { + t.Logf("Listen failed: %s", err) + } + }() + + secure := "" + if useTLS { + secure = "s" + } + return fmt.Sprintf("http%s://localhost:%d", secure, port), func() { + _ = srv.Shutdown(context.Background()) + wg.Wait() + } +} diff --git a/userapi/userapi_test.go b/userapi/userapi_test.go index 423a86125..a46163637 100644 --- a/userapi/userapi_test.go +++ b/userapi/userapi_test.go @@ -3,16 +3,15 @@ package userapi_test import ( "context" "fmt" - "net" "net/http" "reflect" - "sync" "testing" "github.com/gorilla/mux" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/internal/test" "github.com/matrix-org/dendrite/userapi" "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/dendrite/userapi/inthttp" @@ -99,7 +98,7 @@ func TestQueryProfile(t *testing.T) { t.Run("HTTP API", func(t *testing.T) { router := mux.NewRouter().PathPrefix(httputil.InternalPathPrefix).Subrouter() userapi.AddInternalRoutes(router, userAPI) - apiURL, cancel := listenAndServe(t, router) + apiURL, cancel := test.ListenAndServe(t, router, false) defer cancel() httpAPI, err := inthttp.NewUserAPIClient(apiURL, &http.Client{}) if err != nil { @@ -111,28 +110,3 @@ func TestQueryProfile(t *testing.T) { runCases(userAPI) }) } - -func listenAndServe(t *testing.T, router *mux.Router) (apiURL string, cancel func()) { - listener, err := net.Listen("tcp", ":0") - if err != nil { - t.Fatalf("failed to listen: %s", err) - } - port := listener.Addr().(*net.TCPAddr).Port - srv := http.Server{} - - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() - srv.Handler = router - err := srv.Serve(listener) - if err != nil && err != http.ErrServerClosed { - t.Logf("Listen failed: %s", err) - } - }() - - return fmt.Sprintf("http://localhost:%d", port), func() { - srv.Shutdown(context.Background()) - wg.Wait() - } -} From 67ad6618139a495a80800a2145d9ba319c5d0c5d Mon Sep 17 00:00:00 2001 From: Kegsay Date: Mon, 15 Jun 2020 16:58:22 +0100 Subject: [PATCH 164/200] Unbreak HTTP mode (#1131) --- cmd/dendrite-monolith-server/main.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index d7b0bf480..3a0a84ef6 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -74,11 +74,13 @@ func main() { } keyRing := serverKeyAPI.KeyRing() - rsAPI := roomserver.NewInternalAPI( + rsImpl := roomserver.NewInternalAPI( base, keyRing, federation, ) + // call functions directly on the impl unless running in HTTP mode + rsAPI := rsImpl if base.UseHTTPAPIs { - roomserver.AddInternalRoutes(base.InternalAPIMux, rsAPI) + roomserver.AddInternalRoutes(base.InternalAPIMux, rsImpl) rsAPI = base.RoomserverHTTPClient() } if traceInternal { @@ -108,7 +110,9 @@ func main() { federationsender.AddInternalRoutes(base.InternalAPIMux, fsAPI) fsAPI = base.FederationSenderHTTPClient() } - rsAPI.SetFederationSenderAPI(fsAPI) + // The underlying roomserver implementation needs to be able to call the fedsender. + // This is different to rsAPI which can be the http client which doesn't need this dependency + rsImpl.SetFederationSenderAPI(fsAPI) publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties(), cfg.Matrix.ServerName) if err != nil { From 57b7fa3db801c27190bfd143cfebe98e3d76a6ae Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 16 Jun 2020 13:11:20 +0100 Subject: [PATCH 165/200] More server key updates, tests (#1129) * More key tweaks * Start testing stuff * Move responsibility for generating local keys into server key API, don't register prom in caches unless needed, start tests * Don't store our own keys in the database * Don't store our own keys in the database * Don't run tests for now * Tweak caching behaviour, update tests * Update comments, add fixes from forward-merge * Debug logging * Debug logging * Perform final comparison against original set of requests * oops * Fetcher timeouts * Fetcher timeouts * missing func * Tweaks * Update gomatrixserverlib * Fix Federation API test * Break up FetchKeys * Add comments to caching * Add URL check in test * Partially revert "Move responsibility for generating local keys into server key API, don't register prom in caches unless needed, start tests" This reverts commit d7eb54c5b30b2f6a9d6514b643e32e6ad2b602f3. * Fix federation API test * Fix internal cache stuff again * Fix server key API test * Update comments * Update comments from review * Fix lint --- cmd/roomserver-integration-tests/main.go | 2 +- go.mod | 3 +- go.sum | 13 +- internal/caching/cache_serverkeys.go | 20 +- internal/caching/impl_inmemorylru.go | 22 +- internal/setup/base.go | 2 +- serverkeyapi/internal/api.go | 217 +++++++++++++--- serverkeyapi/inthttp/client.go | 2 +- serverkeyapi/serverkeyapi.go | 6 +- serverkeyapi/serverkeyapi_test.go | 315 +++++++++++++++++++++++ serverkeyapi/storage/cache/keydb.go | 4 +- serverkeyapi/storage/postgres/keydb.go | 23 -- serverkeyapi/storage/sqlite3/keydb.go | 20 -- 13 files changed, 538 insertions(+), 111 deletions(-) create mode 100644 serverkeyapi/serverkeyapi_test.go diff --git a/cmd/roomserver-integration-tests/main.go b/cmd/roomserver-integration-tests/main.go index 43aca0789..3860ca1f7 100644 --- a/cmd/roomserver-integration-tests/main.go +++ b/cmd/roomserver-integration-tests/main.go @@ -255,7 +255,7 @@ func testRoomserver(input []string, wantOutput []string, checkQueries func(api.R panic(err) } - cache, err := caching.NewInMemoryLRUCache() + cache, err := caching.NewInMemoryLRUCache(false) if err != nil { panic(err) } diff --git a/go.mod b/go.mod index 2f7874082..b2451d85f 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200608125510-defe251235b1 + github.com/matrix-org/gomatrixserverlib v0.0.0-20200615161710-f69539c86ea5 github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible @@ -38,7 +38,6 @@ require ( github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20200530233943-aec82d7a391b go.uber.org/atomic v1.4.0 golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d - golang.org/x/tools v0.0.0-20200612022331-742c5eb664c2 // indirect gopkg.in/h2non/bimg.v1 v1.0.18 gopkg.in/yaml.v2 v2.2.8 ) diff --git a/go.sum b/go.sum index 301066b90..2578e1750 100644 --- a/go.sum +++ b/go.sum @@ -131,7 +131,6 @@ github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uG github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hjson/hjson-go v3.0.1-0.20190209023717-9147687966d9+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio= github.com/hjson/hjson-go v3.0.2-0.20200316202735-d5d0e8b0617d+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -372,8 +371,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 h1:Yb+Wlf github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bhrnp3Ky1qgx/fzCtCALOoGYylh2tpS9K4= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200608125510-defe251235b1 h1:BfrvDrbjoPBvYua/3F/FmrqiZTRGrvtoMRgCVnrufMI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200608125510-defe251235b1/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200615161710-f69539c86ea5 h1:VN7DoSFVkQF9Bv+TWuBWHLgAz9Nw9UiahFfe2oE6uiQ= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200615161710-f69539c86ea5/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f/go.mod h1:y0oDTjZDv5SM9a2rp3bl+CU+bvTRINQsdb7YlDql5Go= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= @@ -566,11 +565,8 @@ github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhe github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yggdrasil-network/yggdrasil-extras v0.0.0-20200525205615-6c8a4a2e8855/go.mod h1:xQdsh08Io6nV4WRnOVTe6gI8/2iTvfLDQ0CYa5aMt+I= -github.com/yggdrasil-network/yggdrasil-go v0.3.14 h1:vWzYzCQxOruS+J5FkLfXOS0JhCJx1yI9Erj/h2wfZ/E= -github.com/yggdrasil-network/yggdrasil-go v0.3.14/go.mod h1:rkQzLzVHlFdzsEMG+bDdTI+KeWPCZq1HpXRFzwinf6M= github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20200530233943-aec82d7a391b h1:ELOisSxFXCcptRs4LFub+Hz5fYUvV12wZrTps99Eb3E= github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20200530233943-aec82d7a391b/go.mod h1:d+Nz6SPeG6kmeSPFL0cvfWfgwEql75fUnZiAONgvyBE= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -604,8 +600,6 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -675,9 +669,6 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd h1:/e+gpKk9r3dJobndpTytxS2gOy6m5uvpg+ISQoEcusQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200612022331-742c5eb664c2 h1:DVqHa33CzfnTKwUV6be+I4hp31W6iXn3ZiEcdKGzLyI= -golang.org/x/tools v0.0.0-20200612022331-742c5eb664c2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= diff --git a/internal/caching/cache_serverkeys.go b/internal/caching/cache_serverkeys.go index b5e315758..4697fb4d2 100644 --- a/internal/caching/cache_serverkeys.go +++ b/internal/caching/cache_serverkeys.go @@ -2,7 +2,6 @@ package caching import ( "fmt" - "time" "github.com/matrix-org/gomatrixserverlib" ) @@ -16,22 +15,29 @@ const ( // ServerKeyCache contains the subset of functions needed for // a server key cache. type ServerKeyCache interface { - GetServerKey(request gomatrixserverlib.PublicKeyLookupRequest) (response gomatrixserverlib.PublicKeyLookupResult, ok bool) + // request -> timestamp is emulating gomatrixserverlib.FetchKeys: + // https://github.com/matrix-org/gomatrixserverlib/blob/f69539c86ea55d1e2cc76fd8e944e2d82d30397c/keyring.go#L95 + // The timestamp should be the timestamp of the event that is being + // verified. We will not return keys from the cache that are not valid + // at this timestamp. + GetServerKey(request gomatrixserverlib.PublicKeyLookupRequest, timestamp gomatrixserverlib.Timestamp) (response gomatrixserverlib.PublicKeyLookupResult, ok bool) + + // request -> result is emulating gomatrixserverlib.StoreKeys: + // https://github.com/matrix-org/gomatrixserverlib/blob/f69539c86ea55d1e2cc76fd8e944e2d82d30397c/keyring.go#L112 StoreServerKey(request gomatrixserverlib.PublicKeyLookupRequest, response gomatrixserverlib.PublicKeyLookupResult) } func (c Caches) GetServerKey( request gomatrixserverlib.PublicKeyLookupRequest, + timestamp gomatrixserverlib.Timestamp, ) (gomatrixserverlib.PublicKeyLookupResult, bool) { key := fmt.Sprintf("%s/%s", request.ServerName, request.KeyID) - now := gomatrixserverlib.AsTimestamp(time.Now()) val, found := c.ServerKeys.Get(key) if found && val != nil { if keyLookupResult, ok := val.(gomatrixserverlib.PublicKeyLookupResult); ok { - if !keyLookupResult.WasValidAt(now, true) { - // We appear to be past the key validity so don't return this - // with the results. This ensures that the cache doesn't return - // values that are not useful to us. + if !keyLookupResult.WasValidAt(timestamp, true) { + // The key wasn't valid at the requested timestamp so don't + // return it. The caller will have to work out what to do. c.ServerKeys.Unset(key) return gomatrixserverlib.PublicKeyLookupResult{}, false } diff --git a/internal/caching/impl_inmemorylru.go b/internal/caching/impl_inmemorylru.go index 158deca49..7bb791dd8 100644 --- a/internal/caching/impl_inmemorylru.go +++ b/internal/caching/impl_inmemorylru.go @@ -8,11 +8,12 @@ import ( "github.com/prometheus/client_golang/prometheus/promauto" ) -func NewInMemoryLRUCache() (*Caches, error) { +func NewInMemoryLRUCache(enablePrometheus bool) (*Caches, error) { roomVersions, err := NewInMemoryLRUCachePartition( RoomVersionCacheName, RoomVersionCacheMutable, RoomVersionCacheMaxEntries, + enablePrometheus, ) if err != nil { return nil, err @@ -21,6 +22,7 @@ func NewInMemoryLRUCache() (*Caches, error) { ServerKeyCacheName, ServerKeyCacheMutable, ServerKeyCacheMaxEntries, + enablePrometheus, ) if err != nil { return nil, err @@ -38,7 +40,7 @@ type InMemoryLRUCachePartition struct { lru *lru.Cache } -func NewInMemoryLRUCachePartition(name string, mutable bool, maxEntries int) (*InMemoryLRUCachePartition, error) { +func NewInMemoryLRUCachePartition(name string, mutable bool, maxEntries int, enablePrometheus bool) (*InMemoryLRUCachePartition, error) { var err error cache := InMemoryLRUCachePartition{ name: name, @@ -49,13 +51,15 @@ func NewInMemoryLRUCachePartition(name string, mutable bool, maxEntries int) (*I if err != nil { return nil, err } - promauto.NewGaugeFunc(prometheus.GaugeOpts{ - Namespace: "dendrite", - Subsystem: "caching_in_memory_lru", - Name: name, - }, func() float64 { - return float64(cache.lru.Len()) - }) + if enablePrometheus { + promauto.NewGaugeFunc(prometheus.GaugeOpts{ + Namespace: "dendrite", + Subsystem: "caching_in_memory_lru", + Name: name, + }, func() float64 { + return float64(cache.lru.Len()) + }) + } return &cache, nil } diff --git a/internal/setup/base.go b/internal/setup/base.go index 414b7964a..59bdfd2e4 100644 --- a/internal/setup/base.go +++ b/internal/setup/base.go @@ -96,7 +96,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo kafkaConsumer, kafkaProducer = setupKafka(cfg) } - cache, err := caching.NewInMemoryLRUCache() + cache, err := caching.NewInMemoryLRUCache(true) if err != nil { logrus.WithError(err).Warnf("Failed to create cache") } diff --git a/serverkeyapi/internal/api.go b/serverkeyapi/internal/api.go index 7a35aa8e7..02028c60e 100644 --- a/serverkeyapi/internal/api.go +++ b/serverkeyapi/internal/api.go @@ -2,16 +2,23 @@ package internal import ( "context" + "crypto/ed25519" "fmt" "time" "github.com/matrix-org/dendrite/serverkeyapi/api" "github.com/matrix-org/gomatrixserverlib" + "github.com/sirupsen/logrus" ) type ServerKeyAPI struct { api.ServerKeyInternalAPI + ServerName gomatrixserverlib.ServerName + ServerPublicKey ed25519.PublicKey + ServerKeyID gomatrixserverlib.KeyID + ServerKeyValidity time.Duration + OurKeyRing gomatrixserverlib.KeyRing FedClient *gomatrixserverlib.FederationClient } @@ -33,6 +40,7 @@ func (s *ServerKeyAPI) StoreKeys( // Run in a background context - we don't want to stop this work just // because the caller gives up waiting. ctx := context.Background() + // Store any keys that we were given in our database. return s.OurKeyRing.KeyDatabase.StoreKeys(ctx, results) } @@ -44,52 +52,57 @@ func (s *ServerKeyAPI) FetchKeys( // Run in a background context - we don't want to stop this work just // because the caller gives up waiting. ctx := context.Background() - results := map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult{} now := gomatrixserverlib.AsTimestamp(time.Now()) - // First consult our local database and see if we have the requested + results := map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult{} + origRequests := map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp{} + for k, v := range requests { + origRequests[k] = v + } + + // First, check if any of these key checks are for our own keys. If + // they are then we will satisfy them directly. + s.handleLocalKeys(ctx, requests, results) + + // Then consult our local database and see if we have the requested // keys. These might come from a cache, depending on the database // implementation used. - if dbResults, err := s.OurKeyRing.KeyDatabase.FetchKeys(ctx, requests); err == nil { - // We successfully got some keys. Add them to the results and - // remove them from the request list. - for req, res := range dbResults { - if !res.WasValidAt(now, true) { - // We appear to be past the key validity. Don't return this - // key with the results. - continue - } - results[req] = res - delete(requests, req) - } + if err := s.handleDatabaseKeys(ctx, now, requests, results); err != nil { + return nil, err } + // For any key requests that we still have outstanding, next try to // fetch them directly. We'll go through each of the key fetchers to - // ask for the remaining keys. + // ask for the remaining keys for _, fetcher := range s.OurKeyRing.KeyFetchers { + // If there are no more keys to look up then stop. if len(requests) == 0 { break } - if fetcherResults, err := fetcher.FetchKeys(ctx, requests); err == nil { - // We successfully got some keys. Add them to the results and - // remove them from the request list. - for req, res := range fetcherResults { - if !res.WasValidAt(now, true) { - // We appear to be past the key validity. Don't return this - // key with the results. - continue - } - results[req] = res - delete(requests, req) - } - if err = s.OurKeyRing.KeyDatabase.StoreKeys(ctx, fetcherResults); err != nil { - return nil, fmt.Errorf("server key API failed to store retrieved keys: %w", err) - } + + // Ask the fetcher to look up our keys. + if err := s.handleFetcherKeys(ctx, now, fetcher, requests, results); err != nil { + logrus.WithError(err).WithFields(logrus.Fields{ + "fetcher_name": fetcher.FetcherName(), + }).Errorf("Failed to retrieve %d key(s)", len(requests)) + continue } } - // If we failed to fetch any keys then we should report an error. - if len(requests) > 0 { - return results, fmt.Errorf("server key API failed to fetch %d keys", len(requests)) + + // Check that we've actually satisfied all of the key requests that we + // were given. We should report an error if we didn't. + for req := range origRequests { + if _, ok := results[req]; !ok { + // The results don't contain anything for this specific request, so + // we've failed to satisfy it from local keys, database keys or from + // all of the fetchers. Report an error. + logrus.Warnf("Failed to retrieve key %q for server %q", req.KeyID, req.ServerName) + return results, fmt.Errorf( + "server key API failed to satisfy key request for server %q key ID %q", + req.ServerName, req.KeyID, + ) + } } + // Return the keys. return results, nil } @@ -97,3 +110,141 @@ func (s *ServerKeyAPI) FetchKeys( func (s *ServerKeyAPI) FetcherName() string { return fmt.Sprintf("ServerKeyAPI (wrapping %q)", s.OurKeyRing.KeyDatabase.FetcherName()) } + +// handleLocalKeys handles cases where the key request contains +// a request for our own server keys. +func (s *ServerKeyAPI) handleLocalKeys( + _ context.Context, + requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp, + results map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, +) { + for req := range requests { + if req.ServerName == s.ServerName { + // We found a key request that is supposed to be for our own + // keys. Remove it from the request list so we don't hit the + // database or the fetchers for it. + delete(requests, req) + + // Insert our own key into the response. + results[req] = gomatrixserverlib.PublicKeyLookupResult{ + VerifyKey: gomatrixserverlib.VerifyKey{ + Key: gomatrixserverlib.Base64Bytes(s.ServerPublicKey), + }, + ExpiredTS: gomatrixserverlib.PublicKeyNotExpired, + ValidUntilTS: gomatrixserverlib.AsTimestamp(time.Now().Add(s.ServerKeyValidity)), + } + } + } +} + +// handleDatabaseKeys handles cases where the key requests can be +// satisfied from our local database/cache. +func (s *ServerKeyAPI) handleDatabaseKeys( + ctx context.Context, + now gomatrixserverlib.Timestamp, + requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp, + results map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, +) error { + // Ask the database/cache for the keys. + dbResults, err := s.OurKeyRing.KeyDatabase.FetchKeys(ctx, requests) + if err != nil { + return err + } + + // We successfully got some keys. Add them to the results. + for req, res := range dbResults { + // The key we've retrieved from the database/cache might + // have passed its validity period, but right now, it's + // the best thing we've got, and it might be sufficient to + // verify a past event. + results[req] = res + + // If the key is valid right now then we can also remove it + // from the request list as we don't need to fetch it again + // in that case. If the key isn't valid right now, then by + // leaving it in the 'requests' map, we'll try to update the + // key using the fetchers in handleFetcherKeys. + if res.WasValidAt(now, true) { + delete(requests, req) + } + } + return nil +} + +// handleFetcherKeys handles cases where a fetcher can satisfy +// the remaining requests. +func (s *ServerKeyAPI) handleFetcherKeys( + ctx context.Context, + now gomatrixserverlib.Timestamp, + fetcher gomatrixserverlib.KeyFetcher, + requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp, + results map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, +) error { + logrus.WithFields(logrus.Fields{ + "fetcher_name": fetcher.FetcherName(), + }).Infof("Fetching %d key(s)", len(requests)) + + // Create a context that limits our requests to 30 seconds. + fetcherCtx, fetcherCancel := context.WithTimeout(ctx, time.Second*30) + defer fetcherCancel() + + // Try to fetch the keys. + fetcherResults, err := fetcher.FetchKeys(fetcherCtx, requests) + if err != nil { + return err + } + + // Build a map of the results that we want to commit to the + // database. We do this in a separate map because otherwise we + // might end up trying to rewrite database entries. + storeResults := map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult{} + + // Now let's look at the results that we got from this fetcher. + for req, res := range fetcherResults { + if prev, ok := results[req]; ok { + // We've already got a previous entry for this request + // so let's see if the newly retrieved one contains a more + // up-to-date validity period. + if res.ValidUntilTS > prev.ValidUntilTS { + // This key is newer than the one we had so let's store + // it in the database. + if req.ServerName != s.ServerName { + storeResults[req] = res + } + } + } else { + // We didn't already have a previous entry for this request + // so store it in the database anyway for now. + if req.ServerName != s.ServerName { + storeResults[req] = res + } + } + + // Update the results map with this new result. If nothing + // else, we can try verifying against this key. + results[req] = res + + // If the key is valid right now then we can remove it from the + // request list as we won't need to re-fetch it. + if res.WasValidAt(now, true) { + delete(requests, req) + } + } + + // Store the keys from our store map. + if err = s.OurKeyRing.KeyDatabase.StoreKeys(ctx, storeResults); err != nil { + logrus.WithError(err).WithFields(logrus.Fields{ + "fetcher_name": fetcher.FetcherName(), + "database_name": s.OurKeyRing.KeyDatabase.FetcherName(), + }).Errorf("Failed to store keys in the database") + return fmt.Errorf("server key API failed to store retrieved keys: %w", err) + } + + if len(storeResults) > 0 { + logrus.WithFields(logrus.Fields{ + "fetcher_name": fetcher.FetcherName(), + }).Infof("Updated %d of %d key(s) in database", len(storeResults), len(results)) + } + + return nil +} diff --git a/serverkeyapi/inthttp/client.go b/serverkeyapi/inthttp/client.go index e84cf47f6..39ab8c6c5 100644 --- a/serverkeyapi/inthttp/client.go +++ b/serverkeyapi/inthttp/client.go @@ -90,7 +90,7 @@ func (s *httpServerKeyInternalAPI) FetchKeys( Results: make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult), } for req, ts := range requests { - if res, ok := s.cache.GetServerKey(req); ok { + if res, ok := s.cache.GetServerKey(req, ts); ok { result[req] = res continue } diff --git a/serverkeyapi/serverkeyapi.go b/serverkeyapi/serverkeyapi.go index 58ca00b73..cddd392ed 100644 --- a/serverkeyapi/serverkeyapi.go +++ b/serverkeyapi/serverkeyapi.go @@ -46,7 +46,11 @@ func NewInternalAPI( } internalAPI := internal.ServerKeyAPI{ - FedClient: fedClient, + ServerName: cfg.Matrix.ServerName, + ServerPublicKey: cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey), + ServerKeyID: cfg.Matrix.KeyID, + ServerKeyValidity: cfg.Matrix.KeyValidityPeriod, + FedClient: fedClient, OurKeyRing: gomatrixserverlib.KeyRing{ KeyFetchers: []gomatrixserverlib.KeyFetcher{ &gomatrixserverlib.DirectKeyFetcher{ diff --git a/serverkeyapi/serverkeyapi_test.go b/serverkeyapi/serverkeyapi_test.go new file mode 100644 index 000000000..3368f5b2a --- /dev/null +++ b/serverkeyapi/serverkeyapi_test.go @@ -0,0 +1,315 @@ +package serverkeyapi + +import ( + "bytes" + "context" + "crypto/ed25519" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "os" + "reflect" + "testing" + "time" + + "github.com/matrix-org/dendrite/federationapi/routing" + "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/serverkeyapi/api" + "github.com/matrix-org/gomatrixserverlib" +) + +type server struct { + name gomatrixserverlib.ServerName // server name + validity time.Duration // key validity duration from now + config *config.Dendrite // skeleton config, from TestMain + fedclient *gomatrixserverlib.FederationClient // uses MockRoundTripper + cache *caching.Caches // server-specific cache + api api.ServerKeyInternalAPI // server-specific server key API +} + +func (s *server) renew() { + // This updates the validity period to be an hour in the + // future, which is particularly useful in server A and + // server C's cases which have validity either as now or + // in the past. + s.validity = time.Hour + s.config.Matrix.KeyValidityPeriod = s.validity +} + +var ( + serverKeyID = gomatrixserverlib.KeyID("ed25519:auto") + serverA = &server{name: "a.com", validity: time.Duration(0)} // expires now + serverB = &server{name: "b.com", validity: time.Hour} // expires in an hour + serverC = &server{name: "c.com", validity: -time.Hour} // expired an hour ago +) + +var servers = map[string]*server{ + "a.com": serverA, + "b.com": serverB, + "c.com": serverC, +} + +func TestMain(m *testing.M) { + // Set up the server key API for each "server" that we + // will use in our tests. + for _, s := range servers { + // Generate a new key. + _, testPriv, err := ed25519.GenerateKey(nil) + if err != nil { + panic("can't generate identity key: " + err.Error()) + } + + // Create a new cache but don't enable prometheus! + s.cache, err = caching.NewInMemoryLRUCache(false) + if err != nil { + panic("can't create cache: " + err.Error()) + } + + // Draw up just enough Dendrite config for the server key + // API to work. + s.config = &config.Dendrite{} + s.config.SetDefaults() + s.config.Matrix.ServerName = gomatrixserverlib.ServerName(s.name) + s.config.Matrix.PrivateKey = testPriv + s.config.Matrix.KeyID = serverKeyID + s.config.Matrix.KeyValidityPeriod = s.validity + s.config.Database.ServerKey = config.DataSource("file::memory:") + + // Create a transport which redirects federation requests to + // the mock round tripper. Since we're not *really* listening for + // federation requests then this will return the key instead. + transport := &http.Transport{} + transport.RegisterProtocol("matrix", &MockRoundTripper{}) + + // Create the federation client. + s.fedclient = gomatrixserverlib.NewFederationClientWithTransport( + s.config.Matrix.ServerName, serverKeyID, testPriv, transport, + ) + + // Finally, build the server key APIs. + s.api = NewInternalAPI(s.config, s.fedclient, s.cache) + } + + // Now that we have built our server key APIs, start the + // rest of the tests. + os.Exit(m.Run()) +} + +type MockRoundTripper struct{} + +func (m *MockRoundTripper) RoundTrip(req *http.Request) (res *http.Response, err error) { + // Check if the request is looking for keys from a server that + // we know about in the test. The only reason this should go wrong + // is if the test is broken. + s, ok := servers[req.Host] + if !ok { + return nil, fmt.Errorf("server not known: %s", req.Host) + } + + // We're intercepting /matrix/key/v2/server requests here, so check + // that the URL supplied in the request is for that. + if req.URL.Path != "/_matrix/key/v2/server" { + return nil, fmt.Errorf("unexpected request path: %s", req.URL.Path) + } + + // Get the keys and JSON-ify them. + keys := routing.LocalKeys(s.config) + body, err := json.MarshalIndent(keys.JSON, "", " ") + if err != nil { + return nil, err + } + + // And respond. + res = &http.Response{ + StatusCode: 200, + Body: ioutil.NopCloser(bytes.NewReader(body)), + } + return +} + +func TestServersRequestOwnKeys(t *testing.T) { + // Each server will request its own keys. There's no reason + // for this to fail as each server should know its own keys. + + for name, s := range servers { + req := gomatrixserverlib.PublicKeyLookupRequest{ + ServerName: s.name, + KeyID: serverKeyID, + } + res, err := s.api.FetchKeys( + context.Background(), + map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp{ + req: gomatrixserverlib.AsTimestamp(time.Now()), + }, + ) + if err != nil { + t.Fatalf("server could not fetch own key: %s", err) + } + if _, ok := res[req]; !ok { + t.Fatalf("server didn't return its own key in the results") + } + t.Logf("%s's key expires at %s\n", name, res[req].ValidUntilTS.Time()) + } +} + +func TestCachingBehaviour(t *testing.T) { + // Server A will request Server B's key, which has a validity + // period of an hour from now. We should retrieve the key and + // it should make it into the cache automatically. + + req := gomatrixserverlib.PublicKeyLookupRequest{ + ServerName: serverB.name, + KeyID: serverKeyID, + } + ts := gomatrixserverlib.AsTimestamp(time.Now()) + + res, err := serverA.api.FetchKeys( + context.Background(), + map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp{ + req: ts, + }, + ) + if err != nil { + t.Fatalf("server A failed to retrieve server B key: %s", err) + } + if len(res) != 1 { + t.Fatalf("server B should have returned one key but instead returned %d keys", len(res)) + } + if _, ok := res[req]; !ok { + t.Fatalf("server B isn't included in the key fetch response") + } + + // At this point, if the previous key request was a success, + // then the cache should now contain the key. Check if that's + // the case - if it isn't then there's something wrong with + // the cache implementation or we failed to get the key. + + cres, ok := serverA.cache.GetServerKey(req, ts) + if !ok { + t.Fatalf("server B key should be in cache but isn't") + } + if !reflect.DeepEqual(cres, res[req]) { + t.Fatalf("the cached result from server B wasn't what server B gave us") + } + + // If we ask the cache for the same key but this time for an event + // that happened in +30 minutes. Since the validity period is for + // another hour, then we should get a response back from the cache. + + _, ok = serverA.cache.GetServerKey( + req, + gomatrixserverlib.AsTimestamp(time.Now().Add(time.Minute*30)), + ) + if !ok { + t.Fatalf("server B key isn't in cache when it should be (+30 minutes)") + } + + // If we ask the cache for the same key but this time for an event + // that happened in +90 minutes then we should expect to get no + // cache result. This is because the cache shouldn't return a result + // that is obviously past the validity of the event. + + _, ok = serverA.cache.GetServerKey( + req, + gomatrixserverlib.AsTimestamp(time.Now().Add(time.Minute*90)), + ) + if ok { + t.Fatalf("server B key is in cache when it shouldn't be (+90 minutes)") + } +} + +func TestRenewalBehaviour(t *testing.T) { + // Server A will request Server C's key but their validity period + // is an hour in the past. We'll retrieve the key as, even though it's + // past its validity, it will be able to verify past events. + + req := gomatrixserverlib.PublicKeyLookupRequest{ + ServerName: serverC.name, + KeyID: serverKeyID, + } + + res, err := serverA.api.FetchKeys( + context.Background(), + map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp{ + req: gomatrixserverlib.AsTimestamp(time.Now()), + }, + ) + if err != nil { + t.Fatalf("server A failed to retrieve server C key: %s", err) + } + if len(res) != 1 { + t.Fatalf("server C should have returned one key but instead returned %d keys", len(res)) + } + if _, ok := res[req]; !ok { + t.Fatalf("server C isn't included in the key fetch response") + } + + // If we ask the cache for the server key for an event that happened + // 90 minutes ago then we should get a cache result, as the key hadn't + // passed its validity by that point. The fact that the key is now in + // the cache is, in itself, proof that we successfully retrieved the + // key before. + + oldcached, ok := serverA.cache.GetServerKey( + req, + gomatrixserverlib.AsTimestamp(time.Now().Add(-time.Minute*90)), + ) + if !ok { + t.Fatalf("server C key isn't in cache when it should be (-90 minutes)") + } + + // If we now ask the cache for the same key but this time for an event + // that only happened 30 minutes ago then we shouldn't get a cached + // result, as the event happened after the key validity expired. This + // is really just for sanity checking. + + _, ok = serverA.cache.GetServerKey( + req, + gomatrixserverlib.AsTimestamp(time.Now().Add(-time.Minute*30)), + ) + if ok { + t.Fatalf("server B key is in cache when it shouldn't be (-30 minutes)") + } + + // We're now going to kick server C into renewing its key. Since we're + // happy at this point that the key that we already have is from the past + // then repeating a key fetch should cause us to try and renew the key. + // If so, then the new key will end up in our cache. + + serverC.renew() + + res, err = serverA.api.FetchKeys( + context.Background(), + map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp{ + req: gomatrixserverlib.AsTimestamp(time.Now()), + }, + ) + if err != nil { + t.Fatalf("server A failed to retrieve server C key: %s", err) + } + if len(res) != 1 { + t.Fatalf("server C should have returned one key but instead returned %d keys", len(res)) + } + if _, ok = res[req]; !ok { + t.Fatalf("server C isn't included in the key fetch response") + } + + // We're now going to ask the cache what the new key validity is. If + // it is still the same as the previous validity then we've failed to + // retrieve the renewed key. If it's newer then we've successfully got + // the renewed key. + + newcached, ok := serverA.cache.GetServerKey( + req, + gomatrixserverlib.AsTimestamp(time.Now().Add(-time.Minute*30)), + ) + if !ok { + t.Fatalf("server B key isn't in cache when it shouldn't be (post-renewal)") + } + if oldcached.ValidUntilTS >= newcached.ValidUntilTS { + t.Fatalf("the server B key should have been renewed but wasn't") + } + t.Log(res) +} diff --git a/serverkeyapi/storage/cache/keydb.go b/serverkeyapi/storage/cache/keydb.go index b662e4fdb..2063dfc55 100644 --- a/serverkeyapi/storage/cache/keydb.go +++ b/serverkeyapi/storage/cache/keydb.go @@ -39,8 +39,8 @@ func (d *KeyDatabase) FetchKeys( requests map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.Timestamp, ) (map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, error) { results := make(map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult) - for req := range requests { - if res, cached := d.cache.GetServerKey(req); cached { + for req, ts := range requests { + if res, cached := d.cache.GetServerKey(req, ts); cached { results[req] = res delete(requests, req) } diff --git a/serverkeyapi/storage/postgres/keydb.go b/serverkeyapi/storage/postgres/keydb.go index 32cdf951b..aaa4409be 100644 --- a/serverkeyapi/storage/postgres/keydb.go +++ b/serverkeyapi/storage/postgres/keydb.go @@ -17,7 +17,6 @@ package postgres import ( "context" - "time" "golang.org/x/crypto/ed25519" @@ -51,28 +50,6 @@ func NewDatabase( if err != nil { return nil, err } - // Store our own keys so that we don't end up making HTTP requests to find our - // own keys - index := gomatrixserverlib.PublicKeyLookupRequest{ - ServerName: serverName, - KeyID: serverKeyID, - } - value := gomatrixserverlib.PublicKeyLookupResult{ - VerifyKey: gomatrixserverlib.VerifyKey{ - Key: gomatrixserverlib.Base64Bytes(serverKey), - }, - ValidUntilTS: gomatrixserverlib.AsTimestamp(time.Now().Add(100 * 365 * 24 * time.Hour)), - ExpiredTS: gomatrixserverlib.PublicKeyNotExpired, - } - err = d.StoreKeys( - context.Background(), - map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult{ - index: value, - }, - ) - if err != nil { - return nil, err - } return d, nil } diff --git a/serverkeyapi/storage/sqlite3/keydb.go b/serverkeyapi/storage/sqlite3/keydb.go index 268c75420..dc72b79eb 100644 --- a/serverkeyapi/storage/sqlite3/keydb.go +++ b/serverkeyapi/storage/sqlite3/keydb.go @@ -17,7 +17,6 @@ package sqlite3 import ( "context" - "time" "golang.org/x/crypto/ed25519" @@ -56,25 +55,6 @@ func NewDatabase( if err != nil { return nil, err } - // Store our own keys so that we don't end up making HTTP requests to find our - // own keys - index := gomatrixserverlib.PublicKeyLookupRequest{ - ServerName: serverName, - KeyID: serverKeyID, - } - value := gomatrixserverlib.PublicKeyLookupResult{ - VerifyKey: gomatrixserverlib.VerifyKey{ - Key: gomatrixserverlib.Base64Bytes(serverKey), - }, - ValidUntilTS: gomatrixserverlib.AsTimestamp(time.Now().Add(100 * 365 * 24 * time.Hour)), - ExpiredTS: gomatrixserverlib.PublicKeyNotExpired, - } - err = d.StoreKeys( - context.Background(), - map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult{ - index: value, - }, - ) if err != nil { return nil, err } From 9c77022513f400db59409f5b55fc6223d38d6bb8 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 16 Jun 2020 14:10:55 +0100 Subject: [PATCH 166/200] Make userapi responsible for checking access tokens (#1133) * Make userapi responsible for checking access tokens There's still plenty of dependencies on account/device DBs, but this is a start. This is a breaking change as it adds a required config value `listen.user_api`. * Cleanup * Review comments and test fix --- clientapi/auth/auth.go | 125 +++--------------- clientapi/auth/authtypes/device.go | 30 ----- clientapi/auth/storage/devices/interface.go | 10 +- .../storage/devices/postgres/devices_table.go | 20 +-- .../auth/storage/devices/postgres/storage.go | 10 +- .../storage/devices/sqlite3/devices_table.go | 20 +-- .../auth/storage/devices/sqlite3/storage.go | 10 +- clientapi/clientapi.go | 4 +- clientapi/routing/account_data.go | 6 +- clientapi/routing/createroom.go | 6 +- clientapi/routing/device.go | 12 +- clientapi/routing/directory.go | 6 +- clientapi/routing/filter.go | 6 +- clientapi/routing/getevent.go | 6 +- clientapi/routing/joinroom.go | 4 +- clientapi/routing/leaveroom.go | 4 +- clientapi/routing/login.go | 3 +- clientapi/routing/logout.go | 6 +- clientapi/routing/membership.go | 9 +- clientapi/routing/memberships.go | 6 +- clientapi/routing/profile.go | 5 +- clientapi/routing/room_tagging.go | 8 +- clientapi/routing/routing.go | 98 +++++++------- clientapi/routing/sendevent.go | 6 +- clientapi/routing/sendtodevice.go | 4 +- clientapi/routing/sendtyping.go | 4 +- clientapi/routing/threepid.go | 5 +- clientapi/routing/voip.go | 4 +- clientapi/routing/whoami.go | 4 +- clientapi/threepid/invites.go | 9 +- cmd/dendrite-client-api-server/main.go | 3 +- cmd/dendrite-demo-libp2p/main.go | 3 + cmd/dendrite-demo-yggdrasil/main.go | 3 + cmd/dendrite-key-server/main.go | 5 +- cmd/dendrite-media-api-server/main.go | 4 +- cmd/dendrite-monolith-server/main.go | 4 + cmd/dendrite-public-rooms-api-server/main.go | 4 +- cmd/dendrite-sync-api-server/main.go | 4 +- cmd/dendritejs/main.go | 4 + dendrite-config.yaml | 1 + internal/config/config.go | 11 ++ internal/config/config_test.go | 1 + internal/httputil/httpapi.go | 8 +- internal/setup/base.go | 11 ++ internal/setup/monolith.go | 12 +- keyserver/keyserver.go | 9 +- keyserver/routing/routing.go | 18 +-- mediaapi/mediaapi.go | 7 +- mediaapi/routing/routing.go | 17 +-- publicroomsapi/directory/directory.go | 4 +- publicroomsapi/publicroomsapi.go | 6 +- publicroomsapi/routing/routing.go | 14 +- syncapi/routing/routing.go | 16 +-- syncapi/storage/interface.go | 8 +- syncapi/storage/shared/syncserver.go | 17 +-- syncapi/storage/storage_test.go | 4 +- syncapi/sync/notifier_test.go | 5 +- syncapi/sync/request.go | 7 +- syncapi/sync/requestpool.go | 4 +- syncapi/syncapi.go | 6 +- userapi/api/api.go | 41 +++++- userapi/internal/api.go | 68 ++++++++++ userapi/inthttp/client.go | 15 ++- userapi/inthttp/server.go | 13 ++ userapi/userapi.go | 12 +- userapi/userapi_test.go | 2 +- 66 files changed, 421 insertions(+), 400 deletions(-) delete mode 100644 clientapi/auth/authtypes/device.go diff --git a/clientapi/auth/auth.go b/clientapi/auth/auth.go index 3482e5018..b8e408538 100644 --- a/clientapi/auth/auth.go +++ b/clientapi/auth/auth.go @@ -18,17 +18,14 @@ package auth import ( "context" "crypto/rand" - "database/sql" "encoding/base64" "fmt" "net/http" "strings" - "github.com/matrix-org/dendrite/appservice/types" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/clientapi/userutil" - "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/util" ) @@ -39,7 +36,7 @@ var tokenByteLength = 32 // DeviceDatabase represents a device database. type DeviceDatabase interface { // Look up the device matching the given access token. - GetDeviceByAccessToken(ctx context.Context, token string) (*authtypes.Device, error) + GetDeviceByAccessToken(ctx context.Context, token string) (*api.Device, error) } // AccountDatabase represents an account database. @@ -48,22 +45,14 @@ type AccountDatabase interface { GetAccountByLocalpart(ctx context.Context, localpart string) (*authtypes.Account, error) } -// Data contains information required to authenticate a request. -type Data struct { - AccountDB AccountDatabase - DeviceDB DeviceDatabase - // AppServices is the list of all registered AS - AppServices []config.ApplicationService -} - // VerifyUserFromRequest authenticates the HTTP request, // on success returns Device of the requester. // Finds local user or an application service user. // Note: For an AS user, AS dummy device is returned. // On failure returns an JSON error response which can be sent to the client. func VerifyUserFromRequest( - req *http.Request, data Data, -) (*authtypes.Device, *util.JSONResponse) { + req *http.Request, userAPI api.UserInternalAPI, +) (*api.Device, *util.JSONResponse) { // Try to find the Application Service user token, err := ExtractAccessToken(req) if err != nil { @@ -72,105 +61,31 @@ func VerifyUserFromRequest( JSON: jsonerror.MissingToken(err.Error()), } } - - // Search for app service with given access_token - var appService *config.ApplicationService - for _, as := range data.AppServices { - if as.ASToken == token { - appService = &as - break - } + var res api.QueryAccessTokenResponse + err = userAPI.QueryAccessToken(req.Context(), &api.QueryAccessTokenRequest{ + AccessToken: token, + AppServiceUserID: req.URL.Query().Get("user_id"), + }, &res) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("userAPI.QueryAccessToken failed") + jsonErr := jsonerror.InternalServerError() + return nil, &jsonErr } - - if appService != nil { - // Create a dummy device for AS user - dev := authtypes.Device{ - // Use AS dummy device ID - ID: types.AppServiceDeviceID, - // AS dummy device has AS's token. - AccessToken: token, - } - - userID := req.URL.Query().Get("user_id") - localpart, err := userutil.ParseUsernameParam(userID, nil) - if err != nil { - return nil, &util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.InvalidUsername(err.Error()), - } - } - - if localpart != "" { // AS is masquerading as another user - // Verify that the user is registered - account, err := data.AccountDB.GetAccountByLocalpart(req.Context(), localpart) - // Verify that account exists & appServiceID matches - if err == nil && account.AppServiceID == appService.ID { - // Set the userID of dummy device - dev.UserID = userID - return &dev, nil - } - + if res.Err != nil { + if forbidden, ok := res.Err.(*api.ErrorForbidden); ok { return nil, &util.JSONResponse{ Code: http.StatusForbidden, - JSON: jsonerror.Forbidden("Application service has not registered this user"), + JSON: jsonerror.Forbidden(forbidden.Message), } } - - // AS is not masquerading as any user, so use AS's sender_localpart - dev.UserID = appService.SenderLocalpart - return &dev, nil } - - // Try to find local user from device database - dev, devErr := verifyAccessToken(req, data.DeviceDB) - if devErr == nil { - return dev, verifyUserParameters(req) - } - - return nil, &util.JSONResponse{ - Code: http.StatusUnauthorized, - JSON: jsonerror.UnknownToken("Unrecognized access token"), // nolint: misspell - } -} - -// verifyUserParameters ensures that a request coming from a regular user is not -// using any query parameters reserved for an application service -func verifyUserParameters(req *http.Request) *util.JSONResponse { - if req.URL.Query().Get("ts") != "" { - return &util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.Unknown("parameter 'ts' not allowed without valid parameter 'access_token'"), - } - } - return nil -} - -// verifyAccessToken verifies that an access token was supplied in the given HTTP request -// and returns the device it corresponds to. Returns resErr (an error response which can be -// sent to the client) if the token is invalid or there was a problem querying the database. -func verifyAccessToken(req *http.Request, deviceDB DeviceDatabase) (device *authtypes.Device, resErr *util.JSONResponse) { - token, err := ExtractAccessToken(req) - if err != nil { - resErr = &util.JSONResponse{ + if res.Device == nil { + return nil, &util.JSONResponse{ Code: http.StatusUnauthorized, - JSON: jsonerror.MissingToken(err.Error()), - } - return - } - device, err = deviceDB.GetDeviceByAccessToken(req.Context(), token) - if err != nil { - if err == sql.ErrNoRows { - resErr = &util.JSONResponse{ - Code: http.StatusUnauthorized, - JSON: jsonerror.UnknownToken("Unknown token"), - } - } else { - util.GetLogger(req.Context()).WithError(err).Error("deviceDB.GetDeviceByAccessToken failed") - jsonErr := jsonerror.InternalServerError() - resErr = &jsonErr + JSON: jsonerror.UnknownToken("Unknown token"), } } - return + return res.Device, nil } // GenerateAccessToken creates a new access token. Returns an error if failed to generate diff --git a/clientapi/auth/authtypes/device.go b/clientapi/auth/authtypes/device.go deleted file mode 100644 index 299eff036..000000000 --- a/clientapi/auth/authtypes/device.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2017 Vector Creations Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package authtypes - -// Device represents a client's device (mobile, web, etc) -type Device struct { - ID string - UserID string - // The access_token granted to this device. - // This uniquely identifies the device from all other devices and clients. - AccessToken string - // The unique ID of the session identified by the access token. - // Can be used as a secure substitution in places where data needs to be - // associated with access tokens. - SessionID int64 - // TODO: display name, last used timestamp, keys, etc - DisplayName string -} diff --git a/clientapi/auth/storage/devices/interface.go b/clientapi/auth/storage/devices/interface.go index 95291e4a7..fc2f4a320 100644 --- a/clientapi/auth/storage/devices/interface.go +++ b/clientapi/auth/storage/devices/interface.go @@ -17,14 +17,14 @@ package devices import ( "context" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/userapi/api" ) type Database interface { - GetDeviceByAccessToken(ctx context.Context, token string) (*authtypes.Device, error) - GetDeviceByID(ctx context.Context, localpart, deviceID string) (*authtypes.Device, error) - GetDevicesByLocalpart(ctx context.Context, localpart string) ([]authtypes.Device, error) - CreateDevice(ctx context.Context, localpart string, deviceID *string, accessToken string, displayName *string) (dev *authtypes.Device, returnErr error) + GetDeviceByAccessToken(ctx context.Context, token string) (*api.Device, error) + GetDeviceByID(ctx context.Context, localpart, deviceID string) (*api.Device, error) + GetDevicesByLocalpart(ctx context.Context, localpart string) ([]api.Device, error) + CreateDevice(ctx context.Context, localpart string, deviceID *string, accessToken string, displayName *string) (dev *api.Device, returnErr error) UpdateDevice(ctx context.Context, localpart, deviceID string, displayName *string) error RemoveDevice(ctx context.Context, deviceID, localpart string) error RemoveDevices(ctx context.Context, localpart string, devices []string) error diff --git a/clientapi/auth/storage/devices/postgres/devices_table.go b/clientapi/auth/storage/devices/postgres/devices_table.go index 149ca659f..1d036d1b3 100644 --- a/clientapi/auth/storage/devices/postgres/devices_table.go +++ b/clientapi/auth/storage/devices/postgres/devices_table.go @@ -20,10 +20,10 @@ import ( "time" "github.com/lib/pq" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/userutil" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" ) @@ -135,14 +135,14 @@ func (s *devicesStatements) prepare(db *sql.DB, server gomatrixserverlib.ServerN func (s *devicesStatements) insertDevice( ctx context.Context, txn *sql.Tx, id, localpart, accessToken string, displayName *string, -) (*authtypes.Device, error) { +) (*api.Device, error) { createdTimeMS := time.Now().UnixNano() / 1000000 var sessionID int64 stmt := sqlutil.TxStmt(txn, s.insertDeviceStmt) if err := stmt.QueryRowContext(ctx, id, localpart, accessToken, createdTimeMS, displayName).Scan(&sessionID); err != nil { return nil, err } - return &authtypes.Device{ + return &api.Device{ ID: id, UserID: userutil.MakeUserID(localpart, s.serverName), AccessToken: accessToken, @@ -189,8 +189,8 @@ func (s *devicesStatements) updateDeviceName( func (s *devicesStatements) selectDeviceByToken( ctx context.Context, accessToken string, -) (*authtypes.Device, error) { - var dev authtypes.Device +) (*api.Device, error) { + var dev api.Device var localpart string stmt := s.selectDeviceByTokenStmt err := stmt.QueryRowContext(ctx, accessToken).Scan(&dev.SessionID, &dev.ID, &localpart) @@ -205,8 +205,8 @@ func (s *devicesStatements) selectDeviceByToken( // localpart and deviceID func (s *devicesStatements) selectDeviceByID( ctx context.Context, localpart, deviceID string, -) (*authtypes.Device, error) { - var dev authtypes.Device +) (*api.Device, error) { + var dev api.Device stmt := s.selectDeviceByIDStmt err := stmt.QueryRowContext(ctx, localpart, deviceID).Scan(&dev.DisplayName) if err == nil { @@ -218,8 +218,8 @@ func (s *devicesStatements) selectDeviceByID( func (s *devicesStatements) selectDevicesByLocalpart( ctx context.Context, localpart string, -) ([]authtypes.Device, error) { - devices := []authtypes.Device{} +) ([]api.Device, error) { + devices := []api.Device{} rows, err := s.selectDevicesByLocalpartStmt.QueryContext(ctx, localpart) @@ -229,7 +229,7 @@ func (s *devicesStatements) selectDevicesByLocalpart( defer internal.CloseAndLogIfError(ctx, rows, "selectDevicesByLocalpart: rows.close() failed") for rows.Next() { - var dev authtypes.Device + var dev api.Device var id, displayname sql.NullString err = rows.Scan(&id, &displayname) if err != nil { diff --git a/clientapi/auth/storage/devices/postgres/storage.go b/clientapi/auth/storage/devices/postgres/storage.go index 2b9aede2f..801657bd5 100644 --- a/clientapi/auth/storage/devices/postgres/storage.go +++ b/clientapi/auth/storage/devices/postgres/storage.go @@ -20,8 +20,8 @@ import ( "database/sql" "encoding/base64" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" ) @@ -52,7 +52,7 @@ func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties, serve // Returns sql.ErrNoRows if no matching device was found. func (d *Database) GetDeviceByAccessToken( ctx context.Context, token string, -) (*authtypes.Device, error) { +) (*api.Device, error) { return d.devices.selectDeviceByToken(ctx, token) } @@ -60,14 +60,14 @@ func (d *Database) GetDeviceByAccessToken( // Returns sql.ErrNoRows if no matching device was found. func (d *Database) GetDeviceByID( ctx context.Context, localpart, deviceID string, -) (*authtypes.Device, error) { +) (*api.Device, error) { return d.devices.selectDeviceByID(ctx, localpart, deviceID) } // GetDevicesByLocalpart returns the devices matching the given localpart. func (d *Database) GetDevicesByLocalpart( ctx context.Context, localpart string, -) ([]authtypes.Device, error) { +) ([]api.Device, error) { return d.devices.selectDevicesByLocalpart(ctx, localpart) } @@ -80,7 +80,7 @@ func (d *Database) GetDevicesByLocalpart( func (d *Database) CreateDevice( ctx context.Context, localpart string, deviceID *string, accessToken string, displayName *string, -) (dev *authtypes.Device, returnErr error) { +) (dev *api.Device, returnErr error) { if deviceID != nil { returnErr = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { var err error diff --git a/clientapi/auth/storage/devices/sqlite3/devices_table.go b/clientapi/auth/storage/devices/sqlite3/devices_table.go index 4656b0041..07ea5dca3 100644 --- a/clientapi/auth/storage/devices/sqlite3/devices_table.go +++ b/clientapi/auth/storage/devices/sqlite3/devices_table.go @@ -21,8 +21,8 @@ import ( "time" "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/userapi/api" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/userutil" "github.com/matrix-org/gomatrixserverlib" ) @@ -125,7 +125,7 @@ func (s *devicesStatements) prepare(db *sql.DB, server gomatrixserverlib.ServerN func (s *devicesStatements) insertDevice( ctx context.Context, txn *sql.Tx, id, localpart, accessToken string, displayName *string, -) (*authtypes.Device, error) { +) (*api.Device, error) { createdTimeMS := time.Now().UnixNano() / 1000000 var sessionID int64 countStmt := sqlutil.TxStmt(txn, s.selectDevicesCountStmt) @@ -137,7 +137,7 @@ func (s *devicesStatements) insertDevice( if _, err := insertStmt.ExecContext(ctx, id, localpart, accessToken, createdTimeMS, displayName, sessionID); err != nil { return nil, err } - return &authtypes.Device{ + return &api.Device{ ID: id, UserID: userutil.MakeUserID(localpart, s.serverName), AccessToken: accessToken, @@ -190,8 +190,8 @@ func (s *devicesStatements) updateDeviceName( func (s *devicesStatements) selectDeviceByToken( ctx context.Context, accessToken string, -) (*authtypes.Device, error) { - var dev authtypes.Device +) (*api.Device, error) { + var dev api.Device var localpart string stmt := s.selectDeviceByTokenStmt err := stmt.QueryRowContext(ctx, accessToken).Scan(&dev.SessionID, &dev.ID, &localpart) @@ -206,8 +206,8 @@ func (s *devicesStatements) selectDeviceByToken( // localpart and deviceID func (s *devicesStatements) selectDeviceByID( ctx context.Context, localpart, deviceID string, -) (*authtypes.Device, error) { - var dev authtypes.Device +) (*api.Device, error) { + var dev api.Device stmt := s.selectDeviceByIDStmt err := stmt.QueryRowContext(ctx, localpart, deviceID).Scan(&dev.DisplayName) if err == nil { @@ -219,8 +219,8 @@ func (s *devicesStatements) selectDeviceByID( func (s *devicesStatements) selectDevicesByLocalpart( ctx context.Context, localpart string, -) ([]authtypes.Device, error) { - devices := []authtypes.Device{} +) ([]api.Device, error) { + devices := []api.Device{} rows, err := s.selectDevicesByLocalpartStmt.QueryContext(ctx, localpart) @@ -229,7 +229,7 @@ func (s *devicesStatements) selectDevicesByLocalpart( } for rows.Next() { - var dev authtypes.Device + var dev api.Device var id, displayname sql.NullString err = rows.Scan(&id, &displayname) if err != nil { diff --git a/clientapi/auth/storage/devices/sqlite3/storage.go b/clientapi/auth/storage/devices/sqlite3/storage.go index 09e0bc81e..f248abda4 100644 --- a/clientapi/auth/storage/devices/sqlite3/storage.go +++ b/clientapi/auth/storage/devices/sqlite3/storage.go @@ -20,8 +20,8 @@ import ( "database/sql" "encoding/base64" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" _ "github.com/mattn/go-sqlite3" @@ -58,7 +58,7 @@ func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) // Returns sql.ErrNoRows if no matching device was found. func (d *Database) GetDeviceByAccessToken( ctx context.Context, token string, -) (*authtypes.Device, error) { +) (*api.Device, error) { return d.devices.selectDeviceByToken(ctx, token) } @@ -66,14 +66,14 @@ func (d *Database) GetDeviceByAccessToken( // Returns sql.ErrNoRows if no matching device was found. func (d *Database) GetDeviceByID( ctx context.Context, localpart, deviceID string, -) (*authtypes.Device, error) { +) (*api.Device, error) { return d.devices.selectDeviceByID(ctx, localpart, deviceID) } // GetDevicesByLocalpart returns the devices matching the given localpart. func (d *Database) GetDevicesByLocalpart( ctx context.Context, localpart string, -) ([]authtypes.Device, error) { +) ([]api.Device, error) { return d.devices.selectDevicesByLocalpart(ctx, localpart) } @@ -86,7 +86,7 @@ func (d *Database) GetDevicesByLocalpart( func (d *Database) CreateDevice( ctx context.Context, localpart string, deviceID *string, accessToken string, displayName *string, -) (dev *authtypes.Device, returnErr error) { +) (dev *api.Device, returnErr error) { if deviceID != nil { returnErr = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { var err error diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go index 545b95b0e..637e1469e 100644 --- a/clientapi/clientapi.go +++ b/clientapi/clientapi.go @@ -28,6 +28,7 @@ import ( "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/transactions" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" ) @@ -46,6 +47,7 @@ func AddPublicRoutes( asAPI appserviceAPI.AppServiceQueryAPI, transactionsCache *transactions.Cache, fsAPI federationSenderAPI.FederationSenderInternalAPI, + userAPI userapi.UserInternalAPI, ) { syncProducer := &producers.SyncAPIProducer{ Producer: producer, @@ -61,7 +63,7 @@ func AddPublicRoutes( routing.Setup( router, cfg, eduInputAPI, rsAPI, asAPI, - accountsDB, deviceDB, federation, + accountsDB, deviceDB, userAPI, federation, syncProducer, transactionsCache, fsAPI, ) } diff --git a/clientapi/routing/account_data.go b/clientapi/routing/account_data.go index a5d53c326..5e0509a50 100644 --- a/clientapi/routing/account_data.go +++ b/clientapi/routing/account_data.go @@ -19,10 +19,10 @@ import ( "io/ioutil" "net/http" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" + "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -30,7 +30,7 @@ import ( // GetAccountData implements GET /user/{userId}/[rooms/{roomid}/]account_data/{type} func GetAccountData( - req *http.Request, accountDB accounts.Database, device *authtypes.Device, + req *http.Request, accountDB accounts.Database, device *api.Device, userID string, roomID string, dataType string, ) util.JSONResponse { if userID != device.UserID { @@ -63,7 +63,7 @@ func GetAccountData( // SaveAccountData implements PUT /user/{userId}/[rooms/{roomId}/]account_data/{type} func SaveAccountData( - req *http.Request, accountDB accounts.Database, device *authtypes.Device, + req *http.Request, accountDB accounts.Database, device *api.Device, userID string, roomID string, dataType string, syncProducer *producers.SyncAPIProducer, ) util.JSONResponse { if userID != device.UserID { diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index fd91a1060..2bb537b0d 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -24,8 +24,8 @@ import ( appserviceAPI "github.com/matrix-org/dendrite/appservice/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" roomserverVersion "github.com/matrix-org/dendrite/roomserver/version" + "github.com/matrix-org/dendrite/userapi/api" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" @@ -135,7 +135,7 @@ type fledglingEvent struct { // CreateRoom implements /createRoom func CreateRoom( - req *http.Request, device *authtypes.Device, + req *http.Request, device *api.Device, cfg *config.Dendrite, accountDB accounts.Database, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, @@ -149,7 +149,7 @@ func CreateRoom( // createRoom implements /createRoom // nolint: gocyclo func createRoom( - req *http.Request, device *authtypes.Device, + req *http.Request, device *api.Device, cfg *config.Dendrite, roomID string, accountDB accounts.Database, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, diff --git a/clientapi/routing/device.go b/clientapi/routing/device.go index 89c394913..02acb462e 100644 --- a/clientapi/routing/device.go +++ b/clientapi/routing/device.go @@ -19,9 +19,9 @@ import ( "encoding/json" "net/http" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -45,7 +45,7 @@ type devicesDeleteJSON struct { // GetDeviceByID handles /devices/{deviceID} func GetDeviceByID( - req *http.Request, deviceDB devices.Database, device *authtypes.Device, + req *http.Request, deviceDB devices.Database, device *api.Device, deviceID string, ) util.JSONResponse { localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) @@ -77,7 +77,7 @@ func GetDeviceByID( // GetDevicesByLocalpart handles /devices func GetDevicesByLocalpart( - req *http.Request, deviceDB devices.Database, device *authtypes.Device, + req *http.Request, deviceDB devices.Database, device *api.Device, ) util.JSONResponse { localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) if err != nil { @@ -110,7 +110,7 @@ func GetDevicesByLocalpart( // UpdateDeviceByID handles PUT on /devices/{deviceID} func UpdateDeviceByID( - req *http.Request, deviceDB devices.Database, device *authtypes.Device, + req *http.Request, deviceDB devices.Database, device *api.Device, deviceID string, ) util.JSONResponse { localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) @@ -160,7 +160,7 @@ func UpdateDeviceByID( // DeleteDeviceById handles DELETE requests to /devices/{deviceId} func DeleteDeviceById( - req *http.Request, deviceDB devices.Database, device *authtypes.Device, + req *http.Request, deviceDB devices.Database, device *api.Device, deviceID string, ) util.JSONResponse { localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) @@ -185,7 +185,7 @@ func DeleteDeviceById( // DeleteDevices handles POST requests to /delete_devices func DeleteDevices( - req *http.Request, deviceDB devices.Database, device *authtypes.Device, + req *http.Request, deviceDB devices.Database, device *api.Device, ) util.JSONResponse { localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) if err != nil { diff --git a/clientapi/routing/directory.go b/clientapi/routing/directory.go index 3d4b5f5bc..0dc4d5605 100644 --- a/clientapi/routing/directory.go +++ b/clientapi/routing/directory.go @@ -18,12 +18,12 @@ import ( "fmt" "net/http" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/internal/config" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -112,7 +112,7 @@ func DirectoryRoom( // TODO: Check if the user has the power level to set an alias func SetLocalAlias( req *http.Request, - device *authtypes.Device, + device *api.Device, alias string, cfg *config.Dendrite, aliasAPI roomserverAPI.RoomserverInternalAPI, @@ -188,7 +188,7 @@ func SetLocalAlias( // RemoveLocalAlias implements DELETE /directory/room/{roomAlias} func RemoveLocalAlias( req *http.Request, - device *authtypes.Device, + device *api.Device, alias string, aliasAPI roomserverAPI.RoomserverInternalAPI, ) util.JSONResponse { diff --git a/clientapi/routing/filter.go b/clientapi/routing/filter.go index 505e09279..7c583045f 100644 --- a/clientapi/routing/filter.go +++ b/clientapi/routing/filter.go @@ -17,17 +17,17 @@ package routing import ( "net/http" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) // GetFilter implements GET /_matrix/client/r0/user/{userId}/filter/{filterId} func GetFilter( - req *http.Request, device *authtypes.Device, accountDB accounts.Database, userID string, filterID string, + req *http.Request, device *api.Device, accountDB accounts.Database, userID string, filterID string, ) util.JSONResponse { if userID != device.UserID { return util.JSONResponse{ @@ -64,7 +64,7 @@ type filterResponse struct { //PutFilter implements POST /_matrix/client/r0/user/{userId}/filter func PutFilter( - req *http.Request, device *authtypes.Device, accountDB accounts.Database, userID string, + req *http.Request, device *api.Device, accountDB accounts.Database, userID string, ) util.JSONResponse { if userID != device.UserID { return util.JSONResponse{ diff --git a/clientapi/routing/getevent.go b/clientapi/routing/getevent.go index 16f36d661..2a51db730 100644 --- a/clientapi/routing/getevent.go +++ b/clientapi/routing/getevent.go @@ -17,17 +17,17 @@ package routing import ( "net/http" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/roomserver/api" + userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) type getEventRequest struct { req *http.Request - device *authtypes.Device + device *userapi.Device roomID string eventID string cfg *config.Dendrite @@ -39,7 +39,7 @@ type getEventRequest struct { // https://matrix.org/docs/spec/client_server/r0.4.0.html#get-matrix-client-r0-rooms-roomid-event-eventid func GetEvent( req *http.Request, - device *authtypes.Device, + device *userapi.Device, roomID string, eventID string, cfg *config.Dendrite, diff --git a/clientapi/routing/joinroom.go b/clientapi/routing/joinroom.go index a3d676532..a00b34a57 100644 --- a/clientapi/routing/joinroom.go +++ b/clientapi/routing/joinroom.go @@ -17,18 +17,18 @@ package routing import ( "net/http" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) func JoinRoomByIDOrAlias( req *http.Request, - device *authtypes.Device, + device *api.Device, rsAPI roomserverAPI.RoomserverInternalAPI, accountDB accounts.Database, roomIDOrAlias string, diff --git a/clientapi/routing/leaveroom.go b/clientapi/routing/leaveroom.go index bd7696181..38cef118e 100644 --- a/clientapi/routing/leaveroom.go +++ b/clientapi/routing/leaveroom.go @@ -17,15 +17,15 @@ package routing import ( "net/http" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/jsonerror" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/util" ) func LeaveRoomByID( req *http.Request, - device *authtypes.Device, + device *api.Device, rsAPI roomserverAPI.RoomserverInternalAPI, roomID string, ) util.JSONResponse { diff --git a/clientapi/routing/login.go b/clientapi/routing/login.go index c0042fa95..2eb480ef1 100644 --- a/clientapi/routing/login.go +++ b/clientapi/routing/login.go @@ -27,6 +27,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/userutil" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -157,7 +158,7 @@ func getDevice( deviceDB devices.Database, acc *authtypes.Account, token string, -) (dev *authtypes.Device, err error) { +) (dev *api.Device, err error) { dev, err = deviceDB.CreateDevice( ctx, acc.Localpart, r.DeviceID, token, r.InitialDisplayName, ) diff --git a/clientapi/routing/logout.go b/clientapi/routing/logout.go index 26b7f117e..f1276082b 100644 --- a/clientapi/routing/logout.go +++ b/clientapi/routing/logout.go @@ -17,16 +17,16 @@ package routing import ( "net/http" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) // Logout handles POST /logout func Logout( - req *http.Request, deviceDB devices.Database, device *authtypes.Device, + req *http.Request, deviceDB devices.Database, device *api.Device, ) util.JSONResponse { localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) if err != nil { @@ -47,7 +47,7 @@ func Logout( // LogoutAll handles POST /logout/all func LogoutAll( - req *http.Request, deviceDB devices.Database, device *authtypes.Device, + req *http.Request, deviceDB devices.Database, device *api.Device, ) util.JSONResponse { localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) if err != nil { diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index 53484a1a6..c7b91613f 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -30,6 +30,7 @@ import ( "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/roomserver/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -42,7 +43,7 @@ var errMissingUserID = errors.New("'user_id' must be supplied") // TODO: Can we improve the cyclo count here? Separate code paths for invites? // nolint:gocyclo func SendMembership( - req *http.Request, accountDB accounts.Database, device *authtypes.Device, + req *http.Request, accountDB accounts.Database, device *userapi.Device, roomID string, membership string, cfg *config.Dendrite, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, ) util.JSONResponse { @@ -149,7 +150,7 @@ func SendMembership( func buildMembershipEvent( ctx context.Context, body threepid.MembershipRequest, accountDB accounts.Database, - device *authtypes.Device, + device *userapi.Device, membership, roomID string, isDirect bool, cfg *config.Dendrite, evTime time.Time, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, @@ -223,7 +224,7 @@ func loadProfile( // In the latter case, if there was an issue retrieving the user ID from the request body, // returns a JSONResponse with a corresponding error code and message. func getMembershipStateKey( - body threepid.MembershipRequest, device *authtypes.Device, membership string, + body threepid.MembershipRequest, device *userapi.Device, membership string, ) (stateKey string, reason string, err error) { if membership == gomatrixserverlib.Ban || membership == "unban" || membership == "kick" || membership == gomatrixserverlib.Invite { // If we're in this case, the state key is contained in the request body, @@ -245,7 +246,7 @@ func getMembershipStateKey( func checkAndProcessThreepid( req *http.Request, - device *authtypes.Device, + device *userapi.Device, body *threepid.MembershipRequest, cfg *config.Dendrite, rsAPI roomserverAPI.RoomserverInternalAPI, diff --git a/clientapi/routing/memberships.go b/clientapi/routing/memberships.go index 095a85c08..a5bb2a908 100644 --- a/clientapi/routing/memberships.go +++ b/clientapi/routing/memberships.go @@ -19,10 +19,10 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/roomserver/api" + userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -37,7 +37,7 @@ type getJoinedRoomsResponse struct { // GetMemberships implements GET /rooms/{roomId}/members func GetMemberships( - req *http.Request, device *authtypes.Device, roomID string, joinedOnly bool, + req *http.Request, device *userapi.Device, roomID string, joinedOnly bool, _ *config.Dendrite, rsAPI api.RoomserverInternalAPI, ) util.JSONResponse { @@ -67,7 +67,7 @@ func GetMemberships( func GetJoinedRooms( req *http.Request, - device *authtypes.Device, + device *userapi.Device, accountsDB accounts.Database, ) util.JSONResponse { localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index a7a82ed58..a72ad3bcc 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -27,6 +27,7 @@ import ( "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/roomserver/api" + userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrix" @@ -92,7 +93,7 @@ func GetAvatarURL( // SetAvatarURL implements PUT /profile/{userID}/avatar_url // nolint:gocyclo func SetAvatarURL( - req *http.Request, accountDB accounts.Database, device *authtypes.Device, + req *http.Request, accountDB accounts.Database, device *userapi.Device, userID string, cfg *config.Dendrite, rsAPI api.RoomserverInternalAPI, ) util.JSONResponse { if userID != device.UserID { @@ -206,7 +207,7 @@ func GetDisplayName( // SetDisplayName implements PUT /profile/{userID}/displayname // nolint:gocyclo func SetDisplayName( - req *http.Request, accountDB accounts.Database, device *authtypes.Device, + req *http.Request, accountDB accounts.Database, device *userapi.Device, userID string, cfg *config.Dendrite, rsAPI api.RoomserverInternalAPI, ) util.JSONResponse { if userID != device.UserID { diff --git a/clientapi/routing/room_tagging.go b/clientapi/routing/room_tagging.go index 5c68668d0..a3fe0e426 100644 --- a/clientapi/routing/room_tagging.go +++ b/clientapi/routing/room_tagging.go @@ -20,11 +20,11 @@ import ( "github.com/sirupsen/logrus" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" + "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -41,7 +41,7 @@ func newTag() gomatrix.TagContent { func GetTags( req *http.Request, accountDB accounts.Database, - device *authtypes.Device, + device *api.Device, userID string, roomID string, syncProducer *producers.SyncAPIProducer, @@ -79,7 +79,7 @@ func GetTags( func PutTag( req *http.Request, accountDB accounts.Database, - device *authtypes.Device, + device *api.Device, userID string, roomID string, tag string, @@ -139,7 +139,7 @@ func PutTag( func DeleteTag( req *http.Request, accountDB accounts.Database, - device *authtypes.Device, + device *api.Device, userID string, roomID string, tag string, diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 82a80fff9..80d9ab668 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -21,8 +21,6 @@ import ( "github.com/gorilla/mux" appserviceAPI "github.com/matrix-org/dendrite/appservice/api" - "github.com/matrix-org/dendrite/clientapi/auth" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/clientapi/jsonerror" @@ -33,6 +31,7 @@ import ( "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/transactions" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -54,6 +53,7 @@ func Setup( asAPI appserviceAPI.AppServiceQueryAPI, accountDB accounts.Database, deviceDB devices.Database, + userAPI api.UserInternalAPI, federation *gomatrixserverlib.FederationClient, syncProducer *producers.SyncAPIProducer, transactionsCache *transactions.Cache, @@ -80,19 +80,13 @@ func Setup( v1mux := publicAPIMux.PathPrefix(pathPrefixV1).Subrouter() unstableMux := publicAPIMux.PathPrefix(pathPrefixUnstable).Subrouter() - authData := auth.Data{ - AccountDB: accountDB, - DeviceDB: deviceDB, - AppServices: cfg.Derived.ApplicationServices, - } - r0mux.Handle("/createRoom", - httputil.MakeAuthAPI("createRoom", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("createRoom", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { return CreateRoom(req, device, cfg, accountDB, rsAPI, asAPI) }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/join/{roomIDOrAlias}", - httputil.MakeAuthAPI(gomatrixserverlib.Join, authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -103,12 +97,12 @@ func Setup( }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/joined_rooms", - httputil.MakeAuthAPI("joined_rooms", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("joined_rooms", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { return GetJoinedRooms(req, device, accountDB) }), ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/leave", - httputil.MakeAuthAPI("membership", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -119,7 +113,7 @@ func Setup( }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/{membership:(?:join|kick|ban|unban|invite)}", - httputil.MakeAuthAPI("membership", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("membership", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -128,7 +122,7 @@ func Setup( }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/send/{eventType}", - httputil.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -137,7 +131,7 @@ func Setup( }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/send/{eventType}/{txnID}", - httputil.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -148,7 +142,7 @@ func Setup( }), ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/event/{eventID}", - httputil.MakeAuthAPI("rooms_get_event", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("rooms_get_event", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -157,7 +151,7 @@ func Setup( }), ).Methods(http.MethodGet, http.MethodOptions) - r0mux.Handle("/rooms/{roomID}/state", httputil.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + r0mux.Handle("/rooms/{roomID}/state", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -165,7 +159,7 @@ func Setup( return OnIncomingStateRequest(req.Context(), rsAPI, vars["roomID"]) })).Methods(http.MethodGet, http.MethodOptions) - r0mux.Handle("/rooms/{roomID}/state/{type}", httputil.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + r0mux.Handle("/rooms/{roomID}/state/{type}", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -173,7 +167,7 @@ func Setup( return OnIncomingStateTypeRequest(req.Context(), rsAPI, vars["roomID"], vars["type"], "") })).Methods(http.MethodGet, http.MethodOptions) - r0mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", httputil.MakeAuthAPI("room_state", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + r0mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -182,7 +176,7 @@ func Setup( })).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/state/{eventType:[^/]+/?}", - httputil.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -198,7 +192,7 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/state/{eventType}/{stateKey}", - httputil.MakeAuthAPI("send_message", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("send_message", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -231,7 +225,7 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/directory/room/{roomAlias}", - httputil.MakeAuthAPI("directory_room", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("directory_room", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -241,7 +235,7 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/directory/room/{roomAlias}", - httputil.MakeAuthAPI("directory_room", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("directory_room", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -251,19 +245,19 @@ func Setup( ).Methods(http.MethodDelete, http.MethodOptions) r0mux.Handle("/logout", - httputil.MakeAuthAPI("logout", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("logout", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { return Logout(req, deviceDB, device) }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/logout/all", - httputil.MakeAuthAPI("logout", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("logout", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { return LogoutAll(req, deviceDB, device) }), ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/typing/{userID}", - httputil.MakeAuthAPI("rooms_typing", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("rooms_typing", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -273,7 +267,7 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/sendToDevice/{eventType}/{txnID}", - httputil.MakeAuthAPI("send_to_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("send_to_device", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -287,7 +281,7 @@ func Setup( // rather than r0. It's an exact duplicate of the above handler. // TODO: Remove this if/when sytest is fixed! unstableMux.Handle("/sendToDevice/{eventType}/{txnID}", - httputil.MakeAuthAPI("send_to_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("send_to_device", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -298,7 +292,7 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/account/whoami", - httputil.MakeAuthAPI("whoami", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("whoami", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { return Whoami(req, device) }), ).Methods(http.MethodGet, http.MethodOptions) @@ -338,7 +332,7 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/user/{userId}/filter", - httputil.MakeAuthAPI("put_filter", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("put_filter", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -348,7 +342,7 @@ func Setup( ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/user/{userId}/filter/{filterId}", - httputil.MakeAuthAPI("get_filter", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("get_filter", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -380,7 +374,7 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/profile/{userID}/avatar_url", - httputil.MakeAuthAPI("profile_avatar_url", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("profile_avatar_url", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -402,7 +396,7 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/profile/{userID}/displayname", - httputil.MakeAuthAPI("profile_displayname", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("profile_displayname", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -414,19 +408,19 @@ func Setup( // PUT requests, so we need to allow this method r0mux.Handle("/account/3pid", - httputil.MakeAuthAPI("account_3pid", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { return GetAssociated3PIDs(req, accountDB, device) }), ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/account/3pid", - httputil.MakeAuthAPI("account_3pid", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { return CheckAndSave3PIDAssociation(req, accountDB, device, cfg) }), ).Methods(http.MethodPost, http.MethodOptions) unstableMux.Handle("/account/3pid/delete", - httputil.MakeAuthAPI("account_3pid", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("account_3pid", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { return Forget3PID(req, accountDB) }), ).Methods(http.MethodPost, http.MethodOptions) @@ -449,7 +443,7 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/voip/turnServer", - httputil.MakeAuthAPI("turn_server", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("turn_server", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { return RequestTurnServer(req, device, cfg) }), ).Methods(http.MethodGet, http.MethodOptions) @@ -475,7 +469,7 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/user/{userID}/account_data/{type}", - httputil.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -485,7 +479,7 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/user/{userID}/rooms/{roomID}/account_data/{type}", - httputil.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -495,7 +489,7 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/user/{userID}/account_data/{type}", - httputil.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -505,7 +499,7 @@ func Setup( ).Methods(http.MethodGet) r0mux.Handle("/user/{userID}/rooms/{roomID}/account_data/{type}", - httputil.MakeAuthAPI("user_account_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("user_account_data", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -515,7 +509,7 @@ func Setup( ).Methods(http.MethodGet) r0mux.Handle("/rooms/{roomID}/members", - httputil.MakeAuthAPI("rooms_members", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("rooms_members", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -525,7 +519,7 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/joined_members", - httputil.MakeAuthAPI("rooms_members", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("rooms_members", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -542,13 +536,13 @@ func Setup( ).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/devices", - httputil.MakeAuthAPI("get_devices", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("get_devices", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { return GetDevicesByLocalpart(req, deviceDB, device) }), ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/devices/{deviceID}", - httputil.MakeAuthAPI("get_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("get_device", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -558,7 +552,7 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/devices/{deviceID}", - httputil.MakeAuthAPI("device_data", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("device_data", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -568,7 +562,7 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/devices/{deviceID}", - httputil.MakeAuthAPI("delete_device", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("delete_device", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -578,7 +572,7 @@ func Setup( ).Methods(http.MethodDelete, http.MethodOptions) r0mux.Handle("/delete_devices", - httputil.MakeAuthAPI("delete_devices", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("delete_devices", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { return DeleteDevices(req, deviceDB, device) }), ).Methods(http.MethodPost, http.MethodOptions) @@ -603,7 +597,7 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/user/{userId}/rooms/{roomId}/tags", - httputil.MakeAuthAPI("get_tags", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("get_tags", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -613,7 +607,7 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/user/{userId}/rooms/{roomId}/tags/{tag}", - httputil.MakeAuthAPI("put_tag", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("put_tag", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -623,7 +617,7 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/user/{userId}/rooms/{roomId}/tags/{tag}", - httputil.MakeAuthAPI("delete_tag", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("delete_tag", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) @@ -633,7 +627,7 @@ func Setup( ).Methods(http.MethodDelete, http.MethodOptions) r0mux.Handle("/capabilities", - httputil.MakeAuthAPI("capabilities", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("capabilities", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { return GetCapabilities(req, rsAPI) }), ).Methods(http.MethodGet) diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go index 77a370778..d8936f750 100644 --- a/clientapi/routing/sendevent.go +++ b/clientapi/routing/sendevent.go @@ -17,13 +17,13 @@ package routing import ( "net/http" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/internal/transactions" "github.com/matrix-org/dendrite/roomserver/api" + userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" "github.com/sirupsen/logrus" @@ -41,7 +41,7 @@ type sendEventResponse struct { // /rooms/{roomID}/state/{eventType}/{stateKey} func SendEvent( req *http.Request, - device *authtypes.Device, + device *userapi.Device, roomID, eventType string, txnID, stateKey *string, cfg *config.Dendrite, rsAPI api.RoomserverInternalAPI, @@ -110,7 +110,7 @@ func SendEvent( func generateSendEvent( req *http.Request, - device *authtypes.Device, + device *userapi.Device, roomID, eventType string, stateKey *string, cfg *config.Dendrite, rsAPI api.RoomserverInternalAPI, diff --git a/clientapi/routing/sendtodevice.go b/clientapi/routing/sendtodevice.go index dc0a6572b..768e8e0e7 100644 --- a/clientapi/routing/sendtodevice.go +++ b/clientapi/routing/sendtodevice.go @@ -16,18 +16,18 @@ import ( "encoding/json" "net/http" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/internal/transactions" + userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/util" ) // SendToDevice handles PUT /_matrix/client/r0/sendToDevice/{eventType}/{txnId} // sends the device events to the EDU Server func SendToDevice( - req *http.Request, device *authtypes.Device, + req *http.Request, device *userapi.Device, eduAPI api.EDUServerInputAPI, txnCache *transactions.Cache, eventType string, txnID *string, diff --git a/clientapi/routing/sendtyping.go b/clientapi/routing/sendtyping.go index 2eae16582..213e7fdcf 100644 --- a/clientapi/routing/sendtyping.go +++ b/clientapi/routing/sendtyping.go @@ -16,12 +16,12 @@ import ( "database/sql" "net/http" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/userutil" "github.com/matrix-org/dendrite/eduserver/api" + userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/util" ) @@ -33,7 +33,7 @@ type typingContentJSON struct { // SendTyping handles PUT /rooms/{roomID}/typing/{userID} // sends the typing events to client API typingProducer func SendTyping( - req *http.Request, device *authtypes.Device, roomID string, + req *http.Request, device *userapi.Device, roomID string, userID string, accountDB accounts.Database, eduAPI api.EDUServerInputAPI, ) util.JSONResponse { diff --git a/clientapi/routing/threepid.go b/clientapi/routing/threepid.go index 49a9f3047..c712c1c37 100644 --- a/clientapi/routing/threepid.go +++ b/clientapi/routing/threepid.go @@ -23,6 +23,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/threepid" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -84,7 +85,7 @@ func RequestEmailToken(req *http.Request, accountDB accounts.Database, cfg *conf // CheckAndSave3PIDAssociation implements POST /account/3pid func CheckAndSave3PIDAssociation( - req *http.Request, accountDB accounts.Database, device *authtypes.Device, + req *http.Request, accountDB accounts.Database, device *api.Device, cfg *config.Dendrite, ) util.JSONResponse { var body threepid.EmailAssociationCheckRequest @@ -148,7 +149,7 @@ func CheckAndSave3PIDAssociation( // GetAssociated3PIDs implements GET /account/3pid func GetAssociated3PIDs( - req *http.Request, accountDB accounts.Database, device *authtypes.Device, + req *http.Request, accountDB accounts.Database, device *api.Device, ) util.JSONResponse { localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) if err != nil { diff --git a/clientapi/routing/voip.go b/clientapi/routing/voip.go index 212d9b0a4..046e87811 100644 --- a/clientapi/routing/voip.go +++ b/clientapi/routing/voip.go @@ -22,16 +22,16 @@ import ( "net/http" "time" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrix" "github.com/matrix-org/util" ) // RequestTurnServer implements: // GET /voip/turnServer -func RequestTurnServer(req *http.Request, device *authtypes.Device, cfg *config.Dendrite) util.JSONResponse { +func RequestTurnServer(req *http.Request, device *api.Device, cfg *config.Dendrite) util.JSONResponse { turnConfig := cfg.TURN // TODO Guest Support diff --git a/clientapi/routing/whoami.go b/clientapi/routing/whoami.go index 840bcb5f2..26280f6cc 100644 --- a/clientapi/routing/whoami.go +++ b/clientapi/routing/whoami.go @@ -15,7 +15,7 @@ package routing import ( "net/http" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/util" ) @@ -26,7 +26,7 @@ type whoamiResponse struct { // Whoami implements `/account/whoami` which enables client to query their account user id. // https://matrix.org/docs/spec/client_server/r0.3.0.html#get-matrix-client-r0-account-whoami -func Whoami(req *http.Request, device *authtypes.Device) util.JSONResponse { +func Whoami(req *http.Request, device *api.Device) util.JSONResponse { return util.JSONResponse{ Code: http.StatusOK, JSON: whoamiResponse{UserID: device.UserID}, diff --git a/clientapi/threepid/invites.go b/clientapi/threepid/invites.go index 11bf965d4..d9da5c503 100644 --- a/clientapi/threepid/invites.go +++ b/clientapi/threepid/invites.go @@ -29,6 +29,7 @@ import ( "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/roomserver/api" + userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" ) @@ -85,7 +86,7 @@ var ( // can be emitted. func CheckAndProcessInvite( ctx context.Context, - device *authtypes.Device, body *MembershipRequest, cfg *config.Dendrite, + device *userapi.Device, body *MembershipRequest, cfg *config.Dendrite, rsAPI api.RoomserverInternalAPI, db accounts.Database, membership string, roomID string, evTime time.Time, @@ -136,7 +137,7 @@ func CheckAndProcessInvite( // Returns an error if a check or a request failed. func queryIDServer( ctx context.Context, - db accounts.Database, cfg *config.Dendrite, device *authtypes.Device, + db accounts.Database, cfg *config.Dendrite, device *userapi.Device, body *MembershipRequest, roomID string, ) (lookupRes *idServerLookupResponse, storeInviteRes *idServerStoreInviteResponse, err error) { if err = isTrusted(body.IDServer, cfg); err != nil { @@ -205,7 +206,7 @@ func queryIDServerLookup(ctx context.Context, body *MembershipRequest) (*idServe // Returns an error if the request failed to send or if the response couldn't be parsed. func queryIDServerStoreInvite( ctx context.Context, - db accounts.Database, cfg *config.Dendrite, device *authtypes.Device, + db accounts.Database, cfg *config.Dendrite, device *userapi.Device, body *MembershipRequest, roomID string, ) (*idServerStoreInviteResponse, error) { // Retrieve the sender's profile to get their display name @@ -329,7 +330,7 @@ func checkIDServerSignatures( func emit3PIDInviteEvent( ctx context.Context, body *MembershipRequest, res *idServerStoreInviteResponse, - device *authtypes.Device, roomID string, cfg *config.Dendrite, + device *userapi.Device, roomID string, cfg *config.Dendrite, rsAPI api.RoomserverInternalAPI, evTime time.Time, ) error { diff --git a/cmd/dendrite-client-api-server/main.go b/cmd/dendrite-client-api-server/main.go index a28eb8b3b..fe5f30a0e 100644 --- a/cmd/dendrite-client-api-server/main.go +++ b/cmd/dendrite-client-api-server/main.go @@ -34,10 +34,11 @@ func main() { rsAPI := base.RoomserverHTTPClient() fsAPI := base.FederationSenderHTTPClient() eduInputAPI := base.EDUServerClient() + userAPI := base.UserAPIClient() clientapi.AddPublicRoutes( base.PublicAPIMux, base.Cfg, base.KafkaConsumer, base.KafkaProducer, deviceDB, accountDB, federation, - rsAPI, eduInputAPI, asQuery, transactions.New(), fsAPI, + rsAPI, eduInputAPI, asQuery, transactions.New(), fsAPI, userAPI, ) base.SetupAndServeHTTP(string(base.Cfg.Bind.ClientAPI), string(base.Cfg.Listen.ClientAPI)) diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index 0e757de97..51e1e2d5e 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -37,6 +37,7 @@ import ( "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/serverkeyapi" + "github.com/matrix-org/dendrite/userapi" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/dendrite/eduserver/cache" @@ -152,6 +153,7 @@ func main() { if err != nil { logrus.WithError(err).Panicf("failed to connect to public rooms db") } + userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, nil) monolith := setup.Monolith{ Config: base.Base.Cfg, @@ -167,6 +169,7 @@ func main() { FederationSenderAPI: fsAPI, RoomserverAPI: rsAPI, ServerKeyAPI: serverKeyAPI, + UserAPI: userAPI, PublicRoomsDB: publicRoomsDB, } diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index 6923b68b1..c6a7286e0 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -39,6 +39,7 @@ import ( "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/roomserver" + "github.com/matrix-org/dendrite/userapi" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" @@ -155,6 +156,7 @@ func main() { } embed.Embed(*instancePort, "Yggdrasil Demo") + userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, nil) monolith := setup.Monolith{ Config: base.Cfg, @@ -169,6 +171,7 @@ func main() { EDUInternalAPI: eduInputAPI, FederationSenderAPI: fsAPI, RoomserverAPI: rsAPI, + UserAPI: userAPI, //ServerKeyAPI: serverKeyAPI, PublicRoomsDB: publicRoomsDB, diff --git a/cmd/dendrite-key-server/main.go b/cmd/dendrite-key-server/main.go index 06629d39a..b557cbd9e 100644 --- a/cmd/dendrite-key-server/main.go +++ b/cmd/dendrite-key-server/main.go @@ -24,10 +24,9 @@ func main() { base := setup.NewBaseDendrite(cfg, "KeyServer", true) defer base.Close() // nolint: errcheck - accountDB := base.CreateAccountsDB() - deviceDB := base.CreateDeviceDB() + userAPI := base.UserAPIClient() - keyserver.AddPublicRoutes(base.PublicAPIMux, base.Cfg, deviceDB, accountDB) + keyserver.AddPublicRoutes(base.PublicAPIMux, base.Cfg, userAPI) base.SetupAndServeHTTP(string(base.Cfg.Bind.KeyServer), string(base.Cfg.Listen.KeyServer)) diff --git a/cmd/dendrite-media-api-server/main.go b/cmd/dendrite-media-api-server/main.go index 52c760273..8fd80d7b0 100644 --- a/cmd/dendrite-media-api-server/main.go +++ b/cmd/dendrite-media-api-server/main.go @@ -24,9 +24,9 @@ func main() { base := setup.NewBaseDendrite(cfg, "MediaAPI", true) defer base.Close() // nolint: errcheck - deviceDB := base.CreateDeviceDB() + userAPI := base.UserAPIClient() - mediaapi.AddPublicRoutes(base.PublicAPIMux, base.Cfg, deviceDB) + mediaapi.AddPublicRoutes(base.PublicAPIMux, base.Cfg, userAPI) base.SetupAndServeHTTP(string(base.Cfg.Bind.MediaAPI), string(base.Cfg.Listen.MediaAPI)) diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 3a0a84ef6..675474b82 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -30,6 +30,7 @@ import ( "github.com/matrix-org/dendrite/roomserver" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/serverkeyapi" + "github.com/matrix-org/dendrite/userapi" "github.com/sirupsen/logrus" ) @@ -119,6 +120,8 @@ func main() { logrus.WithError(err).Panicf("failed to connect to public rooms db") } + userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, cfg.Derived.ApplicationServices) + monolith := setup.Monolith{ Config: base.Cfg, AccountDB: accountDB, @@ -133,6 +136,7 @@ func main() { FederationSenderAPI: fsAPI, RoomserverAPI: rsAPI, ServerKeyAPI: serverKeyAPI, + UserAPI: userAPI, PublicRoomsDB: publicRoomsDB, } diff --git a/cmd/dendrite-public-rooms-api-server/main.go b/cmd/dendrite-public-rooms-api-server/main.go index 3ba45dc6d..23866b757 100644 --- a/cmd/dendrite-public-rooms-api-server/main.go +++ b/cmd/dendrite-public-rooms-api-server/main.go @@ -26,7 +26,7 @@ func main() { base := setup.NewBaseDendrite(cfg, "PublicRoomsAPI", true) defer base.Close() // nolint: errcheck - deviceDB := base.CreateDeviceDB() + userAPI := base.UserAPIClient() rsAPI := base.RoomserverHTTPClient() @@ -34,7 +34,7 @@ func main() { if err != nil { logrus.WithError(err).Panicf("failed to connect to public rooms db") } - publicroomsapi.AddPublicRoutes(base.PublicAPIMux, base.Cfg, base.KafkaConsumer, deviceDB, publicRoomsDB, rsAPI, nil, nil) + publicroomsapi.AddPublicRoutes(base.PublicAPIMux, base.Cfg, base.KafkaConsumer, userAPI, publicRoomsDB, rsAPI, nil, nil) base.SetupAndServeHTTP(string(base.Cfg.Bind.PublicRoomsAPI), string(base.Cfg.Listen.PublicRoomsAPI)) diff --git a/cmd/dendrite-sync-api-server/main.go b/cmd/dendrite-sync-api-server/main.go index 41e796802..a17b648d6 100644 --- a/cmd/dendrite-sync-api-server/main.go +++ b/cmd/dendrite-sync-api-server/main.go @@ -24,13 +24,13 @@ func main() { base := setup.NewBaseDendrite(cfg, "SyncAPI", true) defer base.Close() // nolint: errcheck - deviceDB := base.CreateDeviceDB() + userAPI := base.UserAPIClient() accountDB := base.CreateAccountsDB() federation := base.CreateFederationClient() rsAPI := base.RoomserverHTTPClient() - syncapi.AddPublicRoutes(base.PublicAPIMux, base.KafkaConsumer, deviceDB, accountDB, rsAPI, federation, cfg) + syncapi.AddPublicRoutes(base.PublicAPIMux, base.KafkaConsumer, userAPI, accountDB, rsAPI, federation, cfg) base.SetupAndServeHTTP(string(base.Cfg.Bind.SyncAPI), string(base.Cfg.Listen.SyncAPI)) diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 8c19eb6d5..7930c28dd 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -30,6 +30,7 @@ import ( "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/roomserver" + "github.com/matrix-org/dendrite/userapi" go_http_js_libp2p "github.com/matrix-org/go-http-js-libp2p" "github.com/matrix-org/gomatrixserverlib" @@ -211,6 +212,8 @@ func main() { logrus.WithError(err).Panicf("failed to connect to public rooms db") } + userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, nil) + monolith := setup.Monolith{ Config: base.Cfg, AccountDB: accountDB, @@ -224,6 +227,7 @@ func main() { EDUInternalAPI: eduInputAPI, FederationSenderAPI: fedSenderAPI, RoomserverAPI: rsAPI, + UserAPI: userAPI, //ServerKeyAPI: serverKeyAPI, PublicRoomsDB: publicRoomsDB, diff --git a/dendrite-config.yaml b/dendrite-config.yaml index a5b295977..52793cda4 100644 --- a/dendrite-config.yaml +++ b/dendrite-config.yaml @@ -140,6 +140,7 @@ listen: edu_server: "localhost:7778" key_server: "localhost:7779" server_key_api: "localhost:7780" + user_api: "localhost:7781" # The configuration for tracing the dendrite components. tracing: diff --git a/internal/config/config.go b/internal/config/config.go index bff4945be..2bd56ad9e 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -241,6 +241,7 @@ type Dendrite struct { ServerKeyAPI Address `yaml:"server_key_api"` AppServiceAPI Address `yaml:"appservice_api"` SyncAPI Address `yaml:"sync_api"` + UserAPI Address `yaml:"user_api"` RoomServer Address `yaml:"room_server"` FederationSender Address `yaml:"federation_sender"` PublicRoomsAPI Address `yaml:"public_rooms_api"` @@ -610,6 +611,7 @@ func (config *Dendrite) checkListen(configErrs *configErrors) { checkNotEmpty(configErrs, "listen.room_server", string(config.Listen.RoomServer)) checkNotEmpty(configErrs, "listen.edu_server", string(config.Listen.EDUServer)) checkNotEmpty(configErrs, "listen.server_key_api", string(config.Listen.EDUServer)) + checkNotEmpty(configErrs, "listen.user_api", string(config.Listen.UserAPI)) } // checkLogging verifies the parameters logging.* are valid. @@ -723,6 +725,15 @@ func (config *Dendrite) RoomServerURL() string { return "http://" + string(config.Listen.RoomServer) } +// UserAPIURL returns an HTTP URL for where the userapi is listening. +func (config *Dendrite) UserAPIURL() string { + // Hard code the userapi to talk HTTP for now. + // If we support HTTPS we need to think of a practical way to do certificate validation. + // People setting up servers shouldn't need to get a certificate valid for the public + // internet for an internal API. + return "http://" + string(config.Listen.UserAPI) +} + // EDUServerURL returns an HTTP URL for where the EDU server is listening. func (config *Dendrite) EDUServerURL() string { // Hard code the EDU server to talk HTTP for now. diff --git a/internal/config/config_test.go b/internal/config/config_test.go index b72f5fad0..9a543e763 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -63,6 +63,7 @@ listen: media_api: "localhost:7774" appservice_api: "localhost:7777" edu_server: "localhost:7778" + user_api: "localhost:7779" logging: - type: "file" level: "info" diff --git a/internal/httputil/httpapi.go b/internal/httputil/httpapi.go index a35a10d68..d371d1728 100644 --- a/internal/httputil/httpapi.go +++ b/internal/httputil/httpapi.go @@ -27,9 +27,9 @@ import ( "github.com/gorilla/mux" "github.com/matrix-org/dendrite/clientapi/auth" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" federationsenderAPI "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/internal/config" + userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" opentracing "github.com/opentracing/opentracing-go" @@ -48,11 +48,11 @@ type BasicAuth struct { // MakeAuthAPI turns a util.JSONRequestHandler function into an http.Handler which authenticates the request. func MakeAuthAPI( - metricsName string, data auth.Data, - f func(*http.Request, *authtypes.Device) util.JSONResponse, + metricsName string, userAPI userapi.UserInternalAPI, + f func(*http.Request, *userapi.Device) util.JSONResponse, ) http.Handler { h := func(req *http.Request) util.JSONResponse { - device, err := auth.VerifyUserFromRequest(req, data) + device, err := auth.VerifyUserFromRequest(req, userAPI) if err != nil { return *err } diff --git a/internal/setup/base.go b/internal/setup/base.go index 59bdfd2e4..e287cfbd0 100644 --- a/internal/setup/base.go +++ b/internal/setup/base.go @@ -46,6 +46,8 @@ import ( rsinthttp "github.com/matrix-org/dendrite/roomserver/inthttp" serverKeyAPI "github.com/matrix-org/dendrite/serverkeyapi/api" skinthttp "github.com/matrix-org/dendrite/serverkeyapi/inthttp" + userapi "github.com/matrix-org/dendrite/userapi/api" + userapiinthttp "github.com/matrix-org/dendrite/userapi/inthttp" "github.com/sirupsen/logrus" _ "net/http/pprof" @@ -160,6 +162,15 @@ func (b *BaseDendrite) RoomserverHTTPClient() roomserverAPI.RoomserverInternalAP return rsAPI } +// UserAPIClient returns UserInternalAPI for hitting the userapi over HTTP. +func (b *BaseDendrite) UserAPIClient() userapi.UserInternalAPI { + userAPI, err := userapiinthttp.NewUserAPIClient(b.Cfg.UserAPIURL(), b.httpClient) + if err != nil { + logrus.WithError(err).Panic("UserAPIClient failed", b.httpClient) + } + return userAPI +} + // EDUServerClient returns EDUServerInputAPI for hitting the EDU server over HTTP func (b *BaseDendrite) EDUServerClient() eduServerAPI.EDUServerInputAPI { e, err := eduinthttp.NewEDUServerClient(b.Cfg.EDUServerURL(), b.httpClient) diff --git a/internal/setup/monolith.go b/internal/setup/monolith.go index 4dfbf7115..f28fea8f3 100644 --- a/internal/setup/monolith.go +++ b/internal/setup/monolith.go @@ -34,6 +34,7 @@ import ( roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" serverKeyAPI "github.com/matrix-org/dendrite/serverkeyapi/api" "github.com/matrix-org/dendrite/syncapi" + userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" ) @@ -53,6 +54,7 @@ type Monolith struct { FederationSenderAPI federationSenderAPI.FederationSenderInternalAPI RoomserverAPI roomserverAPI.RoomserverInternalAPI ServerKeyAPI serverKeyAPI.ServerKeyInternalAPI + UserAPI userapi.UserInternalAPI // TODO: can we remove this? It's weird that we are required the database // yet every other component can do that on its own. libp2p-demo uses a custom @@ -69,21 +71,21 @@ func (m *Monolith) AddAllPublicRoutes(publicMux *mux.Router) { publicMux, m.Config, m.KafkaConsumer, m.KafkaProducer, m.DeviceDB, m.AccountDB, m.FedClient, m.RoomserverAPI, m.EDUInternalAPI, m.AppserviceAPI, transactions.New(), - m.FederationSenderAPI, + m.FederationSenderAPI, m.UserAPI, ) - keyserver.AddPublicRoutes(publicMux, m.Config, m.DeviceDB, m.AccountDB) + keyserver.AddPublicRoutes(publicMux, m.Config, m.UserAPI) federationapi.AddPublicRoutes( publicMux, m.Config, m.AccountDB, m.DeviceDB, m.FedClient, m.KeyRing, m.RoomserverAPI, m.AppserviceAPI, m.FederationSenderAPI, m.EDUInternalAPI, ) - mediaapi.AddPublicRoutes(publicMux, m.Config, m.DeviceDB) + mediaapi.AddPublicRoutes(publicMux, m.Config, m.UserAPI) publicroomsapi.AddPublicRoutes( - publicMux, m.Config, m.KafkaConsumer, m.DeviceDB, m.PublicRoomsDB, m.RoomserverAPI, m.FedClient, + publicMux, m.Config, m.KafkaConsumer, m.UserAPI, m.PublicRoomsDB, m.RoomserverAPI, m.FedClient, m.ExtPublicRoomsProvider, ) syncapi.AddPublicRoutes( - publicMux, m.KafkaConsumer, m.DeviceDB, m.AccountDB, m.RoomserverAPI, m.FedClient, m.Config, + publicMux, m.KafkaConsumer, m.UserAPI, m.AccountDB, m.RoomserverAPI, m.FedClient, m.Config, ) } diff --git a/keyserver/keyserver.go b/keyserver/keyserver.go index 1eb730541..bedc4dfb8 100644 --- a/keyserver/keyserver.go +++ b/keyserver/keyserver.go @@ -16,17 +16,14 @@ package keyserver import ( "github.com/gorilla/mux" - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" - "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/keyserver/routing" + userapi "github.com/matrix-org/dendrite/userapi/api" ) // AddPublicRoutes registers HTTP handlers for CS API calls func AddPublicRoutes( - router *mux.Router, cfg *config.Dendrite, - deviceDB devices.Database, - accountsDB accounts.Database, + router *mux.Router, cfg *config.Dendrite, userAPI userapi.UserInternalAPI, ) { - routing.Setup(router, cfg, accountsDB, deviceDB) + routing.Setup(router, cfg, userAPI) } diff --git a/keyserver/routing/routing.go b/keyserver/routing/routing.go index bce3c46ba..c09031d8a 100644 --- a/keyserver/routing/routing.go +++ b/keyserver/routing/routing.go @@ -18,12 +18,9 @@ import ( "net/http" "github.com/gorilla/mux" - "github.com/matrix-org/dendrite/clientapi/auth" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" - "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/httputil" + userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/util" ) @@ -36,20 +33,11 @@ const pathPrefixR0 = "/client/r0" // applied: // nolint: gocyclo func Setup( - publicAPIMux *mux.Router, cfg *config.Dendrite, - accountDB accounts.Database, - deviceDB devices.Database, + publicAPIMux *mux.Router, cfg *config.Dendrite, userAPI userapi.UserInternalAPI, ) { r0mux := publicAPIMux.PathPrefix(pathPrefixR0).Subrouter() - - authData := auth.Data{ - AccountDB: accountDB, - DeviceDB: deviceDB, - AppServices: cfg.Derived.ApplicationServices, - } - r0mux.Handle("/keys/query", - httputil.MakeAuthAPI("queryKeys", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("queryKeys", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { return QueryKeys(req) }), ).Methods(http.MethodPost, http.MethodOptions) diff --git a/mediaapi/mediaapi.go b/mediaapi/mediaapi.go index d4e260ea4..9219ba20a 100644 --- a/mediaapi/mediaapi.go +++ b/mediaapi/mediaapi.go @@ -16,18 +16,17 @@ package mediaapi import ( "github.com/gorilla/mux" - "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/mediaapi/routing" "github.com/matrix-org/dendrite/mediaapi/storage" + userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" ) // AddPublicRoutes sets up and registers HTTP handlers for the MediaAPI component. func AddPublicRoutes( - router *mux.Router, cfg *config.Dendrite, - deviceDB devices.Database, + router *mux.Router, cfg *config.Dendrite, userAPI userapi.UserInternalAPI, ) { mediaDB, err := storage.Open(string(cfg.Database.MediaAPI), cfg.DbProperties()) if err != nil { @@ -35,6 +34,6 @@ func AddPublicRoutes( } routing.Setup( - router, cfg, mediaDB, deviceDB, gomatrixserverlib.NewClient(), + router, cfg, mediaDB, userAPI, gomatrixserverlib.NewClient(), ) } diff --git a/mediaapi/routing/routing.go b/mediaapi/routing/routing.go index 6bcd3f552..71804606d 100644 --- a/mediaapi/routing/routing.go +++ b/mediaapi/routing/routing.go @@ -17,11 +17,9 @@ package routing import ( "net/http" - "github.com/matrix-org/dendrite/clientapi/auth" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/gorilla/mux" - "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/mediaapi/storage" @@ -44,7 +42,7 @@ func Setup( publicAPIMux *mux.Router, cfg *config.Dendrite, db storage.Database, - deviceDB devices.Database, + userAPI userapi.UserInternalAPI, client *gomatrixserverlib.Client, ) { r0mux := publicAPIMux.PathPrefix(pathPrefixR0).Subrouter() @@ -52,16 +50,9 @@ func Setup( activeThumbnailGeneration := &types.ActiveThumbnailGeneration{ PathToResult: map[string]*types.ThumbnailGenerationResult{}, } - authData := auth.Data{ - AccountDB: nil, - DeviceDB: deviceDB, - AppServices: nil, - } - - // TODO: Add AS support r0mux.Handle("/upload", httputil.MakeAuthAPI( - "upload", authData, - func(req *http.Request, _ *authtypes.Device) util.JSONResponse { + "upload", userAPI, + func(req *http.Request, _ *userapi.Device) util.JSONResponse { return Upload(req, cfg, db, activeThumbnailGeneration) }, )).Methods(http.MethodPost, http.MethodOptions) diff --git a/publicroomsapi/directory/directory.go b/publicroomsapi/directory/directory.go index fe7a67932..8b68279aa 100644 --- a/publicroomsapi/directory/directory.go +++ b/publicroomsapi/directory/directory.go @@ -17,8 +17,8 @@ package directory import ( "net/http" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/roomserver/api" + userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" @@ -59,7 +59,7 @@ func GetVisibility( // SetVisibility implements PUT /directory/list/room/{roomID} // TODO: Allow admin users to edit the room visibility func SetVisibility( - req *http.Request, publicRoomsDatabase storage.Database, rsAPI api.RoomserverInternalAPI, dev *authtypes.Device, + req *http.Request, publicRoomsDatabase storage.Database, rsAPI api.RoomserverInternalAPI, dev *userapi.Device, roomID string, ) util.JSONResponse { queryMembershipReq := api.QueryMembershipForUserRequest{ diff --git a/publicroomsapi/publicroomsapi.go b/publicroomsapi/publicroomsapi.go index 1f98a4e05..b9baa1056 100644 --- a/publicroomsapi/publicroomsapi.go +++ b/publicroomsapi/publicroomsapi.go @@ -17,13 +17,13 @@ package publicroomsapi import ( "github.com/Shopify/sarama" "github.com/gorilla/mux" - "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/publicroomsapi/consumers" "github.com/matrix-org/dendrite/publicroomsapi/routing" "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/publicroomsapi/types" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" ) @@ -34,7 +34,7 @@ func AddPublicRoutes( router *mux.Router, cfg *config.Dendrite, consumer sarama.Consumer, - deviceDB devices.Database, + userAPI userapi.UserInternalAPI, publicRoomsDB storage.Database, rsAPI roomserverAPI.RoomserverInternalAPI, fedClient *gomatrixserverlib.FederationClient, @@ -47,5 +47,5 @@ func AddPublicRoutes( logrus.WithError(err).Panic("failed to start public rooms server consumer") } - routing.Setup(router, deviceDB, publicRoomsDB, rsAPI, fedClient, extRoomsProvider) + routing.Setup(router, userAPI, publicRoomsDB, rsAPI, fedClient, extRoomsProvider) } diff --git a/publicroomsapi/routing/routing.go b/publicroomsapi/routing/routing.go index 2da555f98..9c82d3508 100644 --- a/publicroomsapi/routing/routing.go +++ b/publicroomsapi/routing/routing.go @@ -19,11 +19,9 @@ import ( "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/roomserver/api" + userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/gorilla/mux" - "github.com/matrix-org/dendrite/clientapi/auth" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/publicroomsapi/directory" "github.com/matrix-org/dendrite/publicroomsapi/storage" "github.com/matrix-org/dendrite/publicroomsapi/types" @@ -39,17 +37,11 @@ const pathPrefixR0 = "/client/r0" // applied: // nolint: gocyclo func Setup( - publicAPIMux *mux.Router, deviceDB devices.Database, publicRoomsDB storage.Database, rsAPI api.RoomserverInternalAPI, + publicAPIMux *mux.Router, userAPI userapi.UserInternalAPI, publicRoomsDB storage.Database, rsAPI api.RoomserverInternalAPI, fedClient *gomatrixserverlib.FederationClient, extRoomsProvider types.ExternalPublicRoomsProvider, ) { r0mux := publicAPIMux.PathPrefix(pathPrefixR0).Subrouter() - authData := auth.Data{ - AccountDB: nil, - DeviceDB: deviceDB, - AppServices: nil, - } - r0mux.Handle("/directory/list/room/{roomID}", httputil.MakeExternalAPI("directory_list", func(req *http.Request) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) @@ -61,7 +53,7 @@ func Setup( ).Methods(http.MethodGet, http.MethodOptions) // TODO: Add AS support r0mux.Handle("/directory/list/room/{roomID}", - httputil.MakeAuthAPI("directory_list", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + httputil.MakeAuthAPI("directory_list", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) diff --git a/syncapi/routing/routing.go b/syncapi/routing/routing.go index 50b469177..5744de05a 100644 --- a/syncapi/routing/routing.go +++ b/syncapi/routing/routing.go @@ -18,14 +18,12 @@ import ( "net/http" "github.com/gorilla/mux" - "github.com/matrix-org/dendrite/clientapi/auth" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/sync" + userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -39,24 +37,18 @@ const pathPrefixR0 = "/client/r0" // nolint: gocyclo func Setup( publicAPIMux *mux.Router, srp *sync.RequestPool, syncDB storage.Database, - deviceDB devices.Database, federation *gomatrixserverlib.FederationClient, + userAPI userapi.UserInternalAPI, federation *gomatrixserverlib.FederationClient, rsAPI api.RoomserverInternalAPI, cfg *config.Dendrite, ) { r0mux := publicAPIMux.PathPrefix(pathPrefixR0).Subrouter() - authData := auth.Data{ - AccountDB: nil, - DeviceDB: deviceDB, - AppServices: nil, - } - // TODO: Add AS support for all handlers below. - r0mux.Handle("/sync", httputil.MakeAuthAPI("sync", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + r0mux.Handle("/sync", httputil.MakeAuthAPI("sync", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { return srp.OnIncomingSyncRequest(req, device) })).Methods(http.MethodGet, http.MethodOptions) - r0mux.Handle("/rooms/{roomID}/messages", httputil.MakeAuthAPI("room_messages", authData, func(req *http.Request, device *authtypes.Device) util.JSONResponse { + r0mux.Handle("/rooms/{roomID}/messages", httputil.MakeAuthAPI("room_messages", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index 566e5d589..7b3bd6785 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -18,11 +18,11 @@ import ( "context" "time" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/types" + userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" ) @@ -57,10 +57,10 @@ type Database interface { // from when the device sent the event via an API that included a transaction // ID. A response object must be provided for IncrementaSync to populate - it // will not create one. - IncrementalSync(ctx context.Context, res *types.Response, device authtypes.Device, fromPos, toPos types.StreamingToken, numRecentEventsPerRoom int, wantFullState bool) (*types.Response, error) + IncrementalSync(ctx context.Context, res *types.Response, device userapi.Device, fromPos, toPos types.StreamingToken, numRecentEventsPerRoom int, wantFullState bool) (*types.Response, error) // CompleteSync returns a complete /sync API response for the given user. A response object // must be provided for CompleteSync to populate - it will not create one. - CompleteSync(ctx context.Context, res *types.Response, device authtypes.Device, numRecentEventsPerRoom int) (*types.Response, error) + CompleteSync(ctx context.Context, res *types.Response, device userapi.Device, numRecentEventsPerRoom int) (*types.Response, error) // GetAccountDataInRange returns all account data for a given user inserted or // updated between two given positions // Returns a map following the format data[roomID] = []dataTypes @@ -103,7 +103,7 @@ type Database interface { // StreamEventsToEvents converts streamEvent to Event. If device is non-nil and // matches the streamevent.transactionID device then the transaction ID gets // added to the unsigned section of the output event. - StreamEventsToEvents(device *authtypes.Device, in []types.StreamEvent) []gomatrixserverlib.HeaderedEvent + StreamEventsToEvents(device *userapi.Device, in []types.StreamEvent) []gomatrixserverlib.HeaderedEvent // SyncStreamPosition returns the latest position in the sync stream. Returns 0 if there are no events yet. SyncStreamPosition(ctx context.Context) (types.StreamPosition, error) // AddSendToDevice increases the EDU position in the cache and returns the stream position. diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go index 21d8df375..74ae3eabd 100644 --- a/syncapi/storage/shared/syncserver.go +++ b/syncapi/storage/shared/syncserver.go @@ -21,7 +21,8 @@ import ( "fmt" "time" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" + userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/api" @@ -214,7 +215,7 @@ func (d *Database) UpsertAccountData( return } -func (d *Database) StreamEventsToEvents(device *authtypes.Device, in []types.StreamEvent) []gomatrixserverlib.HeaderedEvent { +func (d *Database) StreamEventsToEvents(device *userapi.Device, in []types.StreamEvent) []gomatrixserverlib.HeaderedEvent { out := make([]gomatrixserverlib.HeaderedEvent, len(in)) for i := 0; i < len(in); i++ { out[i] = in[i].HeaderedEvent @@ -442,7 +443,7 @@ func (d *Database) syncPositionTx( // IDs of all rooms the user joined are returned so EDU deltas can be added for them. func (d *Database) addPDUDeltaToResponse( ctx context.Context, - device authtypes.Device, + device userapi.Device, r types.Range, numRecentEventsPerRoom int, wantFullState bool, @@ -549,7 +550,7 @@ func (d *Database) addEDUDeltaToResponse( func (d *Database) IncrementalSync( ctx context.Context, res *types.Response, - device authtypes.Device, + device userapi.Device, fromPos, toPos types.StreamingToken, numRecentEventsPerRoom int, wantFullState bool, @@ -687,7 +688,7 @@ func (d *Database) getResponseWithPDUsForCompleteSync( func (d *Database) CompleteSync( ctx context.Context, res *types.Response, - device authtypes.Device, numRecentEventsPerRoom int, + device userapi.Device, numRecentEventsPerRoom int, ) (*types.Response, error) { toPos, joinedRoomIDs, err := d.getResponseWithPDUsForCompleteSync( ctx, res, device.UserID, numRecentEventsPerRoom, @@ -758,7 +759,7 @@ func (d *Database) getBackwardTopologyPos( // addRoomDeltaToResponse adds a room state delta to a sync response func (d *Database) addRoomDeltaToResponse( ctx context.Context, - device *authtypes.Device, + device *userapi.Device, txn *sql.Tx, r types.Range, delta stateDelta, @@ -904,7 +905,7 @@ func (d *Database) fetchMissingStateEvents( // the user has new membership events. // A list of joined room IDs is also returned in case the caller needs it. func (d *Database) getStateDeltas( - ctx context.Context, device *authtypes.Device, txn *sql.Tx, + ctx context.Context, device *userapi.Device, txn *sql.Tx, r types.Range, userID string, stateFilter *gomatrixserverlib.StateFilter, ) ([]stateDelta, []string, error) { @@ -979,7 +980,7 @@ func (d *Database) getStateDeltas( // Fetches full state for all joined rooms and uses selectStateInRange to get // updates for other rooms. func (d *Database) getStateDeltasForFullStateSync( - ctx context.Context, device *authtypes.Device, txn *sql.Tx, + ctx context.Context, device *userapi.Device, txn *sql.Tx, r types.Range, userID string, stateFilter *gomatrixserverlib.StateFilter, ) ([]stateDelta, []string, error) { diff --git a/syncapi/storage/storage_test.go b/syncapi/storage/storage_test.go index 4661ede4d..85084facb 100644 --- a/syncapi/storage/storage_test.go +++ b/syncapi/storage/storage_test.go @@ -8,10 +8,10 @@ import ( "testing" "time" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/storage/sqlite3" "github.com/matrix-org/dendrite/syncapi/types" + userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" ) @@ -22,7 +22,7 @@ var ( testRoomID = fmt.Sprintf("!hallownest:%s", testOrigin) testUserIDA = fmt.Sprintf("@hornet:%s", testOrigin) testUserIDB = fmt.Sprintf("@paleking:%s", testOrigin) - testUserDeviceA = authtypes.Device{ + testUserDeviceA = userapi.Device{ UserID: testUserIDA, ID: "device_id_A", DisplayName: "Device A", diff --git a/syncapi/sync/notifier_test.go b/syncapi/sync/notifier_test.go index 132315573..ecc4fcbfc 100644 --- a/syncapi/sync/notifier_test.go +++ b/syncapi/sync/notifier_test.go @@ -22,9 +22,8 @@ import ( "testing" "time" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/syncapi/types" + userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -357,7 +356,7 @@ func lockedFetchUserStream(n *Notifier, userID, deviceID string) *UserDeviceStre func newTestSyncRequest(userID, deviceID string, since types.StreamingToken) syncRequest { return syncRequest{ - device: authtypes.Device{ + device: userapi.Device{ UserID: userID, ID: deviceID, }, diff --git a/syncapi/sync/request.go b/syncapi/sync/request.go index c7796b561..beeaa40f7 100644 --- a/syncapi/sync/request.go +++ b/syncapi/sync/request.go @@ -21,9 +21,8 @@ import ( "strconv" "time" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/syncapi/types" + userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/util" log "github.com/sirupsen/logrus" ) @@ -42,7 +41,7 @@ type filter struct { // syncRequest represents a /sync request, with sensible defaults/sanity checks applied. type syncRequest struct { ctx context.Context - device authtypes.Device + device userapi.Device limit int timeout time.Duration since *types.StreamingToken // nil means that no since token was supplied @@ -50,7 +49,7 @@ type syncRequest struct { log *log.Entry } -func newSyncRequest(req *http.Request, device authtypes.Device) (*syncRequest, error) { +func newSyncRequest(req *http.Request, device userapi.Device) (*syncRequest, error) { timeout := getTimeout(req.URL.Query().Get("timeout")) fullState := req.URL.Query().Get("full_state") wantFullState := fullState != "" && fullState != "false" diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index 8b93cad45..ec22a05f4 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -21,11 +21,11 @@ import ( "net/http" "time" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/types" + userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" log "github.com/sirupsen/logrus" @@ -46,7 +46,7 @@ func NewRequestPool(db storage.Database, n *Notifier, adb accounts.Database) *Re // OnIncomingSyncRequest is called when a client makes a /sync request. This function MUST be // called in a dedicated goroutine for this request. This function will block the goroutine // until a response is ready, or it times out. -func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *authtypes.Device) util.JSONResponse { +func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi.Device) util.JSONResponse { var syncData *types.Response // Extract values from request diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go index 40e652af4..2351ee4d8 100644 --- a/syncapi/syncapi.go +++ b/syncapi/syncapi.go @@ -24,9 +24,9 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/roomserver/api" + userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/syncapi/consumers" "github.com/matrix-org/dendrite/syncapi/routing" "github.com/matrix-org/dendrite/syncapi/storage" @@ -38,7 +38,7 @@ import ( func AddPublicRoutes( router *mux.Router, consumer sarama.Consumer, - deviceDB devices.Database, + userAPI userapi.UserInternalAPI, accountsDB accounts.Database, rsAPI api.RoomserverInternalAPI, federation *gomatrixserverlib.FederationClient, @@ -90,5 +90,5 @@ func AddPublicRoutes( logrus.WithError(err).Panicf("failed to start send-to-device consumer") } - routing.Setup(router, requestPool, syncDB, deviceDB, federation, rsAPI, cfg) + routing.Setup(router, requestPool, syncDB, userAPI, federation, rsAPI, cfg) } diff --git a/userapi/api/api.go b/userapi/api/api.go index 8534fb17e..57b5165a4 100644 --- a/userapi/api/api.go +++ b/userapi/api/api.go @@ -19,6 +19,21 @@ import "context" // UserInternalAPI is the internal API for information about users and devices. type UserInternalAPI interface { QueryProfile(ctx context.Context, req *QueryProfileRequest, res *QueryProfileResponse) error + QueryAccessToken(ctx context.Context, req *QueryAccessTokenRequest, res *QueryAccessTokenResponse) error +} + +// QueryAccessTokenRequest is the request for QueryAccessToken +type QueryAccessTokenRequest struct { + AccessToken string + // optional user ID, valid only if the token is an appservice. + // https://matrix.org/docs/spec/application_service/r0.1.2#using-sync-and-events + AppServiceUserID string +} + +// QueryAccessTokenResponse is the response for QueryAccessToken +type QueryAccessTokenResponse struct { + Device *Device + Err error // e.g ErrorForbidden } // QueryProfileRequest is the request for QueryProfile @@ -29,10 +44,34 @@ type QueryProfileRequest struct { // QueryProfileResponse is the response for QueryProfile type QueryProfileResponse struct { - // True if the user has been created. Querying for a profile does not create them. + // True if the user exists. Querying for a profile does not create them. UserExists bool // The current display name if set. DisplayName string // The current avatar URL if set. AvatarURL string } + +// Device represents a client's device (mobile, web, etc) +type Device struct { + ID string + UserID string + // The access_token granted to this device. + // This uniquely identifies the device from all other devices and clients. + AccessToken string + // The unique ID of the session identified by the access token. + // Can be used as a secure substitution in places where data needs to be + // associated with access tokens. + SessionID int64 + // TODO: display name, last used timestamp, keys, etc + DisplayName string +} + +// ErrorForbidden is an error indicating that the supplied access token is forbidden +type ErrorForbidden struct { + Message string +} + +func (e *ErrorForbidden) Error() string { + return "Forbidden: " + e.Message +} diff --git a/userapi/internal/api.go b/userapi/internal/api.go index 0144526c7..1f0d5c94b 100644 --- a/userapi/internal/api.go +++ b/userapi/internal/api.go @@ -19,8 +19,11 @@ import ( "database/sql" "fmt" + "github.com/matrix-org/dendrite/appservice/types" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" + "github.com/matrix-org/dendrite/clientapi/userutil" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" ) @@ -29,6 +32,8 @@ type UserInternalAPI struct { AccountDB accounts.Database DeviceDB devices.Database ServerName gomatrixserverlib.ServerName + // AppServices is the list of all registered AS + AppServices []config.ApplicationService } func (a *UserInternalAPI) QueryProfile(ctx context.Context, req *api.QueryProfileRequest, res *api.QueryProfileResponse) error { @@ -51,3 +56,66 @@ func (a *UserInternalAPI) QueryProfile(ctx context.Context, req *api.QueryProfil res.DisplayName = prof.DisplayName return nil } + +func (a *UserInternalAPI) QueryAccessToken(ctx context.Context, req *api.QueryAccessTokenRequest, res *api.QueryAccessTokenResponse) error { + if req.AppServiceUserID != "" { + appServiceDevice, err := a.queryAppServiceToken(ctx, req.AccessToken, req.AppServiceUserID) + res.Device = appServiceDevice + res.Err = err + return nil + } + device, err := a.DeviceDB.GetDeviceByAccessToken(ctx, req.AccessToken) + if err != nil { + if err == sql.ErrNoRows { + return nil + } + return err + } + res.Device = device + return nil +} + +// Return the appservice 'device' or nil if the token is not an appservice. Returns an error if there was a problem +// creating a 'device'. +func (a *UserInternalAPI) queryAppServiceToken(ctx context.Context, token, appServiceUserID string) (*api.Device, error) { + // Search for app service with given access_token + var appService *config.ApplicationService + for _, as := range a.AppServices { + if as.ASToken == token { + appService = &as + break + } + } + if appService == nil { + return nil, nil + } + + // Create a dummy device for AS user + dev := api.Device{ + // Use AS dummy device ID + ID: types.AppServiceDeviceID, + // AS dummy device has AS's token. + AccessToken: token, + } + + localpart, err := userutil.ParseUsernameParam(appServiceUserID, &a.ServerName) + if err != nil { + return nil, err + } + + if localpart != "" { // AS is masquerading as another user + // Verify that the user is registered + account, err := a.AccountDB.GetAccountByLocalpart(ctx, localpart) + // Verify that account exists & appServiceID matches + if err == nil && account.AppServiceID == appService.ID { + // Set the userID of dummy device + dev.UserID = appServiceUserID + return &dev, nil + } + return nil, &api.ErrorForbidden{Message: "appservice has not registered this user"} + } + + // AS is not masquerading as any user, so use AS's sender_localpart + dev.UserID = appService.SenderLocalpart + return &dev, nil +} diff --git a/userapi/inthttp/client.go b/userapi/inthttp/client.go index 90cc54a48..022243faa 100644 --- a/userapi/inthttp/client.go +++ b/userapi/inthttp/client.go @@ -26,7 +26,8 @@ import ( // HTTP paths for the internal HTTP APIs const ( - QueryProfilePath = "/userapi/queryProfile" + QueryProfilePath = "/userapi/queryProfile" + QueryAccessTokenPath = "/userapi/queryAccessToken" ) // NewUserAPIClient creates a UserInternalAPI implemented by talking to a HTTP POST API. @@ -60,3 +61,15 @@ func (h *httpUserInternalAPI) QueryProfile( apiURL := h.apiURL + QueryProfilePath return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } + +func (h *httpUserInternalAPI) QueryAccessToken( + ctx context.Context, + request *api.QueryAccessTokenRequest, + response *api.QueryAccessTokenResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "QueryAccessToken") + defer span.Finish() + + apiURL := h.apiURL + QueryAccessTokenPath + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} diff --git a/userapi/inthttp/server.go b/userapi/inthttp/server.go index f3c17ccd4..495b161c7 100644 --- a/userapi/inthttp/server.go +++ b/userapi/inthttp/server.go @@ -38,4 +38,17 @@ func AddRoutes(internalAPIMux *mux.Router, s api.UserInternalAPI) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) + internalAPIMux.Handle(QueryAccessTokenPath, + httputil.MakeInternalAPI("queryAccessToken", func(req *http.Request) util.JSONResponse { + request := api.QueryAccessTokenRequest{} + response := api.QueryAccessTokenResponse{} + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := s.QueryAccessToken(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) } diff --git a/userapi/userapi.go b/userapi/userapi.go index 32f851cc7..961d04fe0 100644 --- a/userapi/userapi.go +++ b/userapi/userapi.go @@ -18,6 +18,7 @@ import ( "github.com/gorilla/mux" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" + "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/dendrite/userapi/internal" "github.com/matrix-org/dendrite/userapi/inthttp" @@ -32,10 +33,13 @@ func AddInternalRoutes(router *mux.Router, intAPI api.UserInternalAPI) { // NewInternalAPI returns a concerete implementation of the internal API. Callers // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. -func NewInternalAPI(accountDB accounts.Database, deviceDB devices.Database, serverName gomatrixserverlib.ServerName) api.UserInternalAPI { +func NewInternalAPI(accountDB accounts.Database, deviceDB devices.Database, + serverName gomatrixserverlib.ServerName, appServices []config.ApplicationService) api.UserInternalAPI { + return &internal.UserInternalAPI{ - AccountDB: accountDB, - DeviceDB: deviceDB, - ServerName: serverName, + AccountDB: accountDB, + DeviceDB: deviceDB, + ServerName: serverName, + AppServices: appServices, } } diff --git a/userapi/userapi_test.go b/userapi/userapi_test.go index a46163637..509bdd7e8 100644 --- a/userapi/userapi_test.go +++ b/userapi/userapi_test.go @@ -32,7 +32,7 @@ func MustMakeInternalAPI(t *testing.T) (api.UserInternalAPI, accounts.Database, t.Fatalf("failed to create device DB: %s", err) } - return userapi.NewInternalAPI(accountDB, deviceDB, serverName), accountDB, deviceDB + return userapi.NewInternalAPI(accountDB, deviceDB, serverName, nil), accountDB, deviceDB } func TestQueryProfile(t *testing.T) { From fc0e74ae0f02b7bb9d71d739660deef824ddbd33 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 16 Jun 2020 14:29:11 +0100 Subject: [PATCH 167/200] Fix media API for demos and possibly Synapse (#1134) * Fix media API for demos and possibly Synapse * User API * goimports --- cmd/dendrite-demo-libp2p/main.go | 13 ++++++++ cmd/dendrite-demo-yggdrasil/main.go | 34 ++++++++++----------- cmd/dendrite-demo-yggdrasil/yggconn/node.go | 18 +++++++++++ cmd/dendrite-media-api-server/main.go | 4 ++- cmd/dendrite-monolith-server/main.go | 2 ++ cmd/dendritejs/main.go | 6 ++++ internal/setup/monolith.go | 3 +- mediaapi/mediaapi.go | 6 ++-- mediaapi/routing/routing.go | 10 ++++-- 9 files changed, 71 insertions(+), 25 deletions(-) diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index 51e1e2d5e..6fb3003c2 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -80,6 +80,17 @@ func createFederationClient( ) } +func createClient( + base *P2PDendrite, +) *gomatrixserverlib.Client { + tr := &http.Transport{} + tr.RegisterProtocol( + "matrix", + p2phttp.NewTransport(base.LibP2P, p2phttp.ProtocolOption("/matrix")), + ) + return gomatrixserverlib.NewClientWithTransport(tr) +} + func main() { instanceName := flag.String("name", "dendrite-p2p", "the name of this P2P demo instance") instancePort := flag.Int("port", 8080, "the port that the client API will listen on") @@ -102,6 +113,7 @@ func main() { } cfg := config.Dendrite{} + cfg.SetDefaults() cfg.Matrix.ServerName = "p2p" cfg.Matrix.PrivateKey = privKey cfg.Matrix.KeyID = gomatrixserverlib.KeyID(fmt.Sprintf("ed25519:%s", *instanceName)) @@ -159,6 +171,7 @@ func main() { Config: base.Base.Cfg, AccountDB: accountDB, DeviceDB: deviceDB, + Client: createClient(base), FedClient: federation, KeyRing: keyRing, KafkaConsumer: base.Base.KafkaConsumer, diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index c6a7286e0..87e2246f2 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -16,18 +16,14 @@ package main import ( "context" - "crypto/ed25519" "crypto/tls" - "encoding/hex" "flag" "fmt" "net" "net/http" - "strings" "time" "github.com/matrix-org/dendrite/appservice" - "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/convert" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/embed" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/yggconn" @@ -63,26 +59,13 @@ func (y *yggroundtripper) RoundTrip(req *http.Request) (*http.Response, error) { func createFederationClient( base *setup.BaseDendrite, n *yggconn.Node, ) *gomatrixserverlib.FederationClient { - yggdialer := func(_, address string) (net.Conn, error) { - tokens := strings.Split(address, ":") - raw, err := hex.DecodeString(tokens[0]) - if err != nil { - return nil, fmt.Errorf("hex.DecodeString: %w", err) - } - converted := convert.Ed25519PublicKeyToCurve25519(ed25519.PublicKey(raw)) - convhex := hex.EncodeToString(converted) - return n.Dial("curve25519", convhex) - } - yggdialerctx := func(ctx context.Context, network, address string) (net.Conn, error) { - return yggdialer(network, address) - } tr := &http.Transport{} tr.RegisterProtocol( "matrix", &yggroundtripper{ inner: &http.Transport{ ResponseHeaderTimeout: 15 * time.Second, IdleConnTimeout: 60 * time.Second, - DialContext: yggdialerctx, + DialContext: n.DialerContext, }, }, ) @@ -91,6 +74,20 @@ func createFederationClient( ) } +func createClient(n *yggconn.Node) *gomatrixserverlib.Client { + tr := &http.Transport{} + tr.RegisterProtocol( + "matrix", &yggroundtripper{ + inner: &http.Transport{ + ResponseHeaderTimeout: 15 * time.Second, + IdleConnTimeout: 60 * time.Second, + DialContext: n.DialerContext, + }, + }, + ) + return gomatrixserverlib.NewClientWithTransport(tr) +} + // nolint:gocyclo func main() { flag.Parse() @@ -162,6 +159,7 @@ func main() { Config: base.Cfg, AccountDB: accountDB, DeviceDB: deviceDB, + Client: createClient(ygg), FedClient: federation, KeyRing: keyRing, KafkaConsumer: base.KafkaConsumer, diff --git a/cmd/dendrite-demo-yggdrasil/yggconn/node.go b/cmd/dendrite-demo-yggdrasil/yggconn/node.go index a625f8d8d..b225e1cf9 100644 --- a/cmd/dendrite-demo-yggdrasil/yggconn/node.go +++ b/cmd/dendrite-demo-yggdrasil/yggconn/node.go @@ -15,13 +15,16 @@ package yggconn import ( + "context" "crypto/ed25519" "encoding/hex" "encoding/json" "fmt" "io/ioutil" "log" + "net" "os" + "strings" "sync" "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/convert" @@ -48,6 +51,21 @@ type Node struct { incoming chan *yamux.Stream } +func (n *Node) Dialer(_, address string) (net.Conn, error) { + tokens := strings.Split(address, ":") + raw, err := hex.DecodeString(tokens[0]) + if err != nil { + return nil, fmt.Errorf("hex.DecodeString: %w", err) + } + converted := convert.Ed25519PublicKeyToCurve25519(ed25519.PublicKey(raw)) + convhex := hex.EncodeToString(converted) + return n.Dial("curve25519", convhex) +} + +func (n *Node) DialerContext(ctx context.Context, network, address string) (net.Conn, error) { + return n.Dialer(network, address) +} + // nolint:gocyclo func Setup(instanceName, instancePeer string) (*Node, error) { n := &Node{ diff --git a/cmd/dendrite-media-api-server/main.go b/cmd/dendrite-media-api-server/main.go index 8fd80d7b0..1582a33a8 100644 --- a/cmd/dendrite-media-api-server/main.go +++ b/cmd/dendrite-media-api-server/main.go @@ -17,6 +17,7 @@ package main import ( "github.com/matrix-org/dendrite/internal/setup" "github.com/matrix-org/dendrite/mediaapi" + "github.com/matrix-org/gomatrixserverlib" ) func main() { @@ -25,8 +26,9 @@ func main() { defer base.Close() // nolint: errcheck userAPI := base.UserAPIClient() + client := gomatrixserverlib.NewClient() - mediaapi.AddPublicRoutes(base.PublicAPIMux, base.Cfg, userAPI) + mediaapi.AddPublicRoutes(base.PublicAPIMux, base.Cfg, userAPI, client) base.SetupAndServeHTTP(string(base.Cfg.Bind.MediaAPI), string(base.Cfg.Listen.MediaAPI)) diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 675474b82..16e274fc9 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -31,6 +31,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/serverkeyapi" "github.com/matrix-org/dendrite/userapi" + "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" ) @@ -126,6 +127,7 @@ func main() { Config: base.Cfg, AccountDB: accountDB, DeviceDB: deviceDB, + Client: gomatrixserverlib.NewClient(), FedClient: federation, KeyRing: keyRing, KafkaConsumer: base.KafkaConsumer, diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 7930c28dd..aa9192129 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -145,6 +145,11 @@ func createFederationClient(cfg *config.Dendrite, node *go_http_js_libp2p.P2pLoc return fed } +func createClient(node *go_http_js_libp2p.P2pLocalNode) *gomatrixserverlib.Client { + tr := go_http_js_libp2p.NewP2pTransport(node) + return gomatrixserverlib.NewClientWithTransport(tr) +} + func createP2PNode(privKey ed25519.PrivateKey) (serverName string, node *go_http_js_libp2p.P2pLocalNode) { hosted := "/dns4/rendezvous.matrix.org/tcp/8443/wss/p2p-websocket-star/" node = go_http_js_libp2p.NewP2pLocalNode("org.matrix.p2p.experiment", privKey.Seed(), []string{hosted}, "p2p") @@ -218,6 +223,7 @@ func main() { Config: base.Cfg, AccountDB: accountDB, DeviceDB: deviceDB, + Client: createClient(node), FedClient: federation, KeyRing: &keyRing, KafkaConsumer: base.KafkaConsumer, diff --git a/internal/setup/monolith.go b/internal/setup/monolith.go index f28fea8f3..b202aa71a 100644 --- a/internal/setup/monolith.go +++ b/internal/setup/monolith.go @@ -45,6 +45,7 @@ type Monolith struct { DeviceDB devices.Database AccountDB accounts.Database KeyRing *gomatrixserverlib.KeyRing + Client *gomatrixserverlib.Client FedClient *gomatrixserverlib.FederationClient KafkaConsumer sarama.Consumer KafkaProducer sarama.SyncProducer @@ -80,7 +81,7 @@ func (m *Monolith) AddAllPublicRoutes(publicMux *mux.Router) { m.KeyRing, m.RoomserverAPI, m.AppserviceAPI, m.FederationSenderAPI, m.EDUInternalAPI, ) - mediaapi.AddPublicRoutes(publicMux, m.Config, m.UserAPI) + mediaapi.AddPublicRoutes(publicMux, m.Config, m.UserAPI, m.Client) publicroomsapi.AddPublicRoutes( publicMux, m.Config, m.KafkaConsumer, m.UserAPI, m.PublicRoomsDB, m.RoomserverAPI, m.FedClient, m.ExtPublicRoomsProvider, diff --git a/mediaapi/mediaapi.go b/mediaapi/mediaapi.go index 9219ba20a..290ef46e1 100644 --- a/mediaapi/mediaapi.go +++ b/mediaapi/mediaapi.go @@ -26,7 +26,9 @@ import ( // AddPublicRoutes sets up and registers HTTP handlers for the MediaAPI component. func AddPublicRoutes( - router *mux.Router, cfg *config.Dendrite, userAPI userapi.UserInternalAPI, + router *mux.Router, cfg *config.Dendrite, + userAPI userapi.UserInternalAPI, + client *gomatrixserverlib.Client, ) { mediaDB, err := storage.Open(string(cfg.Database.MediaAPI), cfg.DbProperties()) if err != nil { @@ -34,6 +36,6 @@ func AddPublicRoutes( } routing.Setup( - router, cfg, mediaDB, userAPI, gomatrixserverlib.NewClient(), + router, cfg, mediaDB, userAPI, client, ) } diff --git a/mediaapi/routing/routing.go b/mediaapi/routing/routing.go index 71804606d..13f84c335 100644 --- a/mediaapi/routing/routing.go +++ b/mediaapi/routing/routing.go @@ -32,6 +32,7 @@ import ( ) const pathPrefixR0 = "/media/r0" +const pathPrefixV1 = "/media/v1" // TODO: remove when synapse is fixed // Setup registers the media API HTTP handlers // @@ -46,6 +47,7 @@ func Setup( client *gomatrixserverlib.Client, ) { r0mux := publicAPIMux.PathPrefix(pathPrefixR0).Subrouter() + v1mux := publicAPIMux.PathPrefix(pathPrefixV1).Subrouter() activeThumbnailGeneration := &types.ActiveThumbnailGeneration{ PathToResult: map[string]*types.ThumbnailGenerationResult{}, @@ -60,9 +62,11 @@ func Setup( activeRemoteRequests := &types.ActiveRemoteRequests{ MXCToResult: map[string]*types.RemoteRequestResult{}, } - r0mux.Handle("/download/{serverName}/{mediaId}", - makeDownloadAPI("download", cfg, db, client, activeRemoteRequests, activeThumbnailGeneration), - ).Methods(http.MethodGet, http.MethodOptions) + + downloadHandler := makeDownloadAPI("download", cfg, db, client, activeRemoteRequests, activeThumbnailGeneration) + r0mux.Handle("/download/{serverName}/{mediaId}", downloadHandler).Methods(http.MethodGet, http.MethodOptions) + v1mux.Handle("/download/{serverName}/{mediaId}", downloadHandler).Methods(http.MethodGet, http.MethodOptions) // TODO: remove when synapse is fixed + r0mux.Handle("/thumbnail/{serverName}/{mediaId}", makeDownloadAPI("thumbnail", cfg, db, client, activeRemoteRequests, activeThumbnailGeneration), ).Methods(http.MethodGet, http.MethodOptions) From 45011579eb65842821dff73fc2028db9d6bf7b93 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 16 Jun 2020 14:47:30 +0100 Subject: [PATCH 168/200] Update sytest-whitelist --- sytest-whitelist | 1 + 1 file changed, 1 insertion(+) diff --git a/sytest-whitelist b/sytest-whitelist index c1c0c13ac..e59d2df16 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -313,3 +313,4 @@ Invalid JSON integers Invalid JSON special values Invalid JSON floats Outbound federation will ignore a missing event with bad JSON for room version 6 +Can download without a file name over federation From 1942928ee5e0398beed45c8b1c63d7b13e89b646 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 16 Jun 2020 14:53:19 +0100 Subject: [PATCH 169/200] Make federationapi use userapi (#1135) Removes dependencies on account DB, device DB and ASAPI. --- cmd/dendrite-federation-api-server/main.go | 8 ++--- federationapi/federationapi.go | 12 +++----- federationapi/federationapi_test.go | 2 +- federationapi/routing/devices.go | 22 +++++--------- federationapi/routing/profile.go | 21 ++++++------- federationapi/routing/routing.go | 14 ++++----- federationapi/routing/threepid.go | 35 +++++++++++----------- internal/setup/monolith.go | 4 +-- userapi/api/api.go | 12 ++++++++ userapi/internal/api.go | 16 ++++++++++ userapi/inthttp/client.go | 9 ++++++ userapi/inthttp/server.go | 13 ++++++++ 12 files changed, 102 insertions(+), 66 deletions(-) diff --git a/cmd/dendrite-federation-api-server/main.go b/cmd/dendrite-federation-api-server/main.go index b8db7927a..e3bf5edc8 100644 --- a/cmd/dendrite-federation-api-server/main.go +++ b/cmd/dendrite-federation-api-server/main.go @@ -24,18 +24,16 @@ func main() { base := setup.NewBaseDendrite(cfg, "FederationAPI", true) defer base.Close() // nolint: errcheck - accountDB := base.CreateAccountsDB() - deviceDB := base.CreateDeviceDB() + userAPI := base.UserAPIClient() federation := base.CreateFederationClient() serverKeyAPI := base.ServerKeyAPIClient() keyRing := serverKeyAPI.KeyRing() fsAPI := base.FederationSenderHTTPClient() rsAPI := base.RoomserverHTTPClient() - asAPI := base.AppserviceHTTPClient() federationapi.AddPublicRoutes( - base.PublicAPIMux, base.Cfg, accountDB, deviceDB, federation, keyRing, - rsAPI, asAPI, fsAPI, base.EDUServerClient(), + base.PublicAPIMux, base.Cfg, userAPI, federation, keyRing, + rsAPI, fsAPI, base.EDUServerClient(), ) base.SetupAndServeHTTP(string(base.Cfg.Bind.FederationAPI), string(base.Cfg.Listen.FederationAPI)) diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go index db272f1c8..c0c000434 100644 --- a/federationapi/federationapi.go +++ b/federationapi/federationapi.go @@ -16,13 +16,11 @@ package federationapi import ( "github.com/gorilla/mux" - appserviceAPI "github.com/matrix-org/dendrite/appservice/api" - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" - "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" eduserverAPI "github.com/matrix-org/dendrite/eduserver/api" federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/internal/config" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/dendrite/federationapi/routing" "github.com/matrix-org/gomatrixserverlib" @@ -32,19 +30,17 @@ import ( func AddPublicRoutes( router *mux.Router, cfg *config.Dendrite, - accountsDB accounts.Database, - deviceDB devices.Database, + userAPI userapi.UserInternalAPI, federation *gomatrixserverlib.FederationClient, keyRing gomatrixserverlib.JSONVerifier, rsAPI roomserverAPI.RoomserverInternalAPI, - asAPI appserviceAPI.AppServiceQueryAPI, federationSenderAPI federationSenderAPI.FederationSenderInternalAPI, eduAPI eduserverAPI.EDUServerInputAPI, ) { routing.Setup( - router, cfg, rsAPI, asAPI, + router, cfg, rsAPI, eduAPI, federationSenderAPI, keyRing, - federation, accountsDB, deviceDB, + federation, userAPI, ) } diff --git a/federationapi/federationapi_test.go b/federationapi/federationapi_test.go index cf7d732bf..cc85c61bf 100644 --- a/federationapi/federationapi_test.go +++ b/federationapi/federationapi_test.go @@ -31,7 +31,7 @@ func TestRoomsV3URLEscapeDoNot404(t *testing.T) { fsAPI := base.FederationSenderHTTPClient() // TODO: This is pretty fragile, as if anything calls anything on these nils this test will break. // Unfortunately, it makes little sense to instantiate these dependencies when we just want to test routing. - federationapi.AddPublicRoutes(base.PublicAPIMux, cfg, nil, nil, nil, keyRing, nil, nil, fsAPI, nil) + federationapi.AddPublicRoutes(base.PublicAPIMux, cfg, nil, nil, keyRing, nil, fsAPI, nil) httputil.SetupHTTPAPI( base.BaseMux, base.PublicAPIMux, diff --git a/federationapi/routing/devices.go b/federationapi/routing/devices.go index caf5fe592..6369c708c 100644 --- a/federationapi/routing/devices.go +++ b/federationapi/routing/devices.go @@ -15,9 +15,8 @@ package routing import ( "net/http" - "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/clientapi/userutil" + userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -25,17 +24,9 @@ import ( // GetUserDevices for the given user id func GetUserDevices( req *http.Request, - deviceDB devices.Database, + userAPI userapi.UserInternalAPI, userID string, ) util.JSONResponse { - localpart, err := userutil.ParseUsernameParam(userID, nil) - if err != nil { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.InvalidArgumentValue("Invalid user ID"), - } - } - response := gomatrixserverlib.RespUserDevices{ UserID: userID, // TODO: we should return an incrementing stream ID each time the device @@ -43,13 +34,16 @@ func GetUserDevices( StreamID: 0, } - devs, err := deviceDB.GetDevicesByLocalpart(req.Context(), localpart) + var res userapi.QueryDevicesResponse + err := userAPI.QueryDevices(req.Context(), &userapi.QueryDevicesRequest{ + UserID: userID, + }, &res) if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("deviceDB.GetDevicesByLocalPart failed") + util.GetLogger(req.Context()).WithError(err).Error("userAPI.QueryDevices failed") return jsonerror.InternalServerError() } - for _, dev := range devs { + for _, dev := range res.Devices { device := gomatrixserverlib.RespUserDevice{ DeviceID: dev.ID, DisplayName: dev.DisplayName, diff --git a/federationapi/routing/profile.go b/federationapi/routing/profile.go index 61d0682bd..a6180ae6d 100644 --- a/federationapi/routing/profile.go +++ b/federationapi/routing/profile.go @@ -18,11 +18,10 @@ import ( "fmt" "net/http" - appserviceAPI "github.com/matrix-org/dendrite/appservice/api" - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/eventutil" + userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -30,9 +29,8 @@ import ( // GetProfile implements GET /_matrix/federation/v1/query/profile func GetProfile( httpReq *http.Request, - accountDB accounts.Database, + userAPI userapi.UserInternalAPI, cfg *config.Dendrite, - asAPI appserviceAPI.AppServiceQueryAPI, ) util.JSONResponse { userID, field := httpReq.FormValue("user_id"), httpReq.FormValue("field") @@ -60,9 +58,12 @@ func GetProfile( } } - profile, err := appserviceAPI.RetrieveUserProfile(httpReq.Context(), userID, asAPI, accountDB) + var profileRes userapi.QueryProfileResponse + err = userAPI.QueryProfile(httpReq.Context(), &userapi.QueryProfileRequest{ + UserID: userID, + }, &profileRes) if err != nil { - util.GetLogger(httpReq.Context()).WithError(err).Error("appserviceAPI.RetrieveUserProfile failed") + util.GetLogger(httpReq.Context()).WithError(err).Error("userAPI.QueryProfile failed") return jsonerror.InternalServerError() } @@ -73,11 +74,11 @@ func GetProfile( switch field { case "displayname": res = eventutil.DisplayName{ - DisplayName: profile.DisplayName, + DisplayName: profileRes.DisplayName, } case "avatar_url": res = eventutil.AvatarURL{ - AvatarURL: profile.AvatarURL, + AvatarURL: profileRes.AvatarURL, } default: code = http.StatusBadRequest @@ -85,8 +86,8 @@ func GetProfile( } } else { res = eventutil.ProfileResponse{ - AvatarURL: profile.AvatarURL, - DisplayName: profile.DisplayName, + AvatarURL: profileRes.AvatarURL, + DisplayName: profileRes.DisplayName, } } diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index 350febbc1..649a43c66 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -18,14 +18,12 @@ import ( "net/http" "github.com/gorilla/mux" - appserviceAPI "github.com/matrix-org/dendrite/appservice/api" - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" - "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" eduserverAPI "github.com/matrix-org/dendrite/eduserver/api" federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/httputil" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) @@ -48,13 +46,11 @@ func Setup( publicAPIMux *mux.Router, cfg *config.Dendrite, rsAPI roomserverAPI.RoomserverInternalAPI, - asAPI appserviceAPI.AppServiceQueryAPI, eduAPI eduserverAPI.EDUServerInputAPI, fsAPI federationSenderAPI.FederationSenderInternalAPI, keys gomatrixserverlib.JSONVerifier, federation *gomatrixserverlib.FederationClient, - accountDB accounts.Database, - deviceDB devices.Database, + userAPI userapi.UserInternalAPI, ) { v2keysmux := publicAPIMux.PathPrefix(pathPrefixV2Keys).Subrouter() v1fedmux := publicAPIMux.PathPrefix(pathPrefixV1Federation).Subrouter() @@ -98,7 +94,7 @@ func Setup( v1fedmux.Handle("/3pid/onbind", httputil.MakeExternalAPI("3pid_onbind", func(req *http.Request) util.JSONResponse { - return CreateInvitesFrom3PIDInvites(req, rsAPI, asAPI, cfg, federation, accountDB) + return CreateInvitesFrom3PIDInvites(req, rsAPI, cfg, federation, userAPI) }, )).Methods(http.MethodPost, http.MethodOptions) @@ -160,7 +156,7 @@ func Setup( "federation_query_profile", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return GetProfile( - httpReq, accountDB, cfg, asAPI, + httpReq, userAPI, cfg, ) }, )).Methods(http.MethodGet) @@ -169,7 +165,7 @@ func Setup( "federation_user_devices", cfg.Matrix.ServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return GetUserDevices( - httpReq, deviceDB, vars["userID"], + httpReq, userAPI, vars["userID"], ) }, )).Methods(http.MethodGet) diff --git a/federationapi/routing/threepid.go b/federationapi/routing/threepid.go index 8f3193870..61788010b 100644 --- a/federationapi/routing/threepid.go +++ b/federationapi/routing/threepid.go @@ -21,13 +21,11 @@ import ( "net/http" "time" - appserviceAPI "github.com/matrix-org/dendrite/appservice/api" - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/roomserver/api" - roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -57,10 +55,10 @@ var ( // CreateInvitesFrom3PIDInvites implements POST /_matrix/federation/v1/3pid/onbind func CreateInvitesFrom3PIDInvites( - req *http.Request, rsAPI roomserverAPI.RoomserverInternalAPI, - asAPI appserviceAPI.AppServiceQueryAPI, cfg *config.Dendrite, + req *http.Request, rsAPI api.RoomserverInternalAPI, + cfg *config.Dendrite, federation *gomatrixserverlib.FederationClient, - accountDB accounts.Database, + userAPI userapi.UserInternalAPI, ) util.JSONResponse { var body invites if reqErr := httputil.UnmarshalJSONRequest(req, &body); reqErr != nil { @@ -79,7 +77,7 @@ func CreateInvitesFrom3PIDInvites( } event, err := createInviteFrom3PIDInvite( - req.Context(), rsAPI, asAPI, cfg, inv, federation, accountDB, + req.Context(), rsAPI, cfg, inv, federation, userAPI, ) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("createInviteFrom3PIDInvite failed") @@ -107,7 +105,7 @@ func ExchangeThirdPartyInvite( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, roomID string, - rsAPI roomserverAPI.RoomserverInternalAPI, + rsAPI api.RoomserverInternalAPI, cfg *config.Dendrite, federation *gomatrixserverlib.FederationClient, ) util.JSONResponse { @@ -197,10 +195,10 @@ func ExchangeThirdPartyInvite( // Returns an error if there was a problem building the event or fetching the // necessary data to do so. func createInviteFrom3PIDInvite( - ctx context.Context, rsAPI roomserverAPI.RoomserverInternalAPI, - asAPI appserviceAPI.AppServiceQueryAPI, cfg *config.Dendrite, + ctx context.Context, rsAPI api.RoomserverInternalAPI, + cfg *config.Dendrite, inv invite, federation *gomatrixserverlib.FederationClient, - accountDB accounts.Database, + userAPI userapi.UserInternalAPI, ) (*gomatrixserverlib.Event, error) { verReq := api.QueryRoomVersionForRoomRequest{RoomID: inv.RoomID} verRes := api.QueryRoomVersionForRoomResponse{} @@ -225,14 +223,17 @@ func createInviteFrom3PIDInvite( StateKey: &inv.MXID, } - profile, err := appserviceAPI.RetrieveUserProfile(ctx, inv.MXID, asAPI, accountDB) + var res userapi.QueryProfileResponse + err = userAPI.QueryProfile(ctx, &userapi.QueryProfileRequest{ + UserID: inv.MXID, + }, &res) if err != nil { return nil, err } content := gomatrixserverlib.MemberContent{ - AvatarURL: profile.AvatarURL, - DisplayName: profile.DisplayName, + AvatarURL: res.AvatarURL, + DisplayName: res.DisplayName, Membership: gomatrixserverlib.Invite, ThirdPartyInvite: &gomatrixserverlib.MemberThirdPartyInvite{ Signed: inv.Signed, @@ -261,7 +262,7 @@ func createInviteFrom3PIDInvite( // Returns an error if something failed during the process. func buildMembershipEvent( ctx context.Context, - builder *gomatrixserverlib.EventBuilder, rsAPI roomserverAPI.RoomserverInternalAPI, + builder *gomatrixserverlib.EventBuilder, rsAPI api.RoomserverInternalAPI, cfg *config.Dendrite, ) (*gomatrixserverlib.Event, error) { eventsNeeded, err := gomatrixserverlib.StateNeededForEventBuilder(builder) @@ -274,11 +275,11 @@ func buildMembershipEvent( } // Ask the roomserver for information about this room - queryReq := roomserverAPI.QueryLatestEventsAndStateRequest{ + queryReq := api.QueryLatestEventsAndStateRequest{ RoomID: builder.RoomID, StateToFetch: eventsNeeded.Tuples(), } - var queryRes roomserverAPI.QueryLatestEventsAndStateResponse + var queryRes api.QueryLatestEventsAndStateResponse if err = rsAPI.QueryLatestEventsAndState(ctx, &queryReq, &queryRes); err != nil { return nil, err } diff --git a/internal/setup/monolith.go b/internal/setup/monolith.go index b202aa71a..aec14aa72 100644 --- a/internal/setup/monolith.go +++ b/internal/setup/monolith.go @@ -77,8 +77,8 @@ func (m *Monolith) AddAllPublicRoutes(publicMux *mux.Router) { keyserver.AddPublicRoutes(publicMux, m.Config, m.UserAPI) federationapi.AddPublicRoutes( - publicMux, m.Config, m.AccountDB, m.DeviceDB, m.FedClient, - m.KeyRing, m.RoomserverAPI, m.AppserviceAPI, m.FederationSenderAPI, + publicMux, m.Config, m.UserAPI, m.FedClient, + m.KeyRing, m.RoomserverAPI, m.FederationSenderAPI, m.EDUInternalAPI, ) mediaapi.AddPublicRoutes(publicMux, m.Config, m.UserAPI, m.Client) diff --git a/userapi/api/api.go b/userapi/api/api.go index 57b5165a4..3ed9252cb 100644 --- a/userapi/api/api.go +++ b/userapi/api/api.go @@ -20,6 +20,7 @@ import "context" type UserInternalAPI interface { QueryProfile(ctx context.Context, req *QueryProfileRequest, res *QueryProfileResponse) error QueryAccessToken(ctx context.Context, req *QueryAccessTokenRequest, res *QueryAccessTokenResponse) error + QueryDevices(ctx context.Context, req *QueryDevicesRequest, res *QueryDevicesResponse) error } // QueryAccessTokenRequest is the request for QueryAccessToken @@ -36,6 +37,17 @@ type QueryAccessTokenResponse struct { Err error // e.g ErrorForbidden } +// QueryDevicesRequest is the request for QueryDevices +type QueryDevicesRequest struct { + UserID string +} + +// QueryDevicesResponse is the response for QueryDevices +type QueryDevicesResponse struct { + UserExists bool + Devices []Device +} + // QueryProfileRequest is the request for QueryProfile type QueryProfileRequest struct { // The user ID to query diff --git a/userapi/internal/api.go b/userapi/internal/api.go index 1f0d5c94b..d8dec11af 100644 --- a/userapi/internal/api.go +++ b/userapi/internal/api.go @@ -57,6 +57,22 @@ func (a *UserInternalAPI) QueryProfile(ctx context.Context, req *api.QueryProfil return nil } +func (a *UserInternalAPI) QueryDevices(ctx context.Context, req *api.QueryDevicesRequest, res *api.QueryDevicesResponse) error { + local, domain, err := gomatrixserverlib.SplitID('@', req.UserID) + if err != nil { + return err + } + if domain != a.ServerName { + return fmt.Errorf("cannot query devices of remote users: got %s want %s", domain, a.ServerName) + } + devs, err := a.DeviceDB.GetDevicesByLocalpart(ctx, local) + if err != nil { + return err + } + res.Devices = devs + return nil +} + func (a *UserInternalAPI) QueryAccessToken(ctx context.Context, req *api.QueryAccessTokenRequest, res *api.QueryAccessTokenResponse) error { if req.AppServiceUserID != "" { appServiceDevice, err := a.queryAppServiceToken(ctx, req.AccessToken, req.AppServiceUserID) diff --git a/userapi/inthttp/client.go b/userapi/inthttp/client.go index 022243faa..638a7e9b6 100644 --- a/userapi/inthttp/client.go +++ b/userapi/inthttp/client.go @@ -28,6 +28,7 @@ import ( const ( QueryProfilePath = "/userapi/queryProfile" QueryAccessTokenPath = "/userapi/queryAccessToken" + QueryDevicesPath = "/userapi/queryDevices" ) // NewUserAPIClient creates a UserInternalAPI implemented by talking to a HTTP POST API. @@ -73,3 +74,11 @@ func (h *httpUserInternalAPI) QueryAccessToken( apiURL := h.apiURL + QueryAccessTokenPath return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } + +func (h *httpUserInternalAPI) QueryDevices(ctx context.Context, req *api.QueryDevicesRequest, res *api.QueryDevicesResponse) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "QueryDevices") + defer span.Finish() + + apiURL := h.apiURL + QueryDevicesPath + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res) +} diff --git a/userapi/inthttp/server.go b/userapi/inthttp/server.go index 495b161c7..19b0e40b4 100644 --- a/userapi/inthttp/server.go +++ b/userapi/inthttp/server.go @@ -51,4 +51,17 @@ func AddRoutes(internalAPIMux *mux.Router, s api.UserInternalAPI) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) + internalAPIMux.Handle(QueryDevicesPath, + httputil.MakeInternalAPI("queryDevices", func(req *http.Request) util.JSONResponse { + request := api.QueryDevicesRequest{} + response := api.QueryDevicesResponse{} + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := s.QueryDevices(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) } From 83391da0e04dda7a52589ee7ec6df2b615571894 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 16 Jun 2020 17:05:38 +0100 Subject: [PATCH 170/200] Make syncapi use userapi (#1136) * Make syncapi use userapi * Unbreak things * Fix tests * Lint --- cmd/dendrite-sync-api-server/main.go | 3 +- docs/WIRING-Current.md | 2 ++ internal/setup/monolith.go | 2 +- syncapi/sync/requestpool.go | 45 +++++++++++++++------------- syncapi/syncapi.go | 4 +-- userapi/api/api.go | 23 +++++++++++++- userapi/internal/api.go | 33 ++++++++++++++++++++ userapi/inthttp/client.go | 9 ++++++ userapi/inthttp/server.go | 13 ++++++++ 9 files changed, 106 insertions(+), 28 deletions(-) diff --git a/cmd/dendrite-sync-api-server/main.go b/cmd/dendrite-sync-api-server/main.go index a17b648d6..d67395fb3 100644 --- a/cmd/dendrite-sync-api-server/main.go +++ b/cmd/dendrite-sync-api-server/main.go @@ -25,12 +25,11 @@ func main() { defer base.Close() // nolint: errcheck userAPI := base.UserAPIClient() - accountDB := base.CreateAccountsDB() federation := base.CreateFederationClient() rsAPI := base.RoomserverHTTPClient() - syncapi.AddPublicRoutes(base.PublicAPIMux, base.KafkaConsumer, userAPI, accountDB, rsAPI, federation, cfg) + syncapi.AddPublicRoutes(base.PublicAPIMux, base.KafkaConsumer, userAPI, rsAPI, federation, cfg) base.SetupAndServeHTTP(string(base.Cfg.Bind.SyncAPI), string(base.Cfg.Listen.SyncAPI)) diff --git a/docs/WIRING-Current.md b/docs/WIRING-Current.md index 62450f2f8..ec539d4e9 100644 --- a/docs/WIRING-Current.md +++ b/docs/WIRING-Current.md @@ -39,6 +39,8 @@ Internal only | `------------------- - 12 (FedSender -> ServerKeyAPI): Verifying event signatures of responses (e.g from send_join) - 13 (Roomserver -> ServerKeyAPI): Verifying event signatures of backfilled events +In addition to this, all public facing components (Tier 1) talk to the `UserAPI` to verify access tokens and extract profile information where needed. + ## Kafka logs ``` diff --git a/internal/setup/monolith.go b/internal/setup/monolith.go index aec14aa72..bb81f7403 100644 --- a/internal/setup/monolith.go +++ b/internal/setup/monolith.go @@ -87,6 +87,6 @@ func (m *Monolith) AddAllPublicRoutes(publicMux *mux.Router) { m.ExtPublicRoomsProvider, ) syncapi.AddPublicRoutes( - publicMux, m.KafkaConsumer, m.UserAPI, m.AccountDB, m.RoomserverAPI, m.FedClient, m.Config, + publicMux, m.KafkaConsumer, m.UserAPI, m.RoomserverAPI, m.FedClient, m.Config, ) } diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index ec22a05f4..26b925eac 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -21,7 +21,6 @@ import ( "net/http" "time" - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/types" @@ -33,14 +32,14 @@ import ( // RequestPool manages HTTP long-poll connections for /sync type RequestPool struct { - db storage.Database - accountDB accounts.Database - notifier *Notifier + db storage.Database + userAPI userapi.UserInternalAPI + notifier *Notifier } // NewRequestPool makes a new RequestPool -func NewRequestPool(db storage.Database, n *Notifier, adb accounts.Database) *RequestPool { - return &RequestPool{db, adb, n} +func NewRequestPool(db storage.Database, n *Notifier, userAPI userapi.UserInternalAPI) *RequestPool { + return &RequestPool{db, userAPI, n} } // OnIncomingSyncRequest is called when a client makes a /sync request. This function MUST be @@ -193,6 +192,7 @@ func (rp *RequestPool) currentSyncForUser(req syncRequest, latestPos types.Strea return } +// nolint:gocyclo func (rp *RequestPool) appendAccountData( data *types.Response, userID string, req syncRequest, currentPos types.StreamPosition, accountDataFilter *gomatrixserverlib.EventFilter, @@ -202,25 +202,21 @@ func (rp *RequestPool) appendAccountData( // data keys were set between two message. This isn't a huge issue since the // duplicate data doesn't represent a huge quantity of data, but an optimisation // here would be making sure each data is sent only once to the client. - localpart, _, err := gomatrixserverlib.SplitID('@', userID) - if err != nil { - return nil, err - } - if req.since == nil { // If this is the initial sync, we don't need to check if a data has // already been sent. Instead, we send the whole batch. - var global []gomatrixserverlib.ClientEvent - var rooms map[string][]gomatrixserverlib.ClientEvent - global, rooms, err = rp.accountDB.GetAccountData(req.ctx, localpart) + var res userapi.QueryAccountDataResponse + err := rp.userAPI.QueryAccountData(req.ctx, &userapi.QueryAccountDataRequest{ + UserID: userID, + }, &res) if err != nil { return nil, err } - data.AccountData.Events = global + data.AccountData.Events = res.GlobalAccountData for r, j := range data.Rooms.Join { - if len(rooms[r]) > 0 { - j.AccountData.Events = rooms[r] + if len(res.RoomAccountData[r]) > 0 { + j.AccountData.Events = res.RoomAccountData[r] data.Rooms.Join[r] = j } } @@ -256,13 +252,20 @@ func (rp *RequestPool) appendAccountData( events := []gomatrixserverlib.ClientEvent{} // Request the missing data from the database for _, dataType := range dataTypes { - event, err := rp.accountDB.GetAccountDataByType( - req.ctx, localpart, roomID, dataType, - ) + var res userapi.QueryAccountDataResponse + err = rp.userAPI.QueryAccountData(req.ctx, &userapi.QueryAccountDataRequest{ + UserID: userID, + RoomID: roomID, + DataType: dataType, + }, &res) if err != nil { return nil, err } - events = append(events, *event) + if len(res.RoomAccountData[roomID]) > 0 { + events = append(events, res.RoomAccountData[roomID]...) + } else if len(res.GlobalAccountData) > 0 { + events = append(events, res.GlobalAccountData...) + } } // Append the data to the response diff --git a/syncapi/syncapi.go b/syncapi/syncapi.go index 2351ee4d8..caf91e27e 100644 --- a/syncapi/syncapi.go +++ b/syncapi/syncapi.go @@ -21,7 +21,6 @@ import ( "github.com/gorilla/mux" "github.com/sirupsen/logrus" - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/roomserver/api" userapi "github.com/matrix-org/dendrite/userapi/api" @@ -39,7 +38,6 @@ func AddPublicRoutes( router *mux.Router, consumer sarama.Consumer, userAPI userapi.UserInternalAPI, - accountsDB accounts.Database, rsAPI api.RoomserverInternalAPI, federation *gomatrixserverlib.FederationClient, cfg *config.Dendrite, @@ -60,7 +58,7 @@ func AddPublicRoutes( logrus.WithError(err).Panicf("failed to start notifier") } - requestPool := sync.NewRequestPool(syncDB, notifier, accountsDB) + requestPool := sync.NewRequestPool(syncDB, notifier, userAPI) roomConsumer := consumers.NewOutputRoomEventConsumer( cfg, consumer, notifier, syncDB, rsAPI, diff --git a/userapi/api/api.go b/userapi/api/api.go index 3ed9252cb..1578268ac 100644 --- a/userapi/api/api.go +++ b/userapi/api/api.go @@ -14,13 +14,18 @@ package api -import "context" +import ( + "context" + + "github.com/matrix-org/gomatrixserverlib" +) // UserInternalAPI is the internal API for information about users and devices. type UserInternalAPI interface { QueryProfile(ctx context.Context, req *QueryProfileRequest, res *QueryProfileResponse) error QueryAccessToken(ctx context.Context, req *QueryAccessTokenRequest, res *QueryAccessTokenResponse) error QueryDevices(ctx context.Context, req *QueryDevicesRequest, res *QueryDevicesResponse) error + QueryAccountData(ctx context.Context, req *QueryAccountDataRequest, res *QueryAccountDataResponse) error } // QueryAccessTokenRequest is the request for QueryAccessToken @@ -37,6 +42,22 @@ type QueryAccessTokenResponse struct { Err error // e.g ErrorForbidden } +// QueryAccountDataRequest is the request for QueryAccountData +type QueryAccountDataRequest struct { + UserID string // required: the user to get account data for. + // TODO: This is a terribly confusing API shape :/ + DataType string // optional: if specified returns only a single event matching this data type. + // optional: Only used if DataType is set. If blank returns global account data matching the data type. + // If set, returns only room account data matching this data type. + RoomID string +} + +// QueryAccountDataResponse is the response for QueryAccountData +type QueryAccountDataResponse struct { + GlobalAccountData []gomatrixserverlib.ClientEvent + RoomAccountData map[string][]gomatrixserverlib.ClientEvent +} + // QueryDevicesRequest is the request for QueryDevices type QueryDevicesRequest struct { UserID string diff --git a/userapi/internal/api.go b/userapi/internal/api.go index d8dec11af..6e737b81f 100644 --- a/userapi/internal/api.go +++ b/userapi/internal/api.go @@ -73,6 +73,39 @@ func (a *UserInternalAPI) QueryDevices(ctx context.Context, req *api.QueryDevice return nil } +func (a *UserInternalAPI) QueryAccountData(ctx context.Context, req *api.QueryAccountDataRequest, res *api.QueryAccountDataResponse) error { + local, domain, err := gomatrixserverlib.SplitID('@', req.UserID) + if err != nil { + return err + } + if domain != a.ServerName { + return fmt.Errorf("cannot query account data of remote users: got %s want %s", domain, a.ServerName) + } + if req.DataType != "" { + var event *gomatrixserverlib.ClientEvent + event, err = a.AccountDB.GetAccountDataByType(ctx, local, req.RoomID, req.DataType) + if err != nil { + return err + } + if event != nil { + if req.RoomID != "" { + res.RoomAccountData = make(map[string][]gomatrixserverlib.ClientEvent) + res.RoomAccountData[req.RoomID] = []gomatrixserverlib.ClientEvent{*event} + } else { + res.GlobalAccountData = append(res.GlobalAccountData, *event) + } + } + return nil + } + global, rooms, err := a.AccountDB.GetAccountData(ctx, local) + if err != nil { + return err + } + res.RoomAccountData = rooms + res.GlobalAccountData = global + return nil +} + func (a *UserInternalAPI) QueryAccessToken(ctx context.Context, req *api.QueryAccessTokenRequest, res *api.QueryAccessTokenResponse) error { if req.AppServiceUserID != "" { appServiceDevice, err := a.queryAppServiceToken(ctx, req.AccessToken, req.AppServiceUserID) diff --git a/userapi/inthttp/client.go b/userapi/inthttp/client.go index 638a7e9b6..48e6d7d72 100644 --- a/userapi/inthttp/client.go +++ b/userapi/inthttp/client.go @@ -29,6 +29,7 @@ const ( QueryProfilePath = "/userapi/queryProfile" QueryAccessTokenPath = "/userapi/queryAccessToken" QueryDevicesPath = "/userapi/queryDevices" + QueryAccountDataPath = "/userapi/queryAccountData" ) // NewUserAPIClient creates a UserInternalAPI implemented by talking to a HTTP POST API. @@ -82,3 +83,11 @@ func (h *httpUserInternalAPI) QueryDevices(ctx context.Context, req *api.QueryDe apiURL := h.apiURL + QueryDevicesPath return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res) } + +func (h *httpUserInternalAPI) QueryAccountData(ctx context.Context, req *api.QueryAccountDataRequest, res *api.QueryAccountDataResponse) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "QueryAccountData") + defer span.Finish() + + apiURL := h.apiURL + QueryAccountDataPath + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res) +} diff --git a/userapi/inthttp/server.go b/userapi/inthttp/server.go index 19b0e40b4..8bf2efc01 100644 --- a/userapi/inthttp/server.go +++ b/userapi/inthttp/server.go @@ -64,4 +64,17 @@ func AddRoutes(internalAPIMux *mux.Router, s api.UserInternalAPI) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) + internalAPIMux.Handle(QueryAccountDataPath, + httputil.MakeInternalAPI("queryAccountData", func(req *http.Request) util.JSONResponse { + request := api.QueryAccountDataRequest{} + response := api.QueryAccountDataResponse{} + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := s.QueryAccountData(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) } From e15a8042a19b270060beef1358f90cda075ddd38 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 16 Jun 2020 17:39:56 +0100 Subject: [PATCH 171/200] BREAKING: Make eduserver/appservice use userapi (#1138) * BREAKING: Make eduserver/appservice use userapi This is a breaking change because this PR restructures how the AS API tracks its position in Kafka streams. Previously, it used the account DB to store partition offsets. However, this is also being used by `clientapi` for the same purpose, which is bad (each component needs to store offsets independently or else you might lose messages across restarts). This PR changes this behaviour to now store partition offsets in the `appservice` database. This means that: - Upon restart, the `appservice` component will attempt to replay all room events from the beginning of time. - An additional table will be created in the appservice database, which in and of itself is backwards compatible. * Return ErrorConflict --- appservice/appservice.go | 38 +++++++------- appservice/consumers/roomserver.go | 6 +-- appservice/storage/interface.go | 2 + appservice/storage/postgres/storage.go | 4 ++ appservice/storage/sqlite3/storage.go | 4 ++ clientapi/auth/storage/accounts/interface.go | 4 ++ clientapi/auth/storage/devices/interface.go | 6 +++ cmd/dendrite-appservice-server/main.go | 5 +- cmd/dendrite-demo-libp2p/main.go | 6 +-- cmd/dendrite-demo-yggdrasil/main.go | 7 +-- cmd/dendrite-edu-server/main.go | 3 +- cmd/dendrite-monolith-server/main.go | 7 ++- cmd/dendritejs/main.go | 7 ++- eduserver/eduserver.go | 6 +-- eduserver/input/input.go | 15 +++--- userapi/api/api.go | 53 ++++++++++++++++++++ userapi/internal/api.go | 34 +++++++++++++ userapi/inthttp/client.go | 27 ++++++++++ userapi/inthttp/server.go | 26 ++++++++++ 19 files changed, 207 insertions(+), 53 deletions(-) diff --git a/appservice/appservice.go b/appservice/appservice.go index bd261ff9b..84a6a9b12 100644 --- a/appservice/appservice.go +++ b/appservice/appservice.go @@ -16,7 +16,6 @@ package appservice import ( "context" - "errors" "net/http" "sync" "time" @@ -29,12 +28,10 @@ import ( "github.com/matrix-org/dendrite/appservice/storage" "github.com/matrix-org/dendrite/appservice/types" "github.com/matrix-org/dendrite/appservice/workers" - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" - "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/setup" - "github.com/matrix-org/dendrite/internal/sqlutil" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/sirupsen/logrus" ) @@ -47,8 +44,7 @@ func AddInternalRoutes(router *mux.Router, queryAPI appserviceAPI.AppServiceQuer // can call functions directly on the returned API or via an HTTP interface using AddInternalRoutes. func NewInternalAPI( base *setup.BaseDendrite, - accountsDB accounts.Database, - deviceDB devices.Database, + userAPI userapi.UserInternalAPI, rsAPI roomserverAPI.RoomserverInternalAPI, ) appserviceAPI.AppServiceQueryAPI { // Create a connection to the appservice postgres DB @@ -70,7 +66,7 @@ func NewInternalAPI( workerStates[i] = ws // Create bot account for this AS if it doesn't already exist - if err = generateAppServiceAccount(accountsDB, deviceDB, appservice); err != nil { + if err = generateAppServiceAccount(userAPI, appservice); err != nil { logrus.WithFields(logrus.Fields{ "appservice": appservice.ID, }).WithError(err).Panicf("failed to generate bot account for appservice") @@ -90,7 +86,7 @@ func NewInternalAPI( // We can't add ASes at runtime so this is safe to do. if len(workerStates) > 0 { consumer := consumers.NewOutputRoomEventConsumer( - base.Cfg, base.KafkaConsumer, accountsDB, appserviceDB, + base.Cfg, base.KafkaConsumer, appserviceDB, rsAPI, workerStates, ) if err := consumer.Start(); err != nil { @@ -109,22 +105,24 @@ func NewInternalAPI( // `sender_localpart` field of each application service if it doesn't // exist already func generateAppServiceAccount( - accountsDB accounts.Database, - deviceDB devices.Database, + userAPI userapi.UserInternalAPI, as config.ApplicationService, ) error { - ctx := context.Background() - - // Create an account for the application service - _, err := accountsDB.CreateAccount(ctx, as.SenderLocalpart, "", as.ID) + var accRes userapi.PerformAccountCreationResponse + err := userAPI.PerformAccountCreation(context.Background(), &userapi.PerformAccountCreationRequest{ + Localpart: as.SenderLocalpart, + AppServiceID: as.ID, + OnConflict: userapi.ConflictUpdate, + }, &accRes) if err != nil { - if errors.Is(err, sqlutil.ErrUserExists) { // This account already exists - return nil - } return err } - - // Create a dummy device with a dummy token for the application service - _, err = deviceDB.CreateDevice(ctx, as.SenderLocalpart, nil, as.ASToken, &as.SenderLocalpart) + var devRes userapi.PerformDeviceCreationResponse + err = userAPI.PerformDeviceCreation(context.Background(), &userapi.PerformDeviceCreationRequest{ + Localpart: as.SenderLocalpart, + AccessToken: as.ASToken, + DeviceID: &as.SenderLocalpart, + DeviceDisplayName: &as.SenderLocalpart, + }, &devRes) return err } diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go index 1657fe542..4c0156b2c 100644 --- a/appservice/consumers/roomserver.go +++ b/appservice/consumers/roomserver.go @@ -20,7 +20,6 @@ import ( "github.com/matrix-org/dendrite/appservice/storage" "github.com/matrix-org/dendrite/appservice/types" - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/roomserver/api" @@ -33,7 +32,6 @@ import ( // OutputRoomEventConsumer consumes events that originated in the room server. type OutputRoomEventConsumer struct { roomServerConsumer *internal.ContinualConsumer - db accounts.Database asDB storage.Database rsAPI api.RoomserverInternalAPI serverName string @@ -45,7 +43,6 @@ type OutputRoomEventConsumer struct { func NewOutputRoomEventConsumer( cfg *config.Dendrite, kafkaConsumer sarama.Consumer, - store accounts.Database, appserviceDB storage.Database, rsAPI api.RoomserverInternalAPI, workerStates []types.ApplicationServiceWorkerState, @@ -53,11 +50,10 @@ func NewOutputRoomEventConsumer( consumer := internal.ContinualConsumer{ Topic: string(cfg.Kafka.Topics.OutputRoomEvent), Consumer: kafkaConsumer, - PartitionStore: store, + PartitionStore: appserviceDB, } s := &OutputRoomEventConsumer{ roomServerConsumer: &consumer, - db: store, asDB: appserviceDB, rsAPI: rsAPI, serverName: string(cfg.Matrix.ServerName), diff --git a/appservice/storage/interface.go b/appservice/storage/interface.go index 25d35af6c..735e2f90a 100644 --- a/appservice/storage/interface.go +++ b/appservice/storage/interface.go @@ -17,10 +17,12 @@ package storage import ( "context" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/gomatrixserverlib" ) type Database interface { + internal.PartitionStorer StoreEvent(ctx context.Context, appServiceID string, event *gomatrixserverlib.HeaderedEvent) error GetEventsWithAppServiceID(ctx context.Context, appServiceID string, limit int) (int, int, []gomatrixserverlib.HeaderedEvent, bool, error) CountEventsWithAppServiceID(ctx context.Context, appServiceID string) (int, error) diff --git a/appservice/storage/postgres/storage.go b/appservice/storage/postgres/storage.go index 3e12f3a0d..03f331d64 100644 --- a/appservice/storage/postgres/storage.go +++ b/appservice/storage/postgres/storage.go @@ -27,6 +27,7 @@ import ( // Database stores events intended to be later sent to application services type Database struct { + sqlutil.PartitionOffsetStatements events eventsStatements txnID txnStatements db *sql.DB @@ -42,6 +43,9 @@ func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties) (*Dat if err = result.prepare(); err != nil { return nil, err } + if err = result.PartitionOffsetStatements.Prepare(result.db, "appservice"); err != nil { + return nil, err + } return &result, nil } diff --git a/appservice/storage/sqlite3/storage.go b/appservice/storage/sqlite3/storage.go index 44dcba4ed..cb55c8d94 100644 --- a/appservice/storage/sqlite3/storage.go +++ b/appservice/storage/sqlite3/storage.go @@ -27,6 +27,7 @@ import ( // Database stores events intended to be later sent to application services type Database struct { + sqlutil.PartitionOffsetStatements events eventsStatements txnID txnStatements db *sql.DB @@ -46,6 +47,9 @@ func NewDatabase(dataSourceName string) (*Database, error) { if err = result.prepare(); err != nil { return nil, err } + if err = result.PartitionOffsetStatements.Prepare(result.db, "appservice"); err != nil { + return nil, err + } return &result, nil } diff --git a/clientapi/auth/storage/accounts/interface.go b/clientapi/auth/storage/accounts/interface.go index 4d1941a23..3391ccbfc 100644 --- a/clientapi/auth/storage/accounts/interface.go +++ b/clientapi/auth/storage/accounts/interface.go @@ -40,6 +40,10 @@ type Database interface { GetMembershipsByLocalpart(ctx context.Context, localpart string) (memberships []authtypes.Membership, err error) SaveAccountData(ctx context.Context, localpart, roomID, dataType, content string) error GetAccountData(ctx context.Context, localpart string) (global []gomatrixserverlib.ClientEvent, rooms map[string][]gomatrixserverlib.ClientEvent, err error) + // GetAccountDataByType returns account data matching a given + // localpart, room ID and type. + // If no account data could be found, returns nil + // Returns an error if there was an issue with the retrieval GetAccountDataByType(ctx context.Context, localpart, roomID, dataType string) (data *gomatrixserverlib.ClientEvent, err error) GetNewNumericLocalpart(ctx context.Context) (int64, error) SaveThreePIDAssociation(ctx context.Context, threepid, localpart, medium string) (err error) diff --git a/clientapi/auth/storage/devices/interface.go b/clientapi/auth/storage/devices/interface.go index fc2f4a320..4bdb57850 100644 --- a/clientapi/auth/storage/devices/interface.go +++ b/clientapi/auth/storage/devices/interface.go @@ -24,6 +24,12 @@ type Database interface { GetDeviceByAccessToken(ctx context.Context, token string) (*api.Device, error) GetDeviceByID(ctx context.Context, localpart, deviceID string) (*api.Device, error) GetDevicesByLocalpart(ctx context.Context, localpart string) ([]api.Device, error) + // CreateDevice makes a new device associated with the given user ID localpart. + // If there is already a device with the same device ID for this user, that access token will be revoked + // and replaced with the given accessToken. If the given accessToken is already in use for another device, + // an error will be returned. + // If no device ID is given one is generated. + // Returns the device on success. CreateDevice(ctx context.Context, localpart string, deviceID *string, accessToken string, displayName *string) (dev *api.Device, returnErr error) UpdateDevice(ctx context.Context, localpart, deviceID string, displayName *string) error RemoveDevice(ctx context.Context, deviceID, localpart string) error diff --git a/cmd/dendrite-appservice-server/main.go b/cmd/dendrite-appservice-server/main.go index ec68940af..6719d0471 100644 --- a/cmd/dendrite-appservice-server/main.go +++ b/cmd/dendrite-appservice-server/main.go @@ -24,11 +24,10 @@ func main() { base := setup.NewBaseDendrite(cfg, "AppServiceAPI", true) defer base.Close() // nolint: errcheck - accountDB := base.CreateAccountsDB() - deviceDB := base.CreateDeviceDB() + userAPI := base.UserAPIClient() rsAPI := base.RoomserverHTTPClient() - intAPI := appservice.NewInternalAPI(base, accountDB, deviceDB, rsAPI) + intAPI := appservice.NewInternalAPI(base, userAPI, rsAPI) appservice.AddInternalRoutes(base.InternalAPIMux, intAPI) base.SetupAndServeHTTP(string(base.Cfg.Bind.AppServiceAPI), string(base.Cfg.Listen.AppServiceAPI)) diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index 6fb3003c2..356ab5a7f 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -141,6 +141,7 @@ func main() { accountDB := base.Base.CreateAccountsDB() deviceDB := base.Base.CreateDeviceDB() federation := createFederationClient(base) + userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, nil) serverKeyAPI := serverkeyapi.NewInternalAPI( base.Base.Cfg, federation, base.Base.Caches, @@ -154,9 +155,9 @@ func main() { &base.Base, keyRing, federation, ) eduInputAPI := eduserver.NewInternalAPI( - &base.Base, cache.New(), deviceDB, + &base.Base, cache.New(), userAPI, ) - asAPI := appservice.NewInternalAPI(&base.Base, accountDB, deviceDB, rsAPI) + asAPI := appservice.NewInternalAPI(&base.Base, userAPI, rsAPI) fsAPI := federationsender.NewInternalAPI( &base.Base, federation, rsAPI, keyRing, ) @@ -165,7 +166,6 @@ func main() { if err != nil { logrus.WithError(err).Panicf("failed to connect to public rooms db") } - userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, nil) monolith := setup.Monolith{ Config: base.Base.Cfg, diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index 87e2246f2..be15da472 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -130,16 +130,18 @@ func main() { serverKeyAPI := &signing.YggdrasilKeys{} keyRing := serverKeyAPI.KeyRing() + userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, nil) + rsComponent := roomserver.NewInternalAPI( base, keyRing, federation, ) rsAPI := rsComponent eduInputAPI := eduserver.NewInternalAPI( - base, cache.New(), deviceDB, + base, cache.New(), userAPI, ) - asAPI := appservice.NewInternalAPI(base, accountDB, deviceDB, rsAPI) + asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI) fsAPI := federationsender.NewInternalAPI( base, federation, rsAPI, keyRing, @@ -153,7 +155,6 @@ func main() { } embed.Embed(*instancePort, "Yggdrasil Demo") - userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, nil) monolith := setup.Monolith{ Config: base.Cfg, diff --git a/cmd/dendrite-edu-server/main.go b/cmd/dendrite-edu-server/main.go index 1ecce884d..6704ebd09 100644 --- a/cmd/dendrite-edu-server/main.go +++ b/cmd/dendrite-edu-server/main.go @@ -29,9 +29,8 @@ func main() { logrus.WithError(err).Warn("BaseDendrite close failed") } }() - deviceDB := base.CreateDeviceDB() - intAPI := eduserver.NewInternalAPI(base, cache.New(), deviceDB) + intAPI := eduserver.NewInternalAPI(base, cache.New(), base.UserAPIClient()) eduserver.AddInternalRoutes(base.InternalAPIMux, intAPI) base.SetupAndServeHTTP(string(base.Cfg.Bind.EDUServer), string(base.Cfg.Listen.EDUServer)) diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index 16e274fc9..339bbe699 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -75,6 +75,7 @@ func main() { serverKeyAPI = base.ServerKeyAPIClient() } keyRing := serverKeyAPI.KeyRing() + userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, cfg.Derived.ApplicationServices) rsImpl := roomserver.NewInternalAPI( base, keyRing, federation, @@ -92,14 +93,14 @@ func main() { } eduInputAPI := eduserver.NewInternalAPI( - base, cache.New(), deviceDB, + base, cache.New(), userAPI, ) if base.UseHTTPAPIs { eduserver.AddInternalRoutes(base.InternalAPIMux, eduInputAPI) eduInputAPI = base.EDUServerClient() } - asAPI := appservice.NewInternalAPI(base, accountDB, deviceDB, rsAPI) + asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI) if base.UseHTTPAPIs { appservice.AddInternalRoutes(base.InternalAPIMux, asAPI) asAPI = base.AppserviceHTTPClient() @@ -121,8 +122,6 @@ func main() { logrus.WithError(err).Panicf("failed to connect to public rooms db") } - userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, cfg.Derived.ApplicationServices) - monolith := setup.Monolith{ Config: base.Cfg, AccountDB: accountDB, diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index aa9192129..883b0fad0 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -194,6 +194,7 @@ func main() { accountDB := base.CreateAccountsDB() deviceDB := base.CreateDeviceDB() federation := createFederationClient(cfg, node) + userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, nil) fetcher := &libp2pKeyFetcher{} keyRing := gomatrixserverlib.KeyRing{ @@ -204,9 +205,9 @@ func main() { } rsAPI := roomserver.NewInternalAPI(base, keyRing, federation) - eduInputAPI := eduserver.NewInternalAPI(base, cache.New(), deviceDB) + eduInputAPI := eduserver.NewInternalAPI(base, cache.New(), userAPI) asQuery := appservice.NewInternalAPI( - base, accountDB, deviceDB, rsAPI, + base, userAPI, rsAPI, ) fedSenderAPI := federationsender.NewInternalAPI(base, federation, rsAPI, &keyRing) rsAPI.SetFederationSenderAPI(fedSenderAPI) @@ -217,8 +218,6 @@ func main() { logrus.WithError(err).Panicf("failed to connect to public rooms db") } - userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, nil) - monolith := setup.Monolith{ Config: base.Cfg, AccountDB: accountDB, diff --git a/eduserver/eduserver.go b/eduserver/eduserver.go index aa65ff239..2e6ba0c85 100644 --- a/eduserver/eduserver.go +++ b/eduserver/eduserver.go @@ -18,12 +18,12 @@ package eduserver import ( "github.com/gorilla/mux" - "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/eduserver/cache" "github.com/matrix-org/dendrite/eduserver/input" "github.com/matrix-org/dendrite/eduserver/inthttp" "github.com/matrix-org/dendrite/internal/setup" + userapi "github.com/matrix-org/dendrite/userapi/api" ) // AddInternalRoutes registers HTTP handlers for the internal API. Invokes functions @@ -37,11 +37,11 @@ func AddInternalRoutes(internalMux *mux.Router, inputAPI api.EDUServerInputAPI) func NewInternalAPI( base *setup.BaseDendrite, eduCache *cache.EDUCache, - deviceDB devices.Database, + userAPI userapi.UserInternalAPI, ) api.EDUServerInputAPI { return &input.EDUServerInputAPI{ Cache: eduCache, - DeviceDB: deviceDB, + UserAPI: userAPI, Producer: base.KafkaProducer, OutputTypingEventTopic: string(base.Cfg.Kafka.Topics.OutputTypingEvent), OutputSendToDeviceEventTopic: string(base.Cfg.Kafka.Topics.OutputSendToDeviceEvent), diff --git a/eduserver/input/input.go b/eduserver/input/input.go index 6eafce42f..e3d2c55e3 100644 --- a/eduserver/input/input.go +++ b/eduserver/input/input.go @@ -22,9 +22,9 @@ import ( "time" "github.com/Shopify/sarama" - "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/eduserver/cache" + userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" ) @@ -39,8 +39,8 @@ type EDUServerInputAPI struct { OutputSendToDeviceEventTopic string // kafka producer Producer sarama.SyncProducer - // device database - DeviceDB devices.Database + // Internal user query API + UserAPI userapi.UserInternalAPI // our server name ServerName gomatrixserverlib.ServerName } @@ -115,7 +115,7 @@ func (t *EDUServerInputAPI) sendTypingEvent(ite *api.InputTypingEvent) error { func (t *EDUServerInputAPI) sendToDeviceEvent(ise *api.InputSendToDeviceEvent) error { devices := []string{} - localpart, domain, err := gomatrixserverlib.SplitID('@', ise.UserID) + _, domain, err := gomatrixserverlib.SplitID('@', ise.UserID) if err != nil { return err } @@ -126,11 +126,14 @@ func (t *EDUServerInputAPI) sendToDeviceEvent(ise *api.InputSendToDeviceEvent) e // wildcard as we don't know about the remote devices, so instead we leave it // as-is, so that the federation sender can send it on with the wildcard intact. if domain == t.ServerName && ise.DeviceID == "*" { - devs, err := t.DeviceDB.GetDevicesByLocalpart(context.TODO(), localpart) + var res userapi.QueryDevicesResponse + err = t.UserAPI.QueryDevices(context.TODO(), &userapi.QueryDevicesRequest{ + UserID: ise.UserID, + }, &res) if err != nil { return err } - for _, dev := range devs { + for _, dev := range res.Devices { devices = append(devices, dev.ID) } } else { diff --git a/userapi/api/api.go b/userapi/api/api.go index 1578268ac..34c74bb3c 100644 --- a/userapi/api/api.go +++ b/userapi/api/api.go @@ -22,6 +22,8 @@ import ( // UserInternalAPI is the internal API for information about users and devices. type UserInternalAPI interface { + PerformAccountCreation(ctx context.Context, req *PerformAccountCreationRequest, res *PerformAccountCreationResponse) error + PerformDeviceCreation(ctx context.Context, req *PerformDeviceCreationRequest, res *PerformDeviceCreationResponse) error QueryProfile(ctx context.Context, req *QueryProfileRequest, res *QueryProfileResponse) error QueryAccessToken(ctx context.Context, req *QueryAccessTokenRequest, res *QueryAccessTokenResponse) error QueryDevices(ctx context.Context, req *QueryDevicesRequest, res *QueryDevicesResponse) error @@ -85,6 +87,38 @@ type QueryProfileResponse struct { AvatarURL string } +// PerformAccountCreationRequest is the request for PerformAccountCreation +type PerformAccountCreationRequest struct { + Localpart string + AppServiceID string + Password string + OnConflict Conflict +} + +// PerformAccountCreationResponse is the response for PerformAccountCreation +type PerformAccountCreationResponse struct { + AccountCreated bool + UserID string +} + +// PerformDeviceCreationRequest is the request for PerformDeviceCreation +type PerformDeviceCreationRequest struct { + Localpart string + AccessToken string // optional: if blank one will be made on your behalf + // optional: if nil an ID is generated for you. If set, replaces any existing device session, + // which will generate a new access token and invalidate the old one. + DeviceID *string + // optional: if nil no display name will be associated with this device. + DeviceDisplayName *string +} + +// PerformDeviceCreationResponse is the response for PerformDeviceCreation +type PerformDeviceCreationResponse struct { + DeviceCreated bool + AccessToken string + DeviceID string +} + // Device represents a client's device (mobile, web, etc) type Device struct { ID string @@ -108,3 +142,22 @@ type ErrorForbidden struct { func (e *ErrorForbidden) Error() string { return "Forbidden: " + e.Message } + +// ErrorConflict is an error indicating that there was a conflict which resulted in the request being aborted. +type ErrorConflict struct { + Message string +} + +func (e *ErrorConflict) Error() string { + return "Conflict: " + e.Message +} + +// Conflict is an enum representing what to do when encountering conflicting when creating profiles/devices +type Conflict int + +const ( + // ConflictUpdate will update matching records returning no error + ConflictUpdate Conflict = 1 + // ConflictAbort will reject the request with ErrorConflict + ConflictAbort Conflict = 2 +) diff --git a/userapi/internal/api.go b/userapi/internal/api.go index 6e737b81f..1b34dc7b7 100644 --- a/userapi/internal/api.go +++ b/userapi/internal/api.go @@ -17,6 +17,7 @@ package internal import ( "context" "database/sql" + "errors" "fmt" "github.com/matrix-org/dendrite/appservice/types" @@ -24,6 +25,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/clientapi/userutil" "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" ) @@ -36,6 +38,38 @@ type UserInternalAPI struct { AppServices []config.ApplicationService } +func (a *UserInternalAPI) PerformAccountCreation(ctx context.Context, req *api.PerformAccountCreationRequest, res *api.PerformAccountCreationResponse) error { + acc, err := a.AccountDB.CreateAccount(ctx, req.Localpart, req.Password, req.AppServiceID) + if err != nil { + if errors.Is(err, sqlutil.ErrUserExists) { // This account already exists + switch req.OnConflict { + case api.ConflictUpdate: + break + case api.ConflictAbort: + return &api.ErrorConflict{ + Message: err.Error(), + } + } + } + res.AccountCreated = false + res.UserID = fmt.Sprintf("@%s:%s", req.Localpart, a.ServerName) + return nil + } + res.AccountCreated = true + res.UserID = acc.UserID + return nil +} +func (a *UserInternalAPI) PerformDeviceCreation(ctx context.Context, req *api.PerformDeviceCreationRequest, res *api.PerformDeviceCreationResponse) error { + dev, err := a.DeviceDB.CreateDevice(ctx, req.Localpart, req.DeviceID, req.AccessToken, req.DeviceDisplayName) + if err != nil { + return err + } + res.DeviceCreated = true + res.AccessToken = dev.AccessToken + res.DeviceID = dev.ID + return nil +} + func (a *UserInternalAPI) QueryProfile(ctx context.Context, req *api.QueryProfileRequest, res *api.QueryProfileResponse) error { local, domain, err := gomatrixserverlib.SplitID('@', req.UserID) if err != nil { diff --git a/userapi/inthttp/client.go b/userapi/inthttp/client.go index 48e6d7d72..0e9628c58 100644 --- a/userapi/inthttp/client.go +++ b/userapi/inthttp/client.go @@ -26,6 +26,9 @@ import ( // HTTP paths for the internal HTTP APIs const ( + PerformDeviceCreationPath = "/userapi/performDeviceCreation" + PerformAccountCreationPath = "/userapi/performAccountCreation" + QueryProfilePath = "/userapi/queryProfile" QueryAccessTokenPath = "/userapi/queryAccessToken" QueryDevicesPath = "/userapi/queryDevices" @@ -52,6 +55,30 @@ type httpUserInternalAPI struct { httpClient *http.Client } +func (h *httpUserInternalAPI) PerformAccountCreation( + ctx context.Context, + request *api.PerformAccountCreationRequest, + response *api.PerformAccountCreationResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "PerformAccountCreation") + defer span.Finish() + + apiURL := h.apiURL + PerformAccountCreationPath + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + +func (h *httpUserInternalAPI) PerformDeviceCreation( + ctx context.Context, + request *api.PerformDeviceCreationRequest, + response *api.PerformDeviceCreationResponse, +) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "PerformDeviceCreation") + defer span.Finish() + + apiURL := h.apiURL + PerformDeviceCreationPath + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) +} + func (h *httpUserInternalAPI) QueryProfile( ctx context.Context, request *api.QueryProfileRequest, diff --git a/userapi/inthttp/server.go b/userapi/inthttp/server.go index 8bf2efc01..8f3be7738 100644 --- a/userapi/inthttp/server.go +++ b/userapi/inthttp/server.go @@ -25,6 +25,32 @@ import ( ) func AddRoutes(internalAPIMux *mux.Router, s api.UserInternalAPI) { + internalAPIMux.Handle(PerformAccountCreationPath, + httputil.MakeInternalAPI("performAccountCreation", func(req *http.Request) util.JSONResponse { + request := api.PerformAccountCreationRequest{} + response := api.PerformAccountCreationResponse{} + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := s.PerformAccountCreation(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) + internalAPIMux.Handle(PerformDeviceCreationPath, + httputil.MakeInternalAPI("performDeviceCreation", func(req *http.Request) util.JSONResponse { + request := api.PerformDeviceCreationRequest{} + response := api.PerformDeviceCreationResponse{} + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if err := s.PerformDeviceCreation(req.Context(), &request, &response); err != nil { + return util.ErrorResponse(err) + } + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) internalAPIMux.Handle(QueryProfilePath, httputil.MakeInternalAPI("queryProfile", func(req *http.Request) util.JSONResponse { request := api.QueryProfileRequest{} From 04c99092a46b2ad0b90645bf6553360b5f1b7da7 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 16 Jun 2020 18:31:38 +0100 Subject: [PATCH 172/200] Update whitelist for sytest media fix (#1137) * Update sytest-whitelist, are-we-synapse-yet.list * Update gomatrixserverlib * Update gomatrixserverlib * Loop avoidance * Return UTF-8 filenames * Replace quotes only, instead of using strconv.Quote * Update sytest-whitelist * Update sytest-whitelist --- are-we-synapse-yet.list | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- mediaapi/routing/download.go | 10 +++++++++- mediaapi/routing/routing.go | 16 +++++++++++++++- sytest-whitelist | 15 ++++++++++++++- 6 files changed, 43 insertions(+), 8 deletions(-) diff --git a/are-we-synapse-yet.list b/are-we-synapse-yet.list index c088c8b5e..f59f80675 100644 --- a/are-we-synapse-yet.list +++ b/are-we-synapse-yet.list @@ -97,8 +97,8 @@ rst PUT power_levels should not explode if the old power levels were empty rst Both GET and PUT work rct POST /rooms/:room_id/receipt can create receipts red POST /rooms/:room_id/read_markers can create read marker -med POST /media/v1/upload can create an upload -med GET /media/v1/download can fetch the value again +med POST /media/r0/upload can create an upload +med GET /media/r0/download can fetch the value again cap GET /capabilities is present and well formed for registered user cap GET /r0/capabilities is not public reg Register with a recaptcha diff --git a/go.mod b/go.mod index b2451d85f..6154d0f32 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200615161710-f69539c86ea5 + github.com/matrix-org/gomatrixserverlib v0.0.0-20200616150727-7ac22b6f8e65 github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible diff --git a/go.sum b/go.sum index 2578e1750..3fa242c78 100644 --- a/go.sum +++ b/go.sum @@ -371,8 +371,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 h1:Yb+Wlf github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bhrnp3Ky1qgx/fzCtCALOoGYylh2tpS9K4= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200615161710-f69539c86ea5 h1:VN7DoSFVkQF9Bv+TWuBWHLgAz9Nw9UiahFfe2oE6uiQ= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200615161710-f69539c86ea5/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200616150727-7ac22b6f8e65 h1:2CcCcBnWdDPDOqFKiGOM+mi/KDDZXSTKmvFy/0/+ZJI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200616150727-7ac22b6f8e65/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f/go.mod h1:y0oDTjZDv5SM9a2rp3bl+CU+bvTRINQsdb7YlDql5Go= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= diff --git a/mediaapi/routing/download.go b/mediaapi/routing/download.go index 1a025f6f0..3ce4ba395 100644 --- a/mediaapi/routing/download.go +++ b/mediaapi/routing/download.go @@ -21,6 +21,7 @@ import ( "io" "mime" "net/http" + "net/url" "os" "path/filepath" "regexp" @@ -302,7 +303,14 @@ func (r *downloadRequest) respondFromLocalFile( responseMetadata = r.MediaMetadata if len(responseMetadata.UploadName) > 0 { - w.Header().Set("Content-Disposition", fmt.Sprintf(`inline; filename*=utf-8"%s"`, responseMetadata.UploadName)) + uploadName, err := url.PathUnescape(string(responseMetadata.UploadName)) + if err != nil { + return nil, fmt.Errorf("url.PathUnescape: %w", err) + } + w.Header().Set("Content-Disposition", fmt.Sprintf( + `inline; filename=utf-8"%s"`, + strings.ReplaceAll(uploadName, `"`, `\"`), // escape quote marks only, as per RFC6266 + )) } } diff --git a/mediaapi/routing/routing.go b/mediaapi/routing/routing.go index 13f84c335..f85778268 100644 --- a/mediaapi/routing/routing.go +++ b/mediaapi/routing/routing.go @@ -16,6 +16,7 @@ package routing import ( "net/http" + "strings" userapi "github.com/matrix-org/dendrite/userapi/api" @@ -94,11 +95,24 @@ func makeDownloadAPI( util.SetCORSHeaders(w) // Content-Type will be overridden in case of returning file data, else we respond with JSON-formatted errors w.Header().Set("Content-Type", "application/json") + vars, _ := httputil.URLDecodeMapValues(mux.Vars(req)) + serverName := gomatrixserverlib.ServerName(vars["serverName"]) + + // For the purposes of loop avoidance, we will return a 404 if allow_remote is set to + // false in the query string and the target server name isn't our own. + // https://github.com/matrix-org/matrix-doc/pull/1265 + if allowRemote := req.URL.Query().Get("allow_remote"); strings.ToLower(allowRemote) == "false" { + if serverName != cfg.Matrix.ServerName { + w.WriteHeader(http.StatusNotFound) + return + } + } + Download( w, req, - gomatrixserverlib.ServerName(vars["serverName"]), + serverName, types.MediaID(vars["mediaId"]), cfg, db, diff --git a/sytest-whitelist b/sytest-whitelist index e59d2df16..04c6f0984 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -128,7 +128,7 @@ Outbound federation can send events # test for now. #Backfill checks the events requested belong to the room Can upload without a file name -Can download without a file name locally +#Can download without a file name locally Can upload with ASCII file name Can send image in room message AS cannot create users outside its own namespace @@ -314,3 +314,16 @@ Invalid JSON special values Invalid JSON floats Outbound federation will ignore a missing event with bad JSON for room version 6 Can download without a file name over federation +POST /media/r0/upload can create an upload +GET /media/r0/download can fetch the value again +Remote users can join room by alias +Alias creators can delete alias with no ops +Alias creators can delete canonical alias with no ops +Room members can override their displayname on a room-specific basis +displayname updates affect room member events +avatar_url updates affect room member events +Real non-joined users can get individual state for world_readable rooms after leaving +Can upload with Unicode file name +POSTed media can be thumbnailed +Remote media can be thumbnailed +Can download with Unicode file name locally From a66a3b830c53c223cb939bd010d3769f02f6ccfb Mon Sep 17 00:00:00 2001 From: Kegsay Date: Wed, 17 Jun 2020 11:22:26 +0100 Subject: [PATCH 173/200] Make userapi control account creation entirely (#1139) This makes a chokepoint with which we can finally fix 'database is locked' errors on sqlite during account creation --- appservice/appservice.go | 1 + clientapi/auth/auth.go | 3 +- clientapi/auth/authtypes/account.go | 31 ------ clientapi/auth/storage/accounts/interface.go | 9 +- .../accounts/postgres/accounts_table.go | 10 +- .../auth/storage/accounts/postgres/storage.go | 11 ++- .../accounts/sqlite3/accounts_table.go | 10 +- .../auth/storage/accounts/sqlite3/storage.go | 11 ++- clientapi/routing/login.go | 5 +- clientapi/routing/register.go | 96 +++++++++++-------- clientapi/routing/routing.go | 4 +- userapi/api/api.go | 31 ++++-- userapi/internal/api.go | 22 ++++- 13 files changed, 131 insertions(+), 113 deletions(-) delete mode 100644 clientapi/auth/authtypes/account.go diff --git a/appservice/appservice.go b/appservice/appservice.go index 84a6a9b12..728690414 100644 --- a/appservice/appservice.go +++ b/appservice/appservice.go @@ -110,6 +110,7 @@ func generateAppServiceAccount( ) error { var accRes userapi.PerformAccountCreationResponse err := userAPI.PerformAccountCreation(context.Background(), &userapi.PerformAccountCreationRequest{ + AccountType: userapi.AccountTypeUser, Localpart: as.SenderLocalpart, AppServiceID: as.ID, OnConflict: userapi.ConflictUpdate, diff --git a/clientapi/auth/auth.go b/clientapi/auth/auth.go index b8e408538..b4c39ae38 100644 --- a/clientapi/auth/auth.go +++ b/clientapi/auth/auth.go @@ -23,7 +23,6 @@ import ( "net/http" "strings" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/util" @@ -42,7 +41,7 @@ type DeviceDatabase interface { // AccountDatabase represents an account database. type AccountDatabase interface { // Look up the account matching the given localpart. - GetAccountByLocalpart(ctx context.Context, localpart string) (*authtypes.Account, error) + GetAccountByLocalpart(ctx context.Context, localpart string) (*api.Account, error) } // VerifyUserFromRequest authenticates the HTTP request, diff --git a/clientapi/auth/authtypes/account.go b/clientapi/auth/authtypes/account.go deleted file mode 100644 index fd3c15a84..000000000 --- a/clientapi/auth/authtypes/account.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2017 Vector Creations Ltd -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package authtypes - -import ( - "github.com/matrix-org/gomatrixserverlib" -) - -// Account represents a Matrix account on this home server. -type Account struct { - UserID string - Localpart string - ServerName gomatrixserverlib.ServerName - Profile *Profile - AppServiceID string - // TODO: Other flags like IsAdmin, IsGuest - // TODO: Devices - // TODO: Associations (e.g. with application services) -} diff --git a/clientapi/auth/storage/accounts/interface.go b/clientapi/auth/storage/accounts/interface.go index 3391ccbfc..13e3e2895 100644 --- a/clientapi/auth/storage/accounts/interface.go +++ b/clientapi/auth/storage/accounts/interface.go @@ -20,20 +20,21 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" ) type Database interface { internal.PartitionStorer - GetAccountByPassword(ctx context.Context, localpart, plaintextPassword string) (*authtypes.Account, error) + GetAccountByPassword(ctx context.Context, localpart, plaintextPassword string) (*api.Account, error) GetProfileByLocalpart(ctx context.Context, localpart string) (*authtypes.Profile, error) SetAvatarURL(ctx context.Context, localpart string, avatarURL string) error SetDisplayName(ctx context.Context, localpart string, displayName string) error // CreateAccount makes a new account with the given login name and password, and creates an empty profile // for this account. If no password is supplied, the account will be a passwordless account. If the // account already exists, it will return nil, ErrUserExists. - CreateAccount(ctx context.Context, localpart, plaintextPassword, appserviceID string) (*authtypes.Account, error) - CreateGuestAccount(ctx context.Context) (*authtypes.Account, error) + CreateAccount(ctx context.Context, localpart, plaintextPassword, appserviceID string) (*api.Account, error) + CreateGuestAccount(ctx context.Context) (*api.Account, error) UpdateMemberships(ctx context.Context, eventsToAdd []gomatrixserverlib.Event, idsToRemove []string) error GetMembershipInRoomByLocalpart(ctx context.Context, localpart, roomID string) (authtypes.Membership, error) GetRoomIDsByLocalPart(ctx context.Context, localpart string) ([]string, error) @@ -53,7 +54,7 @@ type Database interface { GetFilter(ctx context.Context, localpart string, filterID string) (*gomatrixserverlib.Filter, error) PutFilter(ctx context.Context, localpart string, filter *gomatrixserverlib.Filter) (string, error) CheckAccountAvailability(ctx context.Context, localpart string) (bool, error) - GetAccountByLocalpart(ctx context.Context, localpart string) (*authtypes.Account, error) + GetAccountByLocalpart(ctx context.Context, localpart string) (*api.Account, error) } // Err3PIDInUse is the error returned when trying to save an association involving diff --git a/clientapi/auth/storage/accounts/postgres/accounts_table.go b/clientapi/auth/storage/accounts/postgres/accounts_table.go index 85c1938a1..931ffb73d 100644 --- a/clientapi/auth/storage/accounts/postgres/accounts_table.go +++ b/clientapi/auth/storage/accounts/postgres/accounts_table.go @@ -19,8 +19,8 @@ import ( "database/sql" "time" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/userutil" + "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" log "github.com/sirupsen/logrus" @@ -92,7 +92,7 @@ func (s *accountsStatements) prepare(db *sql.DB, server gomatrixserverlib.Server // on success. func (s *accountsStatements) insertAccount( ctx context.Context, txn *sql.Tx, localpart, hash, appserviceID string, -) (*authtypes.Account, error) { +) (*api.Account, error) { createdTimeMS := time.Now().UnixNano() / 1000000 stmt := txn.Stmt(s.insertAccountStmt) @@ -106,7 +106,7 @@ func (s *accountsStatements) insertAccount( return nil, err } - return &authtypes.Account{ + return &api.Account{ Localpart: localpart, UserID: userutil.MakeUserID(localpart, s.serverName), ServerName: s.serverName, @@ -123,9 +123,9 @@ func (s *accountsStatements) selectPasswordHash( func (s *accountsStatements) selectAccountByLocalpart( ctx context.Context, localpart string, -) (*authtypes.Account, error) { +) (*api.Account, error) { var appserviceIDPtr sql.NullString - var acc authtypes.Account + var acc api.Account stmt := s.selectAccountByLocalpartStmt err := stmt.QueryRowContext(ctx, localpart).Scan(&acc.Localpart, &appserviceIDPtr) diff --git a/clientapi/auth/storage/accounts/postgres/storage.go b/clientapi/auth/storage/accounts/postgres/storage.go index fcb592aef..2b88cb70a 100644 --- a/clientapi/auth/storage/accounts/postgres/storage.go +++ b/clientapi/auth/storage/accounts/postgres/storage.go @@ -22,6 +22,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "golang.org/x/crypto/bcrypt" @@ -84,7 +85,7 @@ func NewDatabase(dataSourceName string, dbProperties sqlutil.DbProperties, serve // Returns sql.ErrNoRows if no account exists which matches the given localpart. func (d *Database) GetAccountByPassword( ctx context.Context, localpart, plaintextPassword string, -) (*authtypes.Account, error) { +) (*api.Account, error) { hash, err := d.accounts.selectPasswordHash(ctx, localpart) if err != nil { return nil, err @@ -121,7 +122,7 @@ func (d *Database) SetDisplayName( // CreateGuestAccount makes a new guest account and creates an empty profile // for this account. -func (d *Database) CreateGuestAccount(ctx context.Context) (acc *authtypes.Account, err error) { +func (d *Database) CreateGuestAccount(ctx context.Context) (acc *api.Account, err error) { err = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { var numLocalpart int64 numLocalpart, err = d.accounts.selectNewNumericLocalpart(ctx, txn) @@ -140,7 +141,7 @@ func (d *Database) CreateGuestAccount(ctx context.Context) (acc *authtypes.Accou // account already exists, it will return nil, sqlutil.ErrUserExists. func (d *Database) CreateAccount( ctx context.Context, localpart, plaintextPassword, appserviceID string, -) (acc *authtypes.Account, err error) { +) (acc *api.Account, err error) { err = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { acc, err = d.createAccount(ctx, txn, localpart, plaintextPassword, appserviceID) return err @@ -150,7 +151,7 @@ func (d *Database) CreateAccount( func (d *Database) createAccount( ctx context.Context, txn *sql.Tx, localpart, plaintextPassword, appserviceID string, -) (*authtypes.Account, error) { +) (*api.Account, error) { var err error // Generate a password hash if this is not a password-less user @@ -427,6 +428,6 @@ func (d *Database) CheckAccountAvailability(ctx context.Context, localpart strin // This function assumes the request is authenticated or the account data is used only internally. // Returns sql.ErrNoRows if no account exists which matches the given localpart. func (d *Database) GetAccountByLocalpart(ctx context.Context, localpart string, -) (*authtypes.Account, error) { +) (*api.Account, error) { return d.accounts.selectAccountByLocalpart(ctx, localpart) } diff --git a/clientapi/auth/storage/accounts/sqlite3/accounts_table.go b/clientapi/auth/storage/accounts/sqlite3/accounts_table.go index fd6a09cde..768f536dd 100644 --- a/clientapi/auth/storage/accounts/sqlite3/accounts_table.go +++ b/clientapi/auth/storage/accounts/sqlite3/accounts_table.go @@ -19,8 +19,8 @@ import ( "database/sql" "time" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/userutil" + "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" log "github.com/sirupsen/logrus" @@ -90,7 +90,7 @@ func (s *accountsStatements) prepare(db *sql.DB, server gomatrixserverlib.Server // on success. func (s *accountsStatements) insertAccount( ctx context.Context, txn *sql.Tx, localpart, hash, appserviceID string, -) (*authtypes.Account, error) { +) (*api.Account, error) { createdTimeMS := time.Now().UnixNano() / 1000000 stmt := s.insertAccountStmt @@ -104,7 +104,7 @@ func (s *accountsStatements) insertAccount( return nil, err } - return &authtypes.Account{ + return &api.Account{ Localpart: localpart, UserID: userutil.MakeUserID(localpart, s.serverName), ServerName: s.serverName, @@ -121,9 +121,9 @@ func (s *accountsStatements) selectPasswordHash( func (s *accountsStatements) selectAccountByLocalpart( ctx context.Context, localpart string, -) (*authtypes.Account, error) { +) (*api.Account, error) { var appserviceIDPtr sql.NullString - var acc authtypes.Account + var acc api.Account stmt := s.selectAccountByLocalpartStmt err := stmt.QueryRowContext(ctx, localpart).Scan(&acc.Localpart, &appserviceIDPtr) diff --git a/clientapi/auth/storage/accounts/sqlite3/storage.go b/clientapi/auth/storage/accounts/sqlite3/storage.go index 44245a99d..4dd755a70 100644 --- a/clientapi/auth/storage/accounts/sqlite3/storage.go +++ b/clientapi/auth/storage/accounts/sqlite3/storage.go @@ -23,6 +23,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "golang.org/x/crypto/bcrypt" // Import the sqlite3 database driver. @@ -89,7 +90,7 @@ func NewDatabase(dataSourceName string, serverName gomatrixserverlib.ServerName) // Returns sql.ErrNoRows if no account exists which matches the given localpart. func (d *Database) GetAccountByPassword( ctx context.Context, localpart, plaintextPassword string, -) (*authtypes.Account, error) { +) (*api.Account, error) { hash, err := d.accounts.selectPasswordHash(ctx, localpart) if err != nil { return nil, err @@ -126,7 +127,7 @@ func (d *Database) SetDisplayName( // CreateGuestAccount makes a new guest account and creates an empty profile // for this account. -func (d *Database) CreateGuestAccount(ctx context.Context) (acc *authtypes.Account, err error) { +func (d *Database) CreateGuestAccount(ctx context.Context) (acc *api.Account, err error) { err = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { // We need to lock so we sequentially create numeric localparts. If we don't, two calls to // this function will cause the same number to be selected and one will fail with 'database is locked' @@ -152,7 +153,7 @@ func (d *Database) CreateGuestAccount(ctx context.Context) (acc *authtypes.Accou // account already exists, it will return nil, ErrUserExists. func (d *Database) CreateAccount( ctx context.Context, localpart, plaintextPassword, appserviceID string, -) (acc *authtypes.Account, err error) { +) (acc *api.Account, err error) { err = sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { acc, err = d.createAccount(ctx, txn, localpart, plaintextPassword, appserviceID) return err @@ -162,7 +163,7 @@ func (d *Database) CreateAccount( func (d *Database) createAccount( ctx context.Context, txn *sql.Tx, localpart, plaintextPassword, appserviceID string, -) (*authtypes.Account, error) { +) (*api.Account, error) { var err error // Generate a password hash if this is not a password-less user hash := "" @@ -438,6 +439,6 @@ func (d *Database) CheckAccountAvailability(ctx context.Context, localpart strin // This function assumes the request is authenticated or the account data is used only internally. // Returns sql.ErrNoRows if no account exists which matches the given localpart. func (d *Database) GetAccountByLocalpart(ctx context.Context, localpart string, -) (*authtypes.Account, error) { +) (*api.Account, error) { return d.accounts.selectAccountByLocalpart(ctx, localpart) } diff --git a/clientapi/routing/login.go b/clientapi/routing/login.go index 2eb480ef1..25231a3aa 100644 --- a/clientapi/routing/login.go +++ b/clientapi/routing/login.go @@ -20,7 +20,6 @@ import ( "context" "github.com/matrix-org/dendrite/clientapi/auth" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/clientapi/httputil" @@ -81,7 +80,7 @@ func Login( } } else if req.Method == http.MethodPost { var r passwordRequest - var acc *authtypes.Account + var acc *api.Account resErr := httputil.UnmarshalJSONRequest(req, &r) if resErr != nil { return *resErr @@ -156,7 +155,7 @@ func getDevice( ctx context.Context, r passwordRequest, deviceDB devices.Database, - acc *authtypes.Account, + acc *api.Account, token string, ) (dev *api.Device, err error) { dev, err = deviceDB.CreateDevice( diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index 8988dbd05..fddf9253a 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -34,15 +34,14 @@ import ( "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/eventutil" - "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/clientapi/auth" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" - "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/userutil" + userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/tokens" "github.com/matrix-org/util" @@ -441,8 +440,8 @@ func validateApplicationService( // http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#post-matrix-client-unstable-register func Register( req *http.Request, + userAPI userapi.UserInternalAPI, accountDB accounts.Database, - deviceDB devices.Database, cfg *config.Dendrite, ) util.JSONResponse { var r registerRequest @@ -451,7 +450,7 @@ func Register( return *resErr } if req.URL.Query().Get("kind") == "guest" { - return handleGuestRegistration(req, r, cfg, accountDB, deviceDB) + return handleGuestRegistration(req, r, cfg, userAPI) } // Retrieve or generate the sessionID @@ -507,17 +506,19 @@ func Register( "session_id": r.Auth.Session, }).Info("Processing registration request") - return handleRegistrationFlow(req, r, sessionID, cfg, accountDB, deviceDB) + return handleRegistrationFlow(req, r, sessionID, cfg, userAPI) } func handleGuestRegistration( req *http.Request, r registerRequest, cfg *config.Dendrite, - accountDB accounts.Database, - deviceDB devices.Database, + userAPI userapi.UserInternalAPI, ) util.JSONResponse { - acc, err := accountDB.CreateGuestAccount(req.Context()) + var res userapi.PerformAccountCreationResponse + err := userAPI.PerformAccountCreation(req.Context(), &userapi.PerformAccountCreationRequest{ + AccountType: userapi.AccountTypeGuest, + }, &res) if err != nil { return util.JSONResponse{ Code: http.StatusInternalServerError, @@ -526,8 +527,8 @@ func handleGuestRegistration( } token, err := tokens.GenerateLoginToken(tokens.TokenOptions{ ServerPrivateKey: cfg.Matrix.PrivateKey.Seed(), - ServerName: string(acc.ServerName), - UserID: acc.UserID, + ServerName: string(res.Account.ServerName), + UserID: res.Account.UserID, }) if err != nil { @@ -537,7 +538,12 @@ func handleGuestRegistration( } } //we don't allow guests to specify their own device_id - dev, err := deviceDB.CreateDevice(req.Context(), acc.Localpart, nil, token, r.InitialDisplayName) + var devRes userapi.PerformDeviceCreationResponse + err = userAPI.PerformDeviceCreation(req.Context(), &userapi.PerformDeviceCreationRequest{ + Localpart: res.Account.Localpart, + DeviceDisplayName: r.InitialDisplayName, + AccessToken: token, + }, &devRes) if err != nil { return util.JSONResponse{ Code: http.StatusInternalServerError, @@ -547,10 +553,10 @@ func handleGuestRegistration( return util.JSONResponse{ Code: http.StatusOK, JSON: registerResponse{ - UserID: dev.UserID, - AccessToken: dev.AccessToken, - HomeServer: acc.ServerName, - DeviceID: dev.ID, + UserID: devRes.Device.UserID, + AccessToken: devRes.Device.AccessToken, + HomeServer: res.Account.ServerName, + DeviceID: devRes.Device.ID, }, } } @@ -563,8 +569,7 @@ func handleRegistrationFlow( r registerRequest, sessionID string, cfg *config.Dendrite, - accountDB accounts.Database, - deviceDB devices.Database, + userAPI userapi.UserInternalAPI, ) util.JSONResponse { // TODO: Shared secret registration (create new user scripts) // TODO: Enable registration config flag @@ -615,7 +620,7 @@ func handleRegistrationFlow( // by whether the request contains an access token. if err == nil { return handleApplicationServiceRegistration( - accessToken, err, req, r, cfg, accountDB, deviceDB, + accessToken, err, req, r, cfg, userAPI, ) } @@ -626,7 +631,7 @@ func handleRegistrationFlow( // don't need a condition on that call since the registration is clearly // stated as being AS-related. return handleApplicationServiceRegistration( - accessToken, err, req, r, cfg, accountDB, deviceDB, + accessToken, err, req, r, cfg, userAPI, ) case authtypes.LoginTypeDummy: @@ -645,7 +650,7 @@ func handleRegistrationFlow( // A response with current registration flow and remaining available methods // will be returned if a flow has not been successfully completed yet return checkAndCompleteFlow(sessions.GetCompletedStages(sessionID), - req, r, sessionID, cfg, accountDB, deviceDB) + req, r, sessionID, cfg, userAPI) } // handleApplicationServiceRegistration handles the registration of an @@ -662,8 +667,7 @@ func handleApplicationServiceRegistration( req *http.Request, r registerRequest, cfg *config.Dendrite, - accountDB accounts.Database, - deviceDB devices.Database, + userAPI userapi.UserInternalAPI, ) util.JSONResponse { // Check if we previously had issues extracting the access token from the // request. @@ -687,7 +691,7 @@ func handleApplicationServiceRegistration( // Don't need to worry about appending to registration stages as // application service registration is entirely separate. return completeRegistration( - req.Context(), accountDB, deviceDB, r.Username, "", appserviceID, + req.Context(), userAPI, r.Username, "", appserviceID, r.InhibitLogin, r.InitialDisplayName, r.DeviceID, ) } @@ -701,13 +705,12 @@ func checkAndCompleteFlow( r registerRequest, sessionID string, cfg *config.Dendrite, - accountDB accounts.Database, - deviceDB devices.Database, + userAPI userapi.UserInternalAPI, ) util.JSONResponse { if checkFlowCompleted(flow, cfg.Derived.Registration.Flows) { // This flow was completed, registration can continue return completeRegistration( - req.Context(), accountDB, deviceDB, r.Username, r.Password, "", + req.Context(), userAPI, r.Username, r.Password, "", r.InhibitLogin, r.InitialDisplayName, r.DeviceID, ) } @@ -724,8 +727,7 @@ func checkAndCompleteFlow( // LegacyRegister process register requests from the legacy v1 API func LegacyRegister( req *http.Request, - accountDB accounts.Database, - deviceDB devices.Database, + userAPI userapi.UserInternalAPI, cfg *config.Dendrite, ) util.JSONResponse { var r legacyRegisterRequest @@ -760,10 +762,10 @@ func LegacyRegister( return util.MessageResponse(http.StatusForbidden, "HMAC incorrect") } - return completeRegistration(req.Context(), accountDB, deviceDB, r.Username, r.Password, "", false, nil, nil) + return completeRegistration(req.Context(), userAPI, r.Username, r.Password, "", false, nil, nil) case authtypes.LoginTypeDummy: // there is nothing to do - return completeRegistration(req.Context(), accountDB, deviceDB, r.Username, r.Password, "", false, nil, nil) + return completeRegistration(req.Context(), userAPI, r.Username, r.Password, "", false, nil, nil) default: return util.JSONResponse{ Code: http.StatusNotImplemented, @@ -809,8 +811,7 @@ func parseAndValidateLegacyLogin(req *http.Request, r *legacyRegisterRequest) *u // not all func completeRegistration( ctx context.Context, - accountDB accounts.Database, - deviceDB devices.Database, + userAPI userapi.UserInternalAPI, username, password, appserviceID string, inhibitLogin eventutil.WeakBoolean, displayName, deviceID *string, @@ -829,9 +830,16 @@ func completeRegistration( } } - acc, err := accountDB.CreateAccount(ctx, username, password, appserviceID) + var accRes userapi.PerformAccountCreationResponse + err := userAPI.PerformAccountCreation(ctx, &userapi.PerformAccountCreationRequest{ + AppServiceID: appserviceID, + Localpart: username, + Password: password, + AccountType: userapi.AccountTypeUser, + OnConflict: userapi.ConflictAbort, + }, &accRes) if err != nil { - if errors.Is(err, sqlutil.ErrUserExists) { // user already exists + if _, ok := err.(*userapi.ErrorConflict); ok { // user already exists return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.UserInUse("Desired user ID is already taken."), @@ -852,8 +860,8 @@ func completeRegistration( return util.JSONResponse{ Code: http.StatusOK, JSON: registerResponse{ - UserID: userutil.MakeUserID(username, acc.ServerName), - HomeServer: acc.ServerName, + UserID: userutil.MakeUserID(username, accRes.Account.ServerName), + HomeServer: accRes.Account.ServerName, }, } } @@ -866,7 +874,13 @@ func completeRegistration( } } - dev, err := deviceDB.CreateDevice(ctx, username, deviceID, token, displayName) + var devRes userapi.PerformDeviceCreationResponse + err = userAPI.PerformDeviceCreation(ctx, &userapi.PerformDeviceCreationRequest{ + Localpart: username, + AccessToken: token, + DeviceDisplayName: displayName, + DeviceID: deviceID, + }, &devRes) if err != nil { return util.JSONResponse{ Code: http.StatusInternalServerError, @@ -877,10 +891,10 @@ func completeRegistration( return util.JSONResponse{ Code: http.StatusOK, JSON: registerResponse{ - UserID: dev.UserID, - AccessToken: dev.AccessToken, - HomeServer: acc.ServerName, - DeviceID: dev.ID, + UserID: devRes.Device.UserID, + AccessToken: devRes.Device.AccessToken, + HomeServer: accRes.Account.ServerName, + DeviceID: devRes.Device.ID, }, } } diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 80d9ab668..5e8a606ac 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -203,11 +203,11 @@ func Setup( ).Methods(http.MethodPut, http.MethodOptions) r0mux.Handle("/register", httputil.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse { - return Register(req, accountDB, deviceDB, cfg) + return Register(req, userAPI, accountDB, cfg) })).Methods(http.MethodPost, http.MethodOptions) v1mux.Handle("/register", httputil.MakeExternalAPI("register", func(req *http.Request) util.JSONResponse { - return LegacyRegister(req, accountDB, deviceDB, cfg) + return LegacyRegister(req, userAPI, cfg) })).Methods(http.MethodPost, http.MethodOptions) r0mux.Handle("/register/available", httputil.MakeExternalAPI("registerAvailable", func(req *http.Request) util.JSONResponse { diff --git a/userapi/api/api.go b/userapi/api/api.go index 34c74bb3c..c953a5bac 100644 --- a/userapi/api/api.go +++ b/userapi/api/api.go @@ -89,16 +89,18 @@ type QueryProfileResponse struct { // PerformAccountCreationRequest is the request for PerformAccountCreation type PerformAccountCreationRequest struct { - Localpart string - AppServiceID string - Password string + AccountType AccountType // Required: whether this is a guest or user account + Localpart string // Required: The localpart for this account. Ignored if account type is guest. + + AppServiceID string // optional: the application service ID (not user ID) creating this account, if any. + Password string // optional: if missing then this account will be a passwordless account OnConflict Conflict } // PerformAccountCreationResponse is the response for PerformAccountCreation type PerformAccountCreationResponse struct { AccountCreated bool - UserID string + Account *Account } // PerformDeviceCreationRequest is the request for PerformDeviceCreation @@ -115,8 +117,7 @@ type PerformDeviceCreationRequest struct { // PerformDeviceCreationResponse is the response for PerformDeviceCreation type PerformDeviceCreationResponse struct { DeviceCreated bool - AccessToken string - DeviceID string + Device *Device } // Device represents a client's device (mobile, web, etc) @@ -134,6 +135,16 @@ type Device struct { DisplayName string } +// Account represents a Matrix account on this home server. +type Account struct { + UserID string + Localpart string + ServerName gomatrixserverlib.ServerName + AppServiceID string + // TODO: Other flags like IsAdmin, IsGuest + // TODO: Associations (e.g. with application services) +} + // ErrorForbidden is an error indicating that the supplied access token is forbidden type ErrorForbidden struct { Message string @@ -155,9 +166,17 @@ func (e *ErrorConflict) Error() string { // Conflict is an enum representing what to do when encountering conflicting when creating profiles/devices type Conflict int +// AccountType is an enum representing the kind of account +type AccountType int + const ( // ConflictUpdate will update matching records returning no error ConflictUpdate Conflict = 1 // ConflictAbort will reject the request with ErrorConflict ConflictAbort Conflict = 2 + + // AccountTypeUser indicates this is a user account + AccountTypeUser AccountType = 1 + // AccountTypeGuest indicates this is a guest account + AccountTypeGuest AccountType = 2 ) diff --git a/userapi/internal/api.go b/userapi/internal/api.go index 1b34dc7b7..3a4131666 100644 --- a/userapi/internal/api.go +++ b/userapi/internal/api.go @@ -39,6 +39,15 @@ type UserInternalAPI struct { } func (a *UserInternalAPI) PerformAccountCreation(ctx context.Context, req *api.PerformAccountCreationRequest, res *api.PerformAccountCreationResponse) error { + if req.AccountType == api.AccountTypeGuest { + acc, err := a.AccountDB.CreateGuestAccount(ctx) + if err != nil { + return err + } + res.AccountCreated = true + res.Account = acc + return nil + } acc, err := a.AccountDB.CreateAccount(ctx, req.Localpart, req.Password, req.AppServiceID) if err != nil { if errors.Is(err, sqlutil.ErrUserExists) { // This account already exists @@ -51,12 +60,18 @@ func (a *UserInternalAPI) PerformAccountCreation(ctx context.Context, req *api.P } } } + // account already exists res.AccountCreated = false - res.UserID = fmt.Sprintf("@%s:%s", req.Localpart, a.ServerName) + res.Account = &api.Account{ + AppServiceID: req.AppServiceID, + Localpart: req.Localpart, + ServerName: a.ServerName, + UserID: fmt.Sprintf("@%s:%s", req.Localpart, a.ServerName), + } return nil } res.AccountCreated = true - res.UserID = acc.UserID + res.Account = acc return nil } func (a *UserInternalAPI) PerformDeviceCreation(ctx context.Context, req *api.PerformDeviceCreationRequest, res *api.PerformDeviceCreationResponse) error { @@ -65,8 +80,7 @@ func (a *UserInternalAPI) PerformDeviceCreation(ctx context.Context, req *api.Pe return err } res.DeviceCreated = true - res.AccessToken = dev.AccessToken - res.DeviceID = dev.ID + res.Device = dev return nil } From 5d5aa0a31d60941c7ece95b4b516044cb8a10cce Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 17 Jun 2020 11:53:26 +0100 Subject: [PATCH 174/200] Media filename handling improvements (#1140) * Derive content ID from hash+filename but preserve dedupe, improve Content-Disposition handling and ASCII handling * Linter fix * Some more comments * Update sytest-whitelist --- mediaapi/routing/download.go | 77 +++++++++++++++++++++++++++++++----- mediaapi/routing/routing.go | 5 ++- mediaapi/routing/upload.go | 5 ++- sytest-whitelist | 6 ++- 4 files changed, 80 insertions(+), 13 deletions(-) diff --git a/mediaapi/routing/download.go b/mediaapi/routing/download.go index 3ce4ba395..fa1bb2573 100644 --- a/mediaapi/routing/download.go +++ b/mediaapi/routing/download.go @@ -28,6 +28,7 @@ import ( "strconv" "strings" "sync" + "unicode" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/internal/config" @@ -54,6 +55,7 @@ type downloadRequest struct { IsThumbnailRequest bool ThumbnailSize types.ThumbnailSize Logger *log.Entry + DownloadFilename string } // Download implements GET /download and GET /thumbnail @@ -73,6 +75,7 @@ func Download( activeRemoteRequests *types.ActiveRemoteRequests, activeThumbnailGeneration *types.ActiveThumbnailGeneration, isThumbnailRequest bool, + customFilename string, ) { dReq := &downloadRequest{ MediaMetadata: &types.MediaMetadata{ @@ -84,6 +87,7 @@ func Download( "Origin": origin, "MediaID": mediaID, }), + DownloadFilename: customFilename, } if dReq.IsThumbnailRequest { @@ -301,16 +305,8 @@ func (r *downloadRequest) respondFromLocalFile( }).Info("Responding with file") responseFile = file responseMetadata = r.MediaMetadata - - if len(responseMetadata.UploadName) > 0 { - uploadName, err := url.PathUnescape(string(responseMetadata.UploadName)) - if err != nil { - return nil, fmt.Errorf("url.PathUnescape: %w", err) - } - w.Header().Set("Content-Disposition", fmt.Sprintf( - `inline; filename=utf-8"%s"`, - strings.ReplaceAll(uploadName, `"`, `\"`), // escape quote marks only, as per RFC6266 - )) + if err := r.addDownloadFilenameToHeaders(w, responseMetadata); err != nil { + return nil, err } } @@ -329,6 +325,67 @@ func (r *downloadRequest) respondFromLocalFile( return responseMetadata, nil } +func (r *downloadRequest) addDownloadFilenameToHeaders( + w http.ResponseWriter, + responseMetadata *types.MediaMetadata, +) error { + // If the requestor supplied a filename to name the download then + // use that, otherwise use the filename from the response metadata. + filename := string(responseMetadata.UploadName) + if r.DownloadFilename != "" { + filename = r.DownloadFilename + } + + if len(filename) == 0 { + return nil + } + + unescaped, err := url.PathUnescape(filename) + if err != nil { + return fmt.Errorf("url.PathUnescape: %w", err) + } + + isASCII := true // Is the string ASCII or UTF-8? + quote := `` // Encloses the string (ASCII only) + for i := 0; i < len(unescaped); i++ { + if unescaped[i] > unicode.MaxASCII { + isASCII = false + } + if unescaped[i] == 0x20 || unescaped[i] == 0x3B { + // If the filename contains a space or a semicolon, which + // are special characters in Content-Disposition + quote = `"` + } + } + + // We don't necessarily want a full escape as the Content-Disposition + // can take many of the characters that PathEscape would otherwise and + // browser support for encoding is a bit wild, so we'll escape only + // the characters that we know will mess up the parsing of the + // Content-Disposition header elements themselves + unescaped = strings.ReplaceAll(unescaped, `\`, `\\"`) + unescaped = strings.ReplaceAll(unescaped, `"`, `\"`) + + if isASCII { + // For ASCII filenames, we should only quote the filename if + // it needs to be done, e.g. it contains a space or a character + // that would otherwise be parsed as a control character in the + // Content-Disposition header + w.Header().Set("Content-Disposition", fmt.Sprintf( + `inline; filename=%s%s%s`, + quote, unescaped, quote, + )) + } else { + // For UTF-8 filenames, we quote always, as that's the standard + w.Header().Set("Content-Disposition", fmt.Sprintf( + `inline; filename=utf-8"%s"`, + unescaped, + )) + } + + return nil +} + // Note: Thumbnail generation may be ongoing asynchronously. // If no thumbnail was found then returns nil, nil, nil func (r *downloadRequest) getThumbnailFile( diff --git a/mediaapi/routing/routing.go b/mediaapi/routing/routing.go index f85778268..bc0de0f45 100644 --- a/mediaapi/routing/routing.go +++ b/mediaapi/routing/routing.go @@ -66,7 +66,9 @@ func Setup( downloadHandler := makeDownloadAPI("download", cfg, db, client, activeRemoteRequests, activeThumbnailGeneration) r0mux.Handle("/download/{serverName}/{mediaId}", downloadHandler).Methods(http.MethodGet, http.MethodOptions) - v1mux.Handle("/download/{serverName}/{mediaId}", downloadHandler).Methods(http.MethodGet, http.MethodOptions) // TODO: remove when synapse is fixed + r0mux.Handle("/download/{serverName}/{mediaId}/{downloadName}", downloadHandler).Methods(http.MethodGet, http.MethodOptions) + v1mux.Handle("/download/{serverName}/{mediaId}", downloadHandler).Methods(http.MethodGet, http.MethodOptions) // TODO: remove when synapse is fixed + v1mux.Handle("/download/{serverName}/{mediaId}/{downloadName}", downloadHandler).Methods(http.MethodGet, http.MethodOptions) // TODO: remove when synapse is fixed r0mux.Handle("/thumbnail/{serverName}/{mediaId}", makeDownloadAPI("thumbnail", cfg, db, client, activeRemoteRequests, activeThumbnailGeneration), @@ -120,6 +122,7 @@ func makeDownloadAPI( activeRemoteRequests, activeThumbnailGeneration, name == "thumbnail", + vars["downloadName"], ) } return promhttp.InstrumentHandlerCounter(counterVec, http.HandlerFunc(httpHandler)) diff --git a/mediaapi/routing/upload.go b/mediaapi/routing/upload.go index 022f978d6..9b5dc3df8 100644 --- a/mediaapi/routing/upload.go +++ b/mediaapi/routing/upload.go @@ -16,6 +16,7 @@ package routing import ( "context" + "encoding/base64" "fmt" "io" "net/http" @@ -123,7 +124,9 @@ func (r *uploadRequest) doUpload( r.MediaMetadata.FileSizeBytes = bytesWritten r.MediaMetadata.Base64Hash = hash - r.MediaMetadata.MediaID = types.MediaID(hash) + r.MediaMetadata.MediaID = types.MediaID(base64.RawURLEncoding.EncodeToString( + []byte(string(r.MediaMetadata.UploadName) + string(r.MediaMetadata.Base64Hash)), + )) r.Logger = r.Logger.WithField("MediaID", r.MediaMetadata.MediaID) diff --git a/sytest-whitelist b/sytest-whitelist index 04c6f0984..8f3d128d7 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -128,7 +128,7 @@ Outbound federation can send events # test for now. #Backfill checks the events requested belong to the room Can upload without a file name -#Can download without a file name locally +Can download without a file name locally Can upload with ASCII file name Can send image in room message AS cannot create users outside its own namespace @@ -327,3 +327,7 @@ Can upload with Unicode file name POSTed media can be thumbnailed Remote media can be thumbnailed Can download with Unicode file name locally +Can download file 'ascii' +Can download file 'name with spaces' +Can download file 'name;with;semicolons' +Can download specifying a different ASCII file name From e09d24e7323e73791e7bb31fa7fac1d3acf0c299 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Wed, 17 Jun 2020 12:05:56 +0100 Subject: [PATCH 175/200] Move account/device DBs to userapi (#1141) --- appservice/api/query.go | 2 +- clientapi/clientapi.go | 4 ++-- clientapi/consumers/roomserver.go | 2 +- clientapi/routing/account_data.go | 2 +- clientapi/routing/createroom.go | 2 +- clientapi/routing/device.go | 2 +- clientapi/routing/filter.go | 2 +- clientapi/routing/joinroom.go | 2 +- clientapi/routing/login.go | 4 ++-- clientapi/routing/logout.go | 2 +- clientapi/routing/membership.go | 2 +- clientapi/routing/memberships.go | 2 +- clientapi/routing/profile.go | 2 +- clientapi/routing/register.go | 2 +- clientapi/routing/room_tagging.go | 2 +- clientapi/routing/routing.go | 4 ++-- clientapi/routing/sendtyping.go | 2 +- clientapi/routing/threepid.go | 2 +- clientapi/threepid/invites.go | 2 +- cmd/create-account/main.go | 4 ++-- internal/setup/base.go | 4 ++-- internal/setup/monolith.go | 4 ++-- userapi/internal/api.go | 4 ++-- {clientapi/auth => userapi}/storage/accounts/interface.go | 0 .../storage/accounts/postgres/account_data_table.go | 0 .../storage/accounts/postgres/accounts_table.go | 0 .../storage/accounts/postgres/filter_table.go | 0 .../storage/accounts/postgres/membership_table.go | 0 .../storage/accounts/postgres/profile_table.go | 0 .../auth => userapi}/storage/accounts/postgres/storage.go | 0 .../storage/accounts/postgres/threepid_table.go | 0 .../storage/accounts/sqlite3/account_data_table.go | 0 .../storage/accounts/sqlite3/accounts_table.go | 0 .../auth => userapi}/storage/accounts/sqlite3/constraint.go | 0 .../storage/accounts/sqlite3/constraint_wasm.go | 0 .../auth => userapi}/storage/accounts/sqlite3/filter_table.go | 0 .../storage/accounts/sqlite3/membership_table.go | 0 .../storage/accounts/sqlite3/profile_table.go | 0 .../auth => userapi}/storage/accounts/sqlite3/storage.go | 0 .../storage/accounts/sqlite3/threepid_table.go | 0 {clientapi/auth => userapi}/storage/accounts/storage.go | 4 ++-- {clientapi/auth => userapi}/storage/accounts/storage_wasm.go | 2 +- {clientapi/auth => userapi}/storage/devices/interface.go | 0 .../storage/devices/postgres/devices_table.go | 0 .../auth => userapi}/storage/devices/postgres/storage.go | 0 .../auth => userapi}/storage/devices/sqlite3/devices_table.go | 0 .../auth => userapi}/storage/devices/sqlite3/storage.go | 0 {clientapi/auth => userapi}/storage/devices/storage.go | 4 ++-- {clientapi/auth => userapi}/storage/devices/storage_wasm.go | 2 +- userapi/userapi.go | 4 ++-- userapi/userapi_test.go | 4 ++-- 51 files changed, 40 insertions(+), 40 deletions(-) rename {clientapi/auth => userapi}/storage/accounts/interface.go (100%) rename {clientapi/auth => userapi}/storage/accounts/postgres/account_data_table.go (100%) rename {clientapi/auth => userapi}/storage/accounts/postgres/accounts_table.go (100%) rename {clientapi/auth => userapi}/storage/accounts/postgres/filter_table.go (100%) rename {clientapi/auth => userapi}/storage/accounts/postgres/membership_table.go (100%) rename {clientapi/auth => userapi}/storage/accounts/postgres/profile_table.go (100%) rename {clientapi/auth => userapi}/storage/accounts/postgres/storage.go (100%) rename {clientapi/auth => userapi}/storage/accounts/postgres/threepid_table.go (100%) rename {clientapi/auth => userapi}/storage/accounts/sqlite3/account_data_table.go (100%) rename {clientapi/auth => userapi}/storage/accounts/sqlite3/accounts_table.go (100%) rename {clientapi/auth => userapi}/storage/accounts/sqlite3/constraint.go (100%) rename {clientapi/auth => userapi}/storage/accounts/sqlite3/constraint_wasm.go (100%) rename {clientapi/auth => userapi}/storage/accounts/sqlite3/filter_table.go (100%) rename {clientapi/auth => userapi}/storage/accounts/sqlite3/membership_table.go (100%) rename {clientapi/auth => userapi}/storage/accounts/sqlite3/profile_table.go (100%) rename {clientapi/auth => userapi}/storage/accounts/sqlite3/storage.go (100%) rename {clientapi/auth => userapi}/storage/accounts/sqlite3/threepid_table.go (100%) rename {clientapi/auth => userapi}/storage/accounts/storage.go (90%) rename {clientapi/auth => userapi}/storage/accounts/storage_wasm.go (94%) rename {clientapi/auth => userapi}/storage/devices/interface.go (100%) rename {clientapi/auth => userapi}/storage/devices/postgres/devices_table.go (100%) rename {clientapi/auth => userapi}/storage/devices/postgres/storage.go (100%) rename {clientapi/auth => userapi}/storage/devices/sqlite3/devices_table.go (100%) rename {clientapi/auth => userapi}/storage/devices/sqlite3/storage.go (100%) rename {clientapi/auth => userapi}/storage/devices/storage.go (90%) rename {clientapi/auth => userapi}/storage/devices/storage_wasm.go (94%) diff --git a/appservice/api/query.go b/appservice/api/query.go index 0a5cc9f1d..29e374aca 100644 --- a/appservice/api/query.go +++ b/appservice/api/query.go @@ -22,8 +22,8 @@ import ( "database/sql" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/internal/eventutil" + "github.com/matrix-org/dendrite/userapi/storage/accounts" "github.com/matrix-org/gomatrixserverlib" ) diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go index 637e1469e..174eb1bf1 100644 --- a/clientapi/clientapi.go +++ b/clientapi/clientapi.go @@ -18,8 +18,6 @@ import ( "github.com/Shopify/sarama" "github.com/gorilla/mux" appserviceAPI "github.com/matrix-org/dendrite/appservice/api" - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" - "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/clientapi/consumers" "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/clientapi/routing" @@ -29,6 +27,8 @@ import ( "github.com/matrix-org/dendrite/internal/transactions" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/storage/accounts" + "github.com/matrix-org/dendrite/userapi/storage/devices" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" ) diff --git a/clientapi/consumers/roomserver.go b/clientapi/consumers/roomserver.go index caa028ba3..beeda042b 100644 --- a/clientapi/consumers/roomserver.go +++ b/clientapi/consumers/roomserver.go @@ -18,10 +18,10 @@ import ( "context" "encoding/json" - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/userapi/storage/accounts" "github.com/matrix-org/gomatrixserverlib" "github.com/Shopify/sarama" diff --git a/clientapi/routing/account_data.go b/clientapi/routing/account_data.go index 5e0509a50..875249b32 100644 --- a/clientapi/routing/account_data.go +++ b/clientapi/routing/account_data.go @@ -19,10 +19,10 @@ import ( "io/ioutil" "net/http" - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/storage/accounts" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index 2bb537b0d..be7124828 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -26,12 +26,12 @@ import ( roomserverVersion "github.com/matrix-org/dendrite/roomserver/version" "github.com/matrix-org/dendrite/userapi/api" - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/threepid" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/eventutil" + "github.com/matrix-org/dendrite/userapi/storage/accounts" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" log "github.com/sirupsen/logrus" diff --git a/clientapi/routing/device.go b/clientapi/routing/device.go index 02acb462e..403937c9c 100644 --- a/clientapi/routing/device.go +++ b/clientapi/routing/device.go @@ -19,9 +19,9 @@ import ( "encoding/json" "net/http" - "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/storage/devices" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) diff --git a/clientapi/routing/filter.go b/clientapi/routing/filter.go index 7c583045f..6520e6e40 100644 --- a/clientapi/routing/filter.go +++ b/clientapi/routing/filter.go @@ -17,10 +17,10 @@ package routing import ( "net/http" - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/storage/accounts" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) diff --git a/clientapi/routing/joinroom.go b/clientapi/routing/joinroom.go index a00b34a57..e190beefd 100644 --- a/clientapi/routing/joinroom.go +++ b/clientapi/routing/joinroom.go @@ -17,11 +17,11 @@ package routing import ( "net/http" - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/storage/accounts" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) diff --git a/clientapi/routing/login.go b/clientapi/routing/login.go index 25231a3aa..1b894a566 100644 --- a/clientapi/routing/login.go +++ b/clientapi/routing/login.go @@ -20,13 +20,13 @@ import ( "context" "github.com/matrix-org/dendrite/clientapi/auth" - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" - "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/userutil" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/storage/accounts" + "github.com/matrix-org/dendrite/userapi/storage/devices" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) diff --git a/clientapi/routing/logout.go b/clientapi/routing/logout.go index f1276082b..3ce47169e 100644 --- a/clientapi/routing/logout.go +++ b/clientapi/routing/logout.go @@ -17,9 +17,9 @@ package routing import ( "net/http" - "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/storage/devices" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index c7b91613f..0d4b0d88d 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -22,7 +22,6 @@ import ( appserviceAPI "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/threepid" @@ -31,6 +30,7 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/storage/accounts" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" diff --git a/clientapi/routing/memberships.go b/clientapi/routing/memberships.go index a5bb2a908..574a2c480 100644 --- a/clientapi/routing/memberships.go +++ b/clientapi/routing/memberships.go @@ -17,7 +17,7 @@ package routing import ( "net/http" - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" + "github.com/matrix-org/dendrite/userapi/storage/accounts" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/internal/config" diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index a72ad3bcc..7c2cd19bc 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -21,13 +21,13 @@ import ( appserviceAPI "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/roomserver/api" userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/storage/accounts" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrix" diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index fddf9253a..69ebdfd70 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -37,11 +37,11 @@ import ( "github.com/matrix-org/dendrite/clientapi/auth" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/userutil" userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/storage/accounts" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/tokens" "github.com/matrix-org/util" diff --git a/clientapi/routing/room_tagging.go b/clientapi/routing/room_tagging.go index a3fe0e426..b1cfcca86 100644 --- a/clientapi/routing/room_tagging.go +++ b/clientapi/routing/room_tagging.go @@ -20,11 +20,11 @@ import ( "github.com/sirupsen/logrus" - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/storage/accounts" "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 5e8a606ac..0fe687b30 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -21,8 +21,6 @@ import ( "github.com/gorilla/mux" appserviceAPI "github.com/matrix-org/dendrite/appservice/api" - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" - "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" eduServerAPI "github.com/matrix-org/dendrite/eduserver/api" @@ -32,6 +30,8 @@ import ( "github.com/matrix-org/dendrite/internal/transactions" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/storage/accounts" + "github.com/matrix-org/dendrite/userapi/storage/devices" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) diff --git a/clientapi/routing/sendtyping.go b/clientapi/routing/sendtyping.go index 213e7fdcf..9b6a0b39b 100644 --- a/clientapi/routing/sendtyping.go +++ b/clientapi/routing/sendtyping.go @@ -16,12 +16,12 @@ import ( "database/sql" "net/http" - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/userutil" "github.com/matrix-org/dendrite/eduserver/api" userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/storage/accounts" "github.com/matrix-org/util" ) diff --git a/clientapi/routing/threepid.go b/clientapi/routing/threepid.go index c712c1c37..e7aaadf54 100644 --- a/clientapi/routing/threepid.go +++ b/clientapi/routing/threepid.go @@ -18,12 +18,12 @@ import ( "net/http" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/threepid" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/storage/accounts" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" diff --git a/clientapi/threepid/invites.go b/clientapi/threepid/invites.go index d9da5c503..c308cb1f4 100644 --- a/clientapi/threepid/invites.go +++ b/clientapi/threepid/invites.go @@ -25,11 +25,11 @@ import ( "time" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/roomserver/api" userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/storage/accounts" "github.com/matrix-org/gomatrixserverlib" ) diff --git a/cmd/create-account/main.go b/cmd/create-account/main.go index 9c1e45932..ff022ec3c 100644 --- a/cmd/create-account/main.go +++ b/cmd/create-account/main.go @@ -20,8 +20,8 @@ import ( "fmt" "os" - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" - "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" + "github.com/matrix-org/dendrite/userapi/storage/accounts" + "github.com/matrix-org/dendrite/userapi/storage/devices" "github.com/matrix-org/gomatrixserverlib" ) diff --git a/internal/setup/base.go b/internal/setup/base.go index e287cfbd0..66424a609 100644 --- a/internal/setup/base.go +++ b/internal/setup/base.go @@ -28,9 +28,9 @@ import ( "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/naffka" - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" - "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/userapi/storage/accounts" + "github.com/matrix-org/dendrite/userapi/storage/devices" "github.com/Shopify/sarama" "github.com/gorilla/mux" diff --git a/internal/setup/monolith.go b/internal/setup/monolith.go index bb81f7403..24bee9502 100644 --- a/internal/setup/monolith.go +++ b/internal/setup/monolith.go @@ -19,8 +19,6 @@ import ( "github.com/gorilla/mux" appserviceAPI "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/clientapi" - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" - "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" eduServerAPI "github.com/matrix-org/dendrite/eduserver/api" "github.com/matrix-org/dendrite/federationapi" federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" @@ -35,6 +33,8 @@ import ( serverKeyAPI "github.com/matrix-org/dendrite/serverkeyapi/api" "github.com/matrix-org/dendrite/syncapi" userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/storage/accounts" + "github.com/matrix-org/dendrite/userapi/storage/devices" "github.com/matrix-org/gomatrixserverlib" ) diff --git a/userapi/internal/api.go b/userapi/internal/api.go index 3a4131666..ae021f575 100644 --- a/userapi/internal/api.go +++ b/userapi/internal/api.go @@ -21,12 +21,12 @@ import ( "fmt" "github.com/matrix-org/dendrite/appservice/types" - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" - "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/clientapi/userutil" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/dendrite/userapi/storage/accounts" + "github.com/matrix-org/dendrite/userapi/storage/devices" "github.com/matrix-org/gomatrixserverlib" ) diff --git a/clientapi/auth/storage/accounts/interface.go b/userapi/storage/accounts/interface.go similarity index 100% rename from clientapi/auth/storage/accounts/interface.go rename to userapi/storage/accounts/interface.go diff --git a/clientapi/auth/storage/accounts/postgres/account_data_table.go b/userapi/storage/accounts/postgres/account_data_table.go similarity index 100% rename from clientapi/auth/storage/accounts/postgres/account_data_table.go rename to userapi/storage/accounts/postgres/account_data_table.go diff --git a/clientapi/auth/storage/accounts/postgres/accounts_table.go b/userapi/storage/accounts/postgres/accounts_table.go similarity index 100% rename from clientapi/auth/storage/accounts/postgres/accounts_table.go rename to userapi/storage/accounts/postgres/accounts_table.go diff --git a/clientapi/auth/storage/accounts/postgres/filter_table.go b/userapi/storage/accounts/postgres/filter_table.go similarity index 100% rename from clientapi/auth/storage/accounts/postgres/filter_table.go rename to userapi/storage/accounts/postgres/filter_table.go diff --git a/clientapi/auth/storage/accounts/postgres/membership_table.go b/userapi/storage/accounts/postgres/membership_table.go similarity index 100% rename from clientapi/auth/storage/accounts/postgres/membership_table.go rename to userapi/storage/accounts/postgres/membership_table.go diff --git a/clientapi/auth/storage/accounts/postgres/profile_table.go b/userapi/storage/accounts/postgres/profile_table.go similarity index 100% rename from clientapi/auth/storage/accounts/postgres/profile_table.go rename to userapi/storage/accounts/postgres/profile_table.go diff --git a/clientapi/auth/storage/accounts/postgres/storage.go b/userapi/storage/accounts/postgres/storage.go similarity index 100% rename from clientapi/auth/storage/accounts/postgres/storage.go rename to userapi/storage/accounts/postgres/storage.go diff --git a/clientapi/auth/storage/accounts/postgres/threepid_table.go b/userapi/storage/accounts/postgres/threepid_table.go similarity index 100% rename from clientapi/auth/storage/accounts/postgres/threepid_table.go rename to userapi/storage/accounts/postgres/threepid_table.go diff --git a/clientapi/auth/storage/accounts/sqlite3/account_data_table.go b/userapi/storage/accounts/sqlite3/account_data_table.go similarity index 100% rename from clientapi/auth/storage/accounts/sqlite3/account_data_table.go rename to userapi/storage/accounts/sqlite3/account_data_table.go diff --git a/clientapi/auth/storage/accounts/sqlite3/accounts_table.go b/userapi/storage/accounts/sqlite3/accounts_table.go similarity index 100% rename from clientapi/auth/storage/accounts/sqlite3/accounts_table.go rename to userapi/storage/accounts/sqlite3/accounts_table.go diff --git a/clientapi/auth/storage/accounts/sqlite3/constraint.go b/userapi/storage/accounts/sqlite3/constraint.go similarity index 100% rename from clientapi/auth/storage/accounts/sqlite3/constraint.go rename to userapi/storage/accounts/sqlite3/constraint.go diff --git a/clientapi/auth/storage/accounts/sqlite3/constraint_wasm.go b/userapi/storage/accounts/sqlite3/constraint_wasm.go similarity index 100% rename from clientapi/auth/storage/accounts/sqlite3/constraint_wasm.go rename to userapi/storage/accounts/sqlite3/constraint_wasm.go diff --git a/clientapi/auth/storage/accounts/sqlite3/filter_table.go b/userapi/storage/accounts/sqlite3/filter_table.go similarity index 100% rename from clientapi/auth/storage/accounts/sqlite3/filter_table.go rename to userapi/storage/accounts/sqlite3/filter_table.go diff --git a/clientapi/auth/storage/accounts/sqlite3/membership_table.go b/userapi/storage/accounts/sqlite3/membership_table.go similarity index 100% rename from clientapi/auth/storage/accounts/sqlite3/membership_table.go rename to userapi/storage/accounts/sqlite3/membership_table.go diff --git a/clientapi/auth/storage/accounts/sqlite3/profile_table.go b/userapi/storage/accounts/sqlite3/profile_table.go similarity index 100% rename from clientapi/auth/storage/accounts/sqlite3/profile_table.go rename to userapi/storage/accounts/sqlite3/profile_table.go diff --git a/clientapi/auth/storage/accounts/sqlite3/storage.go b/userapi/storage/accounts/sqlite3/storage.go similarity index 100% rename from clientapi/auth/storage/accounts/sqlite3/storage.go rename to userapi/storage/accounts/sqlite3/storage.go diff --git a/clientapi/auth/storage/accounts/sqlite3/threepid_table.go b/userapi/storage/accounts/sqlite3/threepid_table.go similarity index 100% rename from clientapi/auth/storage/accounts/sqlite3/threepid_table.go rename to userapi/storage/accounts/sqlite3/threepid_table.go diff --git a/clientapi/auth/storage/accounts/storage.go b/userapi/storage/accounts/storage.go similarity index 90% rename from clientapi/auth/storage/accounts/storage.go rename to userapi/storage/accounts/storage.go index 42ec14fc4..87f626bf9 100644 --- a/clientapi/auth/storage/accounts/storage.go +++ b/userapi/storage/accounts/storage.go @@ -19,9 +19,9 @@ package accounts import ( "net/url" - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/postgres" - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/sqlite3" "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/userapi/storage/accounts/postgres" + "github.com/matrix-org/dendrite/userapi/storage/accounts/sqlite3" "github.com/matrix-org/gomatrixserverlib" ) diff --git a/clientapi/auth/storage/accounts/storage_wasm.go b/userapi/storage/accounts/storage_wasm.go similarity index 94% rename from clientapi/auth/storage/accounts/storage_wasm.go rename to userapi/storage/accounts/storage_wasm.go index 6c221ccf5..692567059 100644 --- a/clientapi/auth/storage/accounts/storage_wasm.go +++ b/userapi/storage/accounts/storage_wasm.go @@ -18,8 +18,8 @@ import ( "fmt" "net/url" - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts/sqlite3" "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/userapi/storage/accounts/sqlite3" "github.com/matrix-org/gomatrixserverlib" ) diff --git a/clientapi/auth/storage/devices/interface.go b/userapi/storage/devices/interface.go similarity index 100% rename from clientapi/auth/storage/devices/interface.go rename to userapi/storage/devices/interface.go diff --git a/clientapi/auth/storage/devices/postgres/devices_table.go b/userapi/storage/devices/postgres/devices_table.go similarity index 100% rename from clientapi/auth/storage/devices/postgres/devices_table.go rename to userapi/storage/devices/postgres/devices_table.go diff --git a/clientapi/auth/storage/devices/postgres/storage.go b/userapi/storage/devices/postgres/storage.go similarity index 100% rename from clientapi/auth/storage/devices/postgres/storage.go rename to userapi/storage/devices/postgres/storage.go diff --git a/clientapi/auth/storage/devices/sqlite3/devices_table.go b/userapi/storage/devices/sqlite3/devices_table.go similarity index 100% rename from clientapi/auth/storage/devices/sqlite3/devices_table.go rename to userapi/storage/devices/sqlite3/devices_table.go diff --git a/clientapi/auth/storage/devices/sqlite3/storage.go b/userapi/storage/devices/sqlite3/storage.go similarity index 100% rename from clientapi/auth/storage/devices/sqlite3/storage.go rename to userapi/storage/devices/sqlite3/storage.go diff --git a/clientapi/auth/storage/devices/storage.go b/userapi/storage/devices/storage.go similarity index 90% rename from clientapi/auth/storage/devices/storage.go rename to userapi/storage/devices/storage.go index d0d203427..e094d202a 100644 --- a/clientapi/auth/storage/devices/storage.go +++ b/userapi/storage/devices/storage.go @@ -19,9 +19,9 @@ package devices import ( "net/url" - "github.com/matrix-org/dendrite/clientapi/auth/storage/devices/postgres" - "github.com/matrix-org/dendrite/clientapi/auth/storage/devices/sqlite3" "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/userapi/storage/devices/postgres" + "github.com/matrix-org/dendrite/userapi/storage/devices/sqlite3" "github.com/matrix-org/gomatrixserverlib" ) diff --git a/clientapi/auth/storage/devices/storage_wasm.go b/userapi/storage/devices/storage_wasm.go similarity index 94% rename from clientapi/auth/storage/devices/storage_wasm.go rename to userapi/storage/devices/storage_wasm.go index e32471d8c..a5a515eff 100644 --- a/clientapi/auth/storage/devices/storage_wasm.go +++ b/userapi/storage/devices/storage_wasm.go @@ -18,8 +18,8 @@ import ( "fmt" "net/url" - "github.com/matrix-org/dendrite/clientapi/auth/storage/devices/sqlite3" "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/userapi/storage/devices/sqlite3" "github.com/matrix-org/gomatrixserverlib" ) diff --git a/userapi/userapi.go b/userapi/userapi.go index 961d04fe0..7aadec06a 100644 --- a/userapi/userapi.go +++ b/userapi/userapi.go @@ -16,12 +16,12 @@ package userapi import ( "github.com/gorilla/mux" - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" - "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/internal/config" "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/dendrite/userapi/internal" "github.com/matrix-org/dendrite/userapi/inthttp" + "github.com/matrix-org/dendrite/userapi/storage/accounts" + "github.com/matrix-org/dendrite/userapi/storage/devices" "github.com/matrix-org/gomatrixserverlib" ) diff --git a/userapi/userapi_test.go b/userapi/userapi_test.go index 509bdd7e8..163b10ec7 100644 --- a/userapi/userapi_test.go +++ b/userapi/userapi_test.go @@ -8,13 +8,13 @@ import ( "testing" "github.com/gorilla/mux" - "github.com/matrix-org/dendrite/clientapi/auth/storage/accounts" - "github.com/matrix-org/dendrite/clientapi/auth/storage/devices" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/test" "github.com/matrix-org/dendrite/userapi" "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/dendrite/userapi/inthttp" + "github.com/matrix-org/dendrite/userapi/storage/accounts" + "github.com/matrix-org/dendrite/userapi/storage/devices" "github.com/matrix-org/gomatrixserverlib" ) From c4d9b374927309e327c19f1985ad65c3b78ab5ee Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 17 Jun 2020 13:54:47 +0100 Subject: [PATCH 176/200] add test --- sytest-whitelist | 1 + 1 file changed, 1 insertion(+) diff --git a/sytest-whitelist b/sytest-whitelist index 8f3d128d7..244cfe8cc 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -331,3 +331,4 @@ Can download file 'ascii' Can download file 'name with spaces' Can download file 'name;with;semicolons' Can download specifying a different ASCII file name +Inbound /v1/send_join rejects joins from other servers From 38053a5bb7af9bba522225fc62bee73d363bff85 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Wed, 17 Jun 2020 13:55:55 +0100 Subject: [PATCH 177/200] Do not wrap send_join errors on /v1/send_join (#1143) * Do not wrap v1 send_join errors in [code, body] * Don't wrap errors --- federationapi/routing/routing.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index 649a43c66..645f397de 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -18,6 +18,7 @@ import ( "net/http" "github.com/gorilla/mux" + "github.com/matrix-org/dendrite/clientapi/jsonerror" eduserverAPI "github.com/matrix-org/dendrite/eduserver/api" federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" "github.com/matrix-org/dendrite/internal/config" @@ -203,12 +204,20 @@ func Setup( res := SendJoin( httpReq, request, cfg, rsAPI, keys, roomID, eventID, ) + // not all responses get wrapped in [code, body] + var body interface{} + body = []interface{}{ + res.Code, res.JSON, + } + jerr, ok := res.JSON.(*jsonerror.MatrixError) + if ok { + body = jerr + } + return util.JSONResponse{ Headers: res.Headers, Code: res.Code, - JSON: []interface{}{ - res.Code, res.JSON, - }, + JSON: body, } }, )).Methods(http.MethodPut) From 23bed196e61d4dfb8719e6bfff82888173519cde Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 17 Jun 2020 14:26:45 +0100 Subject: [PATCH 178/200] Fallback parsing for Content-Disposition minefield (#1144) --- mediaapi/routing/download.go | 27 ++++++++++++++++++++++----- sytest-whitelist | 2 ++ 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/mediaapi/routing/download.go b/mediaapi/routing/download.go index fa1bb2573..7e121de3e 100644 --- a/mediaapi/routing/download.go +++ b/mediaapi/routing/download.go @@ -47,6 +47,10 @@ const mediaIDCharacters = "A-Za-z0-9_=-" // Note: unfortunately regex.MustCompile() cannot be assigned to a const var mediaIDRegex = regexp.MustCompile("^[" + mediaIDCharacters + "]+$") +// Regular expressions to help us cope with Content-Disposition parsing +var rfc2183 = regexp.MustCompile(`filename\=utf-8\"(.*)\"`) +var rfc6266 = regexp.MustCompile(`filename\*\=utf-8\'\'(.*)`) + // downloadRequest metadata included in or derivable from a download or thumbnail request // https://matrix.org/docs/spec/client_server/r0.2.0.html#get-matrix-media-r0-download-servername-mediaid // http://matrix.org/docs/spec/client_server/r0.2.0.html#get-matrix-media-r0-thumbnail-servername-mediaid @@ -378,8 +382,8 @@ func (r *downloadRequest) addDownloadFilenameToHeaders( } else { // For UTF-8 filenames, we quote always, as that's the standard w.Header().Set("Content-Disposition", fmt.Sprintf( - `inline; filename=utf-8"%s"`, - unescaped, + `inline; filename*=utf-8''%s`, + url.QueryEscape(unescaped), )) } @@ -700,9 +704,22 @@ func (r *downloadRequest) fetchRemoteFile( } r.MediaMetadata.FileSizeBytes = types.FileSizeBytes(contentLength) r.MediaMetadata.ContentType = types.ContentType(resp.Header.Get("Content-Type")) - _, params, err := mime.ParseMediaType(resp.Header.Get("Content-Disposition")) - if err == nil && params["filename"] != "" { - r.MediaMetadata.UploadName = types.Filename(params["filename"]) + + dispositionHeader := resp.Header.Get("Content-Disposition") + if _, params, e := mime.ParseMediaType(dispositionHeader); e == nil { + if params["filename"] != "" { + r.MediaMetadata.UploadName = types.Filename(params["filename"]) + } else if params["filename*"] != "" { + r.MediaMetadata.UploadName = types.Filename(params["filename*"]) + } + } else { + if matches := rfc6266.FindStringSubmatch(dispositionHeader); len(matches) > 1 { + // Always prefer the RFC6266 UTF-8 name if possible + r.MediaMetadata.UploadName = types.Filename(matches[1]) + } else if matches := rfc2183.FindStringSubmatch(dispositionHeader); len(matches) > 1 { + // Otherwise, see if an RFC2183 name was provided (ASCII only) + r.MediaMetadata.UploadName = types.Filename(matches[1]) + } } r.Logger.Info("Transferring remote file") diff --git a/sytest-whitelist b/sytest-whitelist index 244cfe8cc..e68605628 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -331,4 +331,6 @@ Can download file 'ascii' Can download file 'name with spaces' Can download file 'name;with;semicolons' Can download specifying a different ASCII file name +Can download with Unicode file name over federation +Can download specifying a different Unicode file name Inbound /v1/send_join rejects joins from other servers From c7f7ae69ebd37a4bdb3fdc7c5d0d949445c837dc Mon Sep 17 00:00:00 2001 From: Kegsay Date: Wed, 17 Jun 2020 15:12:09 +0100 Subject: [PATCH 179/200] Are we synapse yet: Accept tests without a group rather than dying (#1142) Produces output like: ``` Non-Spec APIs: 0% (0/52 tests) -------------- Non-Spec API : 0% (0/50 tests) Unknown API (no group specified): 0% (0/2 tests) ``` Co-authored-by: Neil Alexander --- are-we-synapse-yet.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/are-we-synapse-yet.py b/are-we-synapse-yet.py index 5d5128479..30979a129 100755 --- a/are-we-synapse-yet.py +++ b/are-we-synapse-yet.py @@ -33,6 +33,7 @@ import sys test_mappings = { "nsp": "Non-Spec API", + "unk": "Unknown API (no group specified)", "f": "Federation", # flag to mark test involves federation "federation_apis": { @@ -214,7 +215,8 @@ def main(results_tap_path, verbose): # } }, "nonspec": { - "nsp": {} + "nsp": {}, + "unk": {} }, } with open(results_tap_path, "r") as f: @@ -225,7 +227,7 @@ def main(results_tap_path, verbose): name = test_result["name"] group_id = test_name_to_group_id.get(name) if not group_id: - raise Exception("The test '%s' doesn't have a group" % (name,)) + summary["nonspec"]["unk"][name] = test_result["ok"] if group_id == "nsp": summary["nonspec"]["nsp"][name] = test_result["ok"] elif group_id in test_mappings["federation_apis"]: From 8e7c1eda05dddec5842711ce3d80036e7eff70d7 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Wed, 17 Jun 2020 15:40:37 +0100 Subject: [PATCH 180/200] Enable more sytests (#1145) --- go.mod | 2 +- go.sum | 2 ++ sytest-whitelist | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 6154d0f32..ff73ad078 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200616150727-7ac22b6f8e65 + github.com/matrix-org/gomatrixserverlib v0.0.0-20200617141855-5539854e4abc github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible diff --git a/go.sum b/go.sum index 3fa242c78..f62b14b62 100644 --- a/go.sum +++ b/go.sum @@ -373,6 +373,8 @@ github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bh github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrixserverlib v0.0.0-20200616150727-7ac22b6f8e65 h1:2CcCcBnWdDPDOqFKiGOM+mi/KDDZXSTKmvFy/0/+ZJI= github.com/matrix-org/gomatrixserverlib v0.0.0-20200616150727-7ac22b6f8e65/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200617141855-5539854e4abc h1:Oxidxr/H1Nh8aOFWAhTzQYLDOc9OuoJwfDjgEHpyqNE= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200617141855-5539854e4abc/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f/go.mod h1:y0oDTjZDv5SM9a2rp3bl+CU+bvTRINQsdb7YlDql5Go= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= diff --git a/sytest-whitelist b/sytest-whitelist index e68605628..3fb96ff46 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -334,3 +334,5 @@ Can download specifying a different ASCII file name Can download with Unicode file name over federation Can download specifying a different Unicode file name Inbound /v1/send_join rejects joins from other servers +Outbound federation can query v1 /send_join +Inbound /v1/send_join rejects incorrectly-signed joins From 8efeb8eb3b347f1afc16b7fe2e2efe4edd593027 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Wed, 17 Jun 2020 16:21:42 +0100 Subject: [PATCH 181/200] Return the correct /joined_members response and allow ?format=event (#1146) --- clientapi/routing/memberships.go | 27 +++++++++++++++++++++++++++ clientapi/routing/routing.go | 6 ++++-- clientapi/routing/state.go | 12 ++++++++++-- sytest-whitelist | 2 ++ 4 files changed, 43 insertions(+), 4 deletions(-) diff --git a/clientapi/routing/memberships.go b/clientapi/routing/memberships.go index 574a2c480..1c9800b66 100644 --- a/clientapi/routing/memberships.go +++ b/clientapi/routing/memberships.go @@ -15,6 +15,7 @@ package routing import ( + "encoding/json" "net/http" "github.com/matrix-org/dendrite/userapi/storage/accounts" @@ -35,6 +36,16 @@ type getJoinedRoomsResponse struct { JoinedRooms []string `json:"joined_rooms"` } +// https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-rooms-roomid-joined-members +type getJoinedMembersResponse struct { + Joined map[string]joinedMember `json:"joined"` +} + +type joinedMember struct { + DisplayName string `json:"display_name"` + AvatarURL string `json:"avatar_url"` +} + // GetMemberships implements GET /rooms/{roomId}/members func GetMemberships( req *http.Request, device *userapi.Device, roomID string, joinedOnly bool, @@ -59,6 +70,22 @@ func GetMemberships( } } + if joinedOnly { + var res getJoinedMembersResponse + res.Joined = make(map[string]joinedMember) + for _, ev := range queryRes.JoinEvents { + var content joinedMember + if err := json.Unmarshal(ev.Content, &content); err != nil { + util.GetLogger(req.Context()).WithError(err).Error("failed to unmarshal event content") + return jsonerror.InternalServerError() + } + res.Joined[ev.Sender] = content + } + return util.JSONResponse{ + Code: http.StatusOK, + JSON: res, + } + } return util.JSONResponse{ Code: http.StatusOK, JSON: getMembershipResponse{queryRes.JoinEvents}, diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 0fe687b30..41c7fb18e 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -164,7 +164,8 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return OnIncomingStateTypeRequest(req.Context(), rsAPI, vars["roomID"], vars["type"], "") + eventFormat := req.URL.Query().Get("format") == "event" + return OnIncomingStateTypeRequest(req.Context(), rsAPI, vars["roomID"], vars["type"], "", eventFormat) })).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { @@ -172,7 +173,8 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return OnIncomingStateTypeRequest(req.Context(), rsAPI, vars["roomID"], vars["type"], vars["stateKey"]) + eventFormat := req.URL.Query().Get("format") == "event" + return OnIncomingStateTypeRequest(req.Context(), rsAPI, vars["roomID"], vars["type"], vars["stateKey"], eventFormat) })).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/state/{eventType:[^/]+/?}", diff --git a/clientapi/routing/state.go b/clientapi/routing/state.go index e3e5bdb6d..2ec7a33f3 100644 --- a/clientapi/routing/state.go +++ b/clientapi/routing/state.go @@ -98,7 +98,8 @@ func OnIncomingStateRequest(ctx context.Context, rsAPI api.RoomserverInternalAPI // /rooms/{roomID}/state/{type}/{statekey} request. It will look in current // state to see if there is an event with that type and state key, if there // is then (by default) we return the content, otherwise a 404. -func OnIncomingStateTypeRequest(ctx context.Context, rsAPI api.RoomserverInternalAPI, roomID string, evType, stateKey string) util.JSONResponse { +// If eventFormat=true, sends the whole event else just the content. +func OnIncomingStateTypeRequest(ctx context.Context, rsAPI api.RoomserverInternalAPI, roomID, evType, stateKey string, eventFormat bool) util.JSONResponse { // TODO(#287): Auth request and handle the case where the user has left (where // we should return the state at the poin they left) util.GetLogger(ctx).WithFields(log.Fields{ @@ -134,8 +135,15 @@ func OnIncomingStateTypeRequest(ctx context.Context, rsAPI api.RoomserverInterna ClientEvent: gomatrixserverlib.HeaderedToClientEvent(stateRes.StateEvents[0], gomatrixserverlib.FormatAll), } + var res interface{} + if eventFormat { + res = stateEvent + } else { + res = stateEvent.Content + } + return util.JSONResponse{ Code: http.StatusOK, - JSON: stateEvent.Content, + JSON: res, } } diff --git a/sytest-whitelist b/sytest-whitelist index 3fb96ff46..a6427cac5 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -53,6 +53,8 @@ PUT /rooms/:room_id/send/:event_type/:txn_id deduplicates the same txn id GET /rooms/:room_id/state/m.room.power_levels can fetch levels PUT /rooms/:room_id/state/m.room.power_levels can set levels PUT power_levels should not explode if the old power levels were empty +GET /rooms/:room_id/state/m.room.member/:user_id?format=event fetches my membership event +GET /rooms/:room_id/joined_members fetches my membership Both GET and PUT work POST /rooms/:room_id/read_markers can create read marker User signups are forbidden from starting with '_' From 9b408c19fbe3ca3908f9dcf44978d038fe0d230f Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 17 Jun 2020 16:47:21 +0100 Subject: [PATCH 182/200] Missing sytests --- sytest-whitelist | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sytest-whitelist b/sytest-whitelist index a6427cac5..be45b434f 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -338,3 +338,10 @@ Can download specifying a different Unicode file name Inbound /v1/send_join rejects joins from other servers Outbound federation can query v1 /send_join Inbound /v1/send_join rejects incorrectly-signed joins +POST /rooms/:room_id/state/m.room.name sets name +GET /rooms/:room_id/state/m.room.name gets name +POST /rooms/:room_id/state/m.room.topic sets topic +GET /rooms/:room_id/state/m.room.topic gets topic +GET /rooms/:room_id/state fetches entire room state +Setting room topic reports m.room.topic to myself +setting 'm.room.name' respects room powerlevel From 84a7881468a57bd4225f5c990c03b5fce729f914 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Wed, 17 Jun 2020 17:01:03 +0100 Subject: [PATCH 183/200] Make account data sytests pass (#1147) --- clientapi/routing/account_data.go | 2 +- sytest-whitelist | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/clientapi/routing/account_data.go b/clientapi/routing/account_data.go index 875249b32..68e0dc5da 100644 --- a/clientapi/routing/account_data.go +++ b/clientapi/routing/account_data.go @@ -51,7 +51,7 @@ func GetAccountData( ); err == nil { return util.JSONResponse{ Code: http.StatusOK, - JSON: data, + JSON: data.Content, } } diff --git a/sytest-whitelist b/sytest-whitelist index be45b434f..971f8cf0d 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -115,6 +115,8 @@ User can invite local user to room with version 1 Should reject keys claiming to belong to a different user Can add account data Can add account data to room +Can get account data without syncing +Can get room account data without syncing #Latest account data appears in v2 /sync New account data appears in incremental v2 /sync Checking local federation server From ddf1c8adf1fd1441b76834df479d1ab5a132de88 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 17 Jun 2020 17:41:45 +0100 Subject: [PATCH 184/200] Hacks for supporting Riot iOS (#1148) * Join room body is optional * Support deprecated login by user/password * Implement dummy key upload endpoint * Make a very determinate end to /messages if we hit the create event in back-pagination * Linting --- clientapi/routing/joinroom.go | 4 +-- clientapi/routing/login.go | 67 +++++++++++++++++++++++------------ keyserver/routing/routing.go | 10 ++++++ syncapi/routing/messages.go | 45 ++++++++++++++--------- 4 files changed, 84 insertions(+), 42 deletions(-) diff --git a/clientapi/routing/joinroom.go b/clientapi/routing/joinroom.go index e190beefd..3871e4d4e 100644 --- a/clientapi/routing/joinroom.go +++ b/clientapi/routing/joinroom.go @@ -43,9 +43,7 @@ func JoinRoomByIDOrAlias( // If content was provided in the request then incude that // in the request. It'll get used as a part of the membership // event content. - if err := httputil.UnmarshalJSONRequest(req, &joinReq.Content); err != nil { - return *err - } + _ = httputil.UnmarshalJSONRequest(req, &joinReq.Content) // Work out our localpart for the client profile request. localpart, _, err := gomatrixserverlib.SplitID('@', device.UserID) diff --git a/clientapi/routing/login.go b/clientapi/routing/login.go index 1b894a566..dc0180da6 100644 --- a/clientapi/routing/login.go +++ b/clientapi/routing/login.go @@ -47,6 +47,7 @@ type loginIdentifier struct { type passwordRequest struct { Identifier loginIdentifier `json:"identifier"` + User string `json:"user"` // deprecated in favour of identifier Password string `json:"password"` // Both DeviceID and InitialDisplayName can be omitted, or empty strings ("") // Thus a pointer is needed to differentiate between the two @@ -81,6 +82,7 @@ func Login( } else if req.Method == http.MethodPost { var r passwordRequest var acc *api.Account + var errJSON *util.JSONResponse resErr := httputil.UnmarshalJSONRequest(req, &r) if resErr != nil { return *resErr @@ -93,30 +95,22 @@ func Login( JSON: jsonerror.BadJSON("'user' must be supplied."), } } - - util.GetLogger(req.Context()).WithField("user", r.Identifier.User).Info("Processing login request") - - localpart, err := userutil.ParseUsernameParam(r.Identifier.User, &cfg.Matrix.ServerName) - if err != nil { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.InvalidUsername(err.Error()), - } - } - - acc, err = accountDB.GetAccountByPassword(req.Context(), localpart, r.Password) - if err != nil { - // Technically we could tell them if the user does not exist by checking if err == sql.ErrNoRows - // but that would leak the existence of the user. - return util.JSONResponse{ - Code: http.StatusForbidden, - JSON: jsonerror.Forbidden("username or password was incorrect, or the account does not exist"), - } + acc, errJSON = r.processUsernamePasswordLoginRequest(req, accountDB, cfg, r.Identifier.User) + if errJSON != nil { + return *errJSON } default: - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.BadJSON("login identifier '" + r.Identifier.Type + "' not supported"), + // TODO: The below behaviour is deprecated but without it Riot iOS won't log in + if r.User != "" { + acc, errJSON = r.processUsernamePasswordLoginRequest(req, accountDB, cfg, r.User) + if errJSON != nil { + return *errJSON + } + } else { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.BadJSON("login identifier '" + r.Identifier.Type + "' not supported"), + } } } @@ -163,3 +157,32 @@ func getDevice( ) return } + +func (r *passwordRequest) processUsernamePasswordLoginRequest( + req *http.Request, accountDB accounts.Database, + cfg *config.Dendrite, username string, +) (acc *api.Account, errJSON *util.JSONResponse) { + util.GetLogger(req.Context()).WithField("user", username).Info("Processing login request") + + localpart, err := userutil.ParseUsernameParam(username, &cfg.Matrix.ServerName) + if err != nil { + errJSON = &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.InvalidUsername(err.Error()), + } + return + } + + acc, err = accountDB.GetAccountByPassword(req.Context(), localpart, r.Password) + if err != nil { + // Technically we could tell them if the user does not exist by checking if err == sql.ErrNoRows + // but that would leak the existence of the user. + errJSON = &util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("username or password was incorrect, or the account does not exist"), + } + return + } + + return +} diff --git a/keyserver/routing/routing.go b/keyserver/routing/routing.go index c09031d8a..dba43528f 100644 --- a/keyserver/routing/routing.go +++ b/keyserver/routing/routing.go @@ -36,9 +36,19 @@ func Setup( publicAPIMux *mux.Router, cfg *config.Dendrite, userAPI userapi.UserInternalAPI, ) { r0mux := publicAPIMux.PathPrefix(pathPrefixR0).Subrouter() + r0mux.Handle("/keys/query", httputil.MakeAuthAPI("queryKeys", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { return QueryKeys(req) }), ).Methods(http.MethodPost, http.MethodOptions) + + r0mux.Handle("/keys/upload/{keyID}", + httputil.MakeAuthAPI("keys_upload", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + return util.JSONResponse{ + Code: 200, + JSON: map[string]interface{}{}, + } + }), + ).Methods(http.MethodPost, http.MethodOptions) } diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index de5429db4..15add1b45 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -158,6 +158,7 @@ func OnIncomingMessagesRequest( util.GetLogger(req.Context()).WithError(err).Error("mreq.retrieveEvents failed") return jsonerror.InternalServerError() } + util.GetLogger(req.Context()).WithFields(logrus.Fields{ "from": from.String(), "to": to.String(), @@ -246,6 +247,12 @@ func (r *messagesReq) retrieveEvents() ( // change the way topological positions are defined (as depth isn't the most // reliable way to define it), it would be easier and less troublesome to // only have to change it in one place, i.e. the database. + start, end, err = r.getStartEnd(events) + + return clientEvents, start, end, err +} + +func (r *messagesReq) getStartEnd(events []gomatrixserverlib.HeaderedEvent) (start, end types.TopologyToken, err error) { start, err = r.db.EventPositionInTopology( r.ctx, events[0].EventID(), ) @@ -253,24 +260,28 @@ func (r *messagesReq) retrieveEvents() ( err = fmt.Errorf("EventPositionInTopology: for start event %s: %w", events[0].EventID(), err) return } - end, err = r.db.EventPositionInTopology( - r.ctx, events[len(events)-1].EventID(), - ) - if err != nil { - err = fmt.Errorf("EventPositionInTopology: for end event %s: %w", events[len(events)-1].EventID(), err) - return + if r.backwardOrdering && events[len(events)-1].Type() == gomatrixserverlib.MRoomCreate { + // We've hit the beginning of the room so there's really nowhere else + // to go. This seems to fix Riot iOS from looping on /messages endlessly. + end = types.NewTopologyToken(0, 0) + } else { + end, err = r.db.EventPositionInTopology( + r.ctx, events[len(events)-1].EventID(), + ) + if err != nil { + err = fmt.Errorf("EventPositionInTopology: for end event %s: %w", events[len(events)-1].EventID(), err) + return + } + if r.backwardOrdering { + // A stream/topological position is a cursor located between two events. + // While they are identified in the code by the event on their right (if + // we consider a left to right chronological order), tokens need to refer + // to them by the event on their left, therefore we need to decrement the + // end position we send in the response if we're going backward. + end.Decrement() + } } - - if r.backwardOrdering { - // A stream/topological position is a cursor located between two events. - // While they are identified in the code by the event on their right (if - // we consider a left to right chronological order), tokens need to refer - // to them by the event on their left, therefore we need to decrement the - // end position we send in the response if we're going backward. - end.Decrement() - } - - return clientEvents, start, end, err + return } // handleEmptyEventsSlice handles the case where the initial request to the From 3547a1768c36626c672e5c7834f297496f568b2f Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 18 Jun 2020 13:48:47 +0100 Subject: [PATCH 185/200] Fix embed Riot Web into Yggdrasil demo --- cmd/dendrite-demo-yggdrasil/embed/embed_other.go | 4 +++- cmd/dendrite-demo-yggdrasil/embed/embed_riotweb.go | 7 ++++--- cmd/dendrite-demo-yggdrasil/main.go | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/cmd/dendrite-demo-yggdrasil/embed/embed_other.go b/cmd/dendrite-demo-yggdrasil/embed/embed_other.go index a9108fad3..598881148 100644 --- a/cmd/dendrite-demo-yggdrasil/embed/embed_other.go +++ b/cmd/dendrite-demo-yggdrasil/embed/embed_other.go @@ -2,6 +2,8 @@ package embed -func Embed(_ int, _ string) { +import "github.com/gorilla/mux" + +func Embed(_ *mux.Router, _ int, _ string) { } diff --git a/cmd/dendrite-demo-yggdrasil/embed/embed_riotweb.go b/cmd/dendrite-demo-yggdrasil/embed/embed_riotweb.go index 360d0bc5b..a9e04a312 100644 --- a/cmd/dendrite-demo-yggdrasil/embed/embed_riotweb.go +++ b/cmd/dendrite-demo-yggdrasil/embed/embed_riotweb.go @@ -7,19 +7,20 @@ import ( "io" "net/http" + "github.com/gorilla/mux" "github.com/tidwall/sjson" ) // From within the Riot Web directory: // go run github.com/mjibson/esc -o /path/to/dendrite/internal/embed/fs_riotweb.go -private -pkg embed . -func Embed(listenPort int, serverName string) { +func Embed(rootMux *mux.Router, listenPort int, serverName string) { url := fmt.Sprintf("http://localhost:%d", listenPort) embeddedFS := _escFS(false) embeddedServ := http.FileServer(embeddedFS) - http.DefaultServeMux.Handle("/", embeddedServ) - http.DefaultServeMux.HandleFunc("/config.json", func(w http.ResponseWriter, _ *http.Request) { + rootMux.Handle("/", embeddedServ) + rootMux.HandleFunc("/config.json", func(w http.ResponseWriter, _ *http.Request) { configFile, err := embeddedFS.Open("/config.sample.json") if err != nil { w.WriteHeader(500) diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index be15da472..328aa0629 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -154,7 +154,7 @@ func main() { logrus.WithError(err).Panicf("failed to connect to public rooms db") } - embed.Embed(*instancePort, "Yggdrasil Demo") + embed.Embed(base.BaseMux, *instancePort, "Yggdrasil Demo") monolith := setup.Monolith{ Config: base.Cfg, From dc0bac85d5bad933d32ee63f8bc1aef6348ca6e9 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 18 Jun 2020 18:36:03 +0100 Subject: [PATCH 186/200] Refactor account data (#1150) * Refactor account data * Tweak database fetching * Tweaks * Restore syncProducer notification * Various tweaks, update tag behaviour * Fix initial sync --- clientapi/routing/account_data.go | 55 ++++---- clientapi/routing/room_tagging.go | 127 +++++++----------- clientapi/routing/routing.go | 14 +- syncapi/sync/requestpool.go | 75 +++++++---- userapi/api/api.go | 27 ++-- userapi/internal/api.go | 31 ++++- userapi/inthttp/client.go | 10 ++ userapi/storage/accounts/interface.go | 7 +- .../accounts/postgres/account_data_table.go | 48 +++---- userapi/storage/accounts/postgres/storage.go | 13 +- .../accounts/sqlite3/account_data_table.go | 50 +++---- userapi/storage/accounts/sqlite3/storage.go | 13 +- 12 files changed, 248 insertions(+), 222 deletions(-) diff --git a/clientapi/routing/account_data.go b/clientapi/routing/account_data.go index 68e0dc5da..d5fafedb1 100644 --- a/clientapi/routing/account_data.go +++ b/clientapi/routing/account_data.go @@ -16,21 +16,20 @@ package routing import ( "encoding/json" + "fmt" "io/ioutil" "net/http" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/userapi/api" - "github.com/matrix-org/dendrite/userapi/storage/accounts" - "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) // GetAccountData implements GET /user/{userId}/[rooms/{roomid}/]account_data/{type} func GetAccountData( - req *http.Request, accountDB accounts.Database, device *api.Device, + req *http.Request, userAPI api.UserInternalAPI, device *api.Device, userID string, roomID string, dataType string, ) util.JSONResponse { if userID != device.UserID { @@ -40,18 +39,28 @@ func GetAccountData( } } - localpart, _, err := gomatrixserverlib.SplitID('@', userID) - if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") - return jsonerror.InternalServerError() + dataReq := api.QueryAccountDataRequest{ + UserID: userID, + DataType: dataType, + RoomID: roomID, + } + dataRes := api.QueryAccountDataResponse{} + if err := userAPI.QueryAccountData(req.Context(), &dataReq, &dataRes); err != nil { + util.GetLogger(req.Context()).WithError(err).Error("userAPI.QueryAccountData failed") + return util.ErrorResponse(fmt.Errorf("userAPI.QueryAccountData: %w", err)) } - if data, err := accountDB.GetAccountDataByType( - req.Context(), localpart, roomID, dataType, - ); err == nil { + var data json.RawMessage + var ok bool + if roomID != "" { + data, ok = dataRes.RoomAccountData[roomID][dataType] + } else { + data, ok = dataRes.GlobalAccountData[dataType] + } + if ok { return util.JSONResponse{ Code: http.StatusOK, - JSON: data.Content, + JSON: data, } } @@ -63,7 +72,7 @@ func GetAccountData( // SaveAccountData implements PUT /user/{userId}/[rooms/{roomId}/]account_data/{type} func SaveAccountData( - req *http.Request, accountDB accounts.Database, device *api.Device, + req *http.Request, userAPI api.UserInternalAPI, device *api.Device, userID string, roomID string, dataType string, syncProducer *producers.SyncAPIProducer, ) util.JSONResponse { if userID != device.UserID { @@ -73,12 +82,6 @@ func SaveAccountData( } } - localpart, _, err := gomatrixserverlib.SplitID('@', userID) - if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") - return jsonerror.InternalServerError() - } - defer req.Body.Close() // nolint: errcheck if req.Body == http.NoBody { @@ -101,13 +104,19 @@ func SaveAccountData( } } - if err := accountDB.SaveAccountData( - req.Context(), localpart, roomID, dataType, string(body), - ); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("accountDB.SaveAccountData failed") - return jsonerror.InternalServerError() + dataReq := api.InputAccountDataRequest{ + UserID: userID, + DataType: dataType, + RoomID: roomID, + AccountData: json.RawMessage(body), + } + dataRes := api.InputAccountDataResponse{} + if err := userAPI.InputAccountData(req.Context(), &dataReq, &dataRes); err != nil { + util.GetLogger(req.Context()).WithError(err).Error("userAPI.QueryAccountData failed") + return util.ErrorResponse(err) } + // TODO: user API should do this since it's account data if err := syncProducer.SendData(userID, roomID, dataType); err != nil { util.GetLogger(req.Context()).WithError(err).Error("syncProducer.SendData failed") return jsonerror.InternalServerError() diff --git a/clientapi/routing/room_tagging.go b/clientapi/routing/room_tagging.go index b1cfcca86..c683cc949 100644 --- a/clientapi/routing/room_tagging.go +++ b/clientapi/routing/room_tagging.go @@ -24,23 +24,14 @@ import ( "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/userapi/api" - "github.com/matrix-org/dendrite/userapi/storage/accounts" "github.com/matrix-org/gomatrix" - "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) -// newTag creates and returns a new gomatrix.TagContent -func newTag() gomatrix.TagContent { - return gomatrix.TagContent{ - Tags: make(map[string]gomatrix.TagProperties), - } -} - // GetTags implements GET /_matrix/client/r0/user/{userID}/rooms/{roomID}/tags func GetTags( req *http.Request, - accountDB accounts.Database, + userAPI api.UserInternalAPI, device *api.Device, userID string, roomID string, @@ -54,22 +45,15 @@ func GetTags( } } - _, data, err := obtainSavedTags(req, userID, roomID, accountDB) + tagContent, err := obtainSavedTags(req, userID, roomID, userAPI) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("obtainSavedTags failed") return jsonerror.InternalServerError() } - if data == nil { - return util.JSONResponse{ - Code: http.StatusOK, - JSON: struct{}{}, - } - } - return util.JSONResponse{ Code: http.StatusOK, - JSON: data.Content, + JSON: tagContent, } } @@ -78,7 +62,7 @@ func GetTags( // the tag to the "map" and saving the new "map" to the DB func PutTag( req *http.Request, - accountDB accounts.Database, + userAPI api.UserInternalAPI, device *api.Device, userID string, roomID string, @@ -98,34 +82,25 @@ func PutTag( return *reqErr } - localpart, data, err := obtainSavedTags(req, userID, roomID, accountDB) + tagContent, err := obtainSavedTags(req, userID, roomID, userAPI) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("obtainSavedTags failed") return jsonerror.InternalServerError() } - var tagContent gomatrix.TagContent - if data != nil { - if err = json.Unmarshal(data.Content, &tagContent); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("json.Unmarshal failed") - return jsonerror.InternalServerError() - } - } else { - tagContent = newTag() + if tagContent.Tags == nil { + tagContent.Tags = make(map[string]gomatrix.TagProperties) } tagContent.Tags[tag] = properties - if err = saveTagData(req, localpart, roomID, accountDB, tagContent); err != nil { + + if err = saveTagData(req, userID, roomID, userAPI, tagContent); err != nil { util.GetLogger(req.Context()).WithError(err).Error("saveTagData failed") return jsonerror.InternalServerError() } - // Send data to syncProducer in order to inform clients of changes - // Run in a goroutine in order to prevent blocking the tag request response - go func() { - if err := syncProducer.SendData(userID, roomID, "m.tag"); err != nil { - logrus.WithError(err).Error("Failed to send m.tag account data update to syncapi") - } - }() + if err = syncProducer.SendData(userID, roomID, "m.tag"); err != nil { + logrus.WithError(err).Error("Failed to send m.tag account data update to syncapi") + } return util.JSONResponse{ Code: http.StatusOK, @@ -138,7 +113,7 @@ func PutTag( // the "map" and then saving the new "map" in the DB func DeleteTag( req *http.Request, - accountDB accounts.Database, + userAPI api.UserInternalAPI, device *api.Device, userID string, roomID string, @@ -153,28 +128,12 @@ func DeleteTag( } } - localpart, data, err := obtainSavedTags(req, userID, roomID, accountDB) + tagContent, err := obtainSavedTags(req, userID, roomID, userAPI) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("obtainSavedTags failed") return jsonerror.InternalServerError() } - // If there are no tags in the database, exit - if data == nil { - // Spec only defines 200 responses for this endpoint so we don't return anything else. - return util.JSONResponse{ - Code: http.StatusOK, - JSON: struct{}{}, - } - } - - var tagContent gomatrix.TagContent - err = json.Unmarshal(data.Content, &tagContent) - if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("json.Unmarshal failed") - return jsonerror.InternalServerError() - } - // Check whether the tag to be deleted exists if _, ok := tagContent.Tags[tag]; ok { delete(tagContent.Tags, tag) @@ -185,18 +144,16 @@ func DeleteTag( JSON: struct{}{}, } } - if err = saveTagData(req, localpart, roomID, accountDB, tagContent); err != nil { + + if err = saveTagData(req, userID, roomID, userAPI, tagContent); err != nil { util.GetLogger(req.Context()).WithError(err).Error("saveTagData failed") return jsonerror.InternalServerError() } - // Send data to syncProducer in order to inform clients of changes - // Run in a goroutine in order to prevent blocking the tag request response - go func() { - if err := syncProducer.SendData(userID, roomID, "m.tag"); err != nil { - logrus.WithError(err).Error("Failed to send m.tag account data update to syncapi") - } - }() + // TODO: user API should do this since it's account data + if err := syncProducer.SendData(userID, roomID, "m.tag"); err != nil { + logrus.WithError(err).Error("Failed to send m.tag account data update to syncapi") + } return util.JSONResponse{ Code: http.StatusOK, @@ -210,32 +167,46 @@ func obtainSavedTags( req *http.Request, userID string, roomID string, - accountDB accounts.Database, -) (string, *gomatrixserverlib.ClientEvent, error) { - localpart, _, err := gomatrixserverlib.SplitID('@', userID) - if err != nil { - return "", nil, err + userAPI api.UserInternalAPI, +) (tags gomatrix.TagContent, err error) { + dataReq := api.QueryAccountDataRequest{ + UserID: userID, + RoomID: roomID, + DataType: "m.tag", } - - data, err := accountDB.GetAccountDataByType( - req.Context(), localpart, roomID, "m.tag", - ) - - return localpart, data, err + dataRes := api.QueryAccountDataResponse{} + err = userAPI.QueryAccountData(req.Context(), &dataReq, &dataRes) + if err != nil { + return + } + data, ok := dataRes.RoomAccountData[roomID]["m.tag"] + if !ok { + return + } + if err = json.Unmarshal(data, &tags); err != nil { + return + } + return tags, nil } // saveTagData saves the provided tag data into the database func saveTagData( req *http.Request, - localpart string, + userID string, roomID string, - accountDB accounts.Database, + userAPI api.UserInternalAPI, Tag gomatrix.TagContent, ) error { newTagData, err := json.Marshal(Tag) if err != nil { return err } - - return accountDB.SaveAccountData(req.Context(), localpart, roomID, "m.tag", string(newTagData)) + dataReq := api.InputAccountDataRequest{ + UserID: userID, + RoomID: roomID, + DataType: "m.tag", + AccountData: json.RawMessage(newTagData), + } + dataRes := api.InputAccountDataResponse{} + return userAPI.InputAccountData(req.Context(), &dataReq, &dataRes) } diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 41c7fb18e..e91b07ac7 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -476,7 +476,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SaveAccountData(req, accountDB, device, vars["userID"], "", vars["type"], syncProducer) + return SaveAccountData(req, userAPI, device, vars["userID"], "", vars["type"], syncProducer) }), ).Methods(http.MethodPut, http.MethodOptions) @@ -486,7 +486,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return SaveAccountData(req, accountDB, device, vars["userID"], vars["roomID"], vars["type"], syncProducer) + return SaveAccountData(req, userAPI, device, vars["userID"], vars["roomID"], vars["type"], syncProducer) }), ).Methods(http.MethodPut, http.MethodOptions) @@ -496,7 +496,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return GetAccountData(req, accountDB, device, vars["userID"], "", vars["type"]) + return GetAccountData(req, userAPI, device, vars["userID"], "", vars["type"]) }), ).Methods(http.MethodGet) @@ -506,7 +506,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return GetAccountData(req, accountDB, device, vars["userID"], vars["roomID"], vars["type"]) + return GetAccountData(req, userAPI, device, vars["userID"], vars["roomID"], vars["type"]) }), ).Methods(http.MethodGet) @@ -604,7 +604,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return GetTags(req, accountDB, device, vars["userId"], vars["roomId"], syncProducer) + return GetTags(req, userAPI, device, vars["userId"], vars["roomId"], syncProducer) }), ).Methods(http.MethodGet, http.MethodOptions) @@ -614,7 +614,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return PutTag(req, accountDB, device, vars["userId"], vars["roomId"], vars["tag"], syncProducer) + return PutTag(req, userAPI, device, vars["userId"], vars["roomId"], vars["tag"], syncProducer) }), ).Methods(http.MethodPut, http.MethodOptions) @@ -624,7 +624,7 @@ func Setup( if err != nil { return util.ErrorResponse(err) } - return DeleteTag(req, accountDB, device, vars["userId"], vars["roomId"], vars["tag"], syncProducer) + return DeleteTag(req, userAPI, device, vars["userId"], vars["roomId"], vars["tag"], syncProducer) }), ).Methods(http.MethodDelete, http.MethodOptions) diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index 26b925eac..8d51689e3 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -205,22 +205,34 @@ func (rp *RequestPool) appendAccountData( if req.since == nil { // If this is the initial sync, we don't need to check if a data has // already been sent. Instead, we send the whole batch. - var res userapi.QueryAccountDataResponse - err := rp.userAPI.QueryAccountData(req.ctx, &userapi.QueryAccountDataRequest{ + dataReq := &userapi.QueryAccountDataRequest{ UserID: userID, - }, &res) - if err != nil { + } + dataRes := &userapi.QueryAccountDataResponse{} + if err := rp.userAPI.QueryAccountData(req.ctx, dataReq, dataRes); err != nil { return nil, err } - data.AccountData.Events = res.GlobalAccountData - + for datatype, databody := range dataRes.GlobalAccountData { + data.AccountData.Events = append( + data.AccountData.Events, + gomatrixserverlib.ClientEvent{ + Type: datatype, + Content: gomatrixserverlib.RawJSON(databody), + }, + ) + } for r, j := range data.Rooms.Join { - if len(res.RoomAccountData[r]) > 0 { - j.AccountData.Events = res.RoomAccountData[r] + for datatype, databody := range dataRes.RoomAccountData[r] { + j.AccountData.Events = append( + j.AccountData.Events, + gomatrixserverlib.ClientEvent{ + Type: datatype, + Content: gomatrixserverlib.RawJSON(databody), + }, + ) data.Rooms.Join[r] = j } } - return data, nil } @@ -249,33 +261,42 @@ func (rp *RequestPool) appendAccountData( // Iterate over the rooms for roomID, dataTypes := range dataTypes { - events := []gomatrixserverlib.ClientEvent{} // Request the missing data from the database for _, dataType := range dataTypes { - var res userapi.QueryAccountDataResponse - err = rp.userAPI.QueryAccountData(req.ctx, &userapi.QueryAccountDataRequest{ + dataReq := userapi.QueryAccountDataRequest{ UserID: userID, RoomID: roomID, DataType: dataType, - }, &res) + } + dataRes := userapi.QueryAccountDataResponse{} + err = rp.userAPI.QueryAccountData(req.ctx, &dataReq, &dataRes) if err != nil { - return nil, err + continue } - if len(res.RoomAccountData[roomID]) > 0 { - events = append(events, res.RoomAccountData[roomID]...) - } else if len(res.GlobalAccountData) > 0 { - events = append(events, res.GlobalAccountData...) + if roomID == "" { + if globalData, ok := dataRes.GlobalAccountData[dataType]; ok { + data.AccountData.Events = append( + data.AccountData.Events, + gomatrixserverlib.ClientEvent{ + Type: dataType, + Content: gomatrixserverlib.RawJSON(globalData), + }, + ) + } + } else { + if roomData, ok := dataRes.RoomAccountData[roomID][dataType]; ok { + joinData := data.Rooms.Join[roomID] + joinData.AccountData.Events = append( + joinData.AccountData.Events, + gomatrixserverlib.ClientEvent{ + Type: dataType, + Content: gomatrixserverlib.RawJSON(roomData), + }, + ) + data.Rooms.Join[roomID] = joinData + } } } - - // Append the data to the response - if len(roomID) > 0 { - jr := data.Rooms.Join[roomID] - jr.AccountData.Events = events - data.Rooms.Join[roomID] = jr - } else { - data.AccountData.Events = events - } } return data, nil diff --git a/userapi/api/api.go b/userapi/api/api.go index c953a5bac..a80adf2d8 100644 --- a/userapi/api/api.go +++ b/userapi/api/api.go @@ -16,12 +16,14 @@ package api import ( "context" + "encoding/json" "github.com/matrix-org/gomatrixserverlib" ) // UserInternalAPI is the internal API for information about users and devices. type UserInternalAPI interface { + InputAccountData(ctx context.Context, req *InputAccountDataRequest, res *InputAccountDataResponse) error PerformAccountCreation(ctx context.Context, req *PerformAccountCreationRequest, res *PerformAccountCreationResponse) error PerformDeviceCreation(ctx context.Context, req *PerformDeviceCreationRequest, res *PerformDeviceCreationResponse) error QueryProfile(ctx context.Context, req *QueryProfileRequest, res *QueryProfileResponse) error @@ -30,6 +32,18 @@ type UserInternalAPI interface { QueryAccountData(ctx context.Context, req *QueryAccountDataRequest, res *QueryAccountDataResponse) error } +// InputAccountDataRequest is the request for InputAccountData +type InputAccountDataRequest struct { + UserID string // required: the user to set account data for + RoomID string // optional: the room to associate the account data with + DataType string // optional: the data type of the data + AccountData json.RawMessage // required: the message content +} + +// InputAccountDataResponse is the response for InputAccountData +type InputAccountDataResponse struct { +} + // QueryAccessTokenRequest is the request for QueryAccessToken type QueryAccessTokenRequest struct { AccessToken string @@ -46,18 +60,15 @@ type QueryAccessTokenResponse struct { // QueryAccountDataRequest is the request for QueryAccountData type QueryAccountDataRequest struct { - UserID string // required: the user to get account data for. - // TODO: This is a terribly confusing API shape :/ - DataType string // optional: if specified returns only a single event matching this data type. - // optional: Only used if DataType is set. If blank returns global account data matching the data type. - // If set, returns only room account data matching this data type. - RoomID string + UserID string // required: the user to get account data for. + RoomID string // optional: the room ID, or global account data if not specified. + DataType string // optional: the data type, or all types if not specified. } // QueryAccountDataResponse is the response for QueryAccountData type QueryAccountDataResponse struct { - GlobalAccountData []gomatrixserverlib.ClientEvent - RoomAccountData map[string][]gomatrixserverlib.ClientEvent + GlobalAccountData map[string]json.RawMessage // type -> data + RoomAccountData map[string]map[string]json.RawMessage // room -> type -> data } // QueryDevicesRequest is the request for QueryDevices diff --git a/userapi/internal/api.go b/userapi/internal/api.go index ae021f575..b081eca49 100644 --- a/userapi/internal/api.go +++ b/userapi/internal/api.go @@ -17,6 +17,7 @@ package internal import ( "context" "database/sql" + "encoding/json" "errors" "fmt" @@ -38,6 +39,20 @@ type UserInternalAPI struct { AppServices []config.ApplicationService } +func (a *UserInternalAPI) InputAccountData(ctx context.Context, req *api.InputAccountDataRequest, res *api.InputAccountDataResponse) error { + local, domain, err := gomatrixserverlib.SplitID('@', req.UserID) + if err != nil { + return err + } + if domain != a.ServerName { + return fmt.Errorf("cannot query profile of remote users: got %s want %s", domain, a.ServerName) + } + if req.DataType == "" { + return fmt.Errorf("data type must not be empty") + } + return a.AccountDB.SaveAccountData(ctx, local, req.RoomID, req.DataType, req.AccountData) +} + func (a *UserInternalAPI) PerformAccountCreation(ctx context.Context, req *api.PerformAccountCreationRequest, res *api.PerformAccountCreationResponse) error { if req.AccountType == api.AccountTypeGuest { acc, err := a.AccountDB.CreateGuestAccount(ctx) @@ -130,17 +145,21 @@ func (a *UserInternalAPI) QueryAccountData(ctx context.Context, req *api.QueryAc return fmt.Errorf("cannot query account data of remote users: got %s want %s", domain, a.ServerName) } if req.DataType != "" { - var event *gomatrixserverlib.ClientEvent - event, err = a.AccountDB.GetAccountDataByType(ctx, local, req.RoomID, req.DataType) + var data json.RawMessage + data, err = a.AccountDB.GetAccountDataByType(ctx, local, req.RoomID, req.DataType) if err != nil { return err } - if event != nil { + res.RoomAccountData = make(map[string]map[string]json.RawMessage) + res.GlobalAccountData = make(map[string]json.RawMessage) + if data != nil { if req.RoomID != "" { - res.RoomAccountData = make(map[string][]gomatrixserverlib.ClientEvent) - res.RoomAccountData[req.RoomID] = []gomatrixserverlib.ClientEvent{*event} + if _, ok := res.RoomAccountData[req.RoomID]; !ok { + res.RoomAccountData[req.RoomID] = make(map[string]json.RawMessage) + } + res.RoomAccountData[req.RoomID][req.DataType] = data } else { - res.GlobalAccountData = append(res.GlobalAccountData, *event) + res.GlobalAccountData[req.DataType] = data } } return nil diff --git a/userapi/inthttp/client.go b/userapi/inthttp/client.go index 0e9628c58..4ab0d690e 100644 --- a/userapi/inthttp/client.go +++ b/userapi/inthttp/client.go @@ -26,6 +26,8 @@ import ( // HTTP paths for the internal HTTP APIs const ( + InputAccountDataPath = "/userapi/inputAccountData" + PerformDeviceCreationPath = "/userapi/performDeviceCreation" PerformAccountCreationPath = "/userapi/performAccountCreation" @@ -55,6 +57,14 @@ type httpUserInternalAPI struct { httpClient *http.Client } +func (h *httpUserInternalAPI) InputAccountData(ctx context.Context, req *api.InputAccountDataRequest, res *api.InputAccountDataResponse) error { + span, ctx := opentracing.StartSpanFromContext(ctx, "InputAccountData") + defer span.Finish() + + apiURL := h.apiURL + InputAccountDataPath + return httputil.PostJSON(ctx, span, h.httpClient, apiURL, req, res) +} + func (h *httpUserInternalAPI) PerformAccountCreation( ctx context.Context, request *api.PerformAccountCreationRequest, diff --git a/userapi/storage/accounts/interface.go b/userapi/storage/accounts/interface.go index 13e3e2895..c6692879b 100644 --- a/userapi/storage/accounts/interface.go +++ b/userapi/storage/accounts/interface.go @@ -16,6 +16,7 @@ package accounts import ( "context" + "encoding/json" "errors" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" @@ -39,13 +40,13 @@ type Database interface { GetMembershipInRoomByLocalpart(ctx context.Context, localpart, roomID string) (authtypes.Membership, error) GetRoomIDsByLocalPart(ctx context.Context, localpart string) ([]string, error) GetMembershipsByLocalpart(ctx context.Context, localpart string) (memberships []authtypes.Membership, err error) - SaveAccountData(ctx context.Context, localpart, roomID, dataType, content string) error - GetAccountData(ctx context.Context, localpart string) (global []gomatrixserverlib.ClientEvent, rooms map[string][]gomatrixserverlib.ClientEvent, err error) + SaveAccountData(ctx context.Context, localpart, roomID, dataType string, content json.RawMessage) error + GetAccountData(ctx context.Context, localpart string) (global map[string]json.RawMessage, rooms map[string]map[string]json.RawMessage, err error) // GetAccountDataByType returns account data matching a given // localpart, room ID and type. // If no account data could be found, returns nil // Returns an error if there was an issue with the retrieval - GetAccountDataByType(ctx context.Context, localpart, roomID, dataType string) (data *gomatrixserverlib.ClientEvent, err error) + GetAccountDataByType(ctx context.Context, localpart, roomID, dataType string) (data json.RawMessage, err error) GetNewNumericLocalpart(ctx context.Context) (int64, error) SaveThreePIDAssociation(ctx context.Context, threepid, localpart, medium string) (err error) RemoveThreePIDAssociation(ctx context.Context, threepid string, medium string) (err error) diff --git a/userapi/storage/accounts/postgres/account_data_table.go b/userapi/storage/accounts/postgres/account_data_table.go index 2f16c5c02..90c79e878 100644 --- a/userapi/storage/accounts/postgres/account_data_table.go +++ b/userapi/storage/accounts/postgres/account_data_table.go @@ -17,9 +17,9 @@ package postgres import ( "context" "database/sql" + "encoding/json" "github.com/matrix-org/dendrite/internal" - "github.com/matrix-org/gomatrixserverlib" ) const accountDataSchema = ` @@ -73,7 +73,7 @@ func (s *accountDataStatements) prepare(db *sql.DB) (err error) { } func (s *accountDataStatements) insertAccountData( - ctx context.Context, txn *sql.Tx, localpart, roomID, dataType, content string, + ctx context.Context, txn *sql.Tx, localpart, roomID, dataType string, content json.RawMessage, ) (err error) { stmt := txn.Stmt(s.insertAccountDataStmt) _, err = stmt.ExecContext(ctx, localpart, roomID, dataType, content) @@ -83,18 +83,18 @@ func (s *accountDataStatements) insertAccountData( func (s *accountDataStatements) selectAccountData( ctx context.Context, localpart string, ) ( - global []gomatrixserverlib.ClientEvent, - rooms map[string][]gomatrixserverlib.ClientEvent, - err error, + /* global */ map[string]json.RawMessage, + /* rooms */ map[string]map[string]json.RawMessage, + error, ) { rows, err := s.selectAccountDataStmt.QueryContext(ctx, localpart) if err != nil { - return + return nil, nil, err } defer internal.CloseAndLogIfError(ctx, rows, "selectAccountData: rows.close() failed") - global = []gomatrixserverlib.ClientEvent{} - rooms = make(map[string][]gomatrixserverlib.ClientEvent) + global := map[string]json.RawMessage{} + rooms := map[string]map[string]json.RawMessage{} for rows.Next() { var roomID string @@ -102,41 +102,33 @@ func (s *accountDataStatements) selectAccountData( var content []byte if err = rows.Scan(&roomID, &dataType, &content); err != nil { - return + return nil, nil, err } - ac := gomatrixserverlib.ClientEvent{ - Type: dataType, - Content: content, - } - - if len(roomID) > 0 { - rooms[roomID] = append(rooms[roomID], ac) + if roomID != "" { + if _, ok := rooms[roomID]; !ok { + rooms[roomID] = map[string]json.RawMessage{} + } + rooms[roomID][dataType] = content } else { - global = append(global, ac) + global[dataType] = content } } + return global, rooms, rows.Err() } func (s *accountDataStatements) selectAccountDataByType( ctx context.Context, localpart, roomID, dataType string, -) (data *gomatrixserverlib.ClientEvent, err error) { +) (data json.RawMessage, err error) { + var bytes []byte stmt := s.selectAccountDataByTypeStmt - var content []byte - - if err = stmt.QueryRowContext(ctx, localpart, roomID, dataType).Scan(&content); err != nil { + if err = stmt.QueryRowContext(ctx, localpart, roomID, dataType).Scan(&bytes); err != nil { if err == sql.ErrNoRows { return nil, nil } - return } - - data = &gomatrixserverlib.ClientEvent{ - Type: dataType, - Content: content, - } - + data = json.RawMessage(bytes) return } diff --git a/userapi/storage/accounts/postgres/storage.go b/userapi/storage/accounts/postgres/storage.go index 2b88cb70a..e55099800 100644 --- a/userapi/storage/accounts/postgres/storage.go +++ b/userapi/storage/accounts/postgres/storage.go @@ -17,6 +17,7 @@ package postgres import ( "context" "database/sql" + "encoding/json" "errors" "strconv" @@ -169,7 +170,7 @@ func (d *Database) createAccount( return nil, err } - if err := d.accountDatas.insertAccountData(ctx, txn, localpart, "", "m.push_rules", `{ + if err := d.accountDatas.insertAccountData(ctx, txn, localpart, "", "m.push_rules", json.RawMessage(`{ "global": { "content": [], "override": [], @@ -177,7 +178,7 @@ func (d *Database) createAccount( "sender": [], "underride": [] } - }`); err != nil { + }`)); err != nil { return nil, err } return d.accounts.insertAccount(ctx, txn, localpart, hash, appserviceID) @@ -295,7 +296,7 @@ func (d *Database) newMembership( // update the corresponding row with the new content // Returns a SQL error if there was an issue with the insertion/update func (d *Database) SaveAccountData( - ctx context.Context, localpart, roomID, dataType, content string, + ctx context.Context, localpart, roomID, dataType string, content json.RawMessage, ) error { return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { return d.accountDatas.insertAccountData(ctx, txn, localpart, roomID, dataType, content) @@ -306,8 +307,8 @@ func (d *Database) SaveAccountData( // If no account data could be found, returns an empty arrays // Returns an error if there was an issue with the retrieval func (d *Database) GetAccountData(ctx context.Context, localpart string) ( - global []gomatrixserverlib.ClientEvent, - rooms map[string][]gomatrixserverlib.ClientEvent, + global map[string]json.RawMessage, + rooms map[string]map[string]json.RawMessage, err error, ) { return d.accountDatas.selectAccountData(ctx, localpart) @@ -319,7 +320,7 @@ func (d *Database) GetAccountData(ctx context.Context, localpart string) ( // Returns an error if there was an issue with the retrieval func (d *Database) GetAccountDataByType( ctx context.Context, localpart, roomID, dataType string, -) (data *gomatrixserverlib.ClientEvent, err error) { +) (data json.RawMessage, err error) { return d.accountDatas.selectAccountDataByType( ctx, localpart, roomID, dataType, ) diff --git a/userapi/storage/accounts/sqlite3/account_data_table.go b/userapi/storage/accounts/sqlite3/account_data_table.go index b6bb63617..d048dbd19 100644 --- a/userapi/storage/accounts/sqlite3/account_data_table.go +++ b/userapi/storage/accounts/sqlite3/account_data_table.go @@ -17,8 +17,7 @@ package sqlite3 import ( "context" "database/sql" - - "github.com/matrix-org/gomatrixserverlib" + "encoding/json" ) const accountDataSchema = ` @@ -72,7 +71,7 @@ func (s *accountDataStatements) prepare(db *sql.DB) (err error) { } func (s *accountDataStatements) insertAccountData( - ctx context.Context, txn *sql.Tx, localpart, roomID, dataType, content string, + ctx context.Context, txn *sql.Tx, localpart, roomID, dataType string, content json.RawMessage, ) (err error) { _, err = txn.Stmt(s.insertAccountDataStmt).ExecContext(ctx, localpart, roomID, dataType, content) return @@ -81,17 +80,17 @@ func (s *accountDataStatements) insertAccountData( func (s *accountDataStatements) selectAccountData( ctx context.Context, localpart string, ) ( - global []gomatrixserverlib.ClientEvent, - rooms map[string][]gomatrixserverlib.ClientEvent, - err error, + /* global */ map[string]json.RawMessage, + /* rooms */ map[string]map[string]json.RawMessage, + error, ) { rows, err := s.selectAccountDataStmt.QueryContext(ctx, localpart) if err != nil { - return + return nil, nil, err } - global = []gomatrixserverlib.ClientEvent{} - rooms = make(map[string][]gomatrixserverlib.ClientEvent) + global := map[string]json.RawMessage{} + rooms := map[string]map[string]json.RawMessage{} for rows.Next() { var roomID string @@ -99,42 +98,33 @@ func (s *accountDataStatements) selectAccountData( var content []byte if err = rows.Scan(&roomID, &dataType, &content); err != nil { - return + return nil, nil, err } - ac := gomatrixserverlib.ClientEvent{ - Type: dataType, - Content: content, - } - - if len(roomID) > 0 { - rooms[roomID] = append(rooms[roomID], ac) + if roomID != "" { + if _, ok := rooms[roomID]; !ok { + rooms[roomID] = map[string]json.RawMessage{} + } + rooms[roomID][dataType] = content } else { - global = append(global, ac) + global[dataType] = content } } - return + return global, rooms, nil } func (s *accountDataStatements) selectAccountDataByType( ctx context.Context, localpart, roomID, dataType string, -) (data *gomatrixserverlib.ClientEvent, err error) { +) (data json.RawMessage, err error) { + var bytes []byte stmt := s.selectAccountDataByTypeStmt - var content []byte - - if err = stmt.QueryRowContext(ctx, localpart, roomID, dataType).Scan(&content); err != nil { + if err = stmt.QueryRowContext(ctx, localpart, roomID, dataType).Scan(&bytes); err != nil { if err == sql.ErrNoRows { return nil, nil } - return } - - data = &gomatrixserverlib.ClientEvent{ - Type: dataType, - Content: content, - } - + data = json.RawMessage(bytes) return } diff --git a/userapi/storage/accounts/sqlite3/storage.go b/userapi/storage/accounts/sqlite3/storage.go index 4dd755a70..dbf6606c3 100644 --- a/userapi/storage/accounts/sqlite3/storage.go +++ b/userapi/storage/accounts/sqlite3/storage.go @@ -17,6 +17,7 @@ package sqlite3 import ( "context" "database/sql" + "encoding/json" "errors" "strconv" "sync" @@ -180,7 +181,7 @@ func (d *Database) createAccount( return nil, err } - if err := d.accountDatas.insertAccountData(ctx, txn, localpart, "", "m.push_rules", `{ + if err := d.accountDatas.insertAccountData(ctx, txn, localpart, "", "m.push_rules", json.RawMessage(`{ "global": { "content": [], "override": [], @@ -188,7 +189,7 @@ func (d *Database) createAccount( "sender": [], "underride": [] } - }`); err != nil { + }`)); err != nil { return nil, err } return d.accounts.insertAccount(ctx, txn, localpart, hash, appserviceID) @@ -306,7 +307,7 @@ func (d *Database) newMembership( // update the corresponding row with the new content // Returns a SQL error if there was an issue with the insertion/update func (d *Database) SaveAccountData( - ctx context.Context, localpart, roomID, dataType, content string, + ctx context.Context, localpart, roomID, dataType string, content json.RawMessage, ) error { return sqlutil.WithTransaction(d.db, func(txn *sql.Tx) error { return d.accountDatas.insertAccountData(ctx, txn, localpart, roomID, dataType, content) @@ -317,8 +318,8 @@ func (d *Database) SaveAccountData( // If no account data could be found, returns an empty arrays // Returns an error if there was an issue with the retrieval func (d *Database) GetAccountData(ctx context.Context, localpart string) ( - global []gomatrixserverlib.ClientEvent, - rooms map[string][]gomatrixserverlib.ClientEvent, + global map[string]json.RawMessage, + rooms map[string]map[string]json.RawMessage, err error, ) { return d.accountDatas.selectAccountData(ctx, localpart) @@ -330,7 +331,7 @@ func (d *Database) GetAccountData(ctx context.Context, localpart string) ( // Returns an error if there was an issue with the retrieval func (d *Database) GetAccountDataByType( ctx context.Context, localpart, roomID, dataType string, -) (data *gomatrixserverlib.ClientEvent, err error) { +) (data json.RawMessage, err error) { return d.accountDatas.selectAccountDataByType( ctx, localpart, roomID, dataType, ) From 9e3d771a32059b96c5595b08bc1f13a481ca800b Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 19 Jun 2020 09:18:09 +0100 Subject: [PATCH 187/200] Fix comment in InputAccountDataRequest --- userapi/api/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/userapi/api/api.go b/userapi/api/api.go index a80adf2d8..cf0f05633 100644 --- a/userapi/api/api.go +++ b/userapi/api/api.go @@ -36,7 +36,7 @@ type UserInternalAPI interface { type InputAccountDataRequest struct { UserID string // required: the user to set account data for RoomID string // optional: the room to associate the account data with - DataType string // optional: the data type of the data + DataType string // required: the data type of the data AccountData json.RawMessage // required: the message content } From 72444e4a4f31b70dfb7ddd875ef874f637699dce Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 19 Jun 2020 09:37:19 +0100 Subject: [PATCH 188/200] User API polylith component (#1151) * User API polylith component * Add docker-pull.sh --- build/docker/config/dendrite-config.yaml | 1 + build/docker/docker-compose.polylith.yml | 11 +++++++++++ build/docker/images-build.sh | 1 + build/docker/images-pull.sh | 16 ++++++++++++++++ build/docker/images-push.sh | 1 + dendrite-config.yaml | 6 ++++-- docs/INSTALL.md | 10 ++++++++++ internal/config/config.go | 1 + 8 files changed, 45 insertions(+), 2 deletions(-) create mode 100755 build/docker/images-pull.sh diff --git a/build/docker/config/dendrite-config.yaml b/build/docker/config/dendrite-config.yaml index 26dc272ab..f421b2e71 100644 --- a/build/docker/config/dendrite-config.yaml +++ b/build/docker/config/dendrite-config.yaml @@ -117,6 +117,7 @@ listen: federation_sender: "federation_sender:7776" edu_server: "edu_server:7777" key_server: "key_server:7779" + user_api: "user_api:7780" # The configuration for tracing the dendrite components. tracing: diff --git a/build/docker/docker-compose.polylith.yml b/build/docker/docker-compose.polylith.yml index 178604093..af2ae4e81 100644 --- a/build/docker/docker-compose.polylith.yml +++ b/build/docker/docker-compose.polylith.yml @@ -152,6 +152,17 @@ services: networks: - internal + user_api: + hostname: user_api + image: matrixdotorg/dendrite:userapi + command: [ + "--config=dendrite.yaml" + ] + volumes: + - ./config:/etc/dendrite + networks: + - internal + networks: internal: attachable: true diff --git a/build/docker/images-build.sh b/build/docker/images-build.sh index 6bc058962..e43d2d040 100755 --- a/build/docker/images-build.sh +++ b/build/docker/images-build.sh @@ -18,3 +18,4 @@ docker build -t matrixdotorg/dendrite:publicroomsapi --build-arg component=de docker build -t matrixdotorg/dendrite:roomserver --build-arg component=dendrite-room-server -f build/docker/Dockerfile.component . docker build -t matrixdotorg/dendrite:syncapi --build-arg component=dendrite-sync-api-server -f build/docker/Dockerfile.component . docker build -t matrixdotorg/dendrite:serverkeyapi --build-arg component=dendrite-server-key-api-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite:userapi --build-arg component=dendrite-user-api-server -f build/docker/Dockerfile.component . diff --git a/build/docker/images-pull.sh b/build/docker/images-pull.sh new file mode 100755 index 000000000..edccf4a33 --- /dev/null +++ b/build/docker/images-pull.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +docker pull matrixdotorg/dendrite:monolith + +docker pull matrixdotorg/dendrite:clientapi +docker pull matrixdotorg/dendrite:clientproxy +docker pull matrixdotorg/dendrite:eduserver +docker pull matrixdotorg/dendrite:federationapi +docker pull matrixdotorg/dendrite:federationsender +docker pull matrixdotorg/dendrite:federationproxy +docker pull matrixdotorg/dendrite:keyserver +docker pull matrixdotorg/dendrite:mediaapi +docker pull matrixdotorg/dendrite:publicroomsapi +docker pull matrixdotorg/dendrite:roomserver +docker pull matrixdotorg/dendrite:syncapi +docker pull matrixdotorg/dendrite:userapi diff --git a/build/docker/images-push.sh b/build/docker/images-push.sh index b39d98d65..2b4303872 100755 --- a/build/docker/images-push.sh +++ b/build/docker/images-push.sh @@ -13,3 +13,4 @@ docker push matrixdotorg/dendrite:mediaapi docker push matrixdotorg/dendrite:publicroomsapi docker push matrixdotorg/dendrite:roomserver docker push matrixdotorg/dendrite:syncapi +docker push matrixdotorg/dendrite:userapi diff --git a/dendrite-config.yaml b/dendrite-config.yaml index 52793cda4..73bfec247 100644 --- a/dendrite-config.yaml +++ b/dendrite-config.yaml @@ -108,7 +108,9 @@ kafka: output_send_to_device_event: eduServerSendToDeviceOutput user_updates: userUpdates -# The postgres connection configs for connecting to the databases e.g a postgres:// URI +# The postgres connection configs for connecting to the databases, e.g. +# for Postgres: postgres://username:password@hostname/database +# for SQLite: file:filename.db or file:///path/to/filename.db database: account: "postgres://dendrite:itsasecret@localhost/dendrite_account?sslmode=disable" device: "postgres://dendrite:itsasecret@localhost/dendrite_device?sslmode=disable" @@ -122,7 +124,7 @@ database: max_open_conns: 100 max_idle_conns: 2 conn_max_lifetime: -1 - # If using naffka you need to specify a naffka database + # If 'use_naffka: true' set above then you need to specify a naffka database # naffka: "postgres://dendrite:itsasecret@localhost/dendrite_naffka?sslmode=disable" # The TCP host:port pairs to bind the internal HTTP APIs to. diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 184b777d8..b4c81a42b 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -329,3 +329,13 @@ finished). ```bash ./bin/dendrite-key-server --config dendrite.yaml ``` + +### User server + +This manages user accounts, device access tokens and user account data, +amongst other things. + +```bash +./bin/dendrite-user-api-server --config dendrite.yaml +``` + diff --git a/internal/config/config.go b/internal/config/config.go index 2bd56ad9e..baa82be23 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -226,6 +226,7 @@ type Dendrite struct { ServerKeyAPI Address `yaml:"server_key_api"` AppServiceAPI Address `yaml:"appservice_api"` SyncAPI Address `yaml:"sync_api"` + UserAPI Address `yaml:"user_api"` RoomServer Address `yaml:"room_server"` FederationSender Address `yaml:"federation_sender"` PublicRoomsAPI Address `yaml:"public_rooms_api"` From 7f26b0cd13534ba235c2bf0a7220d3462ca8e9da Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 19 Jun 2020 13:29:27 +0100 Subject: [PATCH 189/200] Bind build support, further Yggdrasil demo updates (#1152) * Add gobind builds for Yggdrasil demo * Massage client API a bit * Fix build * Fix gobind build * Fix gobind client API setup * Tweaks * Tweaks * Update sytest-whitelist, add comment * Default to sending push rules on initial sync --- build/gobind/build.sh | 6 + build/gobind/monolith.go | 161 ++++++++++++++++++ build/gobind/platform_ios.go | 25 +++ build/gobind/platform_other.go | 12 ++ clientapi/routing/device.go | 9 +- clientapi/routing/joinroom.go | 1 + cmd/dendrite-demo-yggdrasil/main.go | 49 +----- cmd/dendrite-demo-yggdrasil/yggconn/client.go | 74 ++++++++ cmd/dendrite-demo-yggdrasil/yggconn/node.go | 27 +-- go.mod | 1 + go.sum | 15 ++ syncapi/sync/request.go | 4 + syncapi/sync/requestpool.go | 3 +- syncapi/types/types.go | 11 +- sytest-whitelist | 3 + 15 files changed, 336 insertions(+), 65 deletions(-) create mode 100644 build/gobind/build.sh create mode 100644 build/gobind/monolith.go create mode 100644 build/gobind/platform_ios.go create mode 100644 build/gobind/platform_other.go create mode 100644 cmd/dendrite-demo-yggdrasil/yggconn/client.go diff --git a/build/gobind/build.sh b/build/gobind/build.sh new file mode 100644 index 000000000..3a80d374a --- /dev/null +++ b/build/gobind/build.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +gomobile bind -v \ + -ldflags "-X $github.com/yggdrasil-network/yggdrasil-go/src/version.buildName=riot-ios-p2p" \ + -target ios \ + github.com/matrix-org/dendrite/build/gobind \ No newline at end of file diff --git a/build/gobind/monolith.go b/build/gobind/monolith.go new file mode 100644 index 000000000..750babad8 --- /dev/null +++ b/build/gobind/monolith.go @@ -0,0 +1,161 @@ +package gobind + +import ( + "context" + "crypto/tls" + "fmt" + "net" + "net/http" + "time" + + "github.com/matrix-org/dendrite/appservice" + "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/signing" + "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/yggconn" + "github.com/matrix-org/dendrite/eduserver" + "github.com/matrix-org/dendrite/eduserver/cache" + "github.com/matrix-org/dendrite/federationsender" + "github.com/matrix-org/dendrite/internal/config" + "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/internal/setup" + "github.com/matrix-org/dendrite/publicroomsapi/storage" + "github.com/matrix-org/dendrite/roomserver" + "github.com/matrix-org/dendrite/userapi" + "github.com/matrix-org/gomatrixserverlib" + "github.com/sirupsen/logrus" +) + +type DendriteMonolith struct { + StorageDirectory string + listener net.Listener +} + +func (m *DendriteMonolith) BaseURL() string { + return fmt.Sprintf("http://%s", m.listener.Addr().String()) +} + +func (m *DendriteMonolith) Start() { + logger := logrus.Logger{ + Out: BindLogger{}, + } + logrus.SetOutput(BindLogger{}) + + var err error + m.listener, err = net.Listen("tcp", "localhost:65432") + if err != nil { + panic(err) + } + + ygg, err := yggconn.Setup("dendrite", "", m.StorageDirectory) + if err != nil { + panic(err) + } + + cfg := &config.Dendrite{} + cfg.SetDefaults() + cfg.Matrix.ServerName = gomatrixserverlib.ServerName(ygg.DerivedServerName()) + cfg.Matrix.PrivateKey = ygg.SigningPrivateKey() + cfg.Matrix.KeyID = gomatrixserverlib.KeyID(signing.KeyID) + cfg.Kafka.UseNaffka = true + cfg.Kafka.Topics.OutputRoomEvent = "roomserverOutput" + cfg.Kafka.Topics.OutputClientData = "clientapiOutput" + cfg.Kafka.Topics.OutputTypingEvent = "typingServerOutput" + cfg.Kafka.Topics.OutputSendToDeviceEvent = "sendToDeviceOutput" + cfg.Database.Account = config.DataSource(fmt.Sprintf("file:%s/dendrite-account.db", m.StorageDirectory)) + cfg.Database.Device = config.DataSource(fmt.Sprintf("file:%s/dendrite-device.db", m.StorageDirectory)) + cfg.Database.MediaAPI = config.DataSource(fmt.Sprintf("file:%s/dendrite-mediaapi.db", m.StorageDirectory)) + cfg.Database.SyncAPI = config.DataSource(fmt.Sprintf("file:%s/dendrite-syncapi.db", m.StorageDirectory)) + cfg.Database.RoomServer = config.DataSource(fmt.Sprintf("file:%s/dendrite-roomserver.db", m.StorageDirectory)) + cfg.Database.ServerKey = config.DataSource(fmt.Sprintf("file:%s/dendrite-serverkey.db", m.StorageDirectory)) + cfg.Database.FederationSender = config.DataSource(fmt.Sprintf("file:%s/dendrite-federationsender.db", m.StorageDirectory)) + cfg.Database.AppService = config.DataSource(fmt.Sprintf("file:%s/dendrite-appservice.db", m.StorageDirectory)) + cfg.Database.PublicRoomsAPI = config.DataSource(fmt.Sprintf("file:%s/dendrite-publicroomsa.db", m.StorageDirectory)) + cfg.Database.Naffka = config.DataSource(fmt.Sprintf("file:%s/dendrite-naffka.db", m.StorageDirectory)) + if err = cfg.Derive(); err != nil { + panic(err) + } + + base := setup.NewBaseDendrite(cfg, "Monolith", false) + defer base.Close() // nolint: errcheck + + accountDB := base.CreateAccountsDB() + deviceDB := base.CreateDeviceDB() + federation := ygg.CreateFederationClient(base) + + serverKeyAPI := &signing.YggdrasilKeys{} + keyRing := serverKeyAPI.KeyRing() + userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, cfg.Derived.ApplicationServices) + + rsAPI := roomserver.NewInternalAPI( + base, keyRing, federation, + ) + + eduInputAPI := eduserver.NewInternalAPI( + base, cache.New(), userAPI, + ) + + asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI) + + fsAPI := federationsender.NewInternalAPI( + base, federation, rsAPI, keyRing, + ) + + // The underlying roomserver implementation needs to be able to call the fedsender. + // This is different to rsAPI which can be the http client which doesn't need this dependency + rsAPI.SetFederationSenderAPI(fsAPI) + + publicRoomsDB, err := storage.NewPublicRoomsServerDatabase(string(base.Cfg.Database.PublicRoomsAPI), base.Cfg.DbProperties(), cfg.Matrix.ServerName) + if err != nil { + logrus.WithError(err).Panicf("failed to connect to public rooms db") + } + + monolith := setup.Monolith{ + Config: base.Cfg, + AccountDB: accountDB, + DeviceDB: deviceDB, + Client: ygg.CreateClient(base), + FedClient: federation, + KeyRing: keyRing, + KafkaConsumer: base.KafkaConsumer, + KafkaProducer: base.KafkaProducer, + + AppserviceAPI: asAPI, + EDUInternalAPI: eduInputAPI, + FederationSenderAPI: fsAPI, + RoomserverAPI: rsAPI, + UserAPI: userAPI, + //ServerKeyAPI: serverKeyAPI, + + PublicRoomsDB: publicRoomsDB, + } + monolith.AddAllPublicRoutes(base.PublicAPIMux) + + httputil.SetupHTTPAPI( + base.BaseMux, + base.PublicAPIMux, + base.InternalAPIMux, + cfg, + base.UseHTTPAPIs, + ) + + // Build both ends of a HTTP multiplex. + httpServer := &http.Server{ + Addr: ":0", + TLSNextProto: map[string]func(*http.Server, *tls.Conn, http.Handler){}, + ReadTimeout: 15 * time.Second, + WriteTimeout: 45 * time.Second, + IdleTimeout: 60 * time.Second, + BaseContext: func(_ net.Listener) context.Context { + return context.Background() + }, + Handler: base.BaseMux, + } + + go func() { + logger.Info("Listening on ", ygg.DerivedServerName()) + logger.Fatal(httpServer.Serve(ygg)) + }() + go func() { + logger.Info("Listening on ", m.BaseURL()) + logger.Fatal(httpServer.Serve(m.listener)) + }() +} diff --git a/build/gobind/platform_ios.go b/build/gobind/platform_ios.go new file mode 100644 index 000000000..01f8a6a04 --- /dev/null +++ b/build/gobind/platform_ios.go @@ -0,0 +1,25 @@ +// +build ios + +package gobind + +/* +#cgo CFLAGS: -x objective-c +#cgo LDFLAGS: -framework Foundation +#import +void Log(const char *text) { + NSString *nss = [NSString stringWithUTF8String:text]; + NSLog(@"%@", nss); +} +*/ +import "C" +import "unsafe" + +type BindLogger struct { +} + +func (nsl BindLogger) Write(p []byte) (n int, err error) { + p = append(p, 0) + cstr := (*C.char)(unsafe.Pointer(&p[0])) + C.Log(cstr) + return len(p), nil +} diff --git a/build/gobind/platform_other.go b/build/gobind/platform_other.go new file mode 100644 index 000000000..fdfb13bc0 --- /dev/null +++ b/build/gobind/platform_other.go @@ -0,0 +1,12 @@ +// +build !ios + +package gobind + +import "log" + +type BindLogger struct{} + +func (nsl BindLogger) Write(p []byte) (n int, err error) { + log.Println(string(p)) + return len(p), nil +} diff --git a/clientapi/routing/device.go b/clientapi/routing/device.go index 403937c9c..51a15a882 100644 --- a/clientapi/routing/device.go +++ b/clientapi/routing/device.go @@ -26,9 +26,12 @@ import ( "github.com/matrix-org/util" ) +// https://matrix.org/docs/spec/client_server/r0.6.1#get-matrix-client-r0-devices type deviceJSON struct { - DeviceID string `json:"device_id"` - UserID string `json:"user_id"` + DeviceID string `json:"device_id"` + DisplayName string `json:"display_name"` + LastSeenIP string `json:"last_seen_ip"` + LastSeenTS uint64 `json:"last_seen_ts"` } type devicesJSON struct { @@ -70,7 +73,6 @@ func GetDeviceByID( Code: http.StatusOK, JSON: deviceJSON{ DeviceID: dev.ID, - UserID: dev.UserID, }, } } @@ -98,7 +100,6 @@ func GetDevicesByLocalpart( for _, dev := range deviceList { res.Devices = append(res.Devices, deviceJSON{ DeviceID: dev.ID, - UserID: dev.UserID, }) } diff --git a/clientapi/routing/joinroom.go b/clientapi/routing/joinroom.go index 3871e4d4e..db890d03f 100644 --- a/clientapi/routing/joinroom.go +++ b/clientapi/routing/joinroom.go @@ -37,6 +37,7 @@ func JoinRoomByIDOrAlias( joinReq := roomserverAPI.PerformJoinRequest{ RoomIDOrAlias: roomIDOrAlias, UserID: device.UserID, + Content: map[string]interface{}{}, } joinRes := roomserverAPI.PerformJoinResponse{} diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index 328aa0629..db05ecb76 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -47,52 +47,11 @@ var ( instancePeer = flag.String("peer", "", "an internet Yggdrasil peer to connect to") ) -type yggroundtripper struct { - inner *http.Transport -} - -func (y *yggroundtripper) RoundTrip(req *http.Request) (*http.Response, error) { - req.URL.Scheme = "http" - return y.inner.RoundTrip(req) -} - -func createFederationClient( - base *setup.BaseDendrite, n *yggconn.Node, -) *gomatrixserverlib.FederationClient { - tr := &http.Transport{} - tr.RegisterProtocol( - "matrix", &yggroundtripper{ - inner: &http.Transport{ - ResponseHeaderTimeout: 15 * time.Second, - IdleConnTimeout: 60 * time.Second, - DialContext: n.DialerContext, - }, - }, - ) - return gomatrixserverlib.NewFederationClientWithTransport( - base.Cfg.Matrix.ServerName, base.Cfg.Matrix.KeyID, base.Cfg.Matrix.PrivateKey, tr, - ) -} - -func createClient(n *yggconn.Node) *gomatrixserverlib.Client { - tr := &http.Transport{} - tr.RegisterProtocol( - "matrix", &yggroundtripper{ - inner: &http.Transport{ - ResponseHeaderTimeout: 15 * time.Second, - IdleConnTimeout: 60 * time.Second, - DialContext: n.DialerContext, - }, - }, - ) - return gomatrixserverlib.NewClientWithTransport(tr) -} - // nolint:gocyclo func main() { flag.Parse() - ygg, err := yggconn.Setup(*instanceName, *instancePeer) + ygg, err := yggconn.Setup(*instanceName, *instancePeer, ".") if err != nil { panic(err) } @@ -125,7 +84,7 @@ func main() { accountDB := base.CreateAccountsDB() deviceDB := base.CreateDeviceDB() - federation := createFederationClient(base, ygg) + federation := ygg.CreateFederationClient(base) serverKeyAPI := &signing.YggdrasilKeys{} keyRing := serverKeyAPI.KeyRing() @@ -160,7 +119,7 @@ func main() { Config: base.Cfg, AccountDB: accountDB, DeviceDB: deviceDB, - Client: createClient(ygg), + Client: ygg.CreateClient(base), FedClient: federation, KeyRing: keyRing, KafkaConsumer: base.KafkaConsumer, @@ -203,7 +162,7 @@ func main() { logrus.Fatal(httpServer.Serve(ygg)) }() go func() { - httpBindAddr := fmt.Sprintf("localhost:%d", *instancePort) + httpBindAddr := fmt.Sprintf(":%d", *instancePort) logrus.Info("Listening on ", httpBindAddr) logrus.Fatal(http.ListenAndServe(httpBindAddr, base.BaseMux)) }() diff --git a/cmd/dendrite-demo-yggdrasil/yggconn/client.go b/cmd/dendrite-demo-yggdrasil/yggconn/client.go new file mode 100644 index 000000000..399993e3e --- /dev/null +++ b/cmd/dendrite-demo-yggdrasil/yggconn/client.go @@ -0,0 +1,74 @@ +package yggconn + +import ( + "context" + "crypto/ed25519" + "encoding/hex" + "fmt" + "net" + "net/http" + "strings" + "time" + + "github.com/matrix-org/dendrite/cmd/dendrite-demo-yggdrasil/convert" + "github.com/matrix-org/dendrite/internal/setup" + "github.com/matrix-org/gomatrixserverlib" +) + +func (n *Node) yggdialer(_, address string) (net.Conn, error) { + tokens := strings.Split(address, ":") + raw, err := hex.DecodeString(tokens[0]) + if err != nil { + return nil, fmt.Errorf("hex.DecodeString: %w", err) + } + converted := convert.Ed25519PublicKeyToCurve25519(ed25519.PublicKey(raw)) + convhex := hex.EncodeToString(converted) + return n.Dial("curve25519", convhex) +} + +func (n *Node) yggdialerctx(ctx context.Context, network, address string) (net.Conn, error) { + return n.yggdialer(network, address) +} + +type yggroundtripper struct { + inner *http.Transport +} + +func (y *yggroundtripper) RoundTrip(req *http.Request) (*http.Response, error) { + req.URL.Scheme = "http" + return y.inner.RoundTrip(req) +} + +func (n *Node) CreateClient( + base *setup.BaseDendrite, +) *gomatrixserverlib.Client { + tr := &http.Transport{} + tr.RegisterProtocol( + "matrix", &yggroundtripper{ + inner: &http.Transport{ + ResponseHeaderTimeout: 15 * time.Second, + IdleConnTimeout: 60 * time.Second, + DialContext: n.yggdialerctx, + }, + }, + ) + return gomatrixserverlib.NewClientWithTransport(tr) +} + +func (n *Node) CreateFederationClient( + base *setup.BaseDendrite, +) *gomatrixserverlib.FederationClient { + tr := &http.Transport{} + tr.RegisterProtocol( + "matrix", &yggroundtripper{ + inner: &http.Transport{ + ResponseHeaderTimeout: 15 * time.Second, + IdleConnTimeout: 60 * time.Second, + DialContext: n.yggdialerctx, + }, + }, + ) + return gomatrixserverlib.NewFederationClientWithTransport( + base.Cfg.Matrix.ServerName, base.Cfg.Matrix.KeyID, base.Cfg.Matrix.PrivateKey, tr, + ) +} diff --git a/cmd/dendrite-demo-yggdrasil/yggconn/node.go b/cmd/dendrite-demo-yggdrasil/yggconn/node.go index b225e1cf9..c335f2eac 100644 --- a/cmd/dendrite-demo-yggdrasil/yggconn/node.go +++ b/cmd/dendrite-demo-yggdrasil/yggconn/node.go @@ -67,7 +67,7 @@ func (n *Node) DialerContext(ctx context.Context, network, address string) (net. } // nolint:gocyclo -func Setup(instanceName, instancePeer string) (*Node, error) { +func Setup(instanceName, instancePeer, storageDirectory string) (*Node, error) { n := &Node{ core: &yggdrasil.Core{}, config: yggdrasilconfig.GenerateConfig(), @@ -77,7 +77,7 @@ func Setup(instanceName, instancePeer string) (*Node, error) { incoming: make(chan *yamux.Stream), } - yggfile := fmt.Sprintf("%s-yggdrasil.conf", instanceName) + yggfile := fmt.Sprintf("%s/%s-yggdrasil.conf", storageDirectory, instanceName) if _, err := os.Stat(yggfile); !os.IsNotExist(err) { yggconf, e := ioutil.ReadFile(yggfile) if e != nil { @@ -87,7 +87,7 @@ func Setup(instanceName, instancePeer string) (*Node, error) { panic(err) } } else { - n.config.AdminListen = fmt.Sprintf("unix://./%s-yggdrasil.sock", instanceName) + n.config.AdminListen = "none" // fmt.Sprintf("unix://%s/%s-yggdrasil.sock", storageDirectory, instanceName) n.config.MulticastInterfaces = []string{".*"} n.config.EncryptionPrivateKey = hex.EncodeToString(n.EncryptionPrivateKey()) n.config.EncryptionPublicKey = hex.EncodeToString(n.EncryptionPublicKey()) @@ -114,20 +114,22 @@ func Setup(instanceName, instancePeer string) (*Node, error) { panic(err) } } - if err = n.admin.Init(n.core, n.state, n.log, nil); err != nil { - panic(err) - } - if err = n.admin.Start(); err != nil { - panic(err) - } + /* + if err = n.admin.Init(n.core, n.state, n.log, nil); err != nil { + panic(err) + } + if err = n.admin.Start(); err != nil { + panic(err) + } + */ if err = n.multicast.Init(n.core, n.state, n.log, nil); err != nil { panic(err) } if err = n.multicast.Start(); err != nil { panic(err) } - n.admin.SetupAdminHandlers(n.admin) - n.multicast.SetupAdminHandlers(n.admin) + //n.admin.SetupAdminHandlers(n.admin) + //n.multicast.SetupAdminHandlers(n.admin) n.listener, err = n.core.ConnListen() if err != nil { panic(err) @@ -137,6 +139,9 @@ func Setup(instanceName, instancePeer string) (*Node, error) { panic(err) } + n.log.Println("Public curve25519:", n.core.EncryptionPublicKey()) + n.log.Println("Public ed25519:", n.core.SigningPublicKey()) + go n.listenFromYgg() return n, nil diff --git a/go.mod b/go.mod index ff73ad078..4473dc97f 100644 --- a/go.mod +++ b/go.mod @@ -38,6 +38,7 @@ require ( github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20200530233943-aec82d7a391b go.uber.org/atomic v1.4.0 golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d + golang.org/x/mobile v0.0.0-20200329125638-4c31acba0007 // indirect gopkg.in/h2non/bimg.v1 v1.0.18 gopkg.in/yaml.v2 v2.2.8 ) diff --git a/go.sum b/go.sum index f62b14b62..97abb7b98 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,7 @@ github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOv github.com/Arceliar/phony v0.0.0-20191006174943-d0c68492aca0 h1:p3puK8Sl2xK+2FnnIvY/C0N1aqJo2kbEsdAzU+Tnv48= github.com/Arceliar/phony v0.0.0-20191006174943-d0c68492aca0/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= github.com/Shopify/sarama v1.26.1 h1:3jnfWKD7gVwbB1KSy/lE0szA9duPuSFLViK0o/d3DgA= github.com/Shopify/sarama v1.26.1/go.mod h1:NbSGBSSndYaIhRcBtY9V0U7AyH+x71bG668AuWys/yU= @@ -589,6 +590,7 @@ golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -599,9 +601,18 @@ golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d h1:1ZiEyfaQIg3Qh0EoqpwAakHVhecoE5wlSg5GjnafJGw= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20200329125638-4c31acba0007 h1:JxsyO7zPDWn1rBZW8FV5RFwCKqYeXnyaS/VQPLpXu6I= +golang.org/x/mobile v0.0.0-20200329125638-4c31acba0007/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -671,6 +682,10 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd h1:/e+gpKk9r3dJobndpTytxS2gOy6m5uvpg+ISQoEcusQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69 h1:yBHHx+XZqXJBm6Exke3N7V9gnlsyXxoCPEb1yVenjfk= +golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= diff --git a/syncapi/sync/request.go b/syncapi/sync/request.go index beeaa40f7..5dd92c853 100644 --- a/syncapi/sync/request.go +++ b/syncapi/sync/request.go @@ -62,6 +62,10 @@ func newSyncRequest(req *http.Request, device userapi.Device) (*syncRequest, err } since = &tok } + if since == nil { + tok := types.NewStreamToken(0, 0) + since = &tok + } timelineLimit := defaultTimelineLimit // TODO: read from stored filters too filterQuery := req.URL.Query().Get("filter") diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index 8d51689e3..743c63a62 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -256,7 +256,8 @@ func (rp *RequestPool) appendAccountData( } if len(dataTypes) == 0 { - return data, nil + // TODO: this fixes the sytest but is it the right thing to do? + dataTypes[""] = []string{"m.push_rules"} } // Iterate over the rooms diff --git a/syncapi/types/types.go b/syncapi/types/types.go index c1f09fba5..1094416a1 100644 --- a/syncapi/types/types.go +++ b/syncapi/types/types.go @@ -98,6 +98,9 @@ func (t *StreamingToken) PDUPosition() StreamPosition { func (t *StreamingToken) EDUPosition() StreamPosition { return t.Positions[1] } +func (t *StreamingToken) String() string { + return t.syncToken.String() +} // IsAfter returns true if ANY position in this token is greater than `other`. func (t *StreamingToken) IsAfter(other StreamingToken) bool { @@ -220,8 +223,8 @@ func NewTopologyTokenFromString(tok string) (token TopologyToken, err error) { err = fmt.Errorf("token %s is not a topology token", tok) return } - if len(t.Positions) != 2 { - err = fmt.Errorf("token %s wrong number of values, got %d want 2", tok, len(t.Positions)) + if len(t.Positions) < 2 { + err = fmt.Errorf("token %s wrong number of values, got %d want at least 2", tok, len(t.Positions)) return } return TopologyToken{ @@ -247,8 +250,8 @@ func NewStreamTokenFromString(tok string) (token StreamingToken, err error) { err = fmt.Errorf("token %s is not a streaming token", tok) return } - if len(t.Positions) != 2 { - err = fmt.Errorf("token %s wrong number of values, got %d want 2", tok, len(t.Positions)) + if len(t.Positions) < 2 { + err = fmt.Errorf("token %s wrong number of values, got %d want at least 2", tok, len(t.Positions)) return } return StreamingToken{ diff --git a/sytest-whitelist b/sytest-whitelist index 971f8cf0d..7edc79e64 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -347,3 +347,6 @@ GET /rooms/:room_id/state/m.room.topic gets topic GET /rooms/:room_id/state fetches entire room state Setting room topic reports m.room.topic to myself setting 'm.room.name' respects room powerlevel +Syncing a new room with a large timeline limit isn't limited +Left rooms appear in the leave section of sync +Banned rooms appear in the leave section of sync From 61e0482fef7aa34c0261e41fd90aa5c5e4890bfc Mon Sep 17 00:00:00 2001 From: Ashley Nelson Date: Sat, 20 Jun 2020 15:28:30 -0500 Subject: [PATCH 190/200] Add appservices component to docker scripts (#1153) Signed-off-by: Ashley Nelson --- build/docker/config/dendrite-config.yaml | 1 + build/docker/docker-compose.polylith.yml | 14 ++++++++++ build/docker/images-build.sh | 1 + build/docker/images-pull.sh | 1 + build/docker/images-push.sh | 1 + build/scripts/build-test-lint.sh | 2 +- cmd/dendrite-user-api-server/main.go | 35 ++++++++++++++++++++++++ 7 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 cmd/dendrite-user-api-server/main.go diff --git a/build/docker/config/dendrite-config.yaml b/build/docker/config/dendrite-config.yaml index f421b2e71..53d9f7b02 100644 --- a/build/docker/config/dendrite-config.yaml +++ b/build/docker/config/dendrite-config.yaml @@ -118,6 +118,7 @@ listen: edu_server: "edu_server:7777" key_server: "key_server:7779" user_api: "user_api:7780" + appservice_api: "appservice_api:7781" # The configuration for tracing the dendrite components. tracing: diff --git a/build/docker/docker-compose.polylith.yml b/build/docker/docker-compose.polylith.yml index af2ae4e81..d424d43b1 100644 --- a/build/docker/docker-compose.polylith.yml +++ b/build/docker/docker-compose.polylith.yml @@ -163,6 +163,20 @@ services: networks: - internal + appservice_api: + hostname: appservice_api + image: matrixdotorg/dendrite:appservice + command: [ + "--config=dendrite.yaml" + ] + volumes: + - ./config:/etc/dendrite + networks: + - internal + depends_on: + - room_server + - user_api + networks: internal: attachable: true diff --git a/build/docker/images-build.sh b/build/docker/images-build.sh index e43d2d040..9ee5a09de 100755 --- a/build/docker/images-build.sh +++ b/build/docker/images-build.sh @@ -6,6 +6,7 @@ docker build -f build/docker/Dockerfile -t matrixdotorg/dendrite:latest . docker build -t matrixdotorg/dendrite:monolith --build-arg component=dendrite-monolith-server -f build/docker/Dockerfile.component . +docker build -t matrixdotorg/dendrite:appservice --build-arg component=dendrite-appservice-server -f build/docker/Dockerfile.component . docker build -t matrixdotorg/dendrite:clientapi --build-arg component=dendrite-client-api-server -f build/docker/Dockerfile.component . docker build -t matrixdotorg/dendrite:clientproxy --build-arg component=client-api-proxy -f build/docker/Dockerfile.component . docker build -t matrixdotorg/dendrite:eduserver --build-arg component=dendrite-edu-server -f build/docker/Dockerfile.component . diff --git a/build/docker/images-pull.sh b/build/docker/images-pull.sh index edccf4a33..da08a7325 100755 --- a/build/docker/images-pull.sh +++ b/build/docker/images-pull.sh @@ -2,6 +2,7 @@ docker pull matrixdotorg/dendrite:monolith +docker pull matrixdotorg/dendrite:appservice docker pull matrixdotorg/dendrite:clientapi docker pull matrixdotorg/dendrite:clientproxy docker pull matrixdotorg/dendrite:eduserver diff --git a/build/docker/images-push.sh b/build/docker/images-push.sh index 2b4303872..d8f8758a0 100755 --- a/build/docker/images-push.sh +++ b/build/docker/images-push.sh @@ -2,6 +2,7 @@ docker push matrixdotorg/dendrite:monolith +docker push matrixdotorg/dendrite:appservice docker push matrixdotorg/dendrite:clientapi docker push matrixdotorg/dendrite:clientproxy docker push matrixdotorg/dendrite:eduserver diff --git a/build/scripts/build-test-lint.sh b/build/scripts/build-test-lint.sh index 4b18ca2f8..8f0b775b1 100755 --- a/build/scripts/build-test-lint.sh +++ b/build/scripts/build-test-lint.sh @@ -10,7 +10,7 @@ set -eu echo "Checking that it builds..." go build ./cmd/... -./scripts/find-lint.sh +./build/scripts/find-lint.sh echo "Testing..." go test -v ./... diff --git a/cmd/dendrite-user-api-server/main.go b/cmd/dendrite-user-api-server/main.go new file mode 100644 index 000000000..4257da3f3 --- /dev/null +++ b/cmd/dendrite-user-api-server/main.go @@ -0,0 +1,35 @@ +// Copyright 2017 Vector Creations Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "github.com/matrix-org/dendrite/internal/setup" + "github.com/matrix-org/dendrite/userapi" +) + +func main() { + cfg := setup.ParseFlags(false) + base := setup.NewBaseDendrite(cfg, "UserAPI", true) + defer base.Close() // nolint: errcheck + + accountDB := base.CreateAccountsDB() + deviceDB := base.CreateDeviceDB() + + userAPI := userapi.NewInternalAPI(accountDB, deviceDB, cfg.Matrix.ServerName, cfg.Derived.ApplicationServices) + + userapi.AddInternalRoutes(base.InternalAPIMux, userAPI) + + base.SetupAndServeHTTP(string(base.Cfg.Bind.UserAPI), string(base.Cfg.Listen.UserAPI)) +} From a1352cbd9ed156da546ac67ae5194b30f607fed6 Mon Sep 17 00:00:00 2001 From: Martin Honermeyer Date: Tue, 23 Jun 2020 10:51:38 +0200 Subject: [PATCH 191/200] Push serverkeyapi docker image to registry as well (#1154) Signed-off-by: Martin Honermeyer --- build/docker/images-push.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/build/docker/images-push.sh b/build/docker/images-push.sh index d8f8758a0..1ac60b921 100755 --- a/build/docker/images-push.sh +++ b/build/docker/images-push.sh @@ -14,4 +14,5 @@ docker push matrixdotorg/dendrite:mediaapi docker push matrixdotorg/dendrite:publicroomsapi docker push matrixdotorg/dendrite:roomserver docker push matrixdotorg/dendrite:syncapi +docker push matrixdotorg/dendrite:serverkeyapi docker push matrixdotorg/dendrite:userapi From 02565c37aa2a03b31b3bd7447c9a8ab90a6cd9e7 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 23 Jun 2020 10:31:17 +0100 Subject: [PATCH 192/200] /send auth errors are silent (#1149) * /send auth errors are silent * Fix test --- federationapi/routing/send.go | 12 ++++++++++-- sytest-blacklist | 4 ++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index cf71b8ba0..c80ab89f3 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -176,9 +176,17 @@ func (t *txnReq) processTransaction() (*gomatrixserverlib.RespSend, error) { util.GetLogger(t.context).Warnf("Processing %s failed fatally: %s", e.EventID(), err) return nil, err } else { - util.GetLogger(t.context).WithError(err).WithField("event_id", e.EventID()).Warn("Failed to process incoming federation event, skipping") + // Auth errors mean the event is 'rejected' which have to be silent to appease sytest + _, rejected := err.(*gomatrixserverlib.NotAllowed) + errMsg := err.Error() + if rejected { + errMsg = "" + } + util.GetLogger(t.context).WithError(err).WithField("event_id", e.EventID()).WithField("rejected", rejected).Warn( + "Failed to process incoming federation event, skipping", + ) results[e.EventID()] = gomatrixserverlib.PDUResult{ - Error: err.Error(), + Error: errMsg, } } } else { diff --git a/sytest-blacklist b/sytest-blacklist index 1efc207f7..9f140ed1c 100644 --- a/sytest-blacklist +++ b/sytest-blacklist @@ -44,3 +44,7 @@ Existing members see new members' join events Can recv device messages over federation Device messages over federation wake up /sync Wildcard device messages over federation wake up /sync + +# We don't implement soft-failed events yet, but because the /send response is vague, +# this test thinks it's all fine... +Inbound federation accepts a second soft-failed event From 4220a374cabbc1a885d9c79037fcf42e14fef677 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 23 Jun 2020 11:47:48 +0100 Subject: [PATCH 193/200] Fix room checks for /state and /state_ids (#1155) We would return a 403 first (as the server would not be allowed to see this event) and only then return a 404 if the event is not in the given room. We now invert those checks for /state and /state_ids to make the tests pass. --- federationapi/routing/events.go | 27 ++++++++++++++++++--------- federationapi/routing/state.go | 8 ++++++-- sytest-whitelist | 2 ++ 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/federationapi/routing/events.go b/federationapi/routing/events.go index ced9e3d53..6fa28f69d 100644 --- a/federationapi/routing/events.go +++ b/federationapi/routing/events.go @@ -33,7 +33,11 @@ func GetEvent( eventID string, origin gomatrixserverlib.ServerName, ) util.JSONResponse { - event, err := getEvent(ctx, request, rsAPI, eventID) + err := allowedToSeeEvent(ctx, request.Origin(), rsAPI, eventID) + if err != nil { + return *err + } + event, err := fetchEvent(ctx, rsAPI, eventID) if err != nil { return *err } @@ -47,35 +51,40 @@ func GetEvent( }} } -// getEvent returns the requested event, +// allowedToSeeEvent returns no error if the server is allowed to see this event, // otherwise it returns an error response which can be sent to the client. -func getEvent( +func allowedToSeeEvent( ctx context.Context, - request *gomatrixserverlib.FederationRequest, + origin gomatrixserverlib.ServerName, rsAPI api.RoomserverInternalAPI, eventID string, -) (*gomatrixserverlib.Event, *util.JSONResponse) { +) *util.JSONResponse { var authResponse api.QueryServerAllowedToSeeEventResponse err := rsAPI.QueryServerAllowedToSeeEvent( ctx, &api.QueryServerAllowedToSeeEventRequest{ EventID: eventID, - ServerName: request.Origin(), + ServerName: origin, }, &authResponse, ) if err != nil { resErr := util.ErrorResponse(err) - return nil, &resErr + return &resErr } if !authResponse.AllowedToSeeEvent { resErr := util.MessageResponse(http.StatusForbidden, "server not allowed to see event") - return nil, &resErr + return &resErr } + return nil +} + +// fetchEvent fetches the event without auth checks. Returns an error if the event cannot be found. +func fetchEvent(ctx context.Context, rsAPI api.RoomserverInternalAPI, eventID string) (*gomatrixserverlib.Event, *util.JSONResponse) { var eventsResponse api.QueryEventsByIDResponse - err = rsAPI.QueryEventsByID( + err := rsAPI.QueryEventsByID( ctx, &api.QueryEventsByIDRequest{EventIDs: []string{eventID}}, &eventsResponse, diff --git a/federationapi/routing/state.go b/federationapi/routing/state.go index 04a18904b..28dfad846 100644 --- a/federationapi/routing/state.go +++ b/federationapi/routing/state.go @@ -98,13 +98,17 @@ func getState( roomID string, eventID string, ) (*gomatrixserverlib.RespState, *util.JSONResponse) { - event, resErr := getEvent(ctx, request, rsAPI, eventID) + event, resErr := fetchEvent(ctx, rsAPI, eventID) if resErr != nil { return nil, resErr } if event.RoomID() != roomID { - return nil, &util.JSONResponse{Code: http.StatusNotFound, JSON: nil} + return nil, &util.JSONResponse{Code: http.StatusNotFound, JSON: jsonerror.NotFound("event does not belong to this room")} + } + resErr = allowedToSeeEvent(ctx, request.Origin(), rsAPI, eventID) + if resErr != nil { + return nil, resErr } var response api.QueryStateAndAuthChainResponse diff --git a/sytest-whitelist b/sytest-whitelist index 7edc79e64..77ec6cd91 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -350,3 +350,5 @@ setting 'm.room.name' respects room powerlevel Syncing a new room with a large timeline limit isn't limited Left rooms appear in the leave section of sync Banned rooms appear in the leave section of sync +Getting state checks the events requested belong to the room +Getting state IDs checks the events requested belong to the room From 914f6cadceebad98ed2a3f134437531a1426fc30 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Tue, 23 Jun 2020 13:15:15 +0100 Subject: [PATCH 194/200] Add /send restrictions and return correct error codes (#1156) * Add /send restrictions and return correct error codes - Max 50 PDUs / 100 EDUs - Fail the transaction when PDUs contain bad JSON * Update whitelist * Unbreak test * Linting --- federationapi/routing/send.go | 37 ++++++++++++++++++++++-------- federationapi/routing/send_test.go | 2 +- go.mod | 3 +-- go.sum | 21 ++--------------- sytest-whitelist | 2 ++ 5 files changed, 34 insertions(+), 31 deletions(-) diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index c80ab89f3..53f951951 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -61,6 +61,14 @@ func Send( JSON: jsonerror.NotJSON("The request body could not be decoded into valid JSON. " + err.Error()), } } + // Transactions are limited in size; they can have at most 50 PDUs and 100 EDUs. + // https://matrix.org/docs/spec/server_server/latest#transactions + if len(txnEvents.PDUs) > 50 || len(txnEvents.EDUs) > 100 { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.BadJSON("max 50 pdus / 100 edus"), + } + } // TODO: Really we should have a function to convert FederationRequest to txnReq t.PDUs = txnEvents.PDUs @@ -71,10 +79,10 @@ func Send( util.GetLogger(httpReq.Context()).Infof("Received transaction %q containing %d PDUs, %d EDUs", txnID, len(t.PDUs), len(t.EDUs)) - resp, err := t.processTransaction() - if err != nil { - util.GetLogger(httpReq.Context()).WithError(err).Error("t.processTransaction failed") - return util.ErrorResponse(err) + resp, jsonErr := t.processTransaction() + if jsonErr != nil { + util.GetLogger(httpReq.Context()).WithField("jsonErr", jsonErr).Error("t.processTransaction failed") + return *jsonErr } // https://matrix.org/docs/spec/server_server/r0.1.3#put-matrix-federation-v1-send-txnid @@ -112,7 +120,7 @@ type txnFederationClient interface { roomVersion gomatrixserverlib.RoomVersion) (res gomatrixserverlib.RespMissingEvents, err error) } -func (t *txnReq) processTransaction() (*gomatrixserverlib.RespSend, error) { +func (t *txnReq) processTransaction() (*gomatrixserverlib.RespSend, *util.JSONResponse) { results := make(map[string]gomatrixserverlib.PDUResult) pdus := []gomatrixserverlib.HeaderedEvent{} @@ -136,10 +144,20 @@ func (t *txnReq) processTransaction() (*gomatrixserverlib.RespSend, error) { } event, err := gomatrixserverlib.NewEventFromUntrustedJSON(pdu, verRes.RoomVersion) if err != nil { - util.GetLogger(t.context).WithError(err).Warnf("Transaction: Failed to parse event JSON of event %q", event.EventID()) - results[event.EventID()] = gomatrixserverlib.PDUResult{ - Error: err.Error(), + if _, ok := err.(gomatrixserverlib.BadJSONError); ok { + // Room version 6 states that homeservers should strictly enforce canonical JSON + // on PDUs. + // + // This enforces that the entire transaction is rejected if a single bad PDU is + // sent. It is unclear if this is the correct behaviour or not. + // + // See https://github.com/matrix-org/synapse/issues/7543 + return nil, &util.JSONResponse{ + Code: 400, + JSON: jsonerror.BadJSON("PDU contains bad JSON"), + } } + util.GetLogger(t.context).WithError(err).Warnf("Transaction: Failed to parse event JSON of event %s", string(pdu)) continue } if err = gomatrixserverlib.VerifyAllEventSignatures(t.context, []gomatrixserverlib.Event{event}, t.keys); err != nil { @@ -174,7 +192,8 @@ func (t *txnReq) processTransaction() (*gomatrixserverlib.RespSend, error) { // Any other error should be the result of a temporary error in // our server so we should bail processing the transaction entirely. util.GetLogger(t.context).Warnf("Processing %s failed fatally: %s", e.EventID(), err) - return nil, err + jsonErr := util.ErrorResponse(err) + return nil, &jsonErr } else { // Auth errors mean the event is 'rejected' which have to be silent to appease sytest _, rejected := err.(*gomatrixserverlib.NotAllowed) diff --git a/federationapi/routing/send_test.go b/federationapi/routing/send_test.go index 6e6606c86..e512f4b4d 100644 --- a/federationapi/routing/send_test.go +++ b/federationapi/routing/send_test.go @@ -337,7 +337,7 @@ func mustCreateTransaction(rsAPI api.RoomserverInternalAPI, fedClient txnFederat func mustProcessTransaction(t *testing.T, txn *txnReq, pdusWithErrors []string) { res, err := txn.processTransaction() if err != nil { - t.Errorf("txn.processTransaction returned an error: %s", err) + t.Errorf("txn.processTransaction returned an error: %v", err) return } if len(res.PDUs) != len(txn.PDUs) { diff --git a/go.mod b/go.mod index 4473dc97f..6bfce8441 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 - github.com/matrix-org/gomatrixserverlib v0.0.0-20200617141855-5539854e4abc + github.com/matrix-org/gomatrixserverlib v0.0.0-20200623103809-13ff8109e137 github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 github.com/mattn/go-sqlite3 v2.0.2+incompatible @@ -38,7 +38,6 @@ require ( github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20200530233943-aec82d7a391b go.uber.org/atomic v1.4.0 golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d - golang.org/x/mobile v0.0.0-20200329125638-4c31acba0007 // indirect gopkg.in/h2non/bimg.v1 v1.0.18 gopkg.in/yaml.v2 v2.2.8 ) diff --git a/go.sum b/go.sum index 97abb7b98..6178f152b 100644 --- a/go.sum +++ b/go.sum @@ -4,7 +4,6 @@ github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOv github.com/Arceliar/phony v0.0.0-20191006174943-d0c68492aca0 h1:p3puK8Sl2xK+2FnnIvY/C0N1aqJo2kbEsdAzU+Tnv48= github.com/Arceliar/phony v0.0.0-20191006174943-d0c68492aca0/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= github.com/Shopify/sarama v1.26.1 h1:3jnfWKD7gVwbB1KSy/lE0szA9duPuSFLViK0o/d3DgA= github.com/Shopify/sarama v1.26.1/go.mod h1:NbSGBSSndYaIhRcBtY9V0U7AyH+x71bG668AuWys/yU= @@ -372,10 +371,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 h1:Yb+Wlf github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26 h1:Hr3zjRsq2bhrnp3Ky1qgx/fzCtCALOoGYylh2tpS9K4= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200616150727-7ac22b6f8e65 h1:2CcCcBnWdDPDOqFKiGOM+mi/KDDZXSTKmvFy/0/+ZJI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200616150727-7ac22b6f8e65/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200617141855-5539854e4abc h1:Oxidxr/H1Nh8aOFWAhTzQYLDOc9OuoJwfDjgEHpyqNE= -github.com/matrix-org/gomatrixserverlib v0.0.0-20200617141855-5539854e4abc/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200623103809-13ff8109e137 h1:+eBh4L04+08IslvFM071TNrQTggU317GsQKzZ1SGEVo= +github.com/matrix-org/gomatrixserverlib v0.0.0-20200623103809-13ff8109e137/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f h1:pRz4VTiRCO4zPlEMc3ESdUOcW4PXHH4Kj+YDz1XyE+Y= github.com/matrix-org/naffka v0.0.0-20200422140631-181f1ee7401f/go.mod h1:y0oDTjZDv5SM9a2rp3bl+CU+bvTRINQsdb7YlDql5Go= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7 h1:ntrLa/8xVzeSs8vHFHK25k0C+NV74sYMJnNSg5NoSRo= @@ -590,7 +587,6 @@ golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -601,18 +597,9 @@ golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d h1:1ZiEyfaQIg3Qh0EoqpwAakHVhecoE5wlSg5GjnafJGw= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20200329125638-4c31acba0007 h1:JxsyO7zPDWn1rBZW8FV5RFwCKqYeXnyaS/VQPLpXu6I= -golang.org/x/mobile v0.0.0-20200329125638-4c31acba0007/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -682,10 +669,6 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd h1:/e+gpKk9r3dJobndpTytxS2gOy6m5uvpg+ISQoEcusQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69 h1:yBHHx+XZqXJBm6Exke3N7V9gnlsyXxoCPEb1yVenjfk= -golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= diff --git a/sytest-whitelist b/sytest-whitelist index 77ec6cd91..ef91507be 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -317,6 +317,8 @@ Invalid JSON integers Invalid JSON special values Invalid JSON floats Outbound federation will ignore a missing event with bad JSON for room version 6 +Server correctly handles transactions that break edu limits +Server rejects invalid JSON in a version 6 room Can download without a file name over federation POST /media/r0/upload can create an upload GET /media/r0/download can fetch the value again From 81beab800260697a094307097e2f19095ba1e95e Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 23 Jun 2020 18:32:22 +0100 Subject: [PATCH 195/200] Add instrumented main for coverage --- cmd/dendrite-monolith-server/main_test.go | 50 +++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 cmd/dendrite-monolith-server/main_test.go diff --git a/cmd/dendrite-monolith-server/main_test.go b/cmd/dendrite-monolith-server/main_test.go new file mode 100644 index 000000000..e504f993b --- /dev/null +++ b/cmd/dendrite-monolith-server/main_test.go @@ -0,0 +1,50 @@ +package main + +import ( + "os" + "os/signal" + "strings" + "syscall" + "testing" +) + +// This is an instrumented main, used when running integration tests (sytest) with code coverage. +// Compile: go test -c -race -cover -covermode=atomic -o monolith.debug -coverpkg "github.com/matrix-org/..." ./cmd/dendrite-monolith-server +// Run the monolith: ./monolith.debug -test.coverprofile=/somewhere/to/dump/integrationcover.out DEVEL --config dendrite.yaml +// Generate HTML with coverage: go tool cover -html=/somewhere/where/there/is/integrationcover.out -o cover.html +// Source: https://dzone.com/articles/measuring-integration-test-coverage-rate-in-pouchc +func TestMain(t *testing.T) { + var ( + args []string + ) + + for _, arg := range os.Args { + switch { + case strings.HasPrefix(arg, "DEVEL"): + case strings.HasPrefix(arg, "-test"): + default: + args = append(args, arg) + } + } + // only run the tests if there are args to be passed + if len(args) <= 1 { + return + } + + waitCh := make(chan int, 1) + os.Args = args + go func() { + main() + close(waitCh) + }() + + signalCh := make(chan os.Signal, 1) + signal.Notify(signalCh, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGHUP) + + select { + case <-signalCh: + return + case <-waitCh: + return + } +} From 1f93427ed906f63925e2768120cf437e0b1e85fb Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 23 Jun 2020 18:44:34 +0100 Subject: [PATCH 196/200] Linting --- cmd/dendrite-monolith-server/main_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/dendrite-monolith-server/main_test.go b/cmd/dendrite-monolith-server/main_test.go index e504f993b..efa1a926c 100644 --- a/cmd/dendrite-monolith-server/main_test.go +++ b/cmd/dendrite-monolith-server/main_test.go @@ -13,7 +13,7 @@ import ( // Run the monolith: ./monolith.debug -test.coverprofile=/somewhere/to/dump/integrationcover.out DEVEL --config dendrite.yaml // Generate HTML with coverage: go tool cover -html=/somewhere/where/there/is/integrationcover.out -o cover.html // Source: https://dzone.com/articles/measuring-integration-test-coverage-rate-in-pouchc -func TestMain(t *testing.T) { +func TestMain(_ *testing.T) { var ( args []string ) From 0577bfca55d1478bd1076d02f0b0623a3d3802a8 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Wed, 24 Jun 2020 09:59:59 +0100 Subject: [PATCH 197/200] Pass join errors through internal API boundaries (#1157) * Pass join errors through internal API boundaries Required for certain invite sytests. We will need to think of a better way of handling this going forwards. * Include m.room.avatar in stripped state; handle trailing slashes when GETing state events * Update whitelist * Update whitelist --- clientapi/routing/joinroom.go | 30 ++++++++++++++++++++++++++--- clientapi/routing/routing.go | 9 +++++++-- roomserver/api/perform.go | 16 +++++++++++++++ roomserver/internal/input_events.go | 3 +++ roomserver/internal/perform_join.go | 10 ++++++++++ roomserver/inthttp/server.go | 2 +- sytest-whitelist | 2 ++ 7 files changed, 66 insertions(+), 6 deletions(-) diff --git a/clientapi/routing/joinroom.go b/clientapi/routing/joinroom.go index db890d03f..e606e35f1 100644 --- a/clientapi/routing/joinroom.go +++ b/clientapi/routing/joinroom.go @@ -15,8 +15,10 @@ package routing import ( + "errors" "net/http" + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" @@ -52,7 +54,8 @@ func JoinRoomByIDOrAlias( util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") } else { // Request our profile content to populate the request content with. - profile, err := accountDB.GetProfileByLocalpart(req.Context(), localpart) + var profile *authtypes.Profile + profile, err = accountDB.GetProfileByLocalpart(req.Context(), localpart) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("accountDB.GetProfileByLocalpart failed") } else { @@ -62,11 +65,32 @@ func JoinRoomByIDOrAlias( } // Ask the roomserver to perform the join. - if err := rsAPI.PerformJoin(req.Context(), &joinReq, &joinRes); err != nil { + err = rsAPI.PerformJoin(req.Context(), &joinReq, &joinRes) + // Handle known errors first, if this is 0 then there will be no matches (eg on success) + switch joinRes.Error { + case roomserverAPI.JoinErrorBadRequest: return util.JSONResponse{ Code: http.StatusBadRequest, - JSON: jsonerror.Unknown(err.Error()), + JSON: jsonerror.Unknown(joinRes.ErrMsg), } + case roomserverAPI.JoinErrorNoRoom: + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound(joinRes.ErrMsg), + } + case roomserverAPI.JoinErrorNotAllowed: + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden(joinRes.ErrMsg), + } + } + // this is always populated on generic errors + if joinRes.ErrMsg != "" { + return util.ErrorResponse(errors.New(joinRes.ErrMsg)) + } + // this is set on network errors in polylith mode + if err != nil { + return util.ErrorResponse(err) } return util.JSONResponse{ diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index e91b07ac7..825ac50f2 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -159,13 +159,18 @@ func Setup( return OnIncomingStateRequest(req.Context(), rsAPI, vars["roomID"]) })).Methods(http.MethodGet, http.MethodOptions) - r0mux.Handle("/rooms/{roomID}/state/{type}", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { + r0mux.Handle("/rooms/{roomID}/state/{type:[^/]+/?}", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) if err != nil { return util.ErrorResponse(err) } + // If there's a trailing slash, remove it + eventType := vars["type"] + if strings.HasSuffix(eventType, "/") { + eventType = eventType[:len(eventType)-1] + } eventFormat := req.URL.Query().Get("format") == "event" - return OnIncomingStateTypeRequest(req.Context(), rsAPI, vars["roomID"], vars["type"], "", eventFormat) + return OnIncomingStateTypeRequest(req.Context(), rsAPI, vars["roomID"], eventType, "", eventFormat) })).Methods(http.MethodGet, http.MethodOptions) r0mux.Handle("/rooms/{roomID}/state/{type}/{stateKey}", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *api.Device) util.JSONResponse { diff --git a/roomserver/api/perform.go b/roomserver/api/perform.go index 3e5cae1b6..0f5394c94 100644 --- a/roomserver/api/perform.go +++ b/roomserver/api/perform.go @@ -5,6 +5,17 @@ import ( "github.com/matrix-org/util" ) +type JoinError int + +const ( + // JoinErrorNotAllowed means the user is not allowed to join this room (e.g join_rule:invite or banned) + JoinErrorNotAllowed JoinError = 1 + // JoinErrorBadRequest means the request was wrong in some way (invalid user ID, wrong server, etc) + JoinErrorBadRequest JoinError = 2 + // JoinErrorNoRoom means that the room being joined doesn't exist. + JoinErrorNoRoom JoinError = 3 +) + type PerformJoinRequest struct { RoomIDOrAlias string `json:"room_id_or_alias"` UserID string `json:"user_id"` @@ -13,7 +24,12 @@ type PerformJoinRequest struct { } type PerformJoinResponse struct { + // The room ID, populated on success. RoomID string `json:"room_id"` + // The reason why the join failed. Can be blank. + Error JoinError `json:"error"` + // Debugging description of the error. Always present on failure. + ErrMsg string `json:"err_msg"` } type PerformLeaveRequest struct { diff --git a/roomserver/internal/input_events.go b/roomserver/internal/input_events.go index 4487aea02..fe3bdf4b8 100644 --- a/roomserver/internal/input_events.go +++ b/roomserver/internal/input_events.go @@ -298,9 +298,12 @@ func buildInviteStrippedState( return nil, fmt.Errorf("room %q unknown", input.Event.RoomID()) } stateWanted := []gomatrixserverlib.StateKeyTuple{} + // "If they are set on the room, at least the state for m.room.avatar, m.room.canonical_alias, m.room.join_rules, and m.room.name SHOULD be included." + // https://matrix.org/docs/spec/client_server/r0.6.0#m-room-member for _, t := range []string{ gomatrixserverlib.MRoomName, gomatrixserverlib.MRoomCanonicalAlias, gomatrixserverlib.MRoomAliases, gomatrixserverlib.MRoomJoinRules, + "m.room.avatar", } { stateWanted = append(stateWanted, gomatrixserverlib.StateKeyTuple{ EventType: t, diff --git a/roomserver/internal/perform_join.go b/roomserver/internal/perform_join.go index 1c951bb15..44f842404 100644 --- a/roomserver/internal/perform_join.go +++ b/roomserver/internal/perform_join.go @@ -2,6 +2,7 @@ package internal import ( "context" + "errors" "fmt" "strings" "time" @@ -21,9 +22,11 @@ func (r *RoomserverInternalAPI) PerformJoin( ) error { _, domain, err := gomatrixserverlib.SplitID('@', req.UserID) if err != nil { + res.Error = api.JoinErrorBadRequest return fmt.Errorf("Supplied user ID %q in incorrect format", req.UserID) } if domain != r.Cfg.Matrix.ServerName { + res.Error = api.JoinErrorBadRequest return fmt.Errorf("User %q does not belong to this homeserver", req.UserID) } if strings.HasPrefix(req.RoomIDOrAlias, "!") { @@ -32,6 +35,7 @@ func (r *RoomserverInternalAPI) PerformJoin( if strings.HasPrefix(req.RoomIDOrAlias, "#") { return r.performJoinRoomByAlias(ctx, req, res) } + res.Error = api.JoinErrorBadRequest return fmt.Errorf("Room ID or alias %q is invalid", req.RoomIDOrAlias) } @@ -99,6 +103,7 @@ func (r *RoomserverInternalAPI) performJoinRoomByID( // Get the domain part of the room ID. _, domain, err := gomatrixserverlib.SplitID('!', req.RoomIDOrAlias) if err != nil { + res.Error = api.JoinErrorBadRequest return fmt.Errorf("Room ID %q is invalid", req.RoomIDOrAlias) } req.ServerNames = append(req.ServerNames, domain) @@ -198,6 +203,10 @@ func (r *RoomserverInternalAPI) performJoinRoomByID( } inputRes := api.InputRoomEventsResponse{} if err = r.InputRoomEvents(ctx, &inputReq, &inputRes); err != nil { + var notAllowed *gomatrixserverlib.NotAllowed + if errors.As(err, ¬Allowed) { + res.Error = api.JoinErrorNotAllowed + } return fmt.Errorf("r.InputRoomEvents: %w", err) } } @@ -207,6 +216,7 @@ func (r *RoomserverInternalAPI) performJoinRoomByID( // room. If it is then there's nothing more to do - the room just // hasn't been created yet. if domain == r.Cfg.Matrix.ServerName { + res.Error = api.JoinErrorNoRoom return fmt.Errorf("Room ID %q does not exist", req.RoomIDOrAlias) } diff --git a/roomserver/inthttp/server.go b/roomserver/inthttp/server.go index 822acd15b..e3b81daa5 100644 --- a/roomserver/inthttp/server.go +++ b/roomserver/inthttp/server.go @@ -34,7 +34,7 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { return util.MessageResponse(http.StatusBadRequest, err.Error()) } if err := r.PerformJoin(req.Context(), &request, &response); err != nil { - return util.ErrorResponse(err) + response.ErrMsg = err.Error() } return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), diff --git a/sytest-whitelist b/sytest-whitelist index ef91507be..ceef20ffd 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -62,6 +62,8 @@ Request to logout with invalid an access token is rejected Request to logout without an access token is rejected Room creation reports m.room.create to myself Room creation reports m.room.member to myself +Invited user can see room metadata +Remote invited user can see room metadata # Blacklisted because these tests call /r0/events which we don't implement # New room members see their own join event # Existing members see new members' join events From c72d23c8eb2f0c8f9e5a301730b308f52490c4e3 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 24 Jun 2020 10:28:03 +0100 Subject: [PATCH 198/200] Update whitelist --- sytest-whitelist | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/sytest-whitelist b/sytest-whitelist index ceef20ffd..0eeccf050 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -356,3 +356,11 @@ Left rooms appear in the leave section of sync Banned rooms appear in the leave section of sync Getting state checks the events requested belong to the room Getting state IDs checks the events requested belong to the room +Can invite users to invite-only rooms +Uninvited users cannot join the room +Invited user can reject invite +Invited user can reject invite for empty room +Invited user can reject local invite after originator leaves +Typing notification sent to local room members +Typing notifications also sent to remote room members +Typing can be explicitly stopped From ebaaf65c54a624e693341e32619806028a45ba2f Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 24 Jun 2020 13:40:50 +0100 Subject: [PATCH 199/200] This doesn't pass --- sytest-whitelist | 1 - 1 file changed, 1 deletion(-) diff --git a/sytest-whitelist b/sytest-whitelist index 0eeccf050..0036d60ea 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -63,7 +63,6 @@ Request to logout without an access token is rejected Room creation reports m.room.create to myself Room creation reports m.room.member to myself Invited user can see room metadata -Remote invited user can see room metadata # Blacklisted because these tests call /r0/events which we don't implement # New room members see their own join event # Existing members see new members' join events From 002fe05a203e316818c108a0dac438e5cd796a68 Mon Sep 17 00:00:00 2001 From: Kegsay Date: Wed, 24 Jun 2020 15:06:14 +0100 Subject: [PATCH 200/200] Add PerformInvite and refactor how errors get handled (#1158) * Add PerformInvite and refactor how errors get handled - Rename `JoinError` to `PerformError` - Remove `error` from the API function signature entirely. This forces errors to be bundled into `PerformError` which makes it easier for callers to detect and handle errors. On network errors, HTTP clients will make a `PerformError`. * Unbreak everything; thanks Go! * Send back JSONResponse according to the PerformError * Update federation invite code too --- clientapi/routing/createroom.go | 8 +- clientapi/routing/joinroom.go | 31 +--- clientapi/routing/membership.go | 8 +- federationapi/routing/invite.go | 6 +- federationapi/routing/send_test.go | 10 +- roomserver/api/api.go | 8 +- roomserver/api/api_trace.go | 16 +- roomserver/api/input.go | 14 +- roomserver/api/perform.go | 71 ++++++-- roomserver/api/wrapper.go | 26 +-- roomserver/internal/input.go | 12 -- roomserver/internal/input_events.go | 195 -------------------- roomserver/internal/perform_invite.go | 249 ++++++++++++++++++++++++++ roomserver/internal/perform_join.go | 104 +++++++---- roomserver/inthttp/client.go | 28 ++- roomserver/inthttp/server.go | 15 +- 16 files changed, 469 insertions(+), 332 deletions(-) create mode 100644 roomserver/internal/perform_invite.go diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index be7124828..8682b03a4 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -403,15 +403,15 @@ func createRoom( } } // Send the invite event to the roomserver. - if err = roomserverAPI.SendInvite( + if perr := roomserverAPI.SendInvite( req.Context(), rsAPI, inviteEvent.Headered(roomVersion), strippedState, // invite room state cfg.Matrix.ServerName, // send as server nil, // transaction ID - ); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("SendInvite failed") - return jsonerror.InternalServerError() + ); perr != nil { + util.GetLogger(req.Context()).WithError(perr).Error("SendInvite failed") + return perr.JSONResponse() } } diff --git a/clientapi/routing/joinroom.go b/clientapi/routing/joinroom.go index e606e35f1..cb68fe196 100644 --- a/clientapi/routing/joinroom.go +++ b/clientapi/routing/joinroom.go @@ -15,12 +15,10 @@ package routing import ( - "errors" "net/http" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/httputil" - "github.com/matrix-org/dendrite/clientapi/jsonerror" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/dendrite/userapi/storage/accounts" @@ -65,32 +63,9 @@ func JoinRoomByIDOrAlias( } // Ask the roomserver to perform the join. - err = rsAPI.PerformJoin(req.Context(), &joinReq, &joinRes) - // Handle known errors first, if this is 0 then there will be no matches (eg on success) - switch joinRes.Error { - case roomserverAPI.JoinErrorBadRequest: - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.Unknown(joinRes.ErrMsg), - } - case roomserverAPI.JoinErrorNoRoom: - return util.JSONResponse{ - Code: http.StatusNotFound, - JSON: jsonerror.NotFound(joinRes.ErrMsg), - } - case roomserverAPI.JoinErrorNotAllowed: - return util.JSONResponse{ - Code: http.StatusForbidden, - JSON: jsonerror.Forbidden(joinRes.ErrMsg), - } - } - // this is always populated on generic errors - if joinRes.ErrMsg != "" { - return util.ErrorResponse(errors.New(joinRes.ErrMsg)) - } - // this is set on network errors in polylith mode - if err != nil { - return util.ErrorResponse(err) + rsAPI.PerformJoin(req.Context(), &joinReq, &joinRes) + if joinRes.Error != nil { + return joinRes.Error.JSONResponse() } return util.JSONResponse{ diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index 0d4b0d88d..aff1730c5 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -111,16 +111,16 @@ func SendMembership( switch membership { case gomatrixserverlib.Invite: // Invites need to be handled specially - err = roomserverAPI.SendInvite( + perr := roomserverAPI.SendInvite( req.Context(), rsAPI, event.Headered(verRes.RoomVersion), nil, // ask the roomserver to draw up invite room state for us cfg.Matrix.ServerName, nil, ) - if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("producer.SendInvite failed") - return jsonerror.InternalServerError() + if perr != nil { + util.GetLogger(req.Context()).WithError(perr).Error("producer.SendInvite failed") + return perr.JSONResponse() } case gomatrixserverlib.Join: // The join membership requires the room id to be sent in the response diff --git a/federationapi/routing/invite.go b/federationapi/routing/invite.go index 7d02bc1d3..b1d84f254 100644 --- a/federationapi/routing/invite.go +++ b/federationapi/routing/invite.go @@ -98,15 +98,15 @@ func Invite( ) // Add the invite event to the roomserver. - if err = api.SendInvite( + if perr := api.SendInvite( httpReq.Context(), rsAPI, signedEvent.Headered(inviteReq.RoomVersion()), inviteReq.InviteRoomState(), event.Origin(), nil, - ); err != nil { + ); perr != nil { util.GetLogger(httpReq.Context()).WithError(err).Error("producer.SendInvite failed") - return jsonerror.InternalServerError() + return perr.JSONResponse() } // Return the signed event to the originating server, it should then tell diff --git a/federationapi/routing/send_test.go b/federationapi/routing/send_test.go index e512f4b4d..3f5d5f4e0 100644 --- a/federationapi/routing/send_test.go +++ b/federationapi/routing/send_test.go @@ -97,12 +97,18 @@ func (t *testRoomserverAPI) InputRoomEvents( return nil } +func (t *testRoomserverAPI) PerformInvite( + ctx context.Context, + req *api.PerformInviteRequest, + res *api.PerformInviteResponse, +) { +} + func (t *testRoomserverAPI) PerformJoin( ctx context.Context, req *api.PerformJoinRequest, res *api.PerformJoinResponse, -) error { - return nil +) { } func (t *testRoomserverAPI) PerformLeave( diff --git a/roomserver/api/api.go b/roomserver/api/api.go index 967f58baf..26ec8ca1d 100644 --- a/roomserver/api/api.go +++ b/roomserver/api/api.go @@ -18,11 +18,17 @@ type RoomserverInternalAPI interface { response *InputRoomEventsResponse, ) error + PerformInvite( + ctx context.Context, + req *PerformInviteRequest, + res *PerformInviteResponse, + ) + PerformJoin( ctx context.Context, req *PerformJoinRequest, res *PerformJoinResponse, - ) error + ) PerformLeave( ctx context.Context, diff --git a/roomserver/api/api_trace.go b/roomserver/api/api_trace.go index a478eeb9a..8645b6f28 100644 --- a/roomserver/api/api_trace.go +++ b/roomserver/api/api_trace.go @@ -29,14 +29,22 @@ func (t *RoomserverInternalAPITrace) InputRoomEvents( return err } +func (t *RoomserverInternalAPITrace) PerformInvite( + ctx context.Context, + req *PerformInviteRequest, + res *PerformInviteResponse, +) { + t.Impl.PerformInvite(ctx, req, res) + util.GetLogger(ctx).Infof("PerformInvite req=%+v res=%+v", js(req), js(res)) +} + func (t *RoomserverInternalAPITrace) PerformJoin( ctx context.Context, req *PerformJoinRequest, res *PerformJoinResponse, -) error { - err := t.Impl.PerformJoin(ctx, req, res) - util.GetLogger(ctx).WithError(err).Infof("PerformJoin req=%+v res=%+v", js(req), js(res)) - return err +) { + t.Impl.PerformJoin(ctx, req, res) + util.GetLogger(ctx).Infof("PerformJoin req=%+v res=%+v", js(req), js(res)) } func (t *RoomserverInternalAPITrace) PerformLeave( diff --git a/roomserver/api/input.go b/roomserver/api/input.go index 6c3c89413..05c981df4 100644 --- a/roomserver/api/input.go +++ b/roomserver/api/input.go @@ -76,21 +76,9 @@ type TransactionID struct { TransactionID string `json:"id"` } -// InputInviteEvent is a matrix invite event received over federation without -// the usual context a matrix room event would have. We usually do not have -// access to the events needed to check the event auth rules for the invite. -type InputInviteEvent struct { - RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"` - Event gomatrixserverlib.HeaderedEvent `json:"event"` - InviteRoomState []gomatrixserverlib.InviteV2StrippedState `json:"invite_room_state"` - SendAsServer string `json:"send_as_server"` - TransactionID *TransactionID `json:"transaction_id"` -} - // InputRoomEventsRequest is a request to InputRoomEvents type InputRoomEventsRequest struct { - InputRoomEvents []InputRoomEvent `json:"input_room_events"` - InputInviteEvents []InputInviteEvent `json:"input_invite_events"` + InputRoomEvents []InputRoomEvent `json:"input_room_events"` } // InputRoomEventsResponse is a response to InputRoomEvents diff --git a/roomserver/api/perform.go b/roomserver/api/perform.go index 0f5394c94..0b8e6df25 100644 --- a/roomserver/api/perform.go +++ b/roomserver/api/perform.go @@ -1,19 +1,57 @@ package api import ( + "fmt" + "net/http" + + "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" ) -type JoinError int +type PerformErrorCode int + +type PerformError struct { + Msg string + Code PerformErrorCode +} + +func (p *PerformError) Error() string { + return fmt.Sprintf("%d : %s", p.Code, p.Msg) +} + +// JSONResponse maps error codes to suitable HTTP error codes, defaulting to 500. +func (p *PerformError) JSONResponse() util.JSONResponse { + switch p.Code { + case PerformErrorBadRequest: + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.Unknown(p.Msg), + } + case PerformErrorNoRoom: + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound(p.Msg), + } + case PerformErrorNotAllowed: + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden(p.Msg), + } + default: + return util.ErrorResponse(p) + } +} const ( - // JoinErrorNotAllowed means the user is not allowed to join this room (e.g join_rule:invite or banned) - JoinErrorNotAllowed JoinError = 1 - // JoinErrorBadRequest means the request was wrong in some way (invalid user ID, wrong server, etc) - JoinErrorBadRequest JoinError = 2 - // JoinErrorNoRoom means that the room being joined doesn't exist. - JoinErrorNoRoom JoinError = 3 + // PerformErrorNotAllowed means the user is not allowed to invite/join/etc this room (e.g join_rule:invite or banned) + PerformErrorNotAllowed PerformErrorCode = 1 + // PerformErrorBadRequest means the request was wrong in some way (invalid user ID, wrong server, etc) + PerformErrorBadRequest PerformErrorCode = 2 + // PerformErrorNoRoom means that the room being joined doesn't exist. + PerformErrorNoRoom PerformErrorCode = 3 + // PerformErrorNoOperation means that the request resulted in nothing happening e.g invite->invite or leave->leave. + PerformErrorNoOperation PerformErrorCode = 4 ) type PerformJoinRequest struct { @@ -26,10 +64,8 @@ type PerformJoinRequest struct { type PerformJoinResponse struct { // The room ID, populated on success. RoomID string `json:"room_id"` - // The reason why the join failed. Can be blank. - Error JoinError `json:"error"` - // Debugging description of the error. Always present on failure. - ErrMsg string `json:"err_msg"` + // If non-nil, the join request failed. Contains more information why it failed. + Error *PerformError } type PerformLeaveRequest struct { @@ -40,6 +76,19 @@ type PerformLeaveRequest struct { type PerformLeaveResponse struct { } +type PerformInviteRequest struct { + RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"` + Event gomatrixserverlib.HeaderedEvent `json:"event"` + InviteRoomState []gomatrixserverlib.InviteV2StrippedState `json:"invite_room_state"` + SendAsServer string `json:"send_as_server"` + TransactionID *TransactionID `json:"transaction_id"` +} + +type PerformInviteResponse struct { + // If non-nil, the invite request failed. Contains more information why it failed. + Error *PerformError +} + // PerformBackfillRequest is a request to PerformBackfill. type PerformBackfillRequest struct { // The room to backfill diff --git a/roomserver/api/wrapper.go b/roomserver/api/wrapper.go index 97940e0c2..b73cd1902 100644 --- a/roomserver/api/wrapper.go +++ b/roomserver/api/wrapper.go @@ -98,16 +98,20 @@ func SendInvite( rsAPI RoomserverInternalAPI, inviteEvent gomatrixserverlib.HeaderedEvent, inviteRoomState []gomatrixserverlib.InviteV2StrippedState, sendAsServer gomatrixserverlib.ServerName, txnID *TransactionID, -) error { - request := InputRoomEventsRequest{ - InputInviteEvents: []InputInviteEvent{{ - Event: inviteEvent, - InviteRoomState: inviteRoomState, - RoomVersion: inviteEvent.RoomVersion, - SendAsServer: string(sendAsServer), - TransactionID: txnID, - }}, +) *PerformError { + request := PerformInviteRequest{ + Event: inviteEvent, + InviteRoomState: inviteRoomState, + RoomVersion: inviteEvent.RoomVersion, + SendAsServer: string(sendAsServer), + TransactionID: txnID, } - var response InputRoomEventsResponse - return rsAPI.InputRoomEvents(ctx, &request, &response) + var response PerformInviteResponse + rsAPI.PerformInvite(ctx, &request, &response) + // we need to do this because many places people will use `var err error` as the return + // arg and a nil interface != nil pointer to a concrete interface (in this case PerformError) + if response.Error != nil && response.Error.Msg != "" { + return response.Error + } + return nil } diff --git a/roomserver/internal/input.go b/roomserver/internal/input.go index e863af953..2af3e62d8 100644 --- a/roomserver/internal/input.go +++ b/roomserver/internal/input.go @@ -74,18 +74,6 @@ func (r *RoomserverInternalAPI) InputRoomEvents( // We lock as processRoomEvent can only be called once at a time r.mutex.Lock() defer r.mutex.Unlock() - for i := range request.InputInviteEvents { - var loopback *api.InputRoomEvent - if loopback, err = r.processInviteEvent(ctx, r, request.InputInviteEvents[i]); err != nil { - return err - } - // The processInviteEvent function can optionally return a - // loopback room event containing the invite, for local invites. - // If it does, we should process it with the room events below. - if loopback != nil { - request.InputRoomEvents = append(request.InputRoomEvents, *loopback) - } - } for i := range request.InputRoomEvents { if response.EventID, err = r.processRoomEvent(ctx, request.InputRoomEvents[i]); err != nil { return err diff --git a/roomserver/internal/input_events.go b/roomserver/internal/input_events.go index fe3bdf4b8..ae57f2e77 100644 --- a/roomserver/internal/input_events.go +++ b/roomserver/internal/input_events.go @@ -18,17 +18,12 @@ package internal import ( "context" - "errors" - "fmt" - "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/state" - "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" - log "github.com/sirupsen/logrus" ) // processRoomEvent can only be called once at a time @@ -148,193 +143,3 @@ func (r *RoomserverInternalAPI) calculateAndSetState( } return r.DB.SetState(ctx, stateAtEvent.EventNID, stateAtEvent.BeforeStateSnapshotNID) } - -func (r *RoomserverInternalAPI) processInviteEvent( - ctx context.Context, - ow *RoomserverInternalAPI, - input api.InputInviteEvent, -) (*api.InputRoomEvent, error) { - if input.Event.StateKey() == nil { - return nil, fmt.Errorf("invite must be a state event") - } - - roomID := input.Event.RoomID() - targetUserID := *input.Event.StateKey() - - log.WithFields(log.Fields{ - "event_id": input.Event.EventID(), - "room_id": roomID, - "room_version": input.RoomVersion, - "target_user_id": targetUserID, - }).Info("processing invite event") - - _, domain, _ := gomatrixserverlib.SplitID('@', targetUserID) - isTargetLocalUser := domain == r.Cfg.Matrix.ServerName - - updater, err := r.DB.MembershipUpdater(ctx, roomID, targetUserID, isTargetLocalUser, input.RoomVersion) - if err != nil { - return nil, err - } - succeeded := false - defer func() { - txerr := sqlutil.EndTransaction(updater, &succeeded) - if err == nil && txerr != nil { - err = txerr - } - }() - - if updater.IsJoin() { - // If the user is joined to the room then that takes precedence over this - // invite event. It makes little sense to move a user that is already - // joined to the room into the invite state. - // This could plausibly happen if an invite request raced with a join - // request for a user. For example if a user was invited to a public - // room and they joined the room at the same time as the invite was sent. - // The other way this could plausibly happen is if an invite raced with - // a kick. For example if a user was kicked from a room in error and in - // response someone else in the room re-invited them then it is possible - // for the invite request to race with the leave event so that the - // target receives invite before it learns that it has been kicked. - // There are a few ways this could be plausibly handled in the roomserver. - // 1) Store the invite, but mark it as retired. That will result in the - // permanent rejection of that invite event. So even if the target - // user leaves the room and the invite is retransmitted it will be - // ignored. However a new invite with a new event ID would still be - // accepted. - // 2) Silently discard the invite event. This means that if the event - // was retransmitted at a later date after the target user had left - // the room we would accept the invite. However since we hadn't told - // the sending server that the invite had been discarded it would - // have no reason to attempt to retry. - // 3) Signal the sending server that the user is already joined to the - // room. - // For now we will implement option 2. Since in the abesence of a retry - // mechanism it will be equivalent to option 1, and we don't have a - // signalling mechanism to implement option 3. - return nil, nil - } - - // Normally, with a federated invite, the federation sender would do - // the /v2/invite request (in which the remote server signs the invite) - // and then the signed event gets sent back to the roomserver as an input - // event. When the invite is local, we don't interact with the federation - // sender therefore we need to generate the loopback invite event for - // the room ourselves. - loopback, err := localInviteLoopback(ow, input) - if err != nil { - return nil, err - } - - event := input.Event.Unwrap() - if len(input.InviteRoomState) > 0 { - // If we were supplied with some invite room state already (which is - // most likely to be if the event came in over federation) then use - // that. - if err = event.SetUnsignedField("invite_room_state", input.InviteRoomState); err != nil { - return nil, err - } - } else { - // There's no invite room state, so let's have a go at building it - // up from local data (which is most likely to be if the event came - // from the CS API). If we know about the room then we can insert - // the invite room state, if we don't then we just fail quietly. - if irs, ierr := buildInviteStrippedState(ctx, r.DB, input); ierr == nil { - if err = event.SetUnsignedField("invite_room_state", irs); err != nil { - return nil, err - } - } - } - - outputUpdates, err := updateToInviteMembership(updater, &event, nil, input.Event.RoomVersion) - if err != nil { - return nil, err - } - - if err = ow.WriteOutputEvents(roomID, outputUpdates); err != nil { - return nil, err - } - - succeeded = true - return loopback, nil -} - -func localInviteLoopback( - ow *RoomserverInternalAPI, - input api.InputInviteEvent, -) (ire *api.InputRoomEvent, err error) { - if input.Event.StateKey() == nil { - return nil, errors.New("no state key on invite event") - } - ourServerName := string(ow.Cfg.Matrix.ServerName) - _, theirServerName, err := gomatrixserverlib.SplitID('@', *input.Event.StateKey()) - if err != nil { - return nil, err - } - // Check if the invite originated locally and is destined locally. - if input.Event.Origin() == ow.Cfg.Matrix.ServerName && string(theirServerName) == ourServerName { - rsEvent := input.Event.Sign( - ourServerName, - ow.Cfg.Matrix.KeyID, - ow.Cfg.Matrix.PrivateKey, - ).Headered(input.RoomVersion) - ire = &api.InputRoomEvent{ - Kind: api.KindNew, - Event: rsEvent, - AuthEventIDs: rsEvent.AuthEventIDs(), - SendAsServer: ourServerName, - TransactionID: nil, - } - } - return ire, nil -} - -func buildInviteStrippedState( - ctx context.Context, - db storage.Database, - input api.InputInviteEvent, -) ([]gomatrixserverlib.InviteV2StrippedState, error) { - roomNID, err := db.RoomNID(ctx, input.Event.RoomID()) - if err != nil || roomNID == 0 { - return nil, fmt.Errorf("room %q unknown", input.Event.RoomID()) - } - stateWanted := []gomatrixserverlib.StateKeyTuple{} - // "If they are set on the room, at least the state for m.room.avatar, m.room.canonical_alias, m.room.join_rules, and m.room.name SHOULD be included." - // https://matrix.org/docs/spec/client_server/r0.6.0#m-room-member - for _, t := range []string{ - gomatrixserverlib.MRoomName, gomatrixserverlib.MRoomCanonicalAlias, - gomatrixserverlib.MRoomAliases, gomatrixserverlib.MRoomJoinRules, - "m.room.avatar", - } { - stateWanted = append(stateWanted, gomatrixserverlib.StateKeyTuple{ - EventType: t, - StateKey: "", - }) - } - _, currentStateSnapshotNID, _, err := db.LatestEventIDs(ctx, roomNID) - if err != nil { - return nil, err - } - roomState := state.NewStateResolution(db) - stateEntries, err := roomState.LoadStateAtSnapshotForStringTuples( - ctx, currentStateSnapshotNID, stateWanted, - ) - if err != nil { - return nil, err - } - stateNIDs := []types.EventNID{} - for _, stateNID := range stateEntries { - stateNIDs = append(stateNIDs, stateNID.EventNID) - } - stateEvents, err := db.Events(ctx, stateNIDs) - if err != nil { - return nil, err - } - inviteState := []gomatrixserverlib.InviteV2StrippedState{ - gomatrixserverlib.NewInviteV2StrippedState(&input.Event.Event), - } - stateEvents = append(stateEvents, types.Event{Event: input.Event.Unwrap()}) - for _, event := range stateEvents { - inviteState = append(inviteState, gomatrixserverlib.NewInviteV2StrippedState(&event.Event)) - } - return inviteState, nil -} diff --git a/roomserver/internal/perform_invite.go b/roomserver/internal/perform_invite.go new file mode 100644 index 000000000..c65c87f91 --- /dev/null +++ b/roomserver/internal/perform_invite.go @@ -0,0 +1,249 @@ +package internal + +import ( + "context" + "errors" + "fmt" + + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/state" + "github.com/matrix-org/dendrite/roomserver/storage" + "github.com/matrix-org/dendrite/roomserver/types" + "github.com/matrix-org/gomatrixserverlib" + log "github.com/sirupsen/logrus" +) + +// PerformInvite handles inviting to matrix rooms, including over federation by talking to the federationsender. +func (r *RoomserverInternalAPI) PerformInvite( + ctx context.Context, + req *api.PerformInviteRequest, + res *api.PerformInviteResponse, +) { + err := r.performInvite(ctx, req) + if err != nil { + perr, ok := err.(*api.PerformError) + if ok { + res.Error = perr + } else { + res.Error = &api.PerformError{ + Msg: err.Error(), + } + } + } +} + +func (r *RoomserverInternalAPI) performInvite(ctx context.Context, + req *api.PerformInviteRequest, +) error { + loopback, err := r.processInviteEvent(ctx, r, req) + if err != nil { + return err + } + // The processInviteEvent function can optionally return a + // loopback room event containing the invite, for local invites. + // If it does, we should process it with the room events below. + if loopback != nil { + var loopbackRes api.InputRoomEventsResponse + err := r.InputRoomEvents(ctx, &api.InputRoomEventsRequest{ + InputRoomEvents: []api.InputRoomEvent{*loopback}, + }, &loopbackRes) + if err != nil { + return err + } + } + return nil +} + +func (r *RoomserverInternalAPI) processInviteEvent( + ctx context.Context, + ow *RoomserverInternalAPI, + input *api.PerformInviteRequest, +) (*api.InputRoomEvent, error) { + if input.Event.StateKey() == nil { + return nil, fmt.Errorf("invite must be a state event") + } + + roomID := input.Event.RoomID() + targetUserID := *input.Event.StateKey() + + log.WithFields(log.Fields{ + "event_id": input.Event.EventID(), + "room_id": roomID, + "room_version": input.RoomVersion, + "target_user_id": targetUserID, + }).Info("processing invite event") + + _, domain, _ := gomatrixserverlib.SplitID('@', targetUserID) + isTargetLocalUser := domain == r.Cfg.Matrix.ServerName + + updater, err := r.DB.MembershipUpdater(ctx, roomID, targetUserID, isTargetLocalUser, input.RoomVersion) + if err != nil { + return nil, err + } + succeeded := false + defer func() { + txerr := sqlutil.EndTransaction(updater, &succeeded) + if err == nil && txerr != nil { + err = txerr + } + }() + + if updater.IsJoin() { + // If the user is joined to the room then that takes precedence over this + // invite event. It makes little sense to move a user that is already + // joined to the room into the invite state. + // This could plausibly happen if an invite request raced with a join + // request for a user. For example if a user was invited to a public + // room and they joined the room at the same time as the invite was sent. + // The other way this could plausibly happen is if an invite raced with + // a kick. For example if a user was kicked from a room in error and in + // response someone else in the room re-invited them then it is possible + // for the invite request to race with the leave event so that the + // target receives invite before it learns that it has been kicked. + // There are a few ways this could be plausibly handled in the roomserver. + // 1) Store the invite, but mark it as retired. That will result in the + // permanent rejection of that invite event. So even if the target + // user leaves the room and the invite is retransmitted it will be + // ignored. However a new invite with a new event ID would still be + // accepted. + // 2) Silently discard the invite event. This means that if the event + // was retransmitted at a later date after the target user had left + // the room we would accept the invite. However since we hadn't told + // the sending server that the invite had been discarded it would + // have no reason to attempt to retry. + // 3) Signal the sending server that the user is already joined to the + // room. + // For now we will implement option 2. Since in the abesence of a retry + // mechanism it will be equivalent to option 1, and we don't have a + // signalling mechanism to implement option 3. + return nil, &api.PerformError{ + Code: api.PerformErrorNoOperation, + Msg: "user is already joined to room", + } + } + + // Normally, with a federated invite, the federation sender would do + // the /v2/invite request (in which the remote server signs the invite) + // and then the signed event gets sent back to the roomserver as an input + // event. When the invite is local, we don't interact with the federation + // sender therefore we need to generate the loopback invite event for + // the room ourselves. + loopback, err := localInviteLoopback(ow, input) + if err != nil { + return nil, err + } + + event := input.Event.Unwrap() + if len(input.InviteRoomState) > 0 { + // If we were supplied with some invite room state already (which is + // most likely to be if the event came in over federation) then use + // that. + if err = event.SetUnsignedField("invite_room_state", input.InviteRoomState); err != nil { + return nil, err + } + } else { + // There's no invite room state, so let's have a go at building it + // up from local data (which is most likely to be if the event came + // from the CS API). If we know about the room then we can insert + // the invite room state, if we don't then we just fail quietly. + if irs, ierr := buildInviteStrippedState(ctx, r.DB, input); ierr == nil { + if err = event.SetUnsignedField("invite_room_state", irs); err != nil { + return nil, err + } + } + } + + outputUpdates, err := updateToInviteMembership(updater, &event, nil, input.Event.RoomVersion) + if err != nil { + return nil, err + } + + if err = ow.WriteOutputEvents(roomID, outputUpdates); err != nil { + return nil, err + } + + succeeded = true + return loopback, nil +} + +func localInviteLoopback( + ow *RoomserverInternalAPI, + input *api.PerformInviteRequest, +) (ire *api.InputRoomEvent, err error) { + if input.Event.StateKey() == nil { + return nil, errors.New("no state key on invite event") + } + ourServerName := string(ow.Cfg.Matrix.ServerName) + _, theirServerName, err := gomatrixserverlib.SplitID('@', *input.Event.StateKey()) + if err != nil { + return nil, err + } + // Check if the invite originated locally and is destined locally. + if input.Event.Origin() == ow.Cfg.Matrix.ServerName && string(theirServerName) == ourServerName { + rsEvent := input.Event.Sign( + ourServerName, + ow.Cfg.Matrix.KeyID, + ow.Cfg.Matrix.PrivateKey, + ).Headered(input.RoomVersion) + ire = &api.InputRoomEvent{ + Kind: api.KindNew, + Event: rsEvent, + AuthEventIDs: rsEvent.AuthEventIDs(), + SendAsServer: ourServerName, + TransactionID: nil, + } + } + return ire, nil +} + +func buildInviteStrippedState( + ctx context.Context, + db storage.Database, + input *api.PerformInviteRequest, +) ([]gomatrixserverlib.InviteV2StrippedState, error) { + roomNID, err := db.RoomNID(ctx, input.Event.RoomID()) + if err != nil || roomNID == 0 { + return nil, fmt.Errorf("room %q unknown", input.Event.RoomID()) + } + stateWanted := []gomatrixserverlib.StateKeyTuple{} + // "If they are set on the room, at least the state for m.room.avatar, m.room.canonical_alias, m.room.join_rules, and m.room.name SHOULD be included." + // https://matrix.org/docs/spec/client_server/r0.6.0#m-room-member + for _, t := range []string{ + gomatrixserverlib.MRoomName, gomatrixserverlib.MRoomCanonicalAlias, + gomatrixserverlib.MRoomAliases, gomatrixserverlib.MRoomJoinRules, + "m.room.avatar", + } { + stateWanted = append(stateWanted, gomatrixserverlib.StateKeyTuple{ + EventType: t, + StateKey: "", + }) + } + _, currentStateSnapshotNID, _, err := db.LatestEventIDs(ctx, roomNID) + if err != nil { + return nil, err + } + roomState := state.NewStateResolution(db) + stateEntries, err := roomState.LoadStateAtSnapshotForStringTuples( + ctx, currentStateSnapshotNID, stateWanted, + ) + if err != nil { + return nil, err + } + stateNIDs := []types.EventNID{} + for _, stateNID := range stateEntries { + stateNIDs = append(stateNIDs, stateNID.EventNID) + } + stateEvents, err := db.Events(ctx, stateNIDs) + if err != nil { + return nil, err + } + inviteState := []gomatrixserverlib.InviteV2StrippedState{ + gomatrixserverlib.NewInviteV2StrippedState(&input.Event.Event), + } + stateEvents = append(stateEvents, types.Event{Event: input.Event.Unwrap()}) + for _, event := range stateEvents { + inviteState = append(inviteState, gomatrixserverlib.NewInviteV2StrippedState(&event.Event)) + } + return inviteState, nil +} diff --git a/roomserver/internal/perform_join.go b/roomserver/internal/perform_join.go index 44f842404..d409b6849 100644 --- a/roomserver/internal/perform_join.go +++ b/roomserver/internal/perform_join.go @@ -14,40 +14,63 @@ import ( "github.com/sirupsen/logrus" ) -// WriteOutputEvents implements OutputRoomEventWriter +// PerformJoin handles joining matrix rooms, including over federation by talking to the federationsender. func (r *RoomserverInternalAPI) PerformJoin( ctx context.Context, req *api.PerformJoinRequest, res *api.PerformJoinResponse, -) error { +) { + roomID, err := r.performJoin(ctx, req) + if err != nil { + perr, ok := err.(*api.PerformError) + if ok { + res.Error = perr + } else { + res.Error = &api.PerformError{ + Msg: err.Error(), + } + } + } + res.RoomID = roomID +} + +func (r *RoomserverInternalAPI) performJoin( + ctx context.Context, + req *api.PerformJoinRequest, +) (string, error) { _, domain, err := gomatrixserverlib.SplitID('@', req.UserID) if err != nil { - res.Error = api.JoinErrorBadRequest - return fmt.Errorf("Supplied user ID %q in incorrect format", req.UserID) + return "", &api.PerformError{ + Code: api.PerformErrorBadRequest, + Msg: fmt.Sprintf("Supplied user ID %q in incorrect format", req.UserID), + } } if domain != r.Cfg.Matrix.ServerName { - res.Error = api.JoinErrorBadRequest - return fmt.Errorf("User %q does not belong to this homeserver", req.UserID) + return "", &api.PerformError{ + Code: api.PerformErrorBadRequest, + Msg: fmt.Sprintf("User %q does not belong to this homeserver", req.UserID), + } } if strings.HasPrefix(req.RoomIDOrAlias, "!") { - return r.performJoinRoomByID(ctx, req, res) + return r.performJoinRoomByID(ctx, req) } if strings.HasPrefix(req.RoomIDOrAlias, "#") { - return r.performJoinRoomByAlias(ctx, req, res) + return r.performJoinRoomByAlias(ctx, req) + } + return "", &api.PerformError{ + Code: api.PerformErrorBadRequest, + Msg: fmt.Sprintf("Room ID or alias %q is invalid", req.RoomIDOrAlias), } - res.Error = api.JoinErrorBadRequest - return fmt.Errorf("Room ID or alias %q is invalid", req.RoomIDOrAlias) } func (r *RoomserverInternalAPI) performJoinRoomByAlias( ctx context.Context, req *api.PerformJoinRequest, - res *api.PerformJoinResponse, -) error { +) (string, error) { // Get the domain part of the room alias. _, domain, err := gomatrixserverlib.SplitID('#', req.RoomIDOrAlias) if err != nil { - return fmt.Errorf("Alias %q is not in the correct format", req.RoomIDOrAlias) + return "", fmt.Errorf("Alias %q is not in the correct format", req.RoomIDOrAlias) } req.ServerNames = append(req.ServerNames, domain) @@ -65,7 +88,7 @@ func (r *RoomserverInternalAPI) performJoinRoomByAlias( err = r.fsAPI.PerformDirectoryLookup(ctx, &dirReq, &dirRes) if err != nil { logrus.WithError(err).Errorf("error looking up alias %q", req.RoomIDOrAlias) - return fmt.Errorf("Looking up alias %q over federation failed: %w", req.RoomIDOrAlias, err) + return "", fmt.Errorf("Looking up alias %q over federation failed: %w", req.RoomIDOrAlias, err) } roomID = dirRes.RoomID req.ServerNames = append(req.ServerNames, dirRes.ServerNames...) @@ -73,18 +96,18 @@ func (r *RoomserverInternalAPI) performJoinRoomByAlias( // Otherwise, look up if we know this room alias locally. roomID, err = r.DB.GetRoomIDForAlias(ctx, req.RoomIDOrAlias) if err != nil { - return fmt.Errorf("Lookup room alias %q failed: %w", req.RoomIDOrAlias, err) + return "", fmt.Errorf("Lookup room alias %q failed: %w", req.RoomIDOrAlias, err) } } // If the room ID is empty then we failed to look up the alias. if roomID == "" { - return fmt.Errorf("Alias %q not found", req.RoomIDOrAlias) + return "", fmt.Errorf("Alias %q not found", req.RoomIDOrAlias) } // If we do, then pluck out the room ID and continue the join. req.RoomIDOrAlias = roomID - return r.performJoinRoomByID(ctx, req, res) + return r.performJoinRoomByID(ctx, req) } // TODO: Break this function up a bit @@ -92,19 +115,14 @@ func (r *RoomserverInternalAPI) performJoinRoomByAlias( func (r *RoomserverInternalAPI) performJoinRoomByID( ctx context.Context, req *api.PerformJoinRequest, - res *api.PerformJoinResponse, // nolint:unparam -) error { - // By this point, if req.RoomIDOrAlias contained an alias, then - // it will have been overwritten with a room ID by performJoinRoomByAlias. - // We should now include this in the response so that the CS API can - // return the right room ID. - res.RoomID = req.RoomIDOrAlias - +) (string, error) { // Get the domain part of the room ID. _, domain, err := gomatrixserverlib.SplitID('!', req.RoomIDOrAlias) if err != nil { - res.Error = api.JoinErrorBadRequest - return fmt.Errorf("Room ID %q is invalid", req.RoomIDOrAlias) + return "", &api.PerformError{ + Code: api.PerformErrorBadRequest, + Msg: fmt.Sprintf("Room ID %q is invalid: %s", req.RoomIDOrAlias, err), + } } req.ServerNames = append(req.ServerNames, domain) @@ -118,7 +136,7 @@ func (r *RoomserverInternalAPI) performJoinRoomByID( Redacts: "", } if err = eb.SetUnsigned(struct{}{}); err != nil { - return fmt.Errorf("eb.SetUnsigned: %w", err) + return "", fmt.Errorf("eb.SetUnsigned: %w", err) } // It is possible for the request to include some "content" for the @@ -129,7 +147,7 @@ func (r *RoomserverInternalAPI) performJoinRoomByID( } req.Content["membership"] = gomatrixserverlib.Join if err = eb.SetContent(req.Content); err != nil { - return fmt.Errorf("eb.SetContent: %w", err) + return "", fmt.Errorf("eb.SetContent: %w", err) } // First work out if this is in response to an existing invite @@ -142,7 +160,7 @@ func (r *RoomserverInternalAPI) performJoinRoomByID( // Check if there's an invite pending. _, inviterDomain, ierr := gomatrixserverlib.SplitID('@', inviteSender) if ierr != nil { - return fmt.Errorf("gomatrixserverlib.SplitID: %w", err) + return "", fmt.Errorf("gomatrixserverlib.SplitID: %w", err) } // Check that the domain isn't ours. If it's local then we don't @@ -154,7 +172,7 @@ func (r *RoomserverInternalAPI) performJoinRoomByID( req.ServerNames = append(req.ServerNames, inviterDomain) // Perform a federated room join. - return r.performFederatedJoinRoomByID(ctx, req, res) + return req.RoomIDOrAlias, r.performFederatedJoinRoomByID(ctx, req) } } @@ -205,9 +223,12 @@ func (r *RoomserverInternalAPI) performJoinRoomByID( if err = r.InputRoomEvents(ctx, &inputReq, &inputRes); err != nil { var notAllowed *gomatrixserverlib.NotAllowed if errors.As(err, ¬Allowed) { - res.Error = api.JoinErrorNotAllowed + return "", &api.PerformError{ + Code: api.PerformErrorNotAllowed, + Msg: fmt.Sprintf("InputRoomEvents auth failed: %s", err), + } } - return fmt.Errorf("r.InputRoomEvents: %w", err) + return "", fmt.Errorf("r.InputRoomEvents: %w", err) } } @@ -216,25 +237,30 @@ func (r *RoomserverInternalAPI) performJoinRoomByID( // room. If it is then there's nothing more to do - the room just // hasn't been created yet. if domain == r.Cfg.Matrix.ServerName { - res.Error = api.JoinErrorNoRoom - return fmt.Errorf("Room ID %q does not exist", req.RoomIDOrAlias) + return "", &api.PerformError{ + Code: api.PerformErrorNoRoom, + Msg: fmt.Sprintf("Room ID %q does not exist", req.RoomIDOrAlias), + } } // Perform a federated room join. - return r.performFederatedJoinRoomByID(ctx, req, res) + return req.RoomIDOrAlias, r.performFederatedJoinRoomByID(ctx, req) default: // Something else went wrong. - return fmt.Errorf("Error joining local room: %q", err) + return "", fmt.Errorf("Error joining local room: %q", err) } - return nil + // By this point, if req.RoomIDOrAlias contained an alias, then + // it will have been overwritten with a room ID by performJoinRoomByAlias. + // We should now include this in the response so that the CS API can + // return the right room ID. + return req.RoomIDOrAlias, nil } func (r *RoomserverInternalAPI) performFederatedJoinRoomByID( ctx context.Context, req *api.PerformJoinRequest, - res *api.PerformJoinResponse, // nolint:unparam ) error { // Try joining by all of the supplied server names. fedReq := fsAPI.PerformJoinRequest{ diff --git a/roomserver/inthttp/client.go b/roomserver/inthttp/client.go index e41adb993..8a2b1204c 100644 --- a/roomserver/inthttp/client.go +++ b/roomserver/inthttp/client.go @@ -3,6 +3,7 @@ package inthttp import ( "context" "errors" + "fmt" "net/http" fsInputAPI "github.com/matrix-org/dendrite/federationsender/api" @@ -24,6 +25,7 @@ const ( RoomserverInputRoomEventsPath = "/roomserver/inputRoomEvents" // Perform operations + RoomserverPerformInvitePath = "/roomserver/performInvite" RoomserverPerformJoinPath = "/roomserver/performJoin" RoomserverPerformLeavePath = "/roomserver/performLeave" RoomserverPerformBackfillPath = "/roomserver/performBackfill" @@ -146,16 +148,38 @@ func (h *httpRoomserverInternalAPI) InputRoomEvents( return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) } +func (h *httpRoomserverInternalAPI) PerformInvite( + ctx context.Context, + request *api.PerformInviteRequest, + response *api.PerformInviteResponse, +) { + span, ctx := opentracing.StartSpanFromContext(ctx, "PerformInvite") + defer span.Finish() + + apiURL := h.roomserverURL + RoomserverPerformInvitePath + err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + if err != nil { + response.Error = &api.PerformError{ + Msg: fmt.Sprintf("failed to communicate with roomserver: %s", err), + } + } +} + func (h *httpRoomserverInternalAPI) PerformJoin( ctx context.Context, request *api.PerformJoinRequest, response *api.PerformJoinResponse, -) error { +) { span, ctx := opentracing.StartSpanFromContext(ctx, "PerformJoin") defer span.Finish() apiURL := h.roomserverURL + RoomserverPerformJoinPath - return httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + err := httputil.PostJSON(ctx, span, h.httpClient, apiURL, request, response) + if err != nil { + response.Error = &api.PerformError{ + Msg: fmt.Sprintf("failed to communicate with roomserver: %s", err), + } + } } func (h *httpRoomserverInternalAPI) PerformLeave( diff --git a/roomserver/inthttp/server.go b/roomserver/inthttp/server.go index e3b81daa5..1c47e87e2 100644 --- a/roomserver/inthttp/server.go +++ b/roomserver/inthttp/server.go @@ -26,6 +26,17 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), ) + internalAPIMux.Handle(RoomserverPerformInvitePath, + httputil.MakeInternalAPI("performInvite", func(req *http.Request) util.JSONResponse { + var request api.PerformInviteRequest + var response api.PerformInviteResponse + if err := json.NewDecoder(req.Body).Decode(&request); err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + r.PerformInvite(req.Context(), &request, &response) + return util.JSONResponse{Code: http.StatusOK, JSON: &response} + }), + ) internalAPIMux.Handle(RoomserverPerformJoinPath, httputil.MakeInternalAPI("performJoin", func(req *http.Request) util.JSONResponse { var request api.PerformJoinRequest @@ -33,9 +44,7 @@ func AddRoutes(r api.RoomserverInternalAPI, internalAPIMux *mux.Router) { if err := json.NewDecoder(req.Body).Decode(&request); err != nil { return util.MessageResponse(http.StatusBadRequest, err.Error()) } - if err := r.PerformJoin(req.Context(), &request, &response); err != nil { - response.ErrMsg = err.Error() - } + r.PerformJoin(req.Context(), &request, &response) return util.JSONResponse{Code: http.StatusOK, JSON: &response} }), )