From 4691adc8f8f7c6d3793a3d5788621ef1ca09ca10 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Tue, 18 May 2021 15:17:37 +0100 Subject: [PATCH 01/26] Update MSC2946 implementation for stable spaces (#1859) Now that MSC1772 passed FCP its identifiers have stabilised This outright drops support for experimental spaces but that's what you get for being on the bleeding edge --- setup/mscs/msc2946/msc2946.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup/mscs/msc2946/msc2946.go b/setup/mscs/msc2946/msc2946.go index 6fdfeed67..08a54f759 100644 --- a/setup/mscs/msc2946/msc2946.go +++ b/setup/mscs/msc2946/msc2946.go @@ -39,9 +39,9 @@ import ( ) const ( - ConstCreateEventContentKey = "org.matrix.msc1772.type" - ConstSpaceChildEventType = "org.matrix.msc1772.space.child" - ConstSpaceParentEventType = "org.matrix.msc1772.space.parent" + ConstCreateEventContentKey = "type" + ConstSpaceChildEventType = "m.space.child" + ConstSpaceParentEventType = "m.space.parent" ) // Defaults sets the request defaults From 140cae81cc0de1465d47b0197e6f50b0f1f26376 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 18 May 2021 15:47:15 +0100 Subject: [PATCH 02/26] Update dendrite-demo-pinecone --- build/gobind-pinecone/build.sh | 2 +- build/gobind-pinecone/monolith.go | 1 + cmd/dendrite-demo-pinecone/main.go | 1 + go.mod | 2 +- go.sum | 12 ++++++++---- 5 files changed, 12 insertions(+), 6 deletions(-) diff --git a/build/gobind-pinecone/build.sh b/build/gobind-pinecone/build.sh index 6fd05a640..0f1b1aab9 100644 --- a/build/gobind-pinecone/build.sh +++ b/build/gobind-pinecone/build.sh @@ -7,7 +7,7 @@ do case "$option" in a) gomobile bind -v -target android -trimpath -ldflags="-s -w" github.com/matrix-org/dendrite/build/gobind-pinecone ;; - i) gomobile bind -v -target ios -trimpath -ldflags="-s -w" github.com/matrix-org/dendrite/build/gobind-pinecone ;; + i) gomobile bind -v -target ios -trimpath -ldflags="" github.com/matrix-org/dendrite/build/gobind-pinecone ;; *) echo "No target specified, specify -a or -i"; exit 1 ;; esac done \ No newline at end of file diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go index b5e1593a1..e69340950 100644 --- a/build/gobind-pinecone/monolith.go +++ b/build/gobind-pinecone/monolith.go @@ -272,6 +272,7 @@ func (m *DendriteMonolith) Start() { cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-appservice.db", m.StorageDirectory, prefix)) cfg.MediaAPI.BasePath = config.Path(fmt.Sprintf("%s/media", m.CacheDirectory)) cfg.MediaAPI.AbsBasePath = config.Path(fmt.Sprintf("%s/media", m.CacheDirectory)) + cfg.MSCs.MSCs = []string{"msc2836", "msc2946"} if err := cfg.Derive(); err != nil { panic(err) } diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go index 46a533f00..cd2832e1e 100644 --- a/cmd/dendrite-demo-pinecone/main.go +++ b/cmd/dendrite-demo-pinecone/main.go @@ -161,6 +161,7 @@ func main() { cfg.FederationSender.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", *instanceName)) cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-appservice.db", *instanceName)) cfg.Global.Kafka.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-naffka.db", *instanceName)) + cfg.MSCs.MSCs = []string{"msc2836", "msc2946"} if err := cfg.Derive(); err != nil { panic(err) } diff --git a/go.mod b/go.mod index 6f6c2379d..e594a55e6 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd github.com/matrix-org/gomatrixserverlib v0.0.0-20210302161955-6142fe3f8c2c github.com/matrix-org/naffka v0.0.0-20201009174903-d26a3b9cb161 - github.com/matrix-org/pinecone v0.0.0-20210510160342-a1dfbcf4bd47 + github.com/matrix-org/pinecone v0.0.0-20210518144623-e76e9ddb89bf github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.7-0.20210414154423-1157a4212dcb github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 diff --git a/go.sum b/go.sum index 132a35ada..dd8df4cb4 100644 --- a/go.sum +++ b/go.sum @@ -705,8 +705,8 @@ github.com/matrix-org/gomatrixserverlib v0.0.0-20210302161955-6142fe3f8c2c h1:vW github.com/matrix-org/gomatrixserverlib v0.0.0-20210302161955-6142fe3f8c2c/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20201009174903-d26a3b9cb161 h1:h1XVh05pLoC+nJjP3GIpj5wUsuC8WdHP3He0RTkRJTs= github.com/matrix-org/naffka v0.0.0-20201009174903-d26a3b9cb161/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= -github.com/matrix-org/pinecone v0.0.0-20210510160342-a1dfbcf4bd47 h1:kWLnEcQoqJWlBtgtooa7ixyGNL3fy4CsyfR8M6b8vdI= -github.com/matrix-org/pinecone v0.0.0-20210510160342-a1dfbcf4bd47/go.mod h1:/75aR0l7umUnMBk02Q5+QC2IPY58S21BLpoPR9BUr0k= +github.com/matrix-org/pinecone v0.0.0-20210518144623-e76e9ddb89bf h1:aM+gb13XAOzVn5n6JshdyCeWO9lqJf29chypqQq3Cm0= +github.com/matrix-org/pinecone v0.0.0-20210518144623-e76e9ddb89bf/go.mod h1:UQzJS6UVyVwfkr+RLrdvBB1vLyECqe3fLYNcbRxv8SA= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= @@ -1036,7 +1036,9 @@ github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= github.com/vishvananda/netlink v1.0.0/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= +github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netns v0.0.0-20190625233234-7109fa855b0f/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= +github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= @@ -1203,6 +1205,7 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190526052359-791d8a0f4d09/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1225,10 +1228,10 @@ golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201231184435-2d18734c6014/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210105210732-16f7687f5001/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210309040221-94ec62e08169 h1:fpeMGRM6A+XFcw4RPCO8s8hH7ppgrGR22pSIjwM7YUI= +golang.org/x/sys v0.0.0-20210309040221-94ec62e08169/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1271,6 +1274,7 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1N golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.zx2c4.com/wireguard v0.0.0-20210203165646-9c7bd73be2cc/go.mod h1:r0ExowOoGFfDoLDxx+M9SYbNVsoZ0xviLL+K4f2mt+A= golang.zx2c4.com/wireguard v0.0.0-20210212170059-7a0fb5bbb172/go.mod h1:r0ExowOoGFfDoLDxx+M9SYbNVsoZ0xviLL+K4f2mt+A= +golang.zx2c4.com/wireguard v0.0.0-20210510202332-9844c74f67ec/go.mod h1:a057zjmoc00UN7gVkaJt2sXVK523kMJcogDTEvPIasg= golang.zx2c4.com/wireguard/windows v0.3.5/go.mod h1:ATrIFNoq3rsK735WJiQzfWYyNFc9xLBhMMjW9DWIvnU= google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= From 08a4370c9447cf7642cb857761b2fcb7e62cd2b5 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 18 May 2021 15:58:50 +0100 Subject: [PATCH 03/26] Update go.mod/go.sum --- go.mod | 3 ++- go.sum | 13 +++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index e594a55e6..d440b79a3 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd github.com/matrix-org/gomatrixserverlib v0.0.0-20210302161955-6142fe3f8c2c github.com/matrix-org/naffka v0.0.0-20201009174903-d26a3b9cb161 - github.com/matrix-org/pinecone v0.0.0-20210518144623-e76e9ddb89bf + github.com/matrix-org/pinecone v0.0.0-20210518145819-76f42940e9b9 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.7-0.20210414154423-1157a4212dcb github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 @@ -42,6 +42,7 @@ require ( github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20210218094457-e77ca8019daa go.uber.org/atomic v1.7.0 golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 + golang.org/x/mobile v0.0.0-20210220033013-bdb1ca9a1e08 // indirect golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69 // indirect gopkg.in/h2non/bimg.v1 v1.1.5 diff --git a/go.sum b/go.sum index dd8df4cb4..4e94f51a9 100644 --- a/go.sum +++ b/go.sum @@ -13,6 +13,7 @@ github.com/Arceliar/phony v0.0.0-20191006174943-d0c68492aca0 h1:p3puK8Sl2xK+2Fnn github.com/Arceliar/phony v0.0.0-20191006174943-d0c68492aca0/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno= github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= @@ -707,6 +708,8 @@ github.com/matrix-org/naffka v0.0.0-20201009174903-d26a3b9cb161 h1:h1XVh05pLoC+n github.com/matrix-org/naffka v0.0.0-20201009174903-d26a3b9cb161/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= github.com/matrix-org/pinecone v0.0.0-20210518144623-e76e9ddb89bf h1:aM+gb13XAOzVn5n6JshdyCeWO9lqJf29chypqQq3Cm0= github.com/matrix-org/pinecone v0.0.0-20210518144623-e76e9ddb89bf/go.mod h1:UQzJS6UVyVwfkr+RLrdvBB1vLyECqe3fLYNcbRxv8SA= +github.com/matrix-org/pinecone v0.0.0-20210518145819-76f42940e9b9 h1:foRhM2ZUQ46Cos6KiKLuIajiuw3db+KzMDRJvQRuHT8= +github.com/matrix-org/pinecone v0.0.0-20210518145819-76f42940e9b9/go.mod h1:UQzJS6UVyVwfkr+RLrdvBB1vLyECqe3fLYNcbRxv8SA= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= @@ -1125,6 +1128,9 @@ golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWP golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -1132,8 +1138,13 @@ golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20210220033013-bdb1ca9a1e08 h1:h+GZ3ubjuWaQjGe8owMGcmMVCqs0xYJtRG5y2bpHaqU= +golang.org/x/mobile v0.0.0-20210220033013-bdb1ca9a1e08/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1254,6 +1265,7 @@ golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -1263,6 +1275,7 @@ golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69 h1:yBHHx+XZqXJBm6Exke3N7V9gnlsyXxoCPEb1yVenjfk= From 2948ffd78207e0fd354be9a3a637cfdaa3c5154a Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 18 May 2021 16:20:04 +0100 Subject: [PATCH 04/26] Update go.mod/go.sum --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index d440b79a3..d7f8bb4dd 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd github.com/matrix-org/gomatrixserverlib v0.0.0-20210302161955-6142fe3f8c2c github.com/matrix-org/naffka v0.0.0-20201009174903-d26a3b9cb161 - github.com/matrix-org/pinecone v0.0.0-20210518145819-76f42940e9b9 + github.com/matrix-org/pinecone v0.0.0-20210518151943-2556e2cf7820 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.7-0.20210414154423-1157a4212dcb github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 diff --git a/go.sum b/go.sum index 4e94f51a9..442f38a88 100644 --- a/go.sum +++ b/go.sum @@ -710,6 +710,8 @@ github.com/matrix-org/pinecone v0.0.0-20210518144623-e76e9ddb89bf h1:aM+gb13XAOz github.com/matrix-org/pinecone v0.0.0-20210518144623-e76e9ddb89bf/go.mod h1:UQzJS6UVyVwfkr+RLrdvBB1vLyECqe3fLYNcbRxv8SA= github.com/matrix-org/pinecone v0.0.0-20210518145819-76f42940e9b9 h1:foRhM2ZUQ46Cos6KiKLuIajiuw3db+KzMDRJvQRuHT8= github.com/matrix-org/pinecone v0.0.0-20210518145819-76f42940e9b9/go.mod h1:UQzJS6UVyVwfkr+RLrdvBB1vLyECqe3fLYNcbRxv8SA= +github.com/matrix-org/pinecone v0.0.0-20210518151943-2556e2cf7820 h1:OflgibJOmt1WvP5TxEES0i1dHj40K+pwmtQ155hZQX8= +github.com/matrix-org/pinecone v0.0.0-20210518151943-2556e2cf7820/go.mod h1:UQzJS6UVyVwfkr+RLrdvBB1vLyECqe3fLYNcbRxv8SA= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= From d2d216402517e613111f6c8abd383c389debe86a Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 24 May 2021 11:17:32 +0100 Subject: [PATCH 05/26] Update pinecone demo --- cmd/dendrite-demo-pinecone/conn/client.go | 14 ++++++++++++-- go.mod | 2 +- go.sum | 2 ++ 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/cmd/dendrite-demo-pinecone/conn/client.go b/cmd/dendrite-demo-pinecone/conn/client.go index 66f9ebfaa..954564d40 100644 --- a/cmd/dendrite-demo-pinecone/conn/client.go +++ b/cmd/dendrite-demo-pinecone/conn/client.go @@ -5,6 +5,7 @@ import ( "net" "net/http" "strings" + "time" "github.com/gorilla/websocket" "github.com/matrix-org/dendrite/setup" @@ -16,15 +17,24 @@ import ( func ConnectToPeer(pRouter *pineconeRouter.Router, peer string) error { var parent net.Conn + dialer := net.Dialer{ + Timeout: time.Second * 5, + KeepAlive: time.Second * 5, + } if strings.HasPrefix(peer, "ws://") || strings.HasPrefix(peer, "wss://") { - c, _, err := websocket.DefaultDialer.Dial(peer, nil) + wsdialer := websocket.Dialer{ + NetDial: dialer.Dial, + NetDialContext: dialer.DialContext, + HandshakeTimeout: time.Second * 5, + } + c, _, err := wsdialer.Dial(peer, nil) if err != nil { return fmt.Errorf("websocket.DefaultDialer.Dial: %w", err) } parent = WrapWebSocketConn(c) } else { var err error - parent, err = net.Dial("tcp", peer) + parent, err = dialer.Dial("tcp", peer) if err != nil { return fmt.Errorf("net.Dial: %w", err) } diff --git a/go.mod b/go.mod index d7f8bb4dd..f41fdc39a 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd github.com/matrix-org/gomatrixserverlib v0.0.0-20210302161955-6142fe3f8c2c github.com/matrix-org/naffka v0.0.0-20201009174903-d26a3b9cb161 - github.com/matrix-org/pinecone v0.0.0-20210518151943-2556e2cf7820 + github.com/matrix-org/pinecone v0.0.0-20210524095705-54dc83854438 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.7-0.20210414154423-1157a4212dcb github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 diff --git a/go.sum b/go.sum index 442f38a88..e7dbe0cb4 100644 --- a/go.sum +++ b/go.sum @@ -712,6 +712,8 @@ github.com/matrix-org/pinecone v0.0.0-20210518145819-76f42940e9b9 h1:foRhM2ZUQ46 github.com/matrix-org/pinecone v0.0.0-20210518145819-76f42940e9b9/go.mod h1:UQzJS6UVyVwfkr+RLrdvBB1vLyECqe3fLYNcbRxv8SA= github.com/matrix-org/pinecone v0.0.0-20210518151943-2556e2cf7820 h1:OflgibJOmt1WvP5TxEES0i1dHj40K+pwmtQ155hZQX8= github.com/matrix-org/pinecone v0.0.0-20210518151943-2556e2cf7820/go.mod h1:UQzJS6UVyVwfkr+RLrdvBB1vLyECqe3fLYNcbRxv8SA= +github.com/matrix-org/pinecone v0.0.0-20210524095705-54dc83854438 h1:pfYBf8db9X8fsieJOUZqv6Evz6YByP+n9rOTzTPC66U= +github.com/matrix-org/pinecone v0.0.0-20210524095705-54dc83854438/go.mod h1:UQzJS6UVyVwfkr+RLrdvBB1vLyECqe3fLYNcbRxv8SA= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= From 79c5485c8d29b103f2e4bc7f55d8d902bccc3d3a Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 24 May 2021 11:43:24 +0100 Subject: [PATCH 06/26] Allow clearing federation blacklist at startup for P2P demos --- build/gobind-pinecone/monolith.go | 2 +- build/gobind-yggdrasil/monolith.go | 2 +- cmd/dendrite-demo-libp2p/main.go | 2 +- cmd/dendrite-demo-pinecone/main.go | 2 +- cmd/dendrite-demo-yggdrasil/main.go | 2 +- cmd/dendrite-monolith-server/main.go | 2 +- .../personalities/federationsender.go | 2 +- cmd/dendritejs/main.go | 2 +- federationsender/federationsender.go | 5 ++++ federationsender/internal/perform.go | 1 + federationsender/storage/interface.go | 1 + .../storage/postgres/blacklist_table.go | 30 ++++++++++++------- federationsender/storage/shared/storage.go | 6 ++++ .../storage/sqlite3/blacklist_table.go | 30 ++++++++++++------- federationsender/storage/tables/interface.go | 1 + 15 files changed, 60 insertions(+), 30 deletions(-) diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go index e69340950..c99ed0b01 100644 --- a/build/gobind-pinecone/monolith.go +++ b/build/gobind-pinecone/monolith.go @@ -291,7 +291,7 @@ func (m *DendriteMonolith) Start() { ) fsAPI := federationsender.NewInternalAPI( - base, federation, rsAPI, keyRing, + base, federation, rsAPI, keyRing, true, ) keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, fsAPI) diff --git a/build/gobind-yggdrasil/monolith.go b/build/gobind-yggdrasil/monolith.go index 332d156bd..eabea5490 100644 --- a/build/gobind-yggdrasil/monolith.go +++ b/build/gobind-yggdrasil/monolith.go @@ -118,7 +118,7 @@ func (m *DendriteMonolith) Start() { ) fsAPI := federationsender.NewInternalAPI( - base, federation, rsAPI, keyRing, + base, federation, rsAPI, keyRing, true, ) keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, federation) diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index 0610ec777..cc7dcf021 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -166,7 +166,7 @@ func main() { asAPI := appservice.NewInternalAPI(&base.Base, userAPI, rsAPI) rsAPI.SetAppserviceAPI(asAPI) fsAPI := federationsender.NewInternalAPI( - &base.Base, federation, rsAPI, keyRing, + &base.Base, federation, rsAPI, keyRing, true, ) rsAPI.SetFederationSenderAPI(fsAPI) provider := newPublicRoomsProvider(base.LibP2PPubsub, rsAPI) diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go index cd2832e1e..7851fdb19 100644 --- a/cmd/dendrite-demo-pinecone/main.go +++ b/cmd/dendrite-demo-pinecone/main.go @@ -180,7 +180,7 @@ func main() { ) rsAPI := rsComponent fsAPI := federationsender.NewInternalAPI( - base, federation, rsAPI, keyRing, + base, federation, rsAPI, keyRing, true, ) keyAPI := keyserver.NewInternalAPI(&base.Cfg.KeyServer, fsAPI) diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index 2a4a335ab..2d710ae79 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -114,7 +114,7 @@ func main() { asAPI := appservice.NewInternalAPI(base, userAPI, rsAPI) rsAPI.SetAppserviceAPI(asAPI) fsAPI := federationsender.NewInternalAPI( - base, federation, rsAPI, keyRing, + base, federation, rsAPI, keyRing, true, ) ygg.SetSessionFunc(func(address string) { diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index b82f73211..ef349505c 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -99,7 +99,7 @@ func main() { } fsAPI := federationsender.NewInternalAPI( - base, federation, rsAPI, keyRing, + base, federation, rsAPI, keyRing, false, ) if base.UseHTTPAPIs { federationsender.AddInternalRoutes(base.InternalAPIMux, fsAPI) diff --git a/cmd/dendrite-polylith-multi/personalities/federationsender.go b/cmd/dendrite-polylith-multi/personalities/federationsender.go index f8b6d3004..ca9a8add0 100644 --- a/cmd/dendrite-polylith-multi/personalities/federationsender.go +++ b/cmd/dendrite-polylith-multi/personalities/federationsender.go @@ -28,7 +28,7 @@ func FederationSender(base *setup.BaseDendrite, cfg *config.Dendrite) { rsAPI := base.RoomserverHTTPClient() fsAPI := federationsender.NewInternalAPI( - base, federation, rsAPI, keyRing, + base, federation, rsAPI, keyRing, false, ) federationsender.AddInternalRoutes(base.InternalAPIMux, fsAPI) diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 0dfa46818..7ece94ff0 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -210,7 +210,7 @@ func main() { base, userAPI, rsAPI, ) rsAPI.SetAppserviceAPI(asQuery) - fedSenderAPI := federationsender.NewInternalAPI(base, federation, rsAPI, &keyRing) + fedSenderAPI := federationsender.NewInternalAPI(base, federation, rsAPI, &keyRing, true) rsAPI.SetFederationSenderAPI(fedSenderAPI) p2pPublicRoomProvider := NewLibP2PPublicRoomsProvider(node, fedSenderAPI, federation) diff --git a/federationsender/federationsender.go b/federationsender/federationsender.go index 9aab91d48..0732c5d38 100644 --- a/federationsender/federationsender.go +++ b/federationsender/federationsender.go @@ -43,6 +43,7 @@ func NewInternalAPI( federation *gomatrixserverlib.FederationClient, rsAPI roomserverAPI.RoomserverInternalAPI, keyRing *gomatrixserverlib.KeyRing, + resetBlacklist bool, ) api.FederationSenderInternalAPI { cfg := &base.Cfg.FederationSender @@ -51,6 +52,10 @@ func NewInternalAPI( logrus.WithError(err).Panic("failed to connect to federation sender db") } + if resetBlacklist { + _ = federationSenderDB.RemoveAllServersFromBlacklist() + } + stats := &statistics.Statistics{ DB: federationSenderDB, FailuresUntilBlacklist: cfg.FederationMaxRetries, diff --git a/federationsender/internal/perform.go b/federationsender/internal/perform.go index 6a2531a03..968df2476 100644 --- a/federationsender/internal/perform.go +++ b/federationsender/internal/perform.go @@ -572,6 +572,7 @@ func (r *FederationSenderInternalAPI) PerformServersAlive( response *api.PerformServersAliveResponse, ) (err error) { for _, srv := range request.Servers { + _ = r.db.RemoveServerFromBlacklist(srv) r.queues.RetryServer(srv) } diff --git a/federationsender/storage/interface.go b/federationsender/storage/interface.go index b83613047..9c5ac0042 100644 --- a/federationsender/storage/interface.go +++ b/federationsender/storage/interface.go @@ -54,6 +54,7 @@ type Database interface { // these don't have contexts passed in as we want things to happen regardless of the request context AddServerToBlacklist(serverName gomatrixserverlib.ServerName) error RemoveServerFromBlacklist(serverName gomatrixserverlib.ServerName) error + RemoveAllServersFromBlacklist() error IsServerBlacklisted(serverName gomatrixserverlib.ServerName) (bool, error) AddOutboundPeek(ctx context.Context, serverName gomatrixserverlib.ServerName, roomID, peekID string, renewalInterval int64) error diff --git a/federationsender/storage/postgres/blacklist_table.go b/federationsender/storage/postgres/blacklist_table.go index f92c59e54..eef37318b 100644 --- a/federationsender/storage/postgres/blacklist_table.go +++ b/federationsender/storage/postgres/blacklist_table.go @@ -40,11 +40,15 @@ const selectBlacklistSQL = "" + const deleteBlacklistSQL = "" + "DELETE FROM federationsender_blacklist WHERE server_name = $1" +const deleteAllBlacklistSQL = "" + + "TRUNCATE federationsender_blacklist" + type blacklistStatements struct { - db *sql.DB - insertBlacklistStmt *sql.Stmt - selectBlacklistStmt *sql.Stmt - deleteBlacklistStmt *sql.Stmt + db *sql.DB + insertBlacklistStmt *sql.Stmt + selectBlacklistStmt *sql.Stmt + deleteBlacklistStmt *sql.Stmt + deleteAllBlacklistStmt *sql.Stmt } func NewPostgresBlacklistTable(db *sql.DB) (s *blacklistStatements, err error) { @@ -65,11 +69,12 @@ func NewPostgresBlacklistTable(db *sql.DB) (s *blacklistStatements, err error) { if s.deleteBlacklistStmt, err = db.Prepare(deleteBlacklistSQL); err != nil { return } + if s.deleteAllBlacklistStmt, err = db.Prepare(deleteAllBlacklistSQL); err != nil { + return + } return } -// insertRoom inserts the room if it didn't already exist. -// If the room didn't exist then last_event_id is set to the empty string. func (s *blacklistStatements) InsertBlacklist( ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName, ) error { @@ -78,9 +83,6 @@ func (s *blacklistStatements) InsertBlacklist( return err } -// selectRoomForUpdate locks the row for the room and returns the last_event_id. -// The row must already exist in the table. Callers can ensure that the row -// exists by calling insertRoom first. func (s *blacklistStatements) SelectBlacklist( ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName, ) (bool, error) { @@ -96,8 +98,6 @@ func (s *blacklistStatements) SelectBlacklist( return res.Next(), nil } -// updateRoom updates the last_event_id for the room. selectRoomForUpdate should -// have already been called earlier within the transaction. func (s *blacklistStatements) DeleteBlacklist( ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName, ) error { @@ -105,3 +105,11 @@ func (s *blacklistStatements) DeleteBlacklist( _, err := stmt.ExecContext(ctx, serverName) return err } + +func (s *blacklistStatements) DeleteAllBlacklist( + ctx context.Context, txn *sql.Tx, +) error { + stmt := sqlutil.TxStmt(txn, s.deleteAllBlacklistStmt) + _, err := stmt.ExecContext(ctx) + return err +} diff --git a/federationsender/storage/shared/storage.go b/federationsender/storage/shared/storage.go index 2e74e9d6a..2d4099229 100644 --- a/federationsender/storage/shared/storage.go +++ b/federationsender/storage/shared/storage.go @@ -148,6 +148,12 @@ func (d *Database) RemoveServerFromBlacklist(serverName gomatrixserverlib.Server }) } +func (d *Database) RemoveAllServersFromBlacklist() error { + return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + return d.FederationSenderBlacklist.DeleteAllBlacklist(context.TODO(), txn) + }) +} + func (d *Database) IsServerBlacklisted(serverName gomatrixserverlib.ServerName) (bool, error) { return d.FederationSenderBlacklist.SelectBlacklist(context.TODO(), nil, serverName) } diff --git a/federationsender/storage/sqlite3/blacklist_table.go b/federationsender/storage/sqlite3/blacklist_table.go index 90b44ac9b..2694e630d 100644 --- a/federationsender/storage/sqlite3/blacklist_table.go +++ b/federationsender/storage/sqlite3/blacklist_table.go @@ -40,11 +40,15 @@ const selectBlacklistSQL = "" + const deleteBlacklistSQL = "" + "DELETE FROM federationsender_blacklist WHERE server_name = $1" +const deleteAllBlacklistSQL = "" + + "DELETE FROM federationsender_blacklist" + type blacklistStatements struct { - db *sql.DB - insertBlacklistStmt *sql.Stmt - selectBlacklistStmt *sql.Stmt - deleteBlacklistStmt *sql.Stmt + db *sql.DB + insertBlacklistStmt *sql.Stmt + selectBlacklistStmt *sql.Stmt + deleteBlacklistStmt *sql.Stmt + deleteAllBlacklistStmt *sql.Stmt } func NewSQLiteBlacklistTable(db *sql.DB) (s *blacklistStatements, err error) { @@ -65,11 +69,12 @@ func NewSQLiteBlacklistTable(db *sql.DB) (s *blacklistStatements, err error) { if s.deleteBlacklistStmt, err = db.Prepare(deleteBlacklistSQL); err != nil { return } + if s.deleteAllBlacklistStmt, err = db.Prepare(deleteAllBlacklistSQL); err != nil { + return + } return } -// insertRoom inserts the room if it didn't already exist. -// If the room didn't exist then last_event_id is set to the empty string. func (s *blacklistStatements) InsertBlacklist( ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName, ) error { @@ -78,9 +83,6 @@ func (s *blacklistStatements) InsertBlacklist( return err } -// selectRoomForUpdate locks the row for the room and returns the last_event_id. -// The row must already exist in the table. Callers can ensure that the row -// exists by calling insertRoom first. func (s *blacklistStatements) SelectBlacklist( ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName, ) (bool, error) { @@ -96,8 +98,6 @@ func (s *blacklistStatements) SelectBlacklist( return res.Next(), nil } -// updateRoom updates the last_event_id for the room. selectRoomForUpdate should -// have already been called earlier within the transaction. func (s *blacklistStatements) DeleteBlacklist( ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName, ) error { @@ -105,3 +105,11 @@ func (s *blacklistStatements) DeleteBlacklist( _, err := stmt.ExecContext(ctx, serverName) return err } + +func (s *blacklistStatements) DeleteAllBlacklist( + ctx context.Context, txn *sql.Tx, +) error { + stmt := sqlutil.TxStmt(txn, s.deleteAllBlacklistStmt) + _, err := stmt.ExecContext(ctx) + return err +} diff --git a/federationsender/storage/tables/interface.go b/federationsender/storage/tables/interface.go index 34ff0b97e..995b6f47a 100644 --- a/federationsender/storage/tables/interface.go +++ b/federationsender/storage/tables/interface.go @@ -60,6 +60,7 @@ type FederationSenderBlacklist interface { InsertBlacklist(ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName) error SelectBlacklist(ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName) (bool, error) DeleteBlacklist(ctx context.Context, txn *sql.Tx, serverName gomatrixserverlib.ServerName) error + DeleteAllBlacklist(ctx context.Context, txn *sql.Tx) error } type FederationSenderOutboundPeeks interface { From a7f2845a6a97af0c1855eb3c8d81cb4ca348d176 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 24 May 2021 13:12:05 +0100 Subject: [PATCH 07/26] Demo tweaks --- build/gobind-pinecone/monolith.go | 2 +- cmd/dendrite-demo-pinecone/conn/client.go | 14 ++------------ go.mod | 2 +- go.sum | 2 ++ 4 files changed, 6 insertions(+), 14 deletions(-) diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go index c99ed0b01..0c26944fc 100644 --- a/build/gobind-pinecone/monolith.go +++ b/build/gobind-pinecone/monolith.go @@ -97,7 +97,7 @@ func (m *DendriteMonolith) SetStaticPeer(uri string) { m.staticPeerMutex.Unlock() m.DisconnectType(pineconeRouter.PeerTypeRemote) if uri != "" { - m.staticPeerConnect() + go m.staticPeerConnect() } } diff --git a/cmd/dendrite-demo-pinecone/conn/client.go b/cmd/dendrite-demo-pinecone/conn/client.go index 954564d40..66f9ebfaa 100644 --- a/cmd/dendrite-demo-pinecone/conn/client.go +++ b/cmd/dendrite-demo-pinecone/conn/client.go @@ -5,7 +5,6 @@ import ( "net" "net/http" "strings" - "time" "github.com/gorilla/websocket" "github.com/matrix-org/dendrite/setup" @@ -17,24 +16,15 @@ import ( func ConnectToPeer(pRouter *pineconeRouter.Router, peer string) error { var parent net.Conn - dialer := net.Dialer{ - Timeout: time.Second * 5, - KeepAlive: time.Second * 5, - } if strings.HasPrefix(peer, "ws://") || strings.HasPrefix(peer, "wss://") { - wsdialer := websocket.Dialer{ - NetDial: dialer.Dial, - NetDialContext: dialer.DialContext, - HandshakeTimeout: time.Second * 5, - } - c, _, err := wsdialer.Dial(peer, nil) + c, _, err := websocket.DefaultDialer.Dial(peer, nil) if err != nil { return fmt.Errorf("websocket.DefaultDialer.Dial: %w", err) } parent = WrapWebSocketConn(c) } else { var err error - parent, err = dialer.Dial("tcp", peer) + parent, err = net.Dial("tcp", peer) if err != nil { return fmt.Errorf("net.Dial: %w", err) } diff --git a/go.mod b/go.mod index f41fdc39a..70f99b82d 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd github.com/matrix-org/gomatrixserverlib v0.0.0-20210302161955-6142fe3f8c2c github.com/matrix-org/naffka v0.0.0-20201009174903-d26a3b9cb161 - github.com/matrix-org/pinecone v0.0.0-20210524095705-54dc83854438 + github.com/matrix-org/pinecone v0.0.0-20210524121048-a698ff38c7c3 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.7-0.20210414154423-1157a4212dcb github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 diff --git a/go.sum b/go.sum index e7dbe0cb4..9ce18c5ec 100644 --- a/go.sum +++ b/go.sum @@ -714,6 +714,8 @@ github.com/matrix-org/pinecone v0.0.0-20210518151943-2556e2cf7820 h1:OflgibJOmt1 github.com/matrix-org/pinecone v0.0.0-20210518151943-2556e2cf7820/go.mod h1:UQzJS6UVyVwfkr+RLrdvBB1vLyECqe3fLYNcbRxv8SA= github.com/matrix-org/pinecone v0.0.0-20210524095705-54dc83854438 h1:pfYBf8db9X8fsieJOUZqv6Evz6YByP+n9rOTzTPC66U= github.com/matrix-org/pinecone v0.0.0-20210524095705-54dc83854438/go.mod h1:UQzJS6UVyVwfkr+RLrdvBB1vLyECqe3fLYNcbRxv8SA= +github.com/matrix-org/pinecone v0.0.0-20210524121048-a698ff38c7c3 h1:uHQiX47DfRK8TSbrTgT3325RSghCVK0y+JPuRuvvIR4= +github.com/matrix-org/pinecone v0.0.0-20210524121048-a698ff38c7c3/go.mod h1:UQzJS6UVyVwfkr+RLrdvBB1vLyECqe3fLYNcbRxv8SA= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= From 30f021700a1c0a9bd095074027f3278ba833846a Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 25 May 2021 09:49:02 +0100 Subject: [PATCH 08/26] Update go.mod/go.sum --- build/gobind-pinecone/monolith.go | 2 ++ build/gobind-yggdrasil/monolith.go | 2 ++ go.mod | 5 ++--- go.sum | 12 ++---------- 4 files changed, 8 insertions(+), 13 deletions(-) diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go index 0c26944fc..c15707e56 100644 --- a/build/gobind-pinecone/monolith.go +++ b/build/gobind-pinecone/monolith.go @@ -46,6 +46,8 @@ import ( pineconeSessions "github.com/matrix-org/pinecone/sessions" "github.com/matrix-org/pinecone/types" pineconeTypes "github.com/matrix-org/pinecone/types" + + _ "golang.org/x/mobile/bind" ) const ( diff --git a/build/gobind-yggdrasil/monolith.go b/build/gobind-yggdrasil/monolith.go index eabea5490..5074f6da4 100644 --- a/build/gobind-yggdrasil/monolith.go +++ b/build/gobind-yggdrasil/monolith.go @@ -25,6 +25,8 @@ import ( "github.com/matrix-org/dendrite/userapi" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" + + _ "golang.org/x/mobile/bind" ) type DendriteMonolith struct { diff --git a/go.mod b/go.mod index 70f99b82d..eacf3663b 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd github.com/matrix-org/gomatrixserverlib v0.0.0-20210302161955-6142fe3f8c2c github.com/matrix-org/naffka v0.0.0-20201009174903-d26a3b9cb161 - github.com/matrix-org/pinecone v0.0.0-20210524121048-a698ff38c7c3 + github.com/matrix-org/pinecone v0.0.0-20210524134733-52de51a41fc4 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.7-0.20210414154423-1157a4212dcb github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 @@ -42,9 +42,8 @@ require ( github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20210218094457-e77ca8019daa go.uber.org/atomic v1.7.0 golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 - golang.org/x/mobile v0.0.0-20210220033013-bdb1ca9a1e08 // indirect + golang.org/x/mobile v0.0.0-20210220033013-bdb1ca9a1e08 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 - golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69 // indirect gopkg.in/h2non/bimg.v1 v1.1.5 gopkg.in/yaml.v2 v2.4.0 ) diff --git a/go.sum b/go.sum index 9ce18c5ec..bfabce915 100644 --- a/go.sum +++ b/go.sum @@ -706,16 +706,8 @@ github.com/matrix-org/gomatrixserverlib v0.0.0-20210302161955-6142fe3f8c2c h1:vW github.com/matrix-org/gomatrixserverlib v0.0.0-20210302161955-6142fe3f8c2c/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20201009174903-d26a3b9cb161 h1:h1XVh05pLoC+nJjP3GIpj5wUsuC8WdHP3He0RTkRJTs= github.com/matrix-org/naffka v0.0.0-20201009174903-d26a3b9cb161/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= -github.com/matrix-org/pinecone v0.0.0-20210518144623-e76e9ddb89bf h1:aM+gb13XAOzVn5n6JshdyCeWO9lqJf29chypqQq3Cm0= -github.com/matrix-org/pinecone v0.0.0-20210518144623-e76e9ddb89bf/go.mod h1:UQzJS6UVyVwfkr+RLrdvBB1vLyECqe3fLYNcbRxv8SA= -github.com/matrix-org/pinecone v0.0.0-20210518145819-76f42940e9b9 h1:foRhM2ZUQ46Cos6KiKLuIajiuw3db+KzMDRJvQRuHT8= -github.com/matrix-org/pinecone v0.0.0-20210518145819-76f42940e9b9/go.mod h1:UQzJS6UVyVwfkr+RLrdvBB1vLyECqe3fLYNcbRxv8SA= -github.com/matrix-org/pinecone v0.0.0-20210518151943-2556e2cf7820 h1:OflgibJOmt1WvP5TxEES0i1dHj40K+pwmtQ155hZQX8= -github.com/matrix-org/pinecone v0.0.0-20210518151943-2556e2cf7820/go.mod h1:UQzJS6UVyVwfkr+RLrdvBB1vLyECqe3fLYNcbRxv8SA= -github.com/matrix-org/pinecone v0.0.0-20210524095705-54dc83854438 h1:pfYBf8db9X8fsieJOUZqv6Evz6YByP+n9rOTzTPC66U= -github.com/matrix-org/pinecone v0.0.0-20210524095705-54dc83854438/go.mod h1:UQzJS6UVyVwfkr+RLrdvBB1vLyECqe3fLYNcbRxv8SA= -github.com/matrix-org/pinecone v0.0.0-20210524121048-a698ff38c7c3 h1:uHQiX47DfRK8TSbrTgT3325RSghCVK0y+JPuRuvvIR4= -github.com/matrix-org/pinecone v0.0.0-20210524121048-a698ff38c7c3/go.mod h1:UQzJS6UVyVwfkr+RLrdvBB1vLyECqe3fLYNcbRxv8SA= +github.com/matrix-org/pinecone v0.0.0-20210524134733-52de51a41fc4 h1:zmpPGvzjD0UEfz6J//sXwhLvvzpwF4/Rz5D43c96paM= +github.com/matrix-org/pinecone v0.0.0-20210524134733-52de51a41fc4/go.mod h1:UQzJS6UVyVwfkr+RLrdvBB1vLyECqe3fLYNcbRxv8SA= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= From 3f84cd6cd0276cee0df0a20abe7f4f841b79cd5a Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 25 May 2021 09:52:55 +0100 Subject: [PATCH 09/26] Update go.mod/go.sum --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index eacf3663b..f13b1530d 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd github.com/matrix-org/gomatrixserverlib v0.0.0-20210302161955-6142fe3f8c2c github.com/matrix-org/naffka v0.0.0-20201009174903-d26a3b9cb161 - github.com/matrix-org/pinecone v0.0.0-20210524134733-52de51a41fc4 + github.com/matrix-org/pinecone v0.0.0-20210525085236-60b91891c8b9 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.7-0.20210414154423-1157a4212dcb github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 diff --git a/go.sum b/go.sum index bfabce915..8b8699751 100644 --- a/go.sum +++ b/go.sum @@ -706,8 +706,8 @@ github.com/matrix-org/gomatrixserverlib v0.0.0-20210302161955-6142fe3f8c2c h1:vW github.com/matrix-org/gomatrixserverlib v0.0.0-20210302161955-6142fe3f8c2c/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20201009174903-d26a3b9cb161 h1:h1XVh05pLoC+nJjP3GIpj5wUsuC8WdHP3He0RTkRJTs= github.com/matrix-org/naffka v0.0.0-20201009174903-d26a3b9cb161/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= -github.com/matrix-org/pinecone v0.0.0-20210524134733-52de51a41fc4 h1:zmpPGvzjD0UEfz6J//sXwhLvvzpwF4/Rz5D43c96paM= -github.com/matrix-org/pinecone v0.0.0-20210524134733-52de51a41fc4/go.mod h1:UQzJS6UVyVwfkr+RLrdvBB1vLyECqe3fLYNcbRxv8SA= +github.com/matrix-org/pinecone v0.0.0-20210525085236-60b91891c8b9 h1:kH/k+TX0nAefivMhhAsyW0SP1B6JV401eXJln8HAfWs= +github.com/matrix-org/pinecone v0.0.0-20210525085236-60b91891c8b9/go.mod h1:UQzJS6UVyVwfkr+RLrdvBB1vLyECqe3fLYNcbRxv8SA= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= From 3797d818c07709ea4942a2cf34f1dcc52a7fce81 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 25 May 2021 12:01:07 +0100 Subject: [PATCH 10/26] Update gomatrixserverlib to matrix-org/gomatrixserverlib#259 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f13b1530d..c030cf47e 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3 github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd - github.com/matrix-org/gomatrixserverlib v0.0.0-20210302161955-6142fe3f8c2c + github.com/matrix-org/gomatrixserverlib v0.0.0-20210525110027-8cb7699aa64a github.com/matrix-org/naffka v0.0.0-20201009174903-d26a3b9cb161 github.com/matrix-org/pinecone v0.0.0-20210525085236-60b91891c8b9 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 diff --git a/go.sum b/go.sum index 8b8699751..445ca89f4 100644 --- a/go.sum +++ b/go.sum @@ -702,8 +702,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20200522092705-bc8506ccbcf3/go.mod h1 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd h1:xVrqJK3xHREMNjwjljkAUaadalWc0rRbmVuQatzmgwg= github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20210302161955-6142fe3f8c2c h1:vW3jBp1PnZ4uUqyS+JatJRYIl73ZLHkUkRzf1JdGxOI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20210302161955-6142fe3f8c2c/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20210525110027-8cb7699aa64a h1:pVhOeJpD0gv5boUnihefPDuYkQ6xSdEVbH5ld5Vvve0= +github.com/matrix-org/gomatrixserverlib v0.0.0-20210525110027-8cb7699aa64a/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20201009174903-d26a3b9cb161 h1:h1XVh05pLoC+nJjP3GIpj5wUsuC8WdHP3He0RTkRJTs= github.com/matrix-org/naffka v0.0.0-20201009174903-d26a3b9cb161/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= github.com/matrix-org/pinecone v0.0.0-20210525085236-60b91891c8b9 h1:kH/k+TX0nAefivMhhAsyW0SP1B6JV401eXJln8HAfWs= From b0aa101dcd568dbb8e6a5db42e358442a6ff1182 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 2 Jun 2021 12:23:01 +0100 Subject: [PATCH 11/26] Update go.mod/go.sum for matrix-org/pinecone (Build 79) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index c030cf47e..7273da647 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd github.com/matrix-org/gomatrixserverlib v0.0.0-20210525110027-8cb7699aa64a github.com/matrix-org/naffka v0.0.0-20201009174903-d26a3b9cb161 - github.com/matrix-org/pinecone v0.0.0-20210525085236-60b91891c8b9 + github.com/matrix-org/pinecone v0.0.0-20210602111459-5cb0e6aa1a6a github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.7-0.20210414154423-1157a4212dcb github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 diff --git a/go.sum b/go.sum index 445ca89f4..eba9a60b1 100644 --- a/go.sum +++ b/go.sum @@ -706,8 +706,8 @@ github.com/matrix-org/gomatrixserverlib v0.0.0-20210525110027-8cb7699aa64a h1:pV github.com/matrix-org/gomatrixserverlib v0.0.0-20210525110027-8cb7699aa64a/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20201009174903-d26a3b9cb161 h1:h1XVh05pLoC+nJjP3GIpj5wUsuC8WdHP3He0RTkRJTs= github.com/matrix-org/naffka v0.0.0-20201009174903-d26a3b9cb161/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= -github.com/matrix-org/pinecone v0.0.0-20210525085236-60b91891c8b9 h1:kH/k+TX0nAefivMhhAsyW0SP1B6JV401eXJln8HAfWs= -github.com/matrix-org/pinecone v0.0.0-20210525085236-60b91891c8b9/go.mod h1:UQzJS6UVyVwfkr+RLrdvBB1vLyECqe3fLYNcbRxv8SA= +github.com/matrix-org/pinecone v0.0.0-20210602111459-5cb0e6aa1a6a h1:BE/cfpyHO2ua1BK4Tibr+2oZCV3H1mC9G7g7Yvl1AmM= +github.com/matrix-org/pinecone v0.0.0-20210602111459-5cb0e6aa1a6a/go.mod h1:UQzJS6UVyVwfkr+RLrdvBB1vLyECqe3fLYNcbRxv8SA= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= From f18001ecbbfe2c70c5bce41e0f8373750f528ace Mon Sep 17 00:00:00 2001 From: bodqhrohro Date: Mon, 7 Jun 2021 10:51:19 +0300 Subject: [PATCH 12/26] Split the select+update query for txn_id counter (#1855) The update part wasn't executed actually for SQLite, so it is moved to a separate statement. Fixes #1852. Signed-off-by: Bohdan Horbeshko --- .../storage/sqlite3/txn_id_counter_table.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/appservice/storage/sqlite3/txn_id_counter_table.go b/appservice/storage/sqlite3/txn_id_counter_table.go index b2940e358..f2e902f98 100644 --- a/appservice/storage/sqlite3/txn_id_counter_table.go +++ b/appservice/storage/sqlite3/txn_id_counter_table.go @@ -32,14 +32,18 @@ INSERT OR IGNORE INTO appservice_counters (name, last_id) VALUES('txn_id', 1); ` const selectTxnIDSQL = ` - SELECT last_id FROM appservice_counters WHERE name='txn_id'; - UPDATE appservice_counters SET last_id=last_id+1 WHERE name='txn_id'; + SELECT last_id FROM appservice_counters WHERE name='txn_id' +` + +const updateTxnIDSQL = ` + UPDATE appservice_counters SET last_id=last_id+1 WHERE name='txn_id' ` type txnStatements struct { db *sql.DB writer sqlutil.Writer selectTxnIDStmt *sql.Stmt + updateTxnIDStmt *sql.Stmt } func (s *txnStatements) prepare(db *sql.DB, writer sqlutil.Writer) (err error) { @@ -54,6 +58,10 @@ func (s *txnStatements) prepare(db *sql.DB, writer sqlutil.Writer) (err error) { return } + if s.updateTxnIDStmt, err = db.Prepare(updateTxnIDSQL); err != nil { + return + } + return } @@ -63,6 +71,11 @@ func (s *txnStatements) selectTxnID( ) (txnID int, err error) { err = s.writer.Do(s.db, nil, func(txn *sql.Tx) error { err := s.selectTxnIDStmt.QueryRowContext(ctx).Scan(&txnID) + if err != nil { + return err + } + + _, err = s.updateTxnIDStmt.ExecContext(ctx) return err }) return From c488d3db752e68c222c2e35a5d5688c399f09268 Mon Sep 17 00:00:00 2001 From: bodqhrohro Date: Mon, 7 Jun 2021 11:13:40 +0300 Subject: [PATCH 13/26] Fix SIGSEGV in IsInterestedInRoomID (#1846) * Avoid crash on non-compiled room regex Roughly fixes #1845 (actual compiling still needed) Signed-off-by: Bohdan Horbeshko * Compile regexes for all namespaces Deadheres the regex compiling from building larger regexes for possibly exclusive namespaces only. A complete fix for #1845, so regexes for rooms namespaces and other non-whitelisted namespaces can be used more safely. Signed-off-by: Bohdan Horbeshko * Appservice config: handle regexp parsing errors Signed-off-by: diamondburned Signed-off-by: Bohdan Horbeshko Co-authored-by: Kegsay --- setup/config/config_appservice.go | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/setup/config/config_appservice.go b/setup/config/config_appservice.go index 4a14672e6..9bd8a1b51 100644 --- a/setup/config/config_appservice.go +++ b/setup/config/config_appservice.go @@ -101,7 +101,7 @@ func (a *ApplicationService) IsInterestedInRoomID( ) bool { if namespaceSlice, ok := a.NamespaceMap["rooms"]; ok { for _, namespace := range namespaceSlice { - if namespace.RegexpObject.MatchString(roomID) { + if namespace.RegexpObject != nil && namespace.RegexpObject.MatchString(roomID) { return true } } @@ -222,6 +222,10 @@ func setupRegexps(asAPI *AppServiceAPI, derived *Derived) (err error) { case "aliases": appendExclusiveNamespaceRegexs(&exclusiveAliasStrings, namespaceSlice) } + + if err = compileNamespaceRegexes(namespaceSlice); err != nil { + return fmt.Errorf("invalid regex in appservice %q, namespace %q: %w", appservice.ID, key, err) + } } } @@ -258,18 +262,31 @@ func setupRegexps(asAPI *AppServiceAPI, derived *Derived) (err error) { func appendExclusiveNamespaceRegexs( exclusiveStrings *[]string, namespaces []ApplicationServiceNamespace, ) { - for index, namespace := range namespaces { + for _, namespace := range namespaces { if namespace.Exclusive { // We append parenthesis to later separate each regex when we compile // i.e. "app1.*", "app2.*" -> "(app1.*)|(app2.*)" *exclusiveStrings = append(*exclusiveStrings, "("+namespace.Regex+")") } - - // Compile this regex into a Regexp object for later use - namespaces[index].RegexpObject, _ = regexp.Compile(namespace.Regex) } } +// compileNamespaceRegexes turns strings into regex objects and complains +// if some of there are bad +func compileNamespaceRegexes(namespaces []ApplicationServiceNamespace) (err error) { + for index, namespace := range namespaces { + // Compile this regex into a Regexp object for later use + r, err := regexp.Compile(namespace.Regex) + if err != nil { + return fmt.Errorf("regex at namespace %d: %w", index, err) + } + + namespaces[index].RegexpObject = r + } + + return nil +} + // checkErrors checks for any configuration errors amongst the loaded // application services according to the application service spec. func checkErrors(config *AppServiceAPI, derived *Derived) (err error) { From 8b22c4270da28bb6e968f0804da8e2dc4a865651 Mon Sep 17 00:00:00 2001 From: S7evinK Date: Mon, 7 Jun 2021 10:17:20 +0200 Subject: [PATCH 14/26] Use LimitReader to prevent DoS risk (#1843) * Use LimitReader to prevent DoS risk Signed-off-by: Till Faelligen * Check if bytesWritten is equal to the maxFileSize Add tests Signed-off-by: Till Faelligen * Use oldschool defer to cleanup after the tests * Let LimitReader read MaxFileSizeBytes + 1 Co-authored-by: Kegsay --- mediaapi/routing/upload.go | 3 +- mediaapi/routing/upload_test.go | 132 ++++++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+), 1 deletion(-) create mode 100644 mediaapi/routing/upload_test.go diff --git a/mediaapi/routing/upload.go b/mediaapi/routing/upload.go index f1dd231df..ada02b11e 100644 --- a/mediaapi/routing/upload.go +++ b/mediaapi/routing/upload.go @@ -147,7 +147,8 @@ func (r *uploadRequest) doUpload( // r.storeFileAndMetadata(ctx, tmpDir, ...) // before you return from doUpload else we will leak a temp file. We could make this nicer with a `WithTransaction` style of // nested function to guarantee either storage or cleanup. - hash, bytesWritten, tmpDir, err := fileutils.WriteTempFile(ctx, reqReader, cfg.AbsBasePath) + lr := io.LimitReader(reqReader, int64(*cfg.MaxFileSizeBytes)+1) + hash, bytesWritten, tmpDir, err := fileutils.WriteTempFile(ctx, lr, cfg.AbsBasePath) if err != nil { r.Logger.WithError(err).WithFields(log.Fields{ "MaxFileSizeBytes": *cfg.MaxFileSizeBytes, diff --git a/mediaapi/routing/upload_test.go b/mediaapi/routing/upload_test.go new file mode 100644 index 000000000..032437b59 --- /dev/null +++ b/mediaapi/routing/upload_test.go @@ -0,0 +1,132 @@ +package routing + +import ( + "context" + "io" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/matrix-org/dendrite/mediaapi/fileutils" + "github.com/matrix-org/dendrite/mediaapi/storage" + "github.com/matrix-org/dendrite/mediaapi/types" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/util" + log "github.com/sirupsen/logrus" +) + +func Test_uploadRequest_doUpload(t *testing.T) { + type fields struct { + MediaMetadata *types.MediaMetadata + Logger *log.Entry + } + type args struct { + ctx context.Context + reqReader io.Reader + cfg *config.MediaAPI + db storage.Database + activeThumbnailGeneration *types.ActiveThumbnailGeneration + } + + wd, err := os.Getwd() + if err != nil { + t.Errorf("failed to get current working directory: %v", err) + } + + maxSize := config.FileSizeBytes(8) + logger := log.New().WithField("mediaapi", "test") + testdataPath := filepath.Join(wd, "./testdata") + + cfg := &config.MediaAPI{ + MaxFileSizeBytes: &maxSize, + BasePath: config.Path(testdataPath), + AbsBasePath: config.Path(testdataPath), + DynamicThumbnails: false, + } + + // create testdata folder and remove when done + _ = os.Mkdir(testdataPath, os.ModePerm) + defer fileutils.RemoveDir(types.Path(testdataPath), nil) + + db, err := storage.Open(&config.DatabaseOptions{ + ConnectionString: "file::memory:?cache=shared", + MaxOpenConnections: 100, + MaxIdleConnections: 2, + ConnMaxLifetimeSeconds: -1, + }) + if err != nil { + t.Errorf("error opening mediaapi database: %v", err) + } + + tests := []struct { + name string + fields fields + args args + want *util.JSONResponse + }{ + { + name: "upload ok", + args: args{ + ctx: context.Background(), + reqReader: strings.NewReader("test"), + cfg: cfg, + db: db, + }, + fields: fields{ + Logger: logger, + MediaMetadata: &types.MediaMetadata{ + MediaID: "1337", + UploadName: "test ok", + }, + }, + want: nil, + }, + { + name: "upload ok (exact size)", + args: args{ + ctx: context.Background(), + reqReader: strings.NewReader("testtest"), + cfg: cfg, + db: db, + }, + fields: fields{ + Logger: logger, + MediaMetadata: &types.MediaMetadata{ + MediaID: "1338", + UploadName: "test ok (exact size)", + }, + }, + want: nil, + }, + { + name: "upload not ok", + args: args{ + ctx: context.Background(), + reqReader: strings.NewReader("test test test"), + cfg: cfg, + db: db, + }, + fields: fields{ + Logger: logger, + MediaMetadata: &types.MediaMetadata{ + MediaID: "1339", + UploadName: "test fail", + }, + }, + want: requestEntityTooLargeJSONResponse(maxSize), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &uploadRequest{ + MediaMetadata: tt.fields.MediaMetadata, + Logger: tt.fields.Logger, + } + if got := r.doUpload(tt.args.ctx, tt.args.reqReader, tt.args.cfg, tt.args.db, tt.args.activeThumbnailGeneration); !reflect.DeepEqual(got, tt.want) { + t.Errorf("doUpload() = %+v, want %+v", got, tt.want) + } + }) + } +} From 89a6787fdb60a3b4e7054726ebe0851128e9e16a Mon Sep 17 00:00:00 2001 From: S7evinK Date: Mon, 7 Jun 2021 10:17:46 +0200 Subject: [PATCH 15/26] Try to optimize SelectOneTimeKeys (#1851) * Try to optimize SelectOneTimeKeys Signed-off-by: Till Faelligen * Use pg.Array when using ANY... Co-authored-by: Kegsay --- .../storage/postgres/one_time_keys_table.go | 24 +++++++------------ 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/keyserver/storage/postgres/one_time_keys_table.go b/keyserver/storage/postgres/one_time_keys_table.go index 6e32838b1..cc397ba84 100644 --- a/keyserver/storage/postgres/one_time_keys_table.go +++ b/keyserver/storage/postgres/one_time_keys_table.go @@ -20,6 +20,7 @@ import ( "encoding/json" "time" + "github.com/lib/pq" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/keyserver/api" @@ -47,7 +48,7 @@ const upsertKeysSQL = "" + " DO UPDATE SET key_json = $6" const selectKeysSQL = "" + - "SELECT key_id, algorithm, key_json FROM keyserver_one_time_keys WHERE user_id=$1 AND device_id=$2" + "SELECT concat(algorithm, ':', key_id) as algorithmwithid, key_json FROM keyserver_one_time_keys WHERE user_id=$1 AND device_id=$2 AND concat(algorithm, ':', key_id) = ANY($3);" const selectKeysCountSQL = "" + "SELECT algorithm, COUNT(key_id) FROM keyserver_one_time_keys WHERE user_id=$1 AND device_id=$2 GROUP BY algorithm" @@ -94,29 +95,22 @@ func NewPostgresOneTimeKeysTable(db *sql.DB) (tables.OneTimeKeys, error) { } func (s *oneTimeKeysStatements) SelectOneTimeKeys(ctx context.Context, userID, deviceID string, keyIDsWithAlgorithms []string) (map[string]json.RawMessage, error) { - rows, err := s.selectKeysStmt.QueryContext(ctx, userID, deviceID) + rows, err := s.selectKeysStmt.QueryContext(ctx, userID, deviceID, pq.Array(keyIDsWithAlgorithms)) if err != nil { return nil, err } defer internal.CloseAndLogIfError(ctx, rows, "selectKeysStmt: rows.close() failed") - wantSet := make(map[string]bool, len(keyIDsWithAlgorithms)) - for _, ka := range keyIDsWithAlgorithms { - wantSet[ka] = true - } - result := make(map[string]json.RawMessage) + var ( + algorithmWithID string + keyJSONStr string + ) for rows.Next() { - var keyID string - var algorithm string - var keyJSONStr string - if err := rows.Scan(&keyID, &algorithm, &keyJSONStr); err != nil { + if err := rows.Scan(&algorithmWithID, &keyJSONStr); err != nil { return nil, err } - keyIDWithAlgo := algorithm + ":" + keyID - if wantSet[keyIDWithAlgo] { - result[keyIDWithAlgo] = json.RawMessage(keyJSONStr) - } + result[algorithmWithID] = json.RawMessage(keyJSONStr) } return result, rows.Err() } From 6fa5743ea04c3838d2336bb0f268bd069f29176a Mon Sep 17 00:00:00 2001 From: Arnaud Venturi Date: Mon, 7 Jun 2021 10:29:54 +0200 Subject: [PATCH 16/26] Add missing IPv6 "ssl" keyword in nginx config examples (#1854) Signed-off-by: Arnaud Venturi --- docs/nginx/monolith-sample.conf | 2 +- docs/nginx/polylith-sample.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/nginx/monolith-sample.conf b/docs/nginx/monolith-sample.conf index 350e83489..f5461d52d 100644 --- a/docs/nginx/monolith-sample.conf +++ b/docs/nginx/monolith-sample.conf @@ -1,6 +1,6 @@ server { listen 443 ssl; # IPv4 - listen [::]:443; # IPv6 + listen [::]:443 ssl; # IPv6 server_name my.hostname.com; ssl_certificate /path/to/fullchain.pem; diff --git a/docs/nginx/polylith-sample.conf b/docs/nginx/polylith-sample.conf index d0d3c98d5..7e8eb4a53 100644 --- a/docs/nginx/polylith-sample.conf +++ b/docs/nginx/polylith-sample.conf @@ -1,6 +1,6 @@ server { listen 443 ssl; # IPv4 - listen [::]:443; # IPv6 + listen [::]:443 ssl; # IPv6 server_name my.hostname.com; ssl_certificate /path/to/fullchain.pem; From caa5c89ecb46f14e612bbabd472b31d767f76a23 Mon Sep 17 00:00:00 2001 From: Ben Langfeld Date: Mon, 7 Jun 2021 06:10:29 -0300 Subject: [PATCH 17/26] Link to Synapse (#1863) Not everyone is very familiar. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7d79bbff0..8c7938412 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # Dendrite [![Build Status](https://badge.buildkite.com/4be40938ab19f2bbc4a6c6724517353ee3ec1422e279faf374.svg?branch=master)](https://buildkite.com/matrix-dot-org/dendrite) [![Dendrite](https://img.shields.io/matrix/dendrite:matrix.org.svg?label=%23dendrite%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite:matrix.org) [![Dendrite Dev](https://img.shields.io/matrix/dendrite-dev:matrix.org.svg?label=%23dendrite-dev%3Amatrix.org&logo=matrix&server_fqdn=matrix.org)](https://matrix.to/#/#dendrite-dev:matrix.org) Dendrite is a second-generation Matrix homeserver written in Go. -It intends to provide an **efficient**, **reliable** and **scalable** alternative to Synapse: +It intends to provide an **efficient**, **reliable** and **scalable** alternative to [Synapse](https://github.com/matrix-org/synapse): - Efficient: A small memory footprint with better baseline performance than an out-of-the-box Synapse. - Reliable: Implements the Matrix specification as written, using the [same test suite](https://github.com/matrix-org/sytest) as Synapse as well as From 2c9a390fa67aa80bd8cfffb02b20739d0b1807d8 Mon Sep 17 00:00:00 2001 From: Rasmus Thomsen Date: Mon, 7 Jun 2021 12:06:57 +0200 Subject: [PATCH 18/26] Add CORS to nginx config (#1791) Without this entry, setups where users have the homeserver on the URL matrix.myurl.com but want the servername to be myurl.com don't work by default since clients like element.io can't connect to the homeserver --- docs/nginx/monolith-sample.conf | 3 +++ docs/nginx/polylith-sample.conf | 3 +++ 2 files changed, 6 insertions(+) diff --git a/docs/nginx/monolith-sample.conf b/docs/nginx/monolith-sample.conf index f5461d52d..0344aa96c 100644 --- a/docs/nginx/monolith-sample.conf +++ b/docs/nginx/monolith-sample.conf @@ -16,6 +16,9 @@ server { } location /.well-known/matrix/client { + # If your sever_name here doesn't match your matrix homeserver URL + # (e.g. hostname.com as server_name and matrix.hostname.com as homeserver URL) + # add_header Access-Control-Allow-Origin '*'; return 200 '{ "m.homeserver": { "base_url": "https://my.hostname.com" } }'; } diff --git a/docs/nginx/polylith-sample.conf b/docs/nginx/polylith-sample.conf index 7e8eb4a53..274d75658 100644 --- a/docs/nginx/polylith-sample.conf +++ b/docs/nginx/polylith-sample.conf @@ -16,6 +16,9 @@ server { } location /.well-known/matrix/client { + # If your sever_name here doesn't match your matrix homeserver URL + # (e.g. hostname.com as server_name and matrix.hostname.com as homeserver URL) + # add_header Access-Control-Allow-Origin '*'; return 200 '{ "m.homeserver": { "base_url": "https://my.hostname.com" } }'; } From bd9dec8e066ebf9af6b25b733c8b4fecd67dddcf Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 14 Jun 2021 13:13:07 +0100 Subject: [PATCH 19/26] Pinecone demo updates --- build/gobind-pinecone/monolith.go | 75 ++++++++++++++++-------------- cmd/dendrite-demo-pinecone/main.go | 37 +++++++-------- go.mod | 2 +- go.sum | 4 +- 4 files changed, 60 insertions(+), 58 deletions(-) diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go index c15707e56..09af80f6c 100644 --- a/build/gobind-pinecone/monolith.go +++ b/build/gobind-pinecone/monolith.go @@ -10,7 +10,6 @@ import ( "io" "io/ioutil" "log" - "math" "net" "net/http" "os" @@ -37,15 +36,14 @@ import ( userapiAPI "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" - "go.uber.org/atomic" "golang.org/x/net/http2" "golang.org/x/net/http2/h2c" pineconeMulticast "github.com/matrix-org/pinecone/multicast" + "github.com/matrix-org/pinecone/router" pineconeRouter "github.com/matrix-org/pinecone/router" pineconeSessions "github.com/matrix-org/pinecone/sessions" "github.com/matrix-org/pinecone/types" - pineconeTypes "github.com/matrix-org/pinecone/types" _ "golang.org/x/mobile/bind" ) @@ -57,19 +55,19 @@ const ( ) type DendriteMonolith struct { - logger logrus.Logger - PineconeRouter *pineconeRouter.Router - PineconeMulticast *pineconeMulticast.Multicast - PineconeQUIC *pineconeSessions.Sessions - StorageDirectory string - CacheDirectory string - staticPeerURI string - staticPeerMutex sync.RWMutex - staticPeerAttempts atomic.Uint32 - listener net.Listener - httpServer *http.Server - processContext *process.ProcessContext - userAPI userapiAPI.UserInternalAPI + logger logrus.Logger + PineconeRouter *pineconeRouter.Router + PineconeMulticast *pineconeMulticast.Multicast + PineconeQUIC *pineconeSessions.Sessions + StorageDirectory string + CacheDirectory string + staticPeerURI string + staticPeerMutex sync.RWMutex + staticPeerAttempt chan struct{} + listener net.Listener + httpServer *http.Server + processContext *process.ProcessContext + userAPI userapiAPI.UserInternalAPI } func (m *DendriteMonolith) BaseURL() string { @@ -99,7 +97,9 @@ func (m *DendriteMonolith) SetStaticPeer(uri string) { m.staticPeerMutex.Unlock() m.DisconnectType(pineconeRouter.PeerTypeRemote) if uri != "" { - go m.staticPeerConnect() + go func() { + m.staticPeerAttempt <- struct{}{} + }() } } @@ -195,17 +195,27 @@ func (m *DendriteMonolith) RegisterDevice(localpart, deviceID string) (string, e } func (m *DendriteMonolith) staticPeerConnect() { - m.staticPeerMutex.RLock() - uri := m.staticPeerURI - m.staticPeerMutex.RUnlock() - if uri == "" { - return + attempt := func() { + if m.PineconeRouter.PeerCount(router.PeerTypeRemote) == 0 { + m.staticPeerMutex.RLock() + uri := m.staticPeerURI + m.staticPeerMutex.RUnlock() + if uri == "" { + return + } + if err := conn.ConnectToPeer(m.PineconeRouter, uri); err != nil { + logrus.WithError(err).Error("Failed to connect to static peer") + } + } } - if err := conn.ConnectToPeer(m.PineconeRouter, uri); err != nil { - exp := time.Second * time.Duration(math.Exp2(float64(m.staticPeerAttempts.Inc()))) - time.AfterFunc(exp, m.staticPeerConnect) - } else { - m.staticPeerAttempts.Store(0) + for { + select { + case <-m.processContext.Context().Done(): + case <-m.staticPeerAttempt: + attempt() + case <-time.After(time.Second * 5): + attempt() + } } } @@ -248,13 +258,6 @@ func (m *DendriteMonolith) Start() { m.PineconeQUIC = pineconeSessions.NewSessions(logger, m.PineconeRouter) m.PineconeMulticast = pineconeMulticast.NewMulticast(logger, m.PineconeRouter) - m.PineconeRouter.SetDisconnectedCallback(func(port pineconeTypes.SwitchPortID, public pineconeTypes.PublicKey, peertype int, err error) { - if peertype == pineconeRouter.PeerTypeRemote { - m.staticPeerAttempts.Store(0) - time.AfterFunc(time.Second, m.staticPeerConnect) - } - }) - prefix := hex.EncodeToString(pk) cfg := &config.Dendrite{} cfg.Defaults() @@ -359,8 +362,12 @@ func (m *DendriteMonolith) Start() { }, Handler: h2c.NewHandler(pMux, h2s), } + m.processContext = base.ProcessContext + m.staticPeerAttempt = make(chan struct{}, 1) + go m.staticPeerConnect() + go func() { m.logger.Info("Listening on ", cfg.Global.ServerName) m.logger.Fatal(m.httpServer.Serve(m.PineconeQUIC)) diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go index 7851fdb19..f40f9190a 100644 --- a/cmd/dendrite-demo-pinecone/main.go +++ b/cmd/dendrite-demo-pinecone/main.go @@ -23,7 +23,6 @@ import ( "fmt" "io/ioutil" "log" - "math" "net" "net/http" "os" @@ -48,12 +47,11 @@ import ( "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/userapi" "github.com/matrix-org/gomatrixserverlib" - "go.uber.org/atomic" pineconeMulticast "github.com/matrix-org/pinecone/multicast" + "github.com/matrix-org/pinecone/router" pineconeRouter "github.com/matrix-org/pinecone/router" pineconeSessions "github.com/matrix-org/pinecone/sessions" - pineconeTypes "github.com/matrix-org/pinecone/types" "github.com/sirupsen/logrus" ) @@ -123,27 +121,23 @@ func main() { pMulticast := pineconeMulticast.NewMulticast(logger, pRouter) pMulticast.Start() - var staticPeerAttempts atomic.Uint32 - var connectToStaticPeer func() - connectToStaticPeer = func() { - uri := *instancePeer - if uri == "" { - return + connectToStaticPeer := func() { + attempt := func() { + if pRouter.PeerCount(router.PeerTypeRemote) == 0 { + uri := *instancePeer + if uri == "" { + return + } + if err := conn.ConnectToPeer(pRouter, uri); err != nil { + logrus.WithError(err).Error("Failed to connect to static peer") + } + } } - if err := conn.ConnectToPeer(pRouter, uri); err != nil { - exp := time.Second * time.Duration(math.Exp2(float64(staticPeerAttempts.Inc()))) - time.AfterFunc(exp, connectToStaticPeer) - } else { - staticPeerAttempts.Store(0) + for { + attempt() + time.Sleep(time.Second * 5) } } - pRouter.SetDisconnectedCallback(func(port pineconeTypes.SwitchPortID, public pineconeTypes.PublicKey, peertype int, err error) { - if peertype == pineconeRouter.PeerTypeRemote && err != nil { - staticPeerAttempts.Store(0) - time.AfterFunc(time.Second, connectToStaticPeer) - } - }) - go connectToStaticPeer() cfg := &config.Dendrite{} cfg.Defaults() @@ -257,6 +251,7 @@ func main() { Handler: pMux, } + go connectToStaticPeer() go func() { pubkey := pRouter.PublicKey() logrus.Info("Listening on ", hex.EncodeToString(pubkey[:])) diff --git a/go.mod b/go.mod index 7273da647..48c8bbf7d 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd github.com/matrix-org/gomatrixserverlib v0.0.0-20210525110027-8cb7699aa64a github.com/matrix-org/naffka v0.0.0-20201009174903-d26a3b9cb161 - github.com/matrix-org/pinecone v0.0.0-20210602111459-5cb0e6aa1a6a + github.com/matrix-org/pinecone v0.0.0-20210614112651-5da1fab608c6 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.7-0.20210414154423-1157a4212dcb github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 diff --git a/go.sum b/go.sum index eba9a60b1..0e7ee0f6c 100644 --- a/go.sum +++ b/go.sum @@ -706,8 +706,8 @@ github.com/matrix-org/gomatrixserverlib v0.0.0-20210525110027-8cb7699aa64a h1:pV github.com/matrix-org/gomatrixserverlib v0.0.0-20210525110027-8cb7699aa64a/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20201009174903-d26a3b9cb161 h1:h1XVh05pLoC+nJjP3GIpj5wUsuC8WdHP3He0RTkRJTs= github.com/matrix-org/naffka v0.0.0-20201009174903-d26a3b9cb161/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= -github.com/matrix-org/pinecone v0.0.0-20210602111459-5cb0e6aa1a6a h1:BE/cfpyHO2ua1BK4Tibr+2oZCV3H1mC9G7g7Yvl1AmM= -github.com/matrix-org/pinecone v0.0.0-20210602111459-5cb0e6aa1a6a/go.mod h1:UQzJS6UVyVwfkr+RLrdvBB1vLyECqe3fLYNcbRxv8SA= +github.com/matrix-org/pinecone v0.0.0-20210614112651-5da1fab608c6 h1:ytVf81AkLmMAs0KeCYW6po0X3foMSKz0HccnipfsOVc= +github.com/matrix-org/pinecone v0.0.0-20210614112651-5da1fab608c6/go.mod h1:UQzJS6UVyVwfkr+RLrdvBB1vLyECqe3fLYNcbRxv8SA= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= From 051cd63f1e43e50c822af98234798616cfb70045 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 14 Jun 2021 13:26:02 +0100 Subject: [PATCH 20/26] Update go.mod/go.sum --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 48c8bbf7d..90bccf4f0 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd github.com/matrix-org/gomatrixserverlib v0.0.0-20210525110027-8cb7699aa64a github.com/matrix-org/naffka v0.0.0-20201009174903-d26a3b9cb161 - github.com/matrix-org/pinecone v0.0.0-20210614112651-5da1fab608c6 + github.com/matrix-org/pinecone v0.0.0-20210614122540-33ce3bd0f3ac github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.7-0.20210414154423-1157a4212dcb github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 diff --git a/go.sum b/go.sum index 0e7ee0f6c..b178ebe4c 100644 --- a/go.sum +++ b/go.sum @@ -706,8 +706,8 @@ github.com/matrix-org/gomatrixserverlib v0.0.0-20210525110027-8cb7699aa64a h1:pV github.com/matrix-org/gomatrixserverlib v0.0.0-20210525110027-8cb7699aa64a/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= github.com/matrix-org/naffka v0.0.0-20201009174903-d26a3b9cb161 h1:h1XVh05pLoC+nJjP3GIpj5wUsuC8WdHP3He0RTkRJTs= github.com/matrix-org/naffka v0.0.0-20201009174903-d26a3b9cb161/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= -github.com/matrix-org/pinecone v0.0.0-20210614112651-5da1fab608c6 h1:ytVf81AkLmMAs0KeCYW6po0X3foMSKz0HccnipfsOVc= -github.com/matrix-org/pinecone v0.0.0-20210614112651-5da1fab608c6/go.mod h1:UQzJS6UVyVwfkr+RLrdvBB1vLyECqe3fLYNcbRxv8SA= +github.com/matrix-org/pinecone v0.0.0-20210614122540-33ce3bd0f3ac h1:qgEfJzulYUVDGh1PGzeGxYMGDtKSxMS+6eQG6E37pgM= +github.com/matrix-org/pinecone v0.0.0-20210614122540-33ce3bd0f3ac/go.mod h1:UQzJS6UVyVwfkr+RLrdvBB1vLyECqe3fLYNcbRxv8SA= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= From eae1c46310e1d6ab2d8b18a61b808a1dd77b45b2 Mon Sep 17 00:00:00 2001 From: Adam Greig Date: Mon, 14 Jun 2021 14:06:14 +0100 Subject: [PATCH 21/26] Use NotFound instead of Forbidden for missing account data (#1872) Signed-off-by: Adam Greig --- clientapi/routing/account_data.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clientapi/routing/account_data.go b/clientapi/routing/account_data.go index 22e635139..03025f1da 100644 --- a/clientapi/routing/account_data.go +++ b/clientapi/routing/account_data.go @@ -69,7 +69,7 @@ func GetAccountData( return util.JSONResponse{ Code: http.StatusNotFound, - JSON: jsonerror.Forbidden("data not found"), + JSON: jsonerror.NotFound("data not found"), } } From 183f43d38da215c922eff025b73417d93661cd49 Mon Sep 17 00:00:00 2001 From: S7evinK Date: Mon, 14 Jun 2021 15:08:29 +0200 Subject: [PATCH 22/26] Add parameters to specify password (#1868) * Add parameters to specify password Signed-off-by: Till Faelligen * Fix typo * Add testdata * Use go1.13 compatible way to read files --- cmd/create-account/main.go | 76 +++++++++++++++++++++++++++-- cmd/create-account/main_test.go | 62 +++++++++++++++++++++++ cmd/create-account/testdata/my.pass | 1 + 3 files changed, 136 insertions(+), 3 deletions(-) create mode 100644 cmd/create-account/main_test.go create mode 100644 cmd/create-account/testdata/my.pass diff --git a/cmd/create-account/main.go b/cmd/create-account/main.go index 060b82f97..a1e254f8d 100644 --- a/cmd/create-account/main.go +++ b/cmd/create-account/main.go @@ -18,13 +18,18 @@ import ( "context" "flag" "fmt" + "io" + "io/ioutil" "os" + "strings" + "syscall" "github.com/matrix-org/dendrite/setup" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/userapi/storage/accounts" "github.com/sirupsen/logrus" "golang.org/x/crypto/bcrypt" + "golang.org/x/term" ) const usage = `Usage: %s @@ -33,7 +38,15 @@ Creates a new user account on the homeserver. Example: - ./create-account --config dendrite.yaml --username alice --password foobarbaz + # provide password by parameter + %s --config dendrite.yaml -username alice -password foobarbaz + # use password from file + %s --config dendrite.yaml -username alice -passwordfile my.pass + # ask user to provide password + %s --config dendrite.yaml -username alice -ask-pass + # read password from stdin + %s --config dendrite.yaml -username alice -passwordstdin < my.pass + cat my.pass | %s --config dendrite.yaml -username alice -passwordstdin Arguments: @@ -42,11 +55,15 @@ Arguments: var ( username = flag.String("username", "", "The username of the account to register (specify the localpart only, e.g. 'alice' for '@alice:domain.com')") password = flag.String("password", "", "The password to associate with the account (optional, account will be password-less if not specified)") + pwdFile = flag.String("passwordfile", "", "The file to use for the password (e.g. for automated account creation)") + pwdStdin = flag.Bool("passwordstdin", false, "Reads the password from stdin") + askPass = flag.Bool("ask-pass", false, "Ask for the password to use") ) func main() { + name := os.Args[0] flag.Usage = func() { - fmt.Fprintf(os.Stderr, usage, os.Args[0]) + _, _ = fmt.Fprintf(os.Stderr, usage, name, name, name, name, name, name) flag.PrintDefaults() } cfg := setup.ParseFlags(true) @@ -56,6 +73,8 @@ func main() { os.Exit(1) } + pass := getPassword(password, pwdFile, pwdStdin, askPass, os.Stdin) + accountDB, err := accounts.NewDatabase(&config.DatabaseOptions{ ConnectionString: cfg.UserAPI.AccountDatabase.ConnectionString, }, cfg.Global.ServerName, bcrypt.DefaultCost, cfg.UserAPI.OpenIDTokenLifetimeMS) @@ -63,10 +82,61 @@ func main() { logrus.Fatalln("Failed to connect to the database:", err.Error()) } - _, err = accountDB.CreateAccount(context.Background(), *username, *password, "") + _, err = accountDB.CreateAccount(context.Background(), *username, pass, "") if err != nil { logrus.Fatalln("Failed to create the account:", err.Error()) } logrus.Infoln("Created account", *username) } + +func getPassword(password, pwdFile *string, pwdStdin, askPass *bool, r io.Reader) string { + // no password option set, use empty password + if password == nil && pwdFile == nil && pwdStdin == nil && askPass == nil { + return "" + } + // password defined as parameter + if password != nil && *password != "" { + return *password + } + + // read password from file + if pwdFile != nil && *pwdFile != "" { + pw, err := ioutil.ReadFile(*pwdFile) + if err != nil { + logrus.Fatalln("Unable to read password from file:", err) + } + return strings.TrimSpace(string(pw)) + } + + // read password from stdin + if pwdStdin != nil && *pwdStdin { + data, err := ioutil.ReadAll(r) + if err != nil { + logrus.Fatalln("Unable to read password from stdin:", err) + } + return strings.TrimSpace(string(data)) + } + + // ask the user to provide the password + if *askPass { + fmt.Print("Enter Password: ") + bytePassword, err := term.ReadPassword(syscall.Stdin) + if err != nil { + logrus.Fatalln("Unable to read password:", err) + } + fmt.Println() + fmt.Print("Confirm Password: ") + bytePassword2, err := term.ReadPassword(syscall.Stdin) + if err != nil { + logrus.Fatalln("Unable to read password:", err) + } + fmt.Println() + if strings.TrimSpace(string(bytePassword)) != strings.TrimSpace(string(bytePassword2)) { + logrus.Fatalln("Entered passwords don't match") + } + return strings.TrimSpace(string(bytePassword)) + } + + return "" +} diff --git a/cmd/create-account/main_test.go b/cmd/create-account/main_test.go new file mode 100644 index 000000000..d06eafe46 --- /dev/null +++ b/cmd/create-account/main_test.go @@ -0,0 +1,62 @@ +package main + +import ( + "bytes" + "io" + "testing" +) + +func Test_getPassword(t *testing.T) { + type args struct { + password *string + pwdFile *string + pwdStdin *bool + askPass *bool + reader io.Reader + } + + pass := "mySecretPass" + passwordFile := "testdata/my.pass" + passwordStdin := true + reader := &bytes.Buffer{} + _, err := reader.WriteString(pass) + if err != nil { + t.Errorf("unable to write to buffer: %+v", err) + } + tests := []struct { + name string + args args + want string + }{ + { + name: "no password defined", + args: args{}, + want: "", + }, + { + name: "password defined", + args: args{password: &pass}, + want: pass, + }, + { + name: "pwdFile defined", + args: args{pwdFile: &passwordFile}, + want: pass, + }, + { + name: "read pass from stdin defined", + args: args{ + pwdStdin: &passwordStdin, + reader: reader, + }, + want: pass, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := getPassword(tt.args.password, tt.args.pwdFile, tt.args.pwdStdin, tt.args.askPass, tt.args.reader); got != tt.want { + t.Errorf("getPassword() = '%v', want '%v'", got, tt.want) + } + }) + } +} diff --git a/cmd/create-account/testdata/my.pass b/cmd/create-account/testdata/my.pass new file mode 100644 index 000000000..c1f7156f0 --- /dev/null +++ b/cmd/create-account/testdata/my.pass @@ -0,0 +1 @@ +mySecretPass \ No newline at end of file From ce574f5e29914e78f0ad1a8bbef64bf29bf0044a Mon Sep 17 00:00:00 2001 From: Ben Yanke Date: Mon, 14 Jun 2021 08:08:59 -0500 Subject: [PATCH 23/26] Fix Key Generation Docs - resolves #1759 (#1865) This fixes the issue found in #1759 which broke due to go changes. The new command allows you to generate keys with docker, and drop them in the current working directory. --- build/docker/README.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/build/docker/README.md b/build/docker/README.md index 818f92d03..c3eb73b2a 100644 --- a/build/docker/README.md +++ b/build/docker/README.md @@ -37,12 +37,17 @@ runtime config should come from. The mounted folder must contain: To generate keys: ``` -go run github.com/matrix-org/dendrite/cmd/generate-keys \ - --private-key=matrix_key.pem \ - --tls-cert=server.crt \ - --tls-key=server.key +docker run --rm --entrypoint="" \ + -v $(pwd):/mnt \ + matrixdotorg/dendrite-monolith:latest \ + /usr/bin/generate-keys \ + -private-key /mnt/matrix_key.pem \ + -tls-cert /mnt/server.crt \ + -tls-key /mnt/server.key ``` +The key files will now exist in your current working directory, and can be mounted into place. + ## Starting Dendrite as a monolith deployment Create your config based on the `dendrite.yaml` configuration file in the `docker/config` From 9633ed747d9629a838e2308530251aadc3585d4c Mon Sep 17 00:00:00 2001 From: Arnaud Venturi Date: Mon, 14 Jun 2021 15:09:35 +0200 Subject: [PATCH 24/26] Remove mention of enabling naffka in install instructions, as it is the default value (#1853) Signed-off-by: Arnaud Venturi --- build/docker/README.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/build/docker/README.md b/build/docker/README.md index c3eb73b2a..6d3cd3dbd 100644 --- a/build/docker/README.md +++ b/build/docker/README.md @@ -51,10 +51,7 @@ The key files will now exist in your current working directory, and can be mount ## Starting Dendrite as a monolith deployment Create your config based on the `dendrite.yaml` configuration file in the `docker/config` -folder in the [Dendrite repository](https://github.com/matrix-org/dendrite). Additionally, -make the following changes to the configuration: - -- Enable Naffka: `use_naffka: true` +folder in the [Dendrite repository](https://github.com/matrix-org/dendrite). Once in place, start the PostgreSQL dependency: From 9ed044042156a9bdfafb4ae9830b3ed41225444e Mon Sep 17 00:00:00 2001 From: S7evinK Date: Mon, 14 Jun 2021 15:12:03 +0200 Subject: [PATCH 25/26] Set MaxFileSizeBytes <= 0 to "unlimited" (#1875) * Set MaxFileSizeBytes < 0 to "unlimited" Signed-off-by: Till Faelligen * int64 overflows later in mediaapi/routing/upload.go[doUpload] * Prevent int overflow when uploading --- mediaapi/routing/upload.go | 12 ++++++++++++ setup/config/config_mediaapi.go | 6 ++++++ 2 files changed, 18 insertions(+) diff --git a/mediaapi/routing/upload.go b/mediaapi/routing/upload.go index ada02b11e..bc0d206b9 100644 --- a/mediaapi/routing/upload.go +++ b/mediaapi/routing/upload.go @@ -147,6 +147,18 @@ func (r *uploadRequest) doUpload( // r.storeFileAndMetadata(ctx, tmpDir, ...) // before you return from doUpload else we will leak a temp file. We could make this nicer with a `WithTransaction` style of // nested function to guarantee either storage or cleanup. + + // should not happen, but prevents any int overflows + if cfg.MaxFileSizeBytes != nil && *cfg.MaxFileSizeBytes+1 <= 0 { + r.Logger.WithFields(log.Fields{ + "MaxFileSizeBytes": *cfg.MaxFileSizeBytes + 1, + }).Error("Error while transferring file, configured max_file_size_bytes overflows int64") + return &util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.Unknown("Failed to upload"), + } + } + lr := io.LimitReader(reqReader, int64(*cfg.MaxFileSizeBytes)+1) hash, bytesWritten, tmpDir, err := fileutils.WriteTempFile(ctx, lr, cfg.AbsBasePath) if err != nil { diff --git a/setup/config/config_mediaapi.go b/setup/config/config_mediaapi.go index 660a508d5..0943a39e2 100644 --- a/setup/config/config_mediaapi.go +++ b/setup/config/config_mediaapi.go @@ -2,6 +2,7 @@ package config import ( "fmt" + "math" ) type MediaAPI struct { @@ -57,6 +58,11 @@ func (c *MediaAPI) Verify(configErrs *ConfigErrors, isMonolith bool) { checkNotEmpty(configErrs, "media_api.database.connection_string", string(c.Database.ConnectionString)) checkNotEmpty(configErrs, "media_api.base_path", string(c.BasePath)) + // allow "unlimited" file size + if c.MaxFileSizeBytes != nil && *c.MaxFileSizeBytes <= 0 { + unlimitedSize := FileSizeBytes(math.MaxInt64 - 1) + c.MaxFileSizeBytes = &unlimitedSize + } checkPositive(configErrs, "media_api.max_file_size_bytes", int64(*c.MaxFileSizeBytes)) checkPositive(configErrs, "media_api.max_thumbnail_generators", int64(c.MaxThumbnailGenerators)) From 5ec7d3a3ecf00614950e4da59f4adc5a12fa55ff Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 14 Jun 2021 15:04:05 +0100 Subject: [PATCH 26/26] Update go.mod/go.sum --- go.mod | 1 + go.sum | 1 + 2 files changed, 2 insertions(+) diff --git a/go.mod b/go.mod index 90bccf4f0..df2bc4c1f 100644 --- a/go.mod +++ b/go.mod @@ -44,6 +44,7 @@ require ( golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 golang.org/x/mobile v0.0.0-20210220033013-bdb1ca9a1e08 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 + golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect gopkg.in/h2non/bimg.v1 v1.1.5 gopkg.in/yaml.v2 v2.4.0 ) diff --git a/go.sum b/go.sum index b178ebe4c..9c8a7dfee 100644 --- a/go.sum +++ b/go.sum @@ -1242,6 +1242,7 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210309040221-94ec62e08169 h1:fpeMGRM6A+XFcw4RPCO8s8hH7ppgrGR22pSIjwM7YUI= golang.org/x/sys v0.0.0-20210309040221-94ec62e08169/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=