From 23c20c40c12dea6b5ae99e8d601baad37c46d0a4 Mon Sep 17 00:00:00 2001 From: kegsay Date: Fri, 29 Oct 2021 12:06:10 +0100 Subject: [PATCH 01/15] generate-config: add flags for db and server name (#2030) --- cmd/generate-config/main.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/cmd/generate-config/main.go b/cmd/generate-config/main.go index bd70cbc5d..e300a1caf 100644 --- a/cmd/generate-config/main.go +++ b/cmd/generate-config/main.go @@ -5,16 +5,35 @@ import ( "fmt" "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/gomatrixserverlib" "golang.org/x/crypto/bcrypt" "gopkg.in/yaml.v2" ) func main() { defaultsForCI := flag.Bool("ci", false, "sane defaults for CI testing") + serverName := flag.String("server", "", "The domain name of the server if not 'localhost'") + dbURI := flag.String("db", "", "The DB URI to use for all components if not SQLite files") flag.Parse() cfg := &config.Dendrite{} cfg.Defaults() + if *serverName != "" { + cfg.Global.ServerName = gomatrixserverlib.ServerName(*serverName) + } + if *dbURI != "" { + cfg.Global.Kafka.Database.ConnectionString = config.DataSource(*dbURI) + cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(*dbURI) + cfg.FederationSender.Database.ConnectionString = config.DataSource(*dbURI) + cfg.KeyServer.Database.ConnectionString = config.DataSource(*dbURI) + cfg.MSCs.Database.ConnectionString = config.DataSource(*dbURI) + cfg.MediaAPI.Database.ConnectionString = config.DataSource(*dbURI) + cfg.RoomServer.Database.ConnectionString = config.DataSource(*dbURI) + cfg.SigningKeyServer.Database.ConnectionString = config.DataSource(*dbURI) + cfg.SyncAPI.Database.ConnectionString = config.DataSource(*dbURI) + cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(*dbURI) + cfg.UserAPI.DeviceDatabase.ConnectionString = config.DataSource(*dbURI) + } cfg.Global.TrustedIDServers = []string{ "matrix.org", "vector.im", From a22a6d360fd7d3b3897da9528f2fe6333689def8 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Fri, 29 Oct 2021 12:06:38 +0100 Subject: [PATCH 02/15] Add 'Federation publicRoom Name/topic keys are correct' to the sytest whitelist (#2029) Sytest had two tests with the name `Name/topic keys are correct`. https://github.com/matrix-org/sytest/pull/1098 edited one so that the names were unique. It seems that this test name was in the dendrite Sytest whitelist. This PR adds the additional, new name to the whitelist as well. Time to see if CI passes! --- sytest-whitelist | 1 + 1 file changed, 1 insertion(+) diff --git a/sytest-whitelist b/sytest-whitelist index c97400497..019caac47 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -565,3 +565,4 @@ User can invite remote user to room with version 7 Remote user can backfill in a room with version 7 Can reject invites over federation for rooms with version 7 Can receive redactions from regular users over federation in room version 7 +Federation publicRoom Name/topic keys are correct From 92026a73ceb41ef2e7e65eac7edb79fa729342ef Mon Sep 17 00:00:00 2001 From: kegsay Date: Fri, 29 Oct 2021 15:01:22 +0100 Subject: [PATCH 03/15] Remove tests from the blacklist (#2031) * Remove tests from the blacklist They seem to be more reliable now * Update whitelist --- sytest-blacklist | 49 ------------------------------------------------ sytest-whitelist | 5 ++++- 2 files changed, 4 insertions(+), 50 deletions(-) diff --git a/sytest-blacklist b/sytest-blacklist index 6ab9b352a..5e562845e 100644 --- a/sytest-blacklist +++ b/sytest-blacklist @@ -1,52 +1,9 @@ -# Blacklisted due to flakiness -Remote users can join room by alias - -# Blacklisted due to flakiness -POST /login can log in as a user with just the local part of the id - -# Blacklisted due to flakiness -avatar_url updates affect room member events - -# Blacklisted due to flakiness -displayname updates affect room member events - -# Blacklisted due to flakiness -Room members can override their displayname on a room-specific basis - -# Blacklisted due to flakiness -Alias creators can delete alias with no ops - -# Blacklisted because matrix-org/dendrite#847 might have broken it but we're not -# really sure and we need it pretty badly anyway -Real non-joined users can get individual state for world_readable rooms after leaving - # Blacklisted until matrix-org/dendrite#862 is reverted due to Riot bug Latest account data appears in v2 /sync -# Blacklisted due to flakiness -Outbound federation can backfill events - -# Blacklisted due to alias work on Synapse -Alias creators can delete canonical alias with no ops - -# Blacklisted because we need to implement v2 invite endpoints for room versions -# to be supported (currently fails with M_UNSUPPORTED_ROOM_VERSION) -Inbound federation rejects invites which are not signed by the sender - # Blacklisted because we don't support ignores yet Ignore invite in incremental sync -# Blacklisted because this test calls /r0/events which we don't implement -New room members see their own join event -Existing members see new members' join events - -# See https://github.com/matrix-org/sytest/pull/901 -Remote invited user can see room metadata - -# We don't implement soft-failed events yet, but because the /send response is vague, -# this test thinks it's all fine... -Inbound federation accepts a second soft-failed event - # Relies on a rejected PL event which will never be accepted into the DAG # Caused by https://github.com/matrix-org/sytest/pull/911 Outbound federation requests missing prev_events and then asks for /state_ids and resolves the state @@ -59,15 +16,9 @@ Invited user can reject local invite after originator leaves Invited user can reject invite for empty room If user leaves room, remote user changes device and rejoins we see update in /sync and /keys/changes -# Blacklisted due to flakiness -A prev_batch token from incremental sync can be used in the v1 messages API - # Blacklisted due to flakiness Forgotten room messages cannot be paginated -# Blacklisted due to flakiness -Can re-join room if re-invited - # Blacklisted due to flakiness after #1774 Local device key changes get to remote servers with correct prev_id diff --git a/sytest-whitelist b/sytest-whitelist index 019caac47..16c7e5443 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -136,7 +136,6 @@ query for user with no keys returns empty key dict Can claim one time key using POST Can claim remote one time key using POST Local device key changes appear in v2 /sync -Local device key changes appear in /keys/changes New users appear in /keys/changes Local delete device changes appear in v2 /sync Local new device changes appear in v2 /sync @@ -566,3 +565,7 @@ Remote user can backfill in a room with version 7 Can reject invites over federation for rooms with version 7 Can receive redactions from regular users over federation in room version 7 Federation publicRoom Name/topic keys are correct +Remote invited user can see room metadata +Can re-join room if re-invited +A prev_batch token from incremental sync can be used in the v1 messages API +Inbound federation rejects invites which are not signed by the sender From 42973e7d65c218eb6771de4707e0aeaa707867fc Mon Sep 17 00:00:00 2001 From: kegsay Date: Mon, 1 Nov 2021 15:05:20 +0000 Subject: [PATCH 04/15] Add part of #1322 tests (#2034) * Add part of #1322 tests * Fix #1322 --- sytest-whitelist | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sytest-whitelist b/sytest-whitelist index 16c7e5443..afe8a7e0d 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -569,3 +569,5 @@ Remote invited user can see room metadata Can re-join room if re-invited A prev_batch token from incremental sync can be used in the v1 messages API Inbound federation rejects invites which are not signed by the sender +Invited user can reject invite over federation several times +Test that we can be reinvited to a room we created From fbd1a0ab13b330026b3893ee40582fcacbfae376 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 2 Nov 2021 10:13:38 +0000 Subject: [PATCH 05/15] Update to matrix-org/gomatrixserverlib@5e02b64 --- federationapi/routing/send.go | 4 ++-- federationsender/internal/perform.go | 8 +------- go.mod | 5 ++--- go.sum | 23 ++++++++++------------- 4 files changed, 15 insertions(+), 25 deletions(-) diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index d31d38dc7..3cae837c9 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -306,7 +306,7 @@ func (t *txnReq) processTransaction(ctx context.Context) (*gomatrixserverlib.Res } continue } - if err = gomatrixserverlib.VerifyAllEventSignatures(ctx, []*gomatrixserverlib.Event{event}, t.keys); err != nil { + if err = event.VerifyEventSignatures(ctx, t.keys); err != nil { util.GetLogger(ctx).WithError(err).Warnf("Transaction: Couldn't validate signature of event %q", event.EventID()) results[event.EventID()] = gomatrixserverlib.PDUResult{ Error: err.Error(), @@ -1358,7 +1358,7 @@ func (t *txnReq) lookupEvent(ctx context.Context, roomVersion gomatrixserverlib. util.GetLogger(ctx).WithField("event_id", missingEventID).Warnf("Failed to get missing /event for event ID from %d server(s)", len(servers)) return nil, fmt.Errorf("wasn't able to find event via %d server(s)", len(servers)) } - if err := gomatrixserverlib.VerifyAllEventSignatures(ctx, []*gomatrixserverlib.Event{event}, t.keys); err != nil { + if err := event.VerifyEventSignatures(ctx, t.keys); err != nil { util.GetLogger(ctx).WithError(err).Warnf("Transaction: Couldn't validate signature of event %q", event.EventID()) return nil, verifySigError{event.EventID(), err} } diff --git a/federationsender/internal/perform.go b/federationsender/internal/perform.go index 2df936e3b..53fa974b2 100644 --- a/federationsender/internal/perform.go +++ b/federationsender/internal/perform.go @@ -713,14 +713,8 @@ func federatedAuthProvider( } // Check the signatures of the event. - if res, err := gomatrixserverlib.VerifyEventSignatures(ctx, []*gomatrixserverlib.Event{ev}, keyRing); err != nil { + if err := ev.VerifyEventSignatures(ctx, keyRing); err != nil { return nil, fmt.Errorf("missingAuth VerifyEventSignatures: %w", err) - } else { - for _, err := range res { - if err != nil { - return nil, fmt.Errorf("missingAuth VerifyEventSignatures: %w", err) - } - } } // If the event is OK then add it to the results and the retry map. diff --git a/go.mod b/go.mod index 72a3a11ba..82cda435e 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 - github.com/matrix-org/gomatrixserverlib v0.0.0-20210817115641-f9416ac1a723 + github.com/matrix-org/gomatrixserverlib v0.0.0-20211102101113-5e02b64e5312 github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 github.com/matrix-org/pinecone v0.0.0-20211022090602-08a50945ac89 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 @@ -46,8 +46,7 @@ require ( github.com/pressly/goose v2.7.0+incompatible github.com/prometheus/client_golang v1.11.0 github.com/sirupsen/logrus v1.8.1 - github.com/tidwall/gjson v1.8.1 - github.com/tidwall/pretty v1.2.0 // indirect + github.com/tidwall/gjson v1.9.3 github.com/tidwall/sjson v1.1.7 github.com/uber/jaeger-client-go v2.29.1+incompatible github.com/uber/jaeger-lib v2.4.1+incompatible diff --git a/go.sum b/go.sum index 268dc6718..1d0dacaca 100644 --- a/go.sum +++ b/go.sum @@ -77,7 +77,6 @@ github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg3 github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg= github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00= github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600= -github.com/Microsoft/hcsshim v0.8.18/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= github.com/Microsoft/hcsshim v0.8.21/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= @@ -218,8 +217,6 @@ github.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI= github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s= github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g= -github.com/containerd/containerd v1.5.5 h1:q1gxsZsGZ8ddVe98yO6pR21b5xQSMiR61lD0W96pgQo= -github.com/containerd/containerd v1.5.5/go.mod h1:oSTh0QpT1w6jYcGmbiSbxv9OSQYaa88mPyWIuU79zyo= github.com/containerd/containerd v1.5.7 h1:rQyoYtj4KddB3bxG6SAqd4+08gePNyJjRqvOIfV3rkM= github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= @@ -996,8 +993,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d/go.mod h1 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20210817115641-f9416ac1a723 h1:b8cyR4aYv9Lmf1lKgASJ+PFSp/GBv8ZFgb/O42ZXLGA= -github.com/matrix-org/gomatrixserverlib v0.0.0-20210817115641-f9416ac1a723/go.mod h1:JsAzE1Ll3+gDWS9JSUHPJiiyAksvOOnGWF2nXdg4ZzU= +github.com/matrix-org/gomatrixserverlib v0.0.0-20211102101113-5e02b64e5312 h1:of3kbkOrjE8793Vsni8ofxMSy/vpO1SB+ZkHSfv4wrs= +github.com/matrix-org/gomatrixserverlib v0.0.0-20211102101113-5e02b64e5312/go.mod h1:rB8tBUUUo1rzUqpzklRDSooxZ6YMhoaEPx4SO5fGeUc= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 h1:HZCzy4oVzz55e+cOMiX/JtSF2UOY1evBl2raaE7ACcU= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= github.com/matrix-org/pinecone v0.0.0-20211022090602-08a50945ac89 h1:6JkIymZ1vxfI0shSpg6gNPTJaF4/95Evy34slPVZGKM= @@ -1036,6 +1033,7 @@ github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyex github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.25/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.28/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.31 h1:sJFOl9BgwbYAWOGEwr61FU28pqsBNdpRBnhGXtO06Oo= github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= @@ -1194,7 +1192,6 @@ github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59P github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84SM2ImC1fxBuqJ/H0= -github.com/opencontainers/runc v1.0.1/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= github.com/opencontainers/runc v1.0.2/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= @@ -1369,15 +1366,12 @@ github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= -github.com/tidwall/gjson v1.6.0/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= github.com/tidwall/gjson v1.8.0/go.mod h1:5/xDoumyyDNerp2U36lyolv46b3uF/9Bu6OfyQ9GImk= -github.com/tidwall/gjson v1.8.1 h1:8j5EE9Hrh3l9Od1OIEDAb7IpezNA20UdRngNAj5N0WU= -github.com/tidwall/gjson v1.8.1/go.mod h1:5/xDoumyyDNerp2U36lyolv46b3uF/9Bu6OfyQ9GImk= -github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= -github.com/tidwall/match v1.0.3 h1:FQUVvBImDutD8wJLN6c5eMzWtjgONK9MwIBCOrUJKeE= +github.com/tidwall/gjson v1.9.3 h1:hqzS9wAHMO+KVBBkLxYdkEeeFHuqr95GfClRLKlgK0E= +github.com/tidwall/gjson v1.9.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= -github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tidwall/pretty v1.0.1/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.1.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= @@ -1504,6 +1498,7 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -1665,6 +1660,7 @@ golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1760,6 +1756,7 @@ golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= From 1ec5b0709b65541c8acee1a34e7f419f8d2b60f1 Mon Sep 17 00:00:00 2001 From: hacktivista Date: Tue, 2 Nov 2021 10:15:13 +0000 Subject: [PATCH 06/15] Support WebP based thumbnails (#2033) Co-authored-by: Felix Freeman --- go.mod | 1 + go.sum | 3 +++ mediaapi/thumbnailer/thumbnailer_nfnt.go | 4 ++++ 3 files changed, 8 insertions(+) diff --git a/go.mod b/go.mod index 82cda435e..6e3589972 100644 --- a/go.mod +++ b/go.mod @@ -53,6 +53,7 @@ require ( github.com/yggdrasil-network/yggdrasil-go v0.4.1-0.20210715083903-52309d094c00 go.uber.org/atomic v1.9.0 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 + golang.org/x/image v0.0.0-20211028202545-6944b10bf410 golang.org/x/mobile v0.0.0-20210716004757-34ab1303b554 golang.org/x/net v0.0.0-20210927181540-4e4d966f7476 golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b diff --git a/go.sum b/go.sum index 1d0dacaca..a36ee34d8 100644 --- a/go.sum +++ b/go.sum @@ -1529,7 +1529,10 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b h1:+qEpEAPhDZ1o0x3tHzZTQDArnOixOzGD9HUJfcg0mb4= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20211028202545-6944b10bf410 h1:hTftEOvwiOq2+O8k2D5/Q7COC7k5Qcrgc2TFURJYnvQ= +golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= diff --git a/mediaapi/thumbnailer/thumbnailer_nfnt.go b/mediaapi/thumbnailer/thumbnailer_nfnt.go index aa9faf4c1..d7aa294ac 100644 --- a/mediaapi/thumbnailer/thumbnailer_nfnt.go +++ b/mediaapi/thumbnailer/thumbnailer_nfnt.go @@ -27,6 +27,10 @@ import ( // Imported for png codec _ "image/png" + + // Imported for webp codec + _ "golang.org/x/image/webp" + "os" "time" From 5b969d172b6dc900e89b77a630a8b1b72dff44a1 Mon Sep 17 00:00:00 2001 From: database64128 Date: Wed, 3 Nov 2021 00:35:15 +0800 Subject: [PATCH 07/15] =?UTF-8?q?0=EF=B8=8F=E2=83=A3=20Media=20API:=20Fix?= =?UTF-8?q?=20maxFileSizeBytes=3D0=20handling=20(#1993)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mediaapi/routing/download.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediaapi/routing/download.go b/mediaapi/routing/download.go index 39b86b443..2358915ee 100644 --- a/mediaapi/routing/download.go +++ b/mediaapi/routing/download.go @@ -737,7 +737,7 @@ func (r *downloadRequest) fetchRemoteFile( return "", false, parseErr } - if contentLength > int64(maxFileSizeBytes) { + if maxFileSizeBytes > 0 && contentLength > int64(maxFileSizeBytes) { // TODO: Bubble up this as a 413 return "", false, fmt.Errorf("remote file is too large (%v > %v bytes)", contentLength, maxFileSizeBytes) } From b99f594a93dda7645fabda4b81498ea4ad40bc97 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 2 Nov 2021 16:47:39 +0000 Subject: [PATCH 08/15] Fix #2028 (#2036) --- roomserver/storage/shared/storage.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index b4fc15f1d..3685332fd 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -165,14 +165,21 @@ func (d *Database) AddState( if berr != nil { return 0, fmt.Errorf("d.StateBlockTable.BulkSelectStateBlockEntries: %w", berr) } + var found bool for i := len(state) - 1; i >= 0; i-- { + found = false for _, events := range blocks { for _, event := range events { if state[i].EventNID == event { - state = append(state[:i], state[i+1:]...) + found = true + break } } } + if found { + state = append(state[:i], state[i+1:]...) + i-- + } } } err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { From dec05c334707346506a979e1cdb3b295dbad5c4f Mon Sep 17 00:00:00 2001 From: PiotrKozimor <37144818+PiotrKozimor@users.noreply.github.com> Date: Tue, 2 Nov 2021 17:48:48 +0100 Subject: [PATCH 09/15] Run gofmt on dendrite - apply go 1.17 preferred build tags (#2021) --- appservice/storage/storage.go | 1 + build/gobind-pinecone/platform_ios.go | 1 + build/gobind-pinecone/platform_other.go | 1 + build/gobind-yggdrasil/platform_ios.go | 1 + build/gobind-yggdrasil/platform_other.go | 1 + cmd/dendrite-demo-pinecone/embed/embed_elementweb.go | 1 + cmd/dendrite-demo-pinecone/embed/embed_other.go | 1 + cmd/dendrite-demo-yggdrasil/embed/embed_elementweb.go | 1 + cmd/dendrite-demo-yggdrasil/embed/embed_other.go | 1 + cmd/dendritejs-pinecone/jsServer.go | 1 + cmd/dendritejs-pinecone/main_noop.go | 1 + cmd/dendritejs-pinecone/main_test.go | 1 + cmd/dendritejs/jsServer.go | 1 + cmd/dendritejs/keyfetcher.go | 1 + cmd/dendritejs/main.go | 1 + cmd/dendritejs/main_noop.go | 1 + cmd/dendritejs/publicrooms.go | 1 + federationsender/storage/storage.go | 1 + internal/sqlutil/postgres.go | 1 + internal/sqlutil/postgres_wasm.go | 1 + internal/sqlutil/trace_driver.go | 1 + internal/sqlutil/trace_driver_wasm.go | 1 + keyserver/storage/storage.go | 1 + mediaapi/storage/storage.go | 1 + mediaapi/thumbnailer/thumbnailer_bimg.go | 1 + mediaapi/thumbnailer/thumbnailer_nfnt.go | 1 + roomserver/storage/storage.go | 1 + signingkeyserver/storage/keydb.go | 1 + signingkeyserver/storage/keydb_wasm.go | 1 + syncapi/storage/storage.go | 1 + userapi/storage/accounts/sqlite3/constraint_wasm.go | 1 + userapi/storage/accounts/storage.go | 1 + userapi/storage/devices/storage.go | 1 + 33 files changed, 33 insertions(+) diff --git a/appservice/storage/storage.go b/appservice/storage/storage.go index b0df2b7dc..97b8501e2 100644 --- a/appservice/storage/storage.go +++ b/appservice/storage/storage.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !wasm // +build !wasm package storage diff --git a/build/gobind-pinecone/platform_ios.go b/build/gobind-pinecone/platform_ios.go index 01f8a6a04..802d7faca 100644 --- a/build/gobind-pinecone/platform_ios.go +++ b/build/gobind-pinecone/platform_ios.go @@ -1,3 +1,4 @@ +//go:build ios // +build ios package gobind diff --git a/build/gobind-pinecone/platform_other.go b/build/gobind-pinecone/platform_other.go index fdfb13bc0..2e81e2f43 100644 --- a/build/gobind-pinecone/platform_other.go +++ b/build/gobind-pinecone/platform_other.go @@ -1,3 +1,4 @@ +//go:build !ios // +build !ios package gobind diff --git a/build/gobind-yggdrasil/platform_ios.go b/build/gobind-yggdrasil/platform_ios.go index 01f8a6a04..802d7faca 100644 --- a/build/gobind-yggdrasil/platform_ios.go +++ b/build/gobind-yggdrasil/platform_ios.go @@ -1,3 +1,4 @@ +//go:build ios // +build ios package gobind diff --git a/build/gobind-yggdrasil/platform_other.go b/build/gobind-yggdrasil/platform_other.go index fdfb13bc0..2e81e2f43 100644 --- a/build/gobind-yggdrasil/platform_other.go +++ b/build/gobind-yggdrasil/platform_other.go @@ -1,3 +1,4 @@ +//go:build !ios // +build !ios package gobind diff --git a/cmd/dendrite-demo-pinecone/embed/embed_elementweb.go b/cmd/dendrite-demo-pinecone/embed/embed_elementweb.go index 4d2da55cb..8b3be72c1 100644 --- a/cmd/dendrite-demo-pinecone/embed/embed_elementweb.go +++ b/cmd/dendrite-demo-pinecone/embed/embed_elementweb.go @@ -1,3 +1,4 @@ +//go:build elementweb // +build elementweb package embed diff --git a/cmd/dendrite-demo-pinecone/embed/embed_other.go b/cmd/dendrite-demo-pinecone/embed/embed_other.go index 04c2188c3..a4b223452 100644 --- a/cmd/dendrite-demo-pinecone/embed/embed_other.go +++ b/cmd/dendrite-demo-pinecone/embed/embed_other.go @@ -1,3 +1,4 @@ +//go:build !elementweb // +build !elementweb package embed diff --git a/cmd/dendrite-demo-yggdrasil/embed/embed_elementweb.go b/cmd/dendrite-demo-yggdrasil/embed/embed_elementweb.go index 8d49c553a..e7725ec83 100644 --- a/cmd/dendrite-demo-yggdrasil/embed/embed_elementweb.go +++ b/cmd/dendrite-demo-yggdrasil/embed/embed_elementweb.go @@ -1,3 +1,4 @@ +//go:build elementweb // +build elementweb package embed diff --git a/cmd/dendrite-demo-yggdrasil/embed/embed_other.go b/cmd/dendrite-demo-yggdrasil/embed/embed_other.go index 04c2188c3..a4b223452 100644 --- a/cmd/dendrite-demo-yggdrasil/embed/embed_other.go +++ b/cmd/dendrite-demo-yggdrasil/embed/embed_other.go @@ -1,3 +1,4 @@ +//go:build !elementweb // +build !elementweb package embed diff --git a/cmd/dendritejs-pinecone/jsServer.go b/cmd/dendritejs-pinecone/jsServer.go index 074d20cba..4298c2ae9 100644 --- a/cmd/dendritejs-pinecone/jsServer.go +++ b/cmd/dendritejs-pinecone/jsServer.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build wasm // +build wasm package main diff --git a/cmd/dendritejs-pinecone/main_noop.go b/cmd/dendritejs-pinecone/main_noop.go index dcea032f2..0cc7e47e5 100644 --- a/cmd/dendritejs-pinecone/main_noop.go +++ b/cmd/dendritejs-pinecone/main_noop.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !wasm // +build !wasm package main diff --git a/cmd/dendritejs-pinecone/main_test.go b/cmd/dendritejs-pinecone/main_test.go index 751700cb2..17fea6cce 100644 --- a/cmd/dendritejs-pinecone/main_test.go +++ b/cmd/dendritejs-pinecone/main_test.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build wasm // +build wasm package main diff --git a/cmd/dendritejs/jsServer.go b/cmd/dendritejs/jsServer.go index 074d20cba..4298c2ae9 100644 --- a/cmd/dendritejs/jsServer.go +++ b/cmd/dendritejs/jsServer.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build wasm // +build wasm package main diff --git a/cmd/dendritejs/keyfetcher.go b/cmd/dendritejs/keyfetcher.go index cef045372..cdf937649 100644 --- a/cmd/dendritejs/keyfetcher.go +++ b/cmd/dendritejs/keyfetcher.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build wasm // +build wasm package main diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index d8fc8b837..10aadb6e5 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build wasm // +build wasm package main diff --git a/cmd/dendritejs/main_noop.go b/cmd/dendritejs/main_noop.go index dcea032f2..0cc7e47e5 100644 --- a/cmd/dendritejs/main_noop.go +++ b/cmd/dendritejs/main_noop.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !wasm // +build !wasm package main diff --git a/cmd/dendritejs/publicrooms.go b/cmd/dendritejs/publicrooms.go index a4623ba32..19afc5bcf 100644 --- a/cmd/dendritejs/publicrooms.go +++ b/cmd/dendritejs/publicrooms.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build wasm // +build wasm package main diff --git a/federationsender/storage/storage.go b/federationsender/storage/storage.go index 5462c3523..46e01f256 100644 --- a/federationsender/storage/storage.go +++ b/federationsender/storage/storage.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !wasm // +build !wasm package storage diff --git a/internal/sqlutil/postgres.go b/internal/sqlutil/postgres.go index 41a5508a1..5e656b1da 100644 --- a/internal/sqlutil/postgres.go +++ b/internal/sqlutil/postgres.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !wasm // +build !wasm package sqlutil diff --git a/internal/sqlutil/postgres_wasm.go b/internal/sqlutil/postgres_wasm.go index c45842f0c..34086f450 100644 --- a/internal/sqlutil/postgres_wasm.go +++ b/internal/sqlutil/postgres_wasm.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build wasm // +build wasm package sqlutil diff --git a/internal/sqlutil/trace_driver.go b/internal/sqlutil/trace_driver.go index f123b1e4d..b7bb36764 100644 --- a/internal/sqlutil/trace_driver.go +++ b/internal/sqlutil/trace_driver.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !wasm // +build !wasm package sqlutil diff --git a/internal/sqlutil/trace_driver_wasm.go b/internal/sqlutil/trace_driver_wasm.go index a3c163f50..51b60c3c8 100644 --- a/internal/sqlutil/trace_driver_wasm.go +++ b/internal/sqlutil/trace_driver_wasm.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build wasm // +build wasm package sqlutil diff --git a/keyserver/storage/storage.go b/keyserver/storage/storage.go index 8f05d0030..742e8463a 100644 --- a/keyserver/storage/storage.go +++ b/keyserver/storage/storage.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !wasm // +build !wasm package storage diff --git a/mediaapi/storage/storage.go b/mediaapi/storage/storage.go index a976f795b..56059f1c8 100644 --- a/mediaapi/storage/storage.go +++ b/mediaapi/storage/storage.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !wasm // +build !wasm package storage diff --git a/mediaapi/thumbnailer/thumbnailer_bimg.go b/mediaapi/thumbnailer/thumbnailer_bimg.go index 087385a76..6ca533176 100644 --- a/mediaapi/thumbnailer/thumbnailer_bimg.go +++ b/mediaapi/thumbnailer/thumbnailer_bimg.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build bimg // +build bimg package thumbnailer diff --git a/mediaapi/thumbnailer/thumbnailer_nfnt.go b/mediaapi/thumbnailer/thumbnailer_nfnt.go index d7aa294ac..beae88c5c 100644 --- a/mediaapi/thumbnailer/thumbnailer_nfnt.go +++ b/mediaapi/thumbnailer/thumbnailer_nfnt.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !bimg // +build !bimg package thumbnailer diff --git a/roomserver/storage/storage.go b/roomserver/storage/storage.go index 9359312db..9f98ea3ed 100644 --- a/roomserver/storage/storage.go +++ b/roomserver/storage/storage.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !wasm // +build !wasm package storage diff --git a/signingkeyserver/storage/keydb.go b/signingkeyserver/storage/keydb.go index aa247f1d8..82b6d0ad5 100644 --- a/signingkeyserver/storage/keydb.go +++ b/signingkeyserver/storage/keydb.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !wasm // +build !wasm package storage diff --git a/signingkeyserver/storage/keydb_wasm.go b/signingkeyserver/storage/keydb_wasm.go index 187d9669f..b112993a1 100644 --- a/signingkeyserver/storage/keydb_wasm.go +++ b/signingkeyserver/storage/keydb_wasm.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build wasm // +build wasm package storage diff --git a/syncapi/storage/storage.go b/syncapi/storage/storage.go index 15386c338..7f9c28e9d 100644 --- a/syncapi/storage/storage.go +++ b/syncapi/storage/storage.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !wasm // +build !wasm package storage diff --git a/userapi/storage/accounts/sqlite3/constraint_wasm.go b/userapi/storage/accounts/sqlite3/constraint_wasm.go index 0dd5b1fea..6c4ee762f 100644 --- a/userapi/storage/accounts/sqlite3/constraint_wasm.go +++ b/userapi/storage/accounts/sqlite3/constraint_wasm.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build wasm // +build wasm package sqlite3 diff --git a/userapi/storage/accounts/storage.go b/userapi/storage/accounts/storage.go index 3489c9d07..a21f7d94e 100644 --- a/userapi/storage/accounts/storage.go +++ b/userapi/storage/accounts/storage.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !wasm // +build !wasm package accounts diff --git a/userapi/storage/devices/storage.go b/userapi/storage/devices/storage.go index bfce924d9..3c2034300 100644 --- a/userapi/storage/devices/storage.go +++ b/userapi/storage/devices/storage.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !wasm // +build !wasm package devices From 0c2872828b7b8aa06a9c6be2c7a5ac0ce119fb6b Mon Sep 17 00:00:00 2001 From: S7evinK Date: Tue, 2 Nov 2021 17:50:59 +0100 Subject: [PATCH 10/15] Fix building on Windows (#2008) Signed-off-by: Till Faelligen --- cmd/create-account/main.go | 5 +-- internal/log.go | 61 --------------------------- internal/log_unix.go | 84 ++++++++++++++++++++++++++++++++++++++ internal/log_windows.go | 48 ++++++++++++++++++++++ 4 files changed, 134 insertions(+), 64 deletions(-) create mode 100644 internal/log_unix.go create mode 100644 internal/log_windows.go diff --git a/cmd/create-account/main.go b/cmd/create-account/main.go index a1e254f8d..3ac077705 100644 --- a/cmd/create-account/main.go +++ b/cmd/create-account/main.go @@ -22,7 +22,6 @@ import ( "io/ioutil" "os" "strings" - "syscall" "github.com/matrix-org/dendrite/setup" "github.com/matrix-org/dendrite/setup/config" @@ -121,13 +120,13 @@ func getPassword(password, pwdFile *string, pwdStdin, askPass *bool, r io.Reader // ask the user to provide the password if *askPass { fmt.Print("Enter Password: ") - bytePassword, err := term.ReadPassword(syscall.Stdin) + bytePassword, err := term.ReadPassword(int(os.Stdin.Fd())) if err != nil { logrus.Fatalln("Unable to read password:", err) } fmt.Println() fmt.Print("Confirm Password: ") - bytePassword2, err := term.ReadPassword(syscall.Stdin) + bytePassword2, err := term.ReadPassword(int(os.Stdin.Fd())) if err != nil { logrus.Fatalln("Unable to read password:", err) } diff --git a/internal/log.go b/internal/log.go index f0656d7d0..bba0ac6e6 100644 --- a/internal/log.go +++ b/internal/log.go @@ -18,7 +18,6 @@ import ( "context" "fmt" "io" - "log/syslog" "net/http" "os" "path" @@ -31,7 +30,6 @@ import ( "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dugong" "github.com/sirupsen/logrus" - lSyslog "github.com/sirupsen/logrus/hooks/syslog" ) type utcFormatter struct { @@ -108,37 +106,6 @@ func SetupStdLogging() { }) } -// SetupHookLogging configures the logging hooks defined in the configuration. -// If something fails here it means that the logging was improperly configured, -// so we just exit with the error -func SetupHookLogging(hooks []config.LogrusHook, componentName string) { - logrus.SetReportCaller(true) - for _, hook := range hooks { - // Check we received a proper logging level - level, err := logrus.ParseLevel(hook.Level) - if err != nil { - logrus.Fatalf("Unrecognised logging level %s: %q", hook.Level, err) - } - - // Perform a first filter on the logs according to the lowest level of all - // (Eg: If we have hook for info and above, prevent logrus from processing debug logs) - if logrus.GetLevel() < level { - logrus.SetLevel(level) - } - - switch hook.Type { - case "file": - checkFileHookParams(hook.Params) - setupFileHook(hook, level, componentName) - case "syslog": - checkSyslogHookParams(hook.Params) - setupSyslogHook(hook, level, componentName) - default: - logrus.Fatalf("Unrecognised logging hook type: %s", hook.Type) - } - } -} - // File type hooks should be provided a path to a directory to store log files func checkFileHookParams(params map[string]interface{}) { path, ok := params["path"] @@ -178,34 +145,6 @@ func setupFileHook(hook config.LogrusHook, level logrus.Level, componentName str }) } -func checkSyslogHookParams(params map[string]interface{}) { - addr, ok := params["address"] - if !ok { - logrus.Fatalf("Expecting a parameter \"address\" for logging hook of type \"syslog\"") - } - - if _, ok := addr.(string); !ok { - logrus.Fatalf("Parameter \"address\" for logging hook of type \"syslog\" should be a string") - } - - proto, ok2 := params["protocol"] - if !ok2 { - logrus.Fatalf("Expecting a parameter \"protocol\" for logging hook of type \"syslog\"") - } - - if _, ok2 := proto.(string); !ok2 { - logrus.Fatalf("Parameter \"protocol\" for logging hook of type \"syslog\" should be a string") - } - -} - -func setupSyslogHook(hook config.LogrusHook, level logrus.Level, componentName string) { - syslogHook, err := lSyslog.NewSyslogHook(hook.Params["protocol"].(string), hook.Params["address"].(string), syslog.LOG_INFO, componentName) - if err == nil { - logrus.AddHook(&logLevelHook{level, syslogHook}) - } -} - //CloseAndLogIfError Closes io.Closer and logs the error if any func CloseAndLogIfError(ctx context.Context, closer io.Closer, message string) { if closer == nil { diff --git a/internal/log_unix.go b/internal/log_unix.go new file mode 100644 index 000000000..25ad04205 --- /dev/null +++ b/internal/log_unix.go @@ -0,0 +1,84 @@ +// Copyright 2021 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !windows + +package internal + +import ( + "log/syslog" + + "github.com/matrix-org/dendrite/setup/config" + "github.com/sirupsen/logrus" + lSyslog "github.com/sirupsen/logrus/hooks/syslog" +) + +// SetupHookLogging configures the logging hooks defined in the configuration. +// If something fails here it means that the logging was improperly configured, +// so we just exit with the error +func SetupHookLogging(hooks []config.LogrusHook, componentName string) { + logrus.SetReportCaller(true) + for _, hook := range hooks { + // Check we received a proper logging level + level, err := logrus.ParseLevel(hook.Level) + if err != nil { + logrus.Fatalf("Unrecognised logging level %s: %q", hook.Level, err) + } + + // Perform a first filter on the logs according to the lowest level of all + // (Eg: If we have hook for info and above, prevent logrus from processing debug logs) + if logrus.GetLevel() < level { + logrus.SetLevel(level) + } + + switch hook.Type { + case "file": + checkFileHookParams(hook.Params) + setupFileHook(hook, level, componentName) + case "syslog": + checkSyslogHookParams(hook.Params) + setupSyslogHook(hook, level, componentName) + default: + logrus.Fatalf("Unrecognised logging hook type: %s", hook.Type) + } + } +} + +func checkSyslogHookParams(params map[string]interface{}) { + addr, ok := params["address"] + if !ok { + logrus.Fatalf("Expecting a parameter \"address\" for logging hook of type \"syslog\"") + } + + if _, ok := addr.(string); !ok { + logrus.Fatalf("Parameter \"address\" for logging hook of type \"syslog\" should be a string") + } + + proto, ok2 := params["protocol"] + if !ok2 { + logrus.Fatalf("Expecting a parameter \"protocol\" for logging hook of type \"syslog\"") + } + + if _, ok2 := proto.(string); !ok2 { + logrus.Fatalf("Parameter \"protocol\" for logging hook of type \"syslog\" should be a string") + } + +} + +func setupSyslogHook(hook config.LogrusHook, level logrus.Level, componentName string) { + syslogHook, err := lSyslog.NewSyslogHook(hook.Params["protocol"].(string), hook.Params["address"].(string), syslog.LOG_INFO, componentName) + if err == nil { + logrus.AddHook(&logLevelHook{level, syslogHook}) + } +} diff --git a/internal/log_windows.go b/internal/log_windows.go new file mode 100644 index 000000000..39562328c --- /dev/null +++ b/internal/log_windows.go @@ -0,0 +1,48 @@ +// Copyright 2021 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package internal + +import ( + "github.com/matrix-org/dendrite/setup/config" + "github.com/sirupsen/logrus" +) + +// SetupHookLogging configures the logging hooks defined in the configuration. +// If something fails here it means that the logging was improperly configured, +// so we just exit with the error +func SetupHookLogging(hooks []config.LogrusHook, componentName string) { + logrus.SetReportCaller(true) + for _, hook := range hooks { + // Check we received a proper logging level + level, err := logrus.ParseLevel(hook.Level) + if err != nil { + logrus.Fatalf("Unrecognised logging level %s: %q", hook.Level, err) + } + + // Perform a first filter on the logs according to the lowest level of all + // (Eg: If we have hook for info and above, prevent logrus from processing debug logs) + if logrus.GetLevel() < level { + logrus.SetLevel(level) + } + + switch hook.Type { + case "file": + checkFileHookParams(hook.Params) + setupFileHook(hook, level, componentName) + default: + logrus.Fatalf("Unrecognised logging hook type: %s", hook.Type) + } + } +} From b4a007ecceafd2b93fee2e775b0a61283d4a3844 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 2 Nov 2021 17:15:06 +0000 Subject: [PATCH 11/15] Support joining version 8 and 9 rooms (update to matrix-org/gomatrixserverlib@13366e7) --- go.mod | 2 +- go.sum | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 6e3589972..0a223f5af 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 - github.com/matrix-org/gomatrixserverlib v0.0.0-20211102101113-5e02b64e5312 + github.com/matrix-org/gomatrixserverlib v0.0.0-20211102131912-13366e7985e1 github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 github.com/matrix-org/pinecone v0.0.0-20211022090602-08a50945ac89 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 diff --git a/go.sum b/go.sum index a36ee34d8..21f4807b4 100644 --- a/go.sum +++ b/go.sum @@ -993,8 +993,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d/go.mod h1 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20211102101113-5e02b64e5312 h1:of3kbkOrjE8793Vsni8ofxMSy/vpO1SB+ZkHSfv4wrs= -github.com/matrix-org/gomatrixserverlib v0.0.0-20211102101113-5e02b64e5312/go.mod h1:rB8tBUUUo1rzUqpzklRDSooxZ6YMhoaEPx4SO5fGeUc= +github.com/matrix-org/gomatrixserverlib v0.0.0-20211102131912-13366e7985e1 h1:Pv7+98sreiHltpamJ4em6RCX/WPVN1wl53Gli9Cz744= +github.com/matrix-org/gomatrixserverlib v0.0.0-20211102131912-13366e7985e1/go.mod h1:rB8tBUUUo1rzUqpzklRDSooxZ6YMhoaEPx4SO5fGeUc= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 h1:HZCzy4oVzz55e+cOMiX/JtSF2UOY1evBl2raaE7ACcU= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= github.com/matrix-org/pinecone v0.0.0-20211022090602-08a50945ac89 h1:6JkIymZ1vxfI0shSpg6gNPTJaF4/95Evy34slPVZGKM= @@ -1529,7 +1529,6 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b h1:+qEpEAPhDZ1o0x3tHzZTQDArnOixOzGD9HUJfcg0mb4= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20211028202545-6944b10bf410 h1:hTftEOvwiOq2+O8k2D5/Q7COC7k5Qcrgc2TFURJYnvQ= golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= From 837f50ac89c54416a3fc8a463055e58e5554b6b5 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 3 Nov 2021 09:53:37 +0000 Subject: [PATCH 12/15] Reduce CPU usage of SelectStateInRange (#2038) --- syncapi/storage/postgres/output_room_events_table.go | 9 +++++---- syncapi/storage/sqlite3/output_room_events_table.go | 9 +++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go index bd7aa018d..44de02c92 100644 --- a/syncapi/storage/postgres/output_room_events_table.go +++ b/syncapi/storage/postgres/output_room_events_table.go @@ -116,7 +116,7 @@ const updateEventJSONSQL = "" + // In order for us to apply the state updates correctly, rows need to be ordered in the order they were received (id). const selectStateInRangeSQL = "" + - "SELECT id, headered_event_json, exclude_from_sync, add_state_ids, remove_state_ids" + + "SELECT event_id, id, headered_event_json, exclude_from_sync, add_state_ids, remove_state_ids" + " FROM syncapi_output_room_events" + " WHERE (id > $1 AND id <= $2) AND (add_state_ids IS NOT NULL OR remove_state_ids IS NOT NULL)" + " AND ( $3::text[] IS NULL OR sender = ANY($3) )" + @@ -221,13 +221,14 @@ func (s *outputRoomEventsStatements) SelectStateInRange( for rows.Next() { var ( + eventID string streamPos types.StreamPosition eventBytes []byte excludeFromSync bool addIDs pq.StringArray delIDs pq.StringArray ) - if err := rows.Scan(&streamPos, &eventBytes, &excludeFromSync, &addIDs, &delIDs); err != nil { + if err := rows.Scan(&eventID, &streamPos, &eventBytes, &excludeFromSync, &addIDs, &delIDs); err != nil { return nil, nil, err } // Sanity check for deleted state and whine if we see it. We don't need to do anything @@ -243,7 +244,7 @@ func (s *outputRoomEventsStatements) SelectStateInRange( // TODO: Handle redacted events var ev gomatrixserverlib.HeaderedEvent - if err := json.Unmarshal(eventBytes, &ev); err != nil { + if err := ev.UnmarshalJSONWithEventID(eventBytes, eventID); err != nil { return nil, nil, err } needSet := stateNeeded[ev.RoomID()] @@ -258,7 +259,7 @@ func (s *outputRoomEventsStatements) SelectStateInRange( } stateNeeded[ev.RoomID()] = needSet - eventIDToEvent[ev.EventID()] = types.StreamEvent{ + eventIDToEvent[eventID] = types.StreamEvent{ HeaderedEvent: &ev, StreamPosition: streamPos, ExcludeFromSync: excludeFromSync, diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index 37f7ea003..afdbe55ce 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -81,7 +81,7 @@ const updateEventJSONSQL = "" + "UPDATE syncapi_output_room_events SET headered_event_json=$1 WHERE event_id=$2" const selectStateInRangeSQL = "" + - "SELECT id, headered_event_json, exclude_from_sync, add_state_ids, remove_state_ids" + + "SELECT event_id, id, headered_event_json, exclude_from_sync, add_state_ids, remove_state_ids" + " FROM syncapi_output_room_events" + " WHERE (id > $1 AND id <= $2)" + " AND ((add_state_ids IS NOT NULL AND add_state_ids != '') OR (remove_state_ids IS NOT NULL AND remove_state_ids != ''))" @@ -173,13 +173,14 @@ func (s *outputRoomEventsStatements) SelectStateInRange( for rows.Next() { var ( + eventID string streamPos types.StreamPosition eventBytes []byte excludeFromSync bool addIDsJSON string delIDsJSON string ) - if err := rows.Scan(&streamPos, &eventBytes, &excludeFromSync, &addIDsJSON, &delIDsJSON); err != nil { + if err := rows.Scan(&eventID, &streamPos, &eventBytes, &excludeFromSync, &addIDsJSON, &delIDsJSON); err != nil { return nil, nil, err } @@ -201,7 +202,7 @@ func (s *outputRoomEventsStatements) SelectStateInRange( // TODO: Handle redacted events var ev gomatrixserverlib.HeaderedEvent - if err := json.Unmarshal(eventBytes, &ev); err != nil { + if err := ev.UnmarshalJSONWithEventID(eventBytes, eventID); err != nil { return nil, nil, err } needSet := stateNeeded[ev.RoomID()] @@ -216,7 +217,7 @@ func (s *outputRoomEventsStatements) SelectStateInRange( } stateNeeded[ev.RoomID()] = needSet - eventIDToEvent[ev.EventID()] = types.StreamEvent{ + eventIDToEvent[eventID] = types.StreamEvent{ HeaderedEvent: &ev, StreamPosition: streamPos, ExcludeFromSync: excludeFromSync, From 3c049c068f6162e3708da9a5375dff3b33ce008a Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 4 Nov 2021 09:09:26 +0000 Subject: [PATCH 13/15] State resolution v2 performance improvements (reduce allocs in Kahn's algorithm, update to matrix-org/gomatrixserverlib@91dadfb) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 0a223f5af..686e96000 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 - github.com/matrix-org/gomatrixserverlib v0.0.0-20211102131912-13366e7985e1 + github.com/matrix-org/gomatrixserverlib v0.0.0-20211104090712-91dadfbef53b github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 github.com/matrix-org/pinecone v0.0.0-20211022090602-08a50945ac89 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 diff --git a/go.sum b/go.sum index 21f4807b4..230457260 100644 --- a/go.sum +++ b/go.sum @@ -993,8 +993,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d/go.mod h1 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20211102131912-13366e7985e1 h1:Pv7+98sreiHltpamJ4em6RCX/WPVN1wl53Gli9Cz744= -github.com/matrix-org/gomatrixserverlib v0.0.0-20211102131912-13366e7985e1/go.mod h1:rB8tBUUUo1rzUqpzklRDSooxZ6YMhoaEPx4SO5fGeUc= +github.com/matrix-org/gomatrixserverlib v0.0.0-20211104090712-91dadfbef53b h1:D2Oyvn+OJOtAHcKDWu5bKo6H92f77dBDncFAFViMxrM= +github.com/matrix-org/gomatrixserverlib v0.0.0-20211104090712-91dadfbef53b/go.mod h1:rB8tBUUUo1rzUqpzklRDSooxZ6YMhoaEPx4SO5fGeUc= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 h1:HZCzy4oVzz55e+cOMiX/JtSF2UOY1evBl2raaE7ACcU= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= github.com/matrix-org/pinecone v0.0.0-20211022090602-08a50945ac89 h1:6JkIymZ1vxfI0shSpg6gNPTJaF4/95Evy34slPVZGKM= From 77096898f0322bb94d0a5594448dc7729856103a Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 4 Nov 2021 10:35:23 +0000 Subject: [PATCH 14/15] Further state resolution v2 optimisations --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 686e96000..b366fba4b 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 - github.com/matrix-org/gomatrixserverlib v0.0.0-20211104090712-91dadfbef53b + github.com/matrix-org/gomatrixserverlib v0.0.0-20211104103430-62945aeee964 github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 github.com/matrix-org/pinecone v0.0.0-20211022090602-08a50945ac89 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 diff --git a/go.sum b/go.sum index 230457260..37667b10d 100644 --- a/go.sum +++ b/go.sum @@ -993,8 +993,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d/go.mod h1 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20211104090712-91dadfbef53b h1:D2Oyvn+OJOtAHcKDWu5bKo6H92f77dBDncFAFViMxrM= -github.com/matrix-org/gomatrixserverlib v0.0.0-20211104090712-91dadfbef53b/go.mod h1:rB8tBUUUo1rzUqpzklRDSooxZ6YMhoaEPx4SO5fGeUc= +github.com/matrix-org/gomatrixserverlib v0.0.0-20211104103430-62945aeee964 h1:YW/wrSyzxB5G0snH6IlJ5u1/HjqtichZj9qem2hqraE= +github.com/matrix-org/gomatrixserverlib v0.0.0-20211104103430-62945aeee964/go.mod h1:rB8tBUUUo1rzUqpzklRDSooxZ6YMhoaEPx4SO5fGeUc= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 h1:HZCzy4oVzz55e+cOMiX/JtSF2UOY1evBl2raaE7ACcU= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= github.com/matrix-org/pinecone v0.0.0-20211022090602-08a50945ac89 h1:6JkIymZ1vxfI0shSpg6gNPTJaF4/95Evy34slPVZGKM= From b9a575919a552ce00be1b65255941874fe87c610 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 4 Nov 2021 10:50:52 +0000 Subject: [PATCH 15/15] Try to reduce re-allocations a bit in resolveConflictsV2 --- roomserver/state/state.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/roomserver/state/state.go b/roomserver/state/state.go index bae8b24c5..78398fc7c 100644 --- a/roomserver/state/state.go +++ b/roomserver/state/state.go @@ -778,7 +778,8 @@ func (v *StateResolution) resolveConflictsV2( ctx context.Context, notConflicted, conflicted []types.StateEntry, ) ([]types.StateEntry, error) { - eventIDMap := make(map[string]types.StateEntry) + estimate := len(conflicted) + len(notConflicted) + eventIDMap := make(map[string]types.StateEntry, estimate) // Load the conflicted events conflictedEvents, conflictedEventMap, err := v.loadStateEvents(ctx, conflicted) @@ -800,18 +801,20 @@ func (v *StateResolution) resolveConflictsV2( // For each conflicted event, we will add a new set of auth events. Auth // events may be duplicated across these sets but that's OK. - authSets := make(map[string][]*gomatrixserverlib.Event) - var authEvents []*gomatrixserverlib.Event - var authDifference []*gomatrixserverlib.Event + authSets := make(map[string][]*gomatrixserverlib.Event, len(conflicted)) + authEvents := make([]*gomatrixserverlib.Event, 0, estimate*3) + authDifference := make([]*gomatrixserverlib.Event, 0, estimate) // For each conflicted event, let's try and get the needed auth events. + neededStateKeys := make([]string, 16) + authEntries := make([]types.StateEntry, 16) for _, conflictedEvent := range conflictedEvents { // Work out which auth events we need to load. key := conflictedEvent.EventID() needed := gomatrixserverlib.StateNeededForAuth([]*gomatrixserverlib.Event{conflictedEvent}) // Find the numeric IDs for the necessary state keys. - var neededStateKeys []string + neededStateKeys = neededStateKeys[:0] neededStateKeys = append(neededStateKeys, needed.Member...) neededStateKeys = append(neededStateKeys, needed.ThirdPartyInvite...) stateKeyNIDMap, err := v.db.EventStateKeyNIDs(ctx, neededStateKeys) @@ -821,7 +824,7 @@ func (v *StateResolution) resolveConflictsV2( // Load the necessary auth events. tuplesNeeded := v.stateKeyTuplesNeeded(stateKeyNIDMap, needed) - var authEntries []types.StateEntry + authEntries = authEntries[:0] for _, tuple := range tuplesNeeded { if eventNID, ok := stateEntryMap(notConflicted).lookup(tuple); ok { authEntries = append(authEntries, types.StateEntry{