From 323a6fb54f023210a213ed8fb2a626358ef00fc3 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 8 Nov 2021 09:24:16 +0000 Subject: [PATCH 1/5] Resume federation sends (#2039) * Resume federation sends * Review comments * Fix build error --- federationapi/routing/send.go | 40 ++++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index 3cae837c9..4b5f0d660 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -148,7 +148,8 @@ type inputWorker struct { input *sendFIFOQueue } -var inputWorkers sync.Map // room ID -> *inputWorker +var inFlightTxnsPerOrigin sync.Map // transaction ID -> chan util.JSONResponse +var inputWorkers sync.Map // room ID -> *inputWorker // Send implements /_matrix/federation/v1/send/{txnID} func Send( @@ -164,6 +165,37 @@ func Send( mu *internal.MutexByRoom, servers federationAPI.ServersInRoomProvider, ) util.JSONResponse { + // First we should check if this origin has already submitted this + // txn ID to us. If they have and the txnIDs map contains an entry, + // the transaction is still being worked on. The new client can wait + // for it to complete rather than creating more work. + index := string(request.Origin()) + "\000" + string(txnID) + v, ok := inFlightTxnsPerOrigin.LoadOrStore(index, make(chan util.JSONResponse, 1)) + ch := v.(chan util.JSONResponse) + if ok { + // This origin already submitted this txn ID to us, and the work + // is still taking place, so we'll just wait for it to finish. + ctx, cancel := context.WithTimeout(httpReq.Context(), time.Minute*5) + defer cancel() + select { + case <-ctx.Done(): + // If the caller gives up then return straight away. We don't + // want to attempt to process what they sent us any further. + return util.JSONResponse{Code: http.StatusRequestTimeout} + case res := <-ch: + // The original task just finished processing so let's return + // the result of it. + if res.Code == 0 { + return util.JSONResponse{Code: http.StatusAccepted} + } + return res + } + } + // Otherwise, store that we're currently working on this txn from + // this origin. When we're done processing, close the channel. + defer close(ch) + defer inFlightTxnsPerOrigin.Delete(index) + t := txnReq{ rsAPI: rsAPI, eduAPI: eduAPI, @@ -205,7 +237,7 @@ func Send( util.GetLogger(httpReq.Context()).Infof("Received transaction %q from %q containing %d PDUs, %d EDUs", txnID, request.Origin(), len(t.PDUs), len(t.EDUs)) - resp, jsonErr := t.processTransaction(httpReq.Context()) + resp, jsonErr := t.processTransaction(context.Background()) if jsonErr != nil { util.GetLogger(httpReq.Context()).WithField("jsonErr", jsonErr).Error("t.processTransaction failed") return *jsonErr @@ -215,10 +247,12 @@ func Send( // Status code 200: // The result of processing the transaction. The server is to use this response // even in the event of one or more PDUs failing to be processed. - return util.JSONResponse{ + res := util.JSONResponse{ Code: http.StatusOK, JSON: resp, } + ch <- res + return res } type txnReq struct { From ee35c15467f9f2e8bb1b3cfe65656b5544ee49a8 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 8 Nov 2021 09:25:01 +0000 Subject: [PATCH 2/5] Update sytest-whitelist --- sytest-whitelist | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/sytest-whitelist b/sytest-whitelist index afe8a7e0d..d074d42b4 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -571,3 +571,19 @@ A prev_batch token from incremental sync can be used in the v1 messages API Inbound federation rejects invites which are not signed by the sender Invited user can reject invite over federation several times Test that we can be reinvited to a room we created +User can create and send/receive messages in a room with version 8 +local user can join room with version 8 +User can invite local user to room with version 8 +remote user can join room with version 8 +User can invite remote user to room with version 8 +Remote user can backfill in a room with version 8 +Can reject invites over federation for rooms with version 8 +Can receive redactions from regular users over federation in room version 8 +User can create and send/receive messages in a room with version 9 +local user can join room with version 9 +User can invite local user to room with version 9 +remote user can join room with version 9 +User can invite remote user to room with version 9 +Remote user can backfill in a room with version 9 +Can reject invites over federation for rooms with version 9 +Can receive redactions from regular users over federation in room version 9 From 9cd4a1f97b9c182f231504816f9815385a21281b Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 8 Nov 2021 09:45:39 +0000 Subject: [PATCH 3/5] Update to matrix-org/gomatrixserverlib@ed70208 (hopefully will fix sytest) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b366fba4b..26cb20325 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 - github.com/matrix-org/gomatrixserverlib v0.0.0-20211104103430-62945aeee964 + github.com/matrix-org/gomatrixserverlib v0.0.0-20211108094458-ed7020868a9a github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 github.com/matrix-org/pinecone v0.0.0-20211022090602-08a50945ac89 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 diff --git a/go.sum b/go.sum index 37667b10d..a5d09ae64 100644 --- a/go.sum +++ b/go.sum @@ -993,8 +993,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d/go.mod h1 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20211104103430-62945aeee964 h1:YW/wrSyzxB5G0snH6IlJ5u1/HjqtichZj9qem2hqraE= -github.com/matrix-org/gomatrixserverlib v0.0.0-20211104103430-62945aeee964/go.mod h1:rB8tBUUUo1rzUqpzklRDSooxZ6YMhoaEPx4SO5fGeUc= +github.com/matrix-org/gomatrixserverlib v0.0.0-20211108094458-ed7020868a9a h1:iBNcIIYr0WYl7PTuin9IXFSmrFGUAjUUyw14p29bkxw= +github.com/matrix-org/gomatrixserverlib v0.0.0-20211108094458-ed7020868a9a/go.mod h1:rB8tBUUUo1rzUqpzklRDSooxZ6YMhoaEPx4SO5fGeUc= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 h1:HZCzy4oVzz55e+cOMiX/JtSF2UOY1evBl2raaE7ACcU= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= github.com/matrix-org/pinecone v0.0.0-20211022090602-08a50945ac89 h1:6JkIymZ1vxfI0shSpg6gNPTJaF4/95Evy34slPVZGKM= From 59cf8e936e2e585a7eda54b0fb7d2b62a8b088c0 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 8 Nov 2021 10:32:21 +0000 Subject: [PATCH 4/5] Fix sytest --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 26cb20325..662114d08 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 - github.com/matrix-org/gomatrixserverlib v0.0.0-20211108094458-ed7020868a9a + github.com/matrix-org/gomatrixserverlib v0.0.0-20211108101507-7a3ac5aa8c82 github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 github.com/matrix-org/pinecone v0.0.0-20211022090602-08a50945ac89 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 diff --git a/go.sum b/go.sum index a5d09ae64..f389495f3 100644 --- a/go.sum +++ b/go.sum @@ -993,8 +993,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d/go.mod h1 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20211108094458-ed7020868a9a h1:iBNcIIYr0WYl7PTuin9IXFSmrFGUAjUUyw14p29bkxw= -github.com/matrix-org/gomatrixserverlib v0.0.0-20211108094458-ed7020868a9a/go.mod h1:rB8tBUUUo1rzUqpzklRDSooxZ6YMhoaEPx4SO5fGeUc= +github.com/matrix-org/gomatrixserverlib v0.0.0-20211108101507-7a3ac5aa8c82 h1:hULlw0rjPgoH53BGSNQZNASJIzYgoW3LH4yJ4Cx8Bak= +github.com/matrix-org/gomatrixserverlib v0.0.0-20211108101507-7a3ac5aa8c82/go.mod h1:rB8tBUUUo1rzUqpzklRDSooxZ6YMhoaEPx4SO5fGeUc= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 h1:HZCzy4oVzz55e+cOMiX/JtSF2UOY1evBl2raaE7ACcU= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= github.com/matrix-org/pinecone v0.0.0-20211022090602-08a50945ac89 h1:6JkIymZ1vxfI0shSpg6gNPTJaF4/95Evy34slPVZGKM= From 5565e403958eb8bced3ee0709620a10d019ac992 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 12 Nov 2021 15:16:22 +0000 Subject: [PATCH 5/5] Fix default power levels (update to matrix-org/gomatrixserverlib@af2616b) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 662114d08..41c66a690 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 - github.com/matrix-org/gomatrixserverlib v0.0.0-20211108101507-7a3ac5aa8c82 + github.com/matrix-org/gomatrixserverlib v0.0.0-20211112151542-af2616bf4c80 github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 github.com/matrix-org/pinecone v0.0.0-20211022090602-08a50945ac89 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 diff --git a/go.sum b/go.sum index f389495f3..ca604c9f7 100644 --- a/go.sum +++ b/go.sum @@ -993,8 +993,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d/go.mod h1 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20211108101507-7a3ac5aa8c82 h1:hULlw0rjPgoH53BGSNQZNASJIzYgoW3LH4yJ4Cx8Bak= -github.com/matrix-org/gomatrixserverlib v0.0.0-20211108101507-7a3ac5aa8c82/go.mod h1:rB8tBUUUo1rzUqpzklRDSooxZ6YMhoaEPx4SO5fGeUc= +github.com/matrix-org/gomatrixserverlib v0.0.0-20211112151542-af2616bf4c80 h1:8Fm/nsRuWsdCWNMfAji/Pi6ynUOch9eLkN15cqXOFx0= +github.com/matrix-org/gomatrixserverlib v0.0.0-20211112151542-af2616bf4c80/go.mod h1:rB8tBUUUo1rzUqpzklRDSooxZ6YMhoaEPx4SO5fGeUc= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 h1:HZCzy4oVzz55e+cOMiX/JtSF2UOY1evBl2raaE7ACcU= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= github.com/matrix-org/pinecone v0.0.0-20211022090602-08a50945ac89 h1:6JkIymZ1vxfI0shSpg6gNPTJaF4/95Evy34slPVZGKM=