From 68d6eb0a6fe911f2e7598bcd8d4f35a3c4e7c1d9 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 29 Sep 2022 13:06:55 +0100 Subject: [PATCH 001/207] P2P demo fixes --- build/gobind-pinecone/monolith.go | 59 ++++++++++++++++++------------- go.mod | 2 +- go.sum | 4 +-- 3 files changed, 37 insertions(+), 28 deletions(-) diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go index 500403ae4..241adcf73 100644 --- a/build/gobind-pinecone/monolith.go +++ b/build/gobind-pinecone/monolith.go @@ -28,6 +28,7 @@ import ( "path/filepath" "strings" "sync" + "sync/atomic" "time" "github.com/gorilla/mux" @@ -66,6 +67,7 @@ const ( PeerTypeRemote = pineconeRouter.PeerTypeRemote PeerTypeMulticast = pineconeRouter.PeerTypeMulticast PeerTypeBluetooth = pineconeRouter.PeerTypeBluetooth + PeerTypeBonjour = pineconeRouter.PeerTypeBonjour ) type DendriteMonolith struct { @@ -82,6 +84,10 @@ type DendriteMonolith struct { userAPI userapiAPI.UserInternalAPI } +func (m *DendriteMonolith) PublicKey() string { + return m.PineconeRouter.PublicKey().String() +} + func (m *DendriteMonolith) BaseURL() string { return fmt.Sprintf("http://%s", m.listener.Addr().String()) } @@ -134,32 +140,21 @@ func (m *DendriteMonolith) Conduit(zone string, peertype int) (*Conduit, error) go func() { conduit.portMutex.Lock() defer conduit.portMutex.Unlock() - loop: - for i := 1; i <= 10; i++ { - logrus.Errorf("Attempting authenticated connect (attempt %d)", i) - var err error - conduit.port, err = m.PineconeRouter.Connect( - l, - pineconeRouter.ConnectionZone(zone), - pineconeRouter.ConnectionPeerType(peertype), - ) - switch err { - case io.ErrClosedPipe: - logrus.Errorf("Authenticated connect failed due to closed pipe (attempt %d)", i) - return - case io.EOF: - logrus.Errorf("Authenticated connect failed due to EOF (attempt %d)", i) - break loop - case nil: - logrus.Errorf("Authenticated connect succeeded, connected to port %d (attempt %d)", conduit.port, i) - return - default: - logrus.WithError(err).Errorf("Authenticated connect failed (attempt %d)", i) - time.Sleep(time.Second) - } + + logrus.Errorf("Attempting authenticated connect") + var err error + if conduit.port, err = m.PineconeRouter.Connect( + l, + pineconeRouter.ConnectionZone(zone), + pineconeRouter.ConnectionPeerType(peertype), + ); err != nil { + logrus.Errorf("Authenticated connect failed: %s", err) + _ = l.Close() + _ = r.Close() + _ = conduit.Close() + return } - _ = l.Close() - _ = r.Close() + logrus.Infof("Authenticated connect succeeded (port %d)", conduit.port) }() return conduit, nil } @@ -395,6 +390,7 @@ func (m *DendriteMonolith) Stop() { const MaxFrameSize = types.MaxFrameSize type Conduit struct { + closed atomic.Bool conn net.Conn port types.SwitchPortID portMutex sync.Mutex @@ -407,10 +403,16 @@ func (c *Conduit) Port() int { } func (c *Conduit) Read(b []byte) (int, error) { + if c.closed.Load() { + return 0, io.EOF + } return c.conn.Read(b) } func (c *Conduit) ReadCopy() ([]byte, error) { + if c.closed.Load() { + return nil, io.EOF + } var buf [65535 * 2]byte n, err := c.conn.Read(buf[:]) if err != nil { @@ -420,9 +422,16 @@ func (c *Conduit) ReadCopy() ([]byte, error) { } func (c *Conduit) Write(b []byte) (int, error) { + if c.closed.Load() { + return 0, io.EOF + } return c.conn.Write(b) } func (c *Conduit) Close() error { + if c.closed.Load() { + return io.ErrClosedPipe + } + c.closed.Store(true) return c.conn.Close() } diff --git a/go.mod b/go.mod index b682d9bc4..b0a41c558 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 github.com/matrix-org/gomatrixserverlib v0.0.0-20220926161602-759a8ee7c4d5 - github.com/matrix-org/pinecone v0.0.0-20220927101513-d0beb180f44d + github.com/matrix-org/pinecone v0.0.0-20220929115107-e6e59c3fc3cd github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.15 github.com/nats-io/nats-server/v2 v2.9.1-0.20220920152220-52d7b481c4b5 diff --git a/go.sum b/go.sum index 1afed73a5..b524efc71 100644 --- a/go.sum +++ b/go.sum @@ -386,8 +386,8 @@ github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5d github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= github.com/matrix-org/gomatrixserverlib v0.0.0-20220926161602-759a8ee7c4d5 h1:cQMA9hip0WSp6cv7CUfButa9Jl/9E6kqWmQyOjx5A5s= github.com/matrix-org/gomatrixserverlib v0.0.0-20220926161602-759a8ee7c4d5/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4= -github.com/matrix-org/pinecone v0.0.0-20220927101513-d0beb180f44d h1:kGPJ6Rg8nn5an2CbCZrRiuTNyNzE0rRMiqm4UXJYrRs= -github.com/matrix-org/pinecone v0.0.0-20220927101513-d0beb180f44d/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k= +github.com/matrix-org/pinecone v0.0.0-20220929115107-e6e59c3fc3cd h1:cmAC/N0QSxbpu8vFyL4x5H+QlwMxnqTwqP+Epvj4VaE= +github.com/matrix-org/pinecone v0.0.0-20220929115107-e6e59c3fc3cd/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= From 6f3dbee5f491118a9105415399712bad87d5811c Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 29 Sep 2022 16:08:00 +0100 Subject: [PATCH 002/207] Fix Go 1.18 --- build/gobind-pinecone/monolith.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go index 241adcf73..5b368a998 100644 --- a/build/gobind-pinecone/monolith.go +++ b/build/gobind-pinecone/monolith.go @@ -28,9 +28,10 @@ import ( "path/filepath" "strings" "sync" - "sync/atomic" "time" + "go.uber.org/atomic" + "github.com/gorilla/mux" "github.com/matrix-org/dendrite/appservice" "github.com/matrix-org/dendrite/clientapi/userutil" From e1bf709eb3e66b5af9f99b9a3d406e3d64ccee44 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 29 Sep 2022 16:08:41 +0100 Subject: [PATCH 003/207] Improve device list update parsing (update to matrix-org/gomatrixserverlib#342) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b0a41c558..99774889c 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 - github.com/matrix-org/gomatrixserverlib v0.0.0-20220926161602-759a8ee7c4d5 + github.com/matrix-org/gomatrixserverlib v0.0.0-20220929150813-b0edbe00432e github.com/matrix-org/pinecone v0.0.0-20220929115107-e6e59c3fc3cd github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.15 diff --git a/go.sum b/go.sum index b524efc71..f64eb1ce8 100644 --- a/go.sum +++ b/go.sum @@ -384,8 +384,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220926161602-759a8ee7c4d5 h1:cQMA9hip0WSp6cv7CUfButa9Jl/9E6kqWmQyOjx5A5s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220926161602-759a8ee7c4d5/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4= +github.com/matrix-org/gomatrixserverlib v0.0.0-20220929150813-b0edbe00432e h1:no8QJnxgvBSDKeFivBPZMCBYdXbUYeAHvzqP9oehz4k= +github.com/matrix-org/gomatrixserverlib v0.0.0-20220929150813-b0edbe00432e/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4= github.com/matrix-org/pinecone v0.0.0-20220929115107-e6e59c3fc3cd h1:cmAC/N0QSxbpu8vFyL4x5H+QlwMxnqTwqP+Epvj4VaE= github.com/matrix-org/pinecone v0.0.0-20220929115107-e6e59c3fc3cd/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= From 1b65c97ad17f95bfffa620a8c5b007d0318c0291 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 29 Sep 2022 16:18:42 +0100 Subject: [PATCH 004/207] Remove `Suppressing send-to-device` log line The behaviour is correct and I have no idea why we are logging it. --- federationapi/consumers/sendtodevice.go | 1 - 1 file changed, 1 deletion(-) diff --git a/federationapi/consumers/sendtodevice.go b/federationapi/consumers/sendtodevice.go index e44bad723..ffc1d8894 100644 --- a/federationapi/consumers/sendtodevice.go +++ b/federationapi/consumers/sendtodevice.go @@ -80,7 +80,6 @@ func (t *OutputSendToDeviceConsumer) onMessage(ctx context.Context, msgs []*nats return true } if originServerName != t.ServerName { - log.WithField("other_server", originServerName).Info("Suppressing send-to-device: originated elsewhere") return true } // Extract the send-to-device event from msg. From b28bbadeb02978634e7d91844568a6b09b79e53b Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 29 Sep 2022 16:55:03 +0100 Subject: [PATCH 005/207] Update to matrix-org/gomatrixserverlib@377b320 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 99774889c..b805a531f 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 - github.com/matrix-org/gomatrixserverlib v0.0.0-20220929150813-b0edbe00432e + github.com/matrix-org/gomatrixserverlib v0.0.0-20220929155203-377b320ad5b1 github.com/matrix-org/pinecone v0.0.0-20220929115107-e6e59c3fc3cd github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.15 diff --git a/go.sum b/go.sum index f64eb1ce8..4e3b6b356 100644 --- a/go.sum +++ b/go.sum @@ -384,8 +384,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220929150813-b0edbe00432e h1:no8QJnxgvBSDKeFivBPZMCBYdXbUYeAHvzqP9oehz4k= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220929150813-b0edbe00432e/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4= +github.com/matrix-org/gomatrixserverlib v0.0.0-20220929155203-377b320ad5b1 h1:kS4jgi/DshFGJArjF14b1u42fVWC+8LMVdSVLdQYq3E= +github.com/matrix-org/gomatrixserverlib v0.0.0-20220929155203-377b320ad5b1/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4= github.com/matrix-org/pinecone v0.0.0-20220929115107-e6e59c3fc3cd h1:cmAC/N0QSxbpu8vFyL4x5H+QlwMxnqTwqP+Epvj4VaE= github.com/matrix-org/pinecone v0.0.0-20220929115107-e6e59c3fc3cd/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= From 90f1985bf30ac9f79ef5a0c7ae8406aca904aaad Mon Sep 17 00:00:00 2001 From: devonh Date: Thu, 29 Sep 2022 16:05:16 +0000 Subject: [PATCH 006/207] Add network interface registration to pinecone demo (#2750) ### Pull Request Checklist No additional tests have been added as this simply extends the API for the embedded pinecone demo. Signed-off-by: `Devon Hudson ` --- build/gobind-pinecone/monolith.go | 14 ++++++++++++++ go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go index 5b368a998..b0d3fa285 100644 --- a/build/gobind-pinecone/monolith.go +++ b/build/gobind-pinecone/monolith.go @@ -101,6 +101,20 @@ func (m *DendriteMonolith) SessionCount() int { return len(m.PineconeQUIC.Protocol("matrix").Sessions()) } +func (m *DendriteMonolith) RegisterNetworkInterface(name string, index int, mtu int, up bool, broadcast bool, loopback bool, pointToPoint bool, multicast bool, addrs string) { + m.PineconeMulticast.RegisterInterface(pineconeMulticast.InterfaceInfo{ + Name: name, + Index: index, + Mtu: mtu, + Up: up, + Broadcast: broadcast, + Loopback: loopback, + PointToPoint: pointToPoint, + Multicast: multicast, + Addrs: addrs, + }) +} + func (m *DendriteMonolith) SetMulticastEnabled(enabled bool) { if enabled { m.PineconeMulticast.Start() diff --git a/go.mod b/go.mod index b805a531f..bd79680ed 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 github.com/matrix-org/gomatrixserverlib v0.0.0-20220929155203-377b320ad5b1 - github.com/matrix-org/pinecone v0.0.0-20220929115107-e6e59c3fc3cd + github.com/matrix-org/pinecone v0.0.0-20220929155234-2ce51dd4a42c github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.15 github.com/nats-io/nats-server/v2 v2.9.1-0.20220920152220-52d7b481c4b5 diff --git a/go.sum b/go.sum index 4e3b6b356..742a52dcb 100644 --- a/go.sum +++ b/go.sum @@ -386,8 +386,8 @@ github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5d github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= github.com/matrix-org/gomatrixserverlib v0.0.0-20220929155203-377b320ad5b1 h1:kS4jgi/DshFGJArjF14b1u42fVWC+8LMVdSVLdQYq3E= github.com/matrix-org/gomatrixserverlib v0.0.0-20220929155203-377b320ad5b1/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4= -github.com/matrix-org/pinecone v0.0.0-20220929115107-e6e59c3fc3cd h1:cmAC/N0QSxbpu8vFyL4x5H+QlwMxnqTwqP+Epvj4VaE= -github.com/matrix-org/pinecone v0.0.0-20220929115107-e6e59c3fc3cd/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k= +github.com/matrix-org/pinecone v0.0.0-20220929155234-2ce51dd4a42c h1:iCHLYwwlPsf4TYFrvhKdhQoAM2lXzcmDZYqwBNWcnVk= +github.com/matrix-org/pinecone v0.0.0-20220929155234-2ce51dd4a42c/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= From e45ba35e979f2f8af1e75f5f16158973b790436b Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 29 Sep 2022 20:05:05 +0100 Subject: [PATCH 007/207] Enable knocking on room versions 8 and 9 (update to matrix-org/gomatrixserverlib@91d455c) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index bd79680ed..0985b986c 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 - github.com/matrix-org/gomatrixserverlib v0.0.0-20220929155203-377b320ad5b1 + github.com/matrix-org/gomatrixserverlib v0.0.0-20220929190355-91d455cd3621 github.com/matrix-org/pinecone v0.0.0-20220929155234-2ce51dd4a42c github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.15 diff --git a/go.sum b/go.sum index 742a52dcb..c3db3e09e 100644 --- a/go.sum +++ b/go.sum @@ -384,8 +384,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220929155203-377b320ad5b1 h1:kS4jgi/DshFGJArjF14b1u42fVWC+8LMVdSVLdQYq3E= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220929155203-377b320ad5b1/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4= +github.com/matrix-org/gomatrixserverlib v0.0.0-20220929190355-91d455cd3621 h1:a8IaoSPDxevkgXnOUrtIW9AqVNvXBJAG0gtnX687S7g= +github.com/matrix-org/gomatrixserverlib v0.0.0-20220929190355-91d455cd3621/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4= github.com/matrix-org/pinecone v0.0.0-20220929155234-2ce51dd4a42c h1:iCHLYwwlPsf4TYFrvhKdhQoAM2lXzcmDZYqwBNWcnVk= github.com/matrix-org/pinecone v0.0.0-20220929155234-2ce51dd4a42c/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= From 9005e5b4a87c0bdd4f19b1dbf46f3093ac07e727 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Fri, 30 Sep 2022 10:32:31 +0200 Subject: [PATCH 008/207] Add `/_dendrite/admin/refreshDevices/{userID}` (#2746) Allows to immediately query `/devices/{userID}` over federation to (hopefully) resolve E2EE issues. --- clientapi/routing/admin.go | 55 ++++++++++++++++++++++++------- clientapi/routing/routing.go | 8 ++++- docs/administration/4_adminapi.md | 5 +++ keyserver/api/api.go | 1 + keyserver/internal/internal.go | 13 ++++++-- 5 files changed, 66 insertions(+), 16 deletions(-) diff --git a/clientapi/routing/admin.go b/clientapi/routing/admin.go index 5089d7c36..89c269f1a 100644 --- a/clientapi/routing/admin.go +++ b/clientapi/routing/admin.go @@ -2,20 +2,23 @@ package routing import ( "encoding/json" + "fmt" "net/http" "time" "github.com/gorilla/mux" - "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/internal/httputil" - roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/setup/config" - "github.com/matrix-org/dendrite/setup/jetstream" - userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" "github.com/nats-io/nats.go" "github.com/sirupsen/logrus" + + "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/internal/httputil" + "github.com/matrix-org/dendrite/keyserver/api" + roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/jetstream" + userapi "github.com/matrix-org/dendrite/userapi/api" ) func AdminEvacuateRoom(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse { @@ -144,12 +147,6 @@ func AdminResetPassword(req *http.Request, cfg *config.ClientAPI, device *userap } func AdminReindex(req *http.Request, cfg *config.ClientAPI, device *userapi.Device, natsClient *nats.Conn) util.JSONResponse { - if device.AccountType != userapi.AccountTypeAdmin { - return util.JSONResponse{ - Code: http.StatusForbidden, - JSON: jsonerror.Forbidden("This API can only be used by admin users."), - } - } _, err := natsClient.RequestMsg(nats.NewMsg(cfg.Matrix.JetStream.Prefixed(jetstream.InputFulltextReindex)), time.Second*10) if err != nil { logrus.WithError(err).Error("failed to publish nats message") @@ -160,3 +157,37 @@ func AdminReindex(req *http.Request, cfg *config.ClientAPI, device *userapi.Devi JSON: struct{}{}, } } + +func AdminMarkAsStale(req *http.Request, cfg *config.ClientAPI, keyAPI api.ClientKeyAPI) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + userID := vars["userID"] + + _, domain, err := gomatrixserverlib.SplitID('@', userID) + if err != nil { + return util.MessageResponse(http.StatusBadRequest, err.Error()) + } + if domain == cfg.Matrix.ServerName { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.InvalidParam("Can not mark local device list as stale"), + } + } + + err = keyAPI.PerformMarkAsStaleIfNeeded(req.Context(), &api.PerformMarkAsStaleRequest{ + UserID: userID, + Domain: domain, + }, &struct{}{}) + if err != nil { + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: jsonerror.Unknown(fmt.Sprintf("Failed to mark device list as stale: %s", err)), + } + } + return util.JSONResponse{ + Code: http.StatusOK, + JSON: struct{}{}, + } +} diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 9c1f8f720..7d1c434c4 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -163,11 +163,17 @@ func Setup( ).Methods(http.MethodPost, http.MethodOptions) dendriteAdminRouter.Handle("/admin/fulltext/reindex", - httputil.MakeAuthAPI("admin_fultext_reindex", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + httputil.MakeAdminAPI("admin_fultext_reindex", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { return AdminReindex(req, cfg, device, natsClient) }), ).Methods(http.MethodGet, http.MethodOptions) + dendriteAdminRouter.Handle("/admin/refreshDevices/{userID}", + httputil.MakeAdminAPI("admin_refresh_devices", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + return AdminMarkAsStale(req, cfg, keyAPI) + }), + ).Methods(http.MethodPost, http.MethodOptions) + // server notifications if cfg.Matrix.ServerNotices.Enabled { logrus.Info("Enabling server notices at /_synapse/admin/v1/send_server_notice") diff --git a/docs/administration/4_adminapi.md b/docs/administration/4_adminapi.md index 1712bb1bf..56e19a8b4 100644 --- a/docs/administration/4_adminapi.md +++ b/docs/administration/4_adminapi.md @@ -62,6 +62,11 @@ the full user ID is `@alice:domain.com` then the local part is `alice`. This endpoint instructs Dendrite to reindex all searchable events (`m.room.message`, `m.room.topic` and `m.room.name`). An empty JSON body will be returned immediately. Indexing is done in the background, the server logs every 1000 events (or below) when they are being indexed. Once reindexing is done, you'll see something along the lines `Indexed 69586 events in 53.68223182s` in your debug logs. +## POST `/_dendrite/admin/refreshDevices/{userID}` + +This endpoint instructs Dendrite to immediately query `/devices/{userID}` on a federated server. An empty JSON body will be returned on success, updating all locally stored user devices/keys. This can be used to possibly resolve E2EE issues, where the remote user can't decrypt messages. + + ## POST `/_synapse/admin/v1/send_server_notice` Request body format: diff --git a/keyserver/api/api.go b/keyserver/api/api.go index c9ec59a75..14fced3e8 100644 --- a/keyserver/api/api.go +++ b/keyserver/api/api.go @@ -45,6 +45,7 @@ type ClientKeyAPI interface { PerformUploadDeviceSignatures(ctx context.Context, req *PerformUploadDeviceSignaturesRequest, res *PerformUploadDeviceSignaturesResponse) error // PerformClaimKeys claims one-time keys for use in pre-key messages PerformClaimKeys(ctx context.Context, req *PerformClaimKeysRequest, res *PerformClaimKeysResponse) error + PerformMarkAsStaleIfNeeded(ctx context.Context, req *PerformMarkAsStaleRequest, res *struct{}) error } // API functions required by the userapi diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go index a8d1128c4..6309066d7 100644 --- a/keyserver/internal/internal.go +++ b/keyserver/internal/internal.go @@ -228,14 +228,21 @@ func (a *KeyInternalAPI) QueryDeviceMessages(ctx context.Context, req *api.Query // PerformMarkAsStaleIfNeeded marks the users device list as stale, if the given deviceID is not present // in our database. func (a *KeyInternalAPI) PerformMarkAsStaleIfNeeded(ctx context.Context, req *api.PerformMarkAsStaleRequest, res *struct{}) error { - knownDevices, err := a.DB.DeviceKeysForUser(ctx, req.UserID, []string{req.DeviceID}, true) + knownDevices, err := a.DB.DeviceKeysForUser(ctx, req.UserID, []string{}, true) if err != nil { return err } if len(knownDevices) == 0 { - return a.Updater.ManualUpdate(ctx, req.Domain, req.UserID) + return fmt.Errorf("unknown user %s", req.UserID) } - return nil + + for i := range knownDevices { + if knownDevices[i].DeviceID == req.DeviceID { + return nil // we already know about this device + } + } + + return a.Updater.ManualUpdate(ctx, req.Domain, req.UserID) } // nolint:gocyclo From 8a82f100460dc5ca7bd39ae2345c251d6622c494 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 30 Sep 2022 09:41:16 +0100 Subject: [PATCH 009/207] Allow more time for device list updates (#2749) This updates the device list updater so that it has a context per-request, rather than a global 30 seconds for the entire server. This could mean that talking to a slow remote server or requesting a lot of user IDs was pretty much guaranteed to fail. It also uses the process context to allow correct cancellation when Dendrite wants to shut down cleanly. --- keyserver/internal/device_list_update.go | 204 ++++++++++-------- keyserver/internal/device_list_update_test.go | 7 +- keyserver/keyserver.go | 2 +- 3 files changed, 117 insertions(+), 96 deletions(-) diff --git a/keyserver/internal/device_list_update.go b/keyserver/internal/device_list_update.go index 525f8a99d..fcfcd092d 100644 --- a/keyserver/internal/device_list_update.go +++ b/keyserver/internal/device_list_update.go @@ -17,6 +17,7 @@ package internal import ( "context" "encoding/json" + "errors" "fmt" "hash/fnv" "net" @@ -31,6 +32,7 @@ import ( fedsenderapi "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/keyserver/api" + "github.com/matrix-org/dendrite/setup/process" ) var ( @@ -45,6 +47,9 @@ var ( ) ) +const defaultWaitTime = time.Minute +const requestTimeout = time.Second * 30 + func init() { prometheus.MustRegister( deviceListUpdateCount, @@ -80,6 +85,7 @@ func init() { // In the event that the query fails, a lock is acquired and the server name along with the time to wait before retrying is // set in a map. A restarter goroutine periodically probes this map and injects servers which are ready to be retried. type DeviceListUpdater struct { + process *process.ProcessContext // A map from user_id to a mutex. Used when we are missing prev IDs so we don't make more than 1 // request to the remote server and race. // TODO: Put in an LRU cache to bound growth @@ -131,10 +137,12 @@ type KeyChangeProducer interface { // NewDeviceListUpdater creates a new updater which fetches fresh device lists when they go stale. func NewDeviceListUpdater( - db DeviceListUpdaterDatabase, api DeviceListUpdaterAPI, producer KeyChangeProducer, + process *process.ProcessContext, db DeviceListUpdaterDatabase, + api DeviceListUpdaterAPI, producer KeyChangeProducer, fedClient fedsenderapi.KeyserverFederationAPI, numWorkers int, ) *DeviceListUpdater { return &DeviceListUpdater{ + process: process, userIDToMutex: make(map[string]*sync.Mutex), mu: &sync.Mutex{}, db: db, @@ -234,7 +242,7 @@ func (u *DeviceListUpdater) update(ctx context.Context, event gomatrixserverlib. "prev_ids": event.PrevID, "display_name": event.DeviceDisplayName, "deleted": event.Deleted, - }).Info("DeviceListUpdater.Update") + }).Trace("DeviceListUpdater.Update") // if we haven't missed anything update the database and notify users if exists || event.Deleted { @@ -378,111 +386,123 @@ func (u *DeviceListUpdater) worker(ch chan gomatrixserverlib.ServerName) { } func (u *DeviceListUpdater) processServer(serverName gomatrixserverlib.ServerName) (time.Duration, bool) { - deviceListUpdateCount.WithLabelValues(string(serverName)).Inc() - requestTimeout := time.Second * 30 // max amount of time we want to spend on each request - ctx, cancel := context.WithTimeout(context.Background(), requestTimeout) - defer cancel() + ctx := u.process.Context() logger := util.GetLogger(ctx).WithField("server_name", serverName) - waitTime := 2 * time.Second - // fetch stale device lists + deviceListUpdateCount.WithLabelValues(string(serverName)).Inc() + + waitTime := defaultWaitTime // How long should we wait to try again? + successCount := 0 // How many user requests failed? + userIDs, err := u.db.StaleDeviceLists(ctx, []gomatrixserverlib.ServerName{serverName}) if err != nil { logger.WithError(err).Error("Failed to load stale device lists") return waitTime, true } - failCount := 0 -userLoop: + defer func() { + for _, userID := range userIDs { + // always clear the channel to unblock Update calls regardless of success/failure + u.clearChannel(userID) + } + }() + for _, userID := range userIDs { - if ctx.Err() != nil { - // we've timed out, give up and go to the back of the queue to let another server be processed. - failCount += 1 - waitTime = time.Minute * 10 + userWait, err := u.processServerUser(ctx, serverName, userID) + if err != nil { + if userWait > waitTime { + waitTime = userWait + } break } - res, err := u.fedClient.GetUserDevices(ctx, serverName, userID) - if err != nil { - failCount += 1 - select { - case <-ctx.Done(): - // we've timed out, give up and go to the back of the queue to let another server be processed. - waitTime = time.Minute * 10 - break userLoop - default: - } - switch e := err.(type) { - case *fedsenderapi.FederationClientError: - if e.RetryAfter > 0 { - waitTime = e.RetryAfter - } else if e.Blacklisted { - waitTime = time.Hour * 8 - break userLoop - } else if e.Code >= 300 { - // We didn't get a real FederationClientError (e.g. in polylith mode, where gomatrix.HTTPError - // are "converted" to FederationClientError), but we probably shouldn't hit them every $waitTime seconds. - waitTime = time.Hour - break userLoop - } - case net.Error: - // Use the default waitTime, if it's a timeout. - // It probably doesn't make sense to try further users. - if !e.Timeout() { - waitTime = time.Minute * 10 - logger.WithError(e).Error("GetUserDevices returned net.Error") - break userLoop - } - case gomatrix.HTTPError: - // The remote server returned an error, give it some time to recover. - // This is to avoid spamming remote servers, which may not be Matrix servers anymore. - if e.Code >= 300 { - waitTime = time.Hour - logger.WithError(e).Error("GetUserDevices returned gomatrix.HTTPError") - break userLoop - } - default: - // Something else failed - waitTime = time.Minute * 10 - logger.WithError(err).WithField("user_id", userID).Debugf("GetUserDevices returned unknown error type: %T", err) - break userLoop - } - continue - } - if res.MasterKey != nil || res.SelfSigningKey != nil { - uploadReq := &api.PerformUploadDeviceKeysRequest{ - UserID: userID, - } - uploadRes := &api.PerformUploadDeviceKeysResponse{} - if res.MasterKey != nil { - if err = sanityCheckKey(*res.MasterKey, userID, gomatrixserverlib.CrossSigningKeyPurposeMaster); err == nil { - uploadReq.MasterKey = *res.MasterKey - } - } - if res.SelfSigningKey != nil { - if err = sanityCheckKey(*res.SelfSigningKey, userID, gomatrixserverlib.CrossSigningKeyPurposeSelfSigning); err == nil { - uploadReq.SelfSigningKey = *res.SelfSigningKey - } - } - _ = u.api.PerformUploadDeviceKeys(ctx, uploadReq, uploadRes) - } - err = u.updateDeviceList(&res) - if err != nil { - logger.WithError(err).WithField("user_id", userID).Error("Fetched device list but failed to store/emit it") - failCount += 1 - } + successCount++ } - if failCount > 0 { + + allUsersSucceeded := successCount == len(userIDs) + if !allUsersSucceeded { logger.WithFields(logrus.Fields{ - "total": len(userIDs), - "failed": failCount, - "skipped": len(userIDs) - failCount, - "waittime": waitTime, + "total": len(userIDs), + "succeeded": successCount, + "failed": len(userIDs) - successCount, + "wait_time": waitTime, }).Warn("Failed to query device keys for some users") } - for _, userID := range userIDs { - // always clear the channel to unblock Update calls regardless of success/failure - u.clearChannel(userID) + return waitTime, !allUsersSucceeded +} + +func (u *DeviceListUpdater) processServerUser(ctx context.Context, serverName gomatrixserverlib.ServerName, userID string) (time.Duration, error) { + ctx, cancel := context.WithTimeout(ctx, requestTimeout) + defer cancel() + logger := util.GetLogger(ctx).WithFields(logrus.Fields{ + "server_name": serverName, + "user_id": userID, + }) + + res, err := u.fedClient.GetUserDevices(ctx, serverName, userID) + if err != nil { + if errors.Is(err, context.DeadlineExceeded) { + return time.Minute * 10, err + } + switch e := err.(type) { + case *json.UnmarshalTypeError, *json.SyntaxError: + logger.WithError(err).Debugf("Device list update for %q contained invalid JSON", userID) + return defaultWaitTime, nil + case *fedsenderapi.FederationClientError: + if e.RetryAfter > 0 { + return e.RetryAfter, err + } else if e.Blacklisted { + return time.Hour * 8, err + } else if e.Code >= 300 { + // We didn't get a real FederationClientError (e.g. in polylith mode, where gomatrix.HTTPError + // are "converted" to FederationClientError), but we probably shouldn't hit them every $waitTime seconds. + return time.Hour, err + } + case net.Error: + // Use the default waitTime, if it's a timeout. + // It probably doesn't make sense to try further users. + if !e.Timeout() { + logger.WithError(e).Debug("GetUserDevices returned net.Error") + return time.Minute * 10, err + } + case gomatrix.HTTPError: + // The remote server returned an error, give it some time to recover. + // This is to avoid spamming remote servers, which may not be Matrix servers anymore. + if e.Code >= 300 { + logger.WithError(e).Debug("GetUserDevices returned gomatrix.HTTPError") + return time.Hour, err + } + default: + // Something else failed + logger.WithError(err).Debugf("GetUserDevices returned unknown error type: %T", err) + return time.Minute * 10, err + } } - return waitTime, failCount > 0 + if res.UserID != userID { + logger.WithError(err).Debugf("User ID %q in device list update response doesn't match expected %q", res.UserID, userID) + return defaultWaitTime, nil + } + if res.MasterKey != nil || res.SelfSigningKey != nil { + uploadReq := &api.PerformUploadDeviceKeysRequest{ + UserID: userID, + } + uploadRes := &api.PerformUploadDeviceKeysResponse{} + if res.MasterKey != nil { + if err = sanityCheckKey(*res.MasterKey, userID, gomatrixserverlib.CrossSigningKeyPurposeMaster); err == nil { + uploadReq.MasterKey = *res.MasterKey + } + } + if res.SelfSigningKey != nil { + if err = sanityCheckKey(*res.SelfSigningKey, userID, gomatrixserverlib.CrossSigningKeyPurposeSelfSigning); err == nil { + uploadReq.SelfSigningKey = *res.SelfSigningKey + } + } + _ = u.api.PerformUploadDeviceKeys(ctx, uploadReq, uploadRes) + } + err = u.updateDeviceList(&res) + if err != nil { + logger.WithError(err).Error("Fetched device list but failed to store/emit it") + return defaultWaitTime, err + } + return defaultWaitTime, nil } func (u *DeviceListUpdater) updateDeviceList(res *gomatrixserverlib.RespUserDevices) error { diff --git a/keyserver/internal/device_list_update_test.go b/keyserver/internal/device_list_update_test.go index 0520a9e66..28a13a0a0 100644 --- a/keyserver/internal/device_list_update_test.go +++ b/keyserver/internal/device_list_update_test.go @@ -30,6 +30,7 @@ import ( "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/dendrite/keyserver/api" + "github.com/matrix-org/dendrite/setup/process" ) var ( @@ -146,7 +147,7 @@ func TestUpdateHavePrevID(t *testing.T) { } ap := &mockDeviceListUpdaterAPI{} producer := &mockKeyChangeProducer{} - updater := NewDeviceListUpdater(db, ap, producer, nil, 1) + updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, nil, 1) event := gomatrixserverlib.DeviceListUpdateEvent{ DeviceDisplayName: "Foo Bar", Deleted: false, @@ -218,7 +219,7 @@ func TestUpdateNoPrevID(t *testing.T) { `)), }, nil }) - updater := NewDeviceListUpdater(db, ap, producer, fedClient, 2) + updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, fedClient, 2) if err := updater.Start(); err != nil { t.Fatalf("failed to start updater: %s", err) } @@ -287,7 +288,7 @@ func TestDebounce(t *testing.T) { close(incomingFedReq) return <-fedCh, nil }) - updater := NewDeviceListUpdater(db, ap, producer, fedClient, 1) + updater := NewDeviceListUpdater(process.NewProcessContext(), db, ap, producer, fedClient, 1) if err := updater.Start(); err != nil { t.Fatalf("failed to start updater: %s", err) } diff --git a/keyserver/keyserver.go b/keyserver/keyserver.go index 5124b777e..9ae4f9ca3 100644 --- a/keyserver/keyserver.go +++ b/keyserver/keyserver.go @@ -58,7 +58,7 @@ func NewInternalAPI( FedClient: fedClient, Producer: keyChangeProducer, } - updater := internal.NewDeviceListUpdater(db, ap, keyChangeProducer, fedClient, 8) // 8 workers TODO: configurable + updater := internal.NewDeviceListUpdater(base.ProcessContext, db, ap, keyChangeProducer, fedClient, 8) // 8 workers TODO: configurable ap.Updater = updater go func() { if err := updater.Start(); err != nil { From 6348486a1365c7469a498101f5035a9b6bd16d22 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 30 Sep 2022 12:48:10 +0100 Subject: [PATCH 010/207] Transactional isolation for `/sync` (#2745) This should transactional snapshot isolation for `/sync` etc requests. For now we don't use repeatable read due to some odd test failures with invites. --- syncapi/consumers/clientapi.go | 5 +- syncapi/consumers/keychange.go | 5 +- syncapi/consumers/presence.go | 5 +- syncapi/consumers/receipts.go | 5 +- syncapi/consumers/roomserver.go | 19 +- syncapi/consumers/sendtodevice.go | 5 +- syncapi/consumers/typing.go | 5 +- syncapi/consumers/userapi.go | 5 +- syncapi/internal/history_visibility.go | 2 +- syncapi/notifier/notifier.go | 19 +- syncapi/routing/context.go | 30 +- syncapi/routing/messages.go | 42 +- syncapi/routing/search.go | 24 +- syncapi/storage/interface.go | 125 +- syncapi/storage/postgres/invites_table.go | 22 +- syncapi/storage/shared/storage_consumer.go | 586 +++++++++ syncapi/storage/shared/storage_sync.go | 574 +++++++++ syncapi/storage/shared/syncserver.go | 1103 ----------------- .../sqlite3/current_room_state_table.go | 8 +- syncapi/storage/sqlite3/invites_table.go | 23 +- syncapi/storage/sqlite3/syncserver.go | 14 + syncapi/storage/storage_test.go | 197 +-- syncapi/storage/tables/interface.go | 2 +- syncapi/streams/stream_accountdata.go | 17 +- syncapi/streams/stream_devicelist.go | 7 +- syncapi/streams/stream_invite.go | 19 +- syncapi/streams/stream_notificationdata.go | 20 +- syncapi/streams/stream_pdu.go | 141 +-- syncapi/streams/stream_presence.go | 22 +- syncapi/streams/stream_receipt.go | 20 +- syncapi/streams/stream_sendtodevice.go | 20 +- syncapi/streams/stream_typing.go | 7 +- syncapi/streams/streamprovider.go | 28 + syncapi/streams/streams.go | 77 +- syncapi/streams/template_stream.go | 10 +- syncapi/sync/requestpool.go | 53 +- syncapi/types/provider.go | 20 - 37 files changed, 1759 insertions(+), 1527 deletions(-) create mode 100644 syncapi/storage/shared/storage_consumer.go create mode 100644 syncapi/storage/shared/storage_sync.go delete mode 100644 syncapi/storage/shared/syncserver.go create mode 100644 syncapi/streams/streamprovider.go diff --git a/syncapi/consumers/clientapi.go b/syncapi/consumers/clientapi.go index 796cc61e1..735f6718c 100644 --- a/syncapi/consumers/clientapi.go +++ b/syncapi/consumers/clientapi.go @@ -34,6 +34,7 @@ import ( "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/dendrite/syncapi/notifier" "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/dendrite/syncapi/streams" "github.com/matrix-org/dendrite/syncapi/types" ) @@ -46,7 +47,7 @@ type OutputClientDataConsumer struct { topic string topicReIndex string db storage.Database - stream types.StreamProvider + stream streams.StreamProvider notifier *notifier.Notifier serverName gomatrixserverlib.ServerName fts *fulltext.Search @@ -61,7 +62,7 @@ func NewOutputClientDataConsumer( nats *nats.Conn, store storage.Database, notifier *notifier.Notifier, - stream types.StreamProvider, + stream streams.StreamProvider, fts *fulltext.Search, ) *OutputClientDataConsumer { return &OutputClientDataConsumer{ diff --git a/syncapi/consumers/keychange.go b/syncapi/consumers/keychange.go index c42e71971..dc7d9e207 100644 --- a/syncapi/consumers/keychange.go +++ b/syncapi/consumers/keychange.go @@ -26,6 +26,7 @@ import ( "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/dendrite/syncapi/notifier" "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/dendrite/syncapi/streams" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" "github.com/nats-io/nats.go" @@ -40,7 +41,7 @@ type OutputKeyChangeEventConsumer struct { topic string db storage.Database notifier *notifier.Notifier - stream types.StreamProvider + stream streams.StreamProvider serverName gomatrixserverlib.ServerName // our server name rsAPI roomserverAPI.SyncRoomserverAPI } @@ -55,7 +56,7 @@ func NewOutputKeyChangeEventConsumer( rsAPI roomserverAPI.SyncRoomserverAPI, store storage.Database, notifier *notifier.Notifier, - stream types.StreamProvider, + stream streams.StreamProvider, ) *OutputKeyChangeEventConsumer { s := &OutputKeyChangeEventConsumer{ ctx: process.Context(), diff --git a/syncapi/consumers/presence.go b/syncapi/consumers/presence.go index 61bdc13de..145059c2d 100644 --- a/syncapi/consumers/presence.go +++ b/syncapi/consumers/presence.go @@ -23,6 +23,7 @@ import ( "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/dendrite/syncapi/notifier" "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/dendrite/syncapi/streams" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" @@ -39,7 +40,7 @@ type PresenceConsumer struct { requestTopic string presenceTopic string db storage.Database - stream types.StreamProvider + stream streams.StreamProvider notifier *notifier.Notifier deviceAPI api.SyncUserAPI cfg *config.SyncAPI @@ -54,7 +55,7 @@ func NewPresenceConsumer( nats *nats.Conn, db storage.Database, notifier *notifier.Notifier, - stream types.StreamProvider, + stream streams.StreamProvider, deviceAPI api.SyncUserAPI, ) *PresenceConsumer { return &PresenceConsumer{ diff --git a/syncapi/consumers/receipts.go b/syncapi/consumers/receipts.go index 4379dd134..8aaa65730 100644 --- a/syncapi/consumers/receipts.go +++ b/syncapi/consumers/receipts.go @@ -28,6 +28,7 @@ import ( "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/dendrite/syncapi/notifier" "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/dendrite/syncapi/streams" "github.com/matrix-org/dendrite/syncapi/types" ) @@ -38,7 +39,7 @@ type OutputReceiptEventConsumer struct { durable string topic string db storage.Database - stream types.StreamProvider + stream streams.StreamProvider notifier *notifier.Notifier serverName gomatrixserverlib.ServerName } @@ -51,7 +52,7 @@ func NewOutputReceiptEventConsumer( js nats.JetStreamContext, store storage.Database, notifier *notifier.Notifier, - stream types.StreamProvider, + stream streams.StreamProvider, ) *OutputReceiptEventConsumer { return &OutputReceiptEventConsumer{ ctx: process.Context(), diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index 3756ad75c..e5e8fe293 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -33,6 +33,7 @@ import ( "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/dendrite/syncapi/notifier" "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/dendrite/syncapi/streams" "github.com/matrix-org/dendrite/syncapi/types" ) @@ -45,8 +46,8 @@ type OutputRoomEventConsumer struct { durable string topic string db storage.Database - pduStream types.StreamProvider - inviteStream types.StreamProvider + pduStream streams.StreamProvider + inviteStream streams.StreamProvider notifier *notifier.Notifier fts *fulltext.Search } @@ -58,8 +59,8 @@ func NewOutputRoomEventConsumer( js nats.JetStreamContext, store storage.Database, notifier *notifier.Notifier, - pduStream types.StreamProvider, - inviteStream types.StreamProvider, + pduStream streams.StreamProvider, + inviteStream streams.StreamProvider, rsAPI api.SyncRoomserverAPI, fts *fulltext.Search, ) *OutputRoomEventConsumer { @@ -449,8 +450,14 @@ func (s *OutputRoomEventConsumer) updateStateEvent(event *gomatrixserverlib.Head } stateKey := *event.StateKey() - prevEvent, err := s.db.GetStateEvent( - context.TODO(), event.RoomID(), event.Type(), stateKey, + snapshot, err := s.db.NewDatabaseSnapshot(s.ctx) + if err != nil { + return nil, err + } + defer snapshot.Rollback() // nolint:errcheck + + prevEvent, err := snapshot.GetStateEvent( + s.ctx, event.RoomID(), event.Type(), stateKey, ) if err != nil { return event, err diff --git a/syncapi/consumers/sendtodevice.go b/syncapi/consumers/sendtodevice.go index c0b432256..49d84cca3 100644 --- a/syncapi/consumers/sendtodevice.go +++ b/syncapi/consumers/sendtodevice.go @@ -31,6 +31,7 @@ import ( "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/dendrite/syncapi/notifier" "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/dendrite/syncapi/streams" "github.com/matrix-org/dendrite/syncapi/types" ) @@ -43,7 +44,7 @@ type OutputSendToDeviceEventConsumer struct { db storage.Database keyAPI keyapi.SyncKeyAPI serverName gomatrixserverlib.ServerName // our server name - stream types.StreamProvider + stream streams.StreamProvider notifier *notifier.Notifier } @@ -56,7 +57,7 @@ func NewOutputSendToDeviceEventConsumer( store storage.Database, keyAPI keyapi.SyncKeyAPI, notifier *notifier.Notifier, - stream types.StreamProvider, + stream streams.StreamProvider, ) *OutputSendToDeviceEventConsumer { return &OutputSendToDeviceEventConsumer{ ctx: process.Context(), diff --git a/syncapi/consumers/typing.go b/syncapi/consumers/typing.go index 88db80f8c..67a26239d 100644 --- a/syncapi/consumers/typing.go +++ b/syncapi/consumers/typing.go @@ -24,6 +24,7 @@ import ( "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/dendrite/syncapi/notifier" + "github.com/matrix-org/dendrite/syncapi/streams" "github.com/matrix-org/dendrite/syncapi/types" "github.com/nats-io/nats.go" log "github.com/sirupsen/logrus" @@ -36,7 +37,7 @@ type OutputTypingEventConsumer struct { durable string topic string eduCache *caching.EDUCache - stream types.StreamProvider + stream streams.StreamProvider notifier *notifier.Notifier } @@ -48,7 +49,7 @@ func NewOutputTypingEventConsumer( js nats.JetStreamContext, eduCache *caching.EDUCache, notifier *notifier.Notifier, - stream types.StreamProvider, + stream streams.StreamProvider, ) *OutputTypingEventConsumer { return &OutputTypingEventConsumer{ ctx: process.Context(), diff --git a/syncapi/consumers/userapi.go b/syncapi/consumers/userapi.go index c9b96f788..3c73dc1fc 100644 --- a/syncapi/consumers/userapi.go +++ b/syncapi/consumers/userapi.go @@ -28,6 +28,7 @@ import ( "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/dendrite/syncapi/notifier" "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/dendrite/syncapi/streams" "github.com/matrix-org/dendrite/syncapi/types" ) @@ -40,7 +41,7 @@ type OutputNotificationDataConsumer struct { topic string db storage.Database notifier *notifier.Notifier - stream types.StreamProvider + stream streams.StreamProvider } // NewOutputNotificationDataConsumer creates a new consumer. Call @@ -51,7 +52,7 @@ func NewOutputNotificationDataConsumer( js nats.JetStreamContext, store storage.Database, notifier *notifier.Notifier, - stream types.StreamProvider, + stream streams.StreamProvider, ) *OutputNotificationDataConsumer { s := &OutputNotificationDataConsumer{ ctx: process.Context(), diff --git a/syncapi/internal/history_visibility.go b/syncapi/internal/history_visibility.go index e73c004e5..bbfe19f4c 100644 --- a/syncapi/internal/history_visibility.go +++ b/syncapi/internal/history_visibility.go @@ -100,7 +100,7 @@ func (ev eventVisibility) allowed() (allowed bool) { // Returns the filtered events and an error, if any. func ApplyHistoryVisibilityFilter( ctx context.Context, - syncDB storage.Database, + syncDB storage.DatabaseTransaction, rsAPI api.SyncRoomserverAPI, events []*gomatrixserverlib.HeaderedEvent, alwaysIncludeEventIDs map[string]struct{}, diff --git a/syncapi/notifier/notifier.go b/syncapi/notifier/notifier.go index 87f0d86d7..a8e5bf9ad 100644 --- a/syncapi/notifier/notifier.go +++ b/syncapi/notifier/notifier.go @@ -318,13 +318,20 @@ func (n *Notifier) GetListener(req types.SyncRequest) UserDeviceStreamListener { func (n *Notifier) Load(ctx context.Context, db storage.Database) error { n.lock.Lock() defer n.lock.Unlock() - roomToUsers, err := db.AllJoinedUsersInRooms(ctx) + + snapshot, err := db.NewDatabaseSnapshot(ctx) + if err != nil { + return err + } + defer snapshot.Rollback() // nolint:errcheck + + roomToUsers, err := snapshot.AllJoinedUsersInRooms(ctx) if err != nil { return err } n.setUsersJoinedToRooms(roomToUsers) - roomToPeekingDevices, err := db.AllPeekingDevicesInRooms(ctx) + roomToPeekingDevices, err := snapshot.AllPeekingDevicesInRooms(ctx) if err != nil { return err } @@ -338,7 +345,13 @@ func (n *Notifier) LoadRooms(ctx context.Context, db storage.Database, roomIDs [ n.lock.Lock() defer n.lock.Unlock() - roomToUsers, err := db.AllJoinedUsersInRoom(ctx, roomIDs) + snapshot, err := db.NewDatabaseSnapshot(ctx) + if err != nil { + return err + } + defer snapshot.Rollback() // nolint:errcheck + + roomToUsers, err := snapshot.AllJoinedUsersInRoom(ctx, roomIDs) if err != nil { return err } diff --git a/syncapi/routing/context.go b/syncapi/routing/context.go index 1ebdfe604..1ce34b85a 100644 --- a/syncapi/routing/context.go +++ b/syncapi/routing/context.go @@ -51,6 +51,12 @@ func Context( roomID, eventID string, lazyLoadCache caching.LazyLoadCache, ) util.JSONResponse { + snapshot, err := syncDB.NewDatabaseSnapshot(req.Context()) + if err != nil { + return jsonerror.InternalServerError() + } + defer snapshot.Rollback() // nolint:errcheck + filter, err := parseRoomEventFilter(req) if err != nil { errMsg := "" @@ -97,7 +103,7 @@ func Context( ContainsURL: filter.ContainsURL, } - id, requestedEvent, err := syncDB.SelectContextEvent(ctx, roomID, eventID) + id, requestedEvent, err := snapshot.SelectContextEvent(ctx, roomID, eventID) if err != nil { if err == sql.ErrNoRows { return util.JSONResponse{ @@ -111,7 +117,7 @@ func Context( // verify the user is allowed to see the context for this room/event startTime := time.Now() - filteredEvents, err := internal.ApplyHistoryVisibilityFilter(ctx, syncDB, rsAPI, []*gomatrixserverlib.HeaderedEvent{&requestedEvent}, nil, device.UserID, "context") + filteredEvents, err := internal.ApplyHistoryVisibilityFilter(ctx, snapshot, rsAPI, []*gomatrixserverlib.HeaderedEvent{&requestedEvent}, nil, device.UserID, "context") if err != nil { logrus.WithError(err).Error("unable to apply history visibility filter") return jsonerror.InternalServerError() @@ -127,20 +133,20 @@ func Context( } } - eventsBefore, err := syncDB.SelectContextBeforeEvent(ctx, id, roomID, filter) + eventsBefore, err := snapshot.SelectContextBeforeEvent(ctx, id, roomID, filter) if err != nil && err != sql.ErrNoRows { logrus.WithError(err).Error("unable to fetch before events") return jsonerror.InternalServerError() } - _, eventsAfter, err := syncDB.SelectContextAfterEvent(ctx, id, roomID, filter) + _, eventsAfter, err := snapshot.SelectContextAfterEvent(ctx, id, roomID, filter) if err != nil && err != sql.ErrNoRows { logrus.WithError(err).Error("unable to fetch after events") return jsonerror.InternalServerError() } startTime = time.Now() - eventsBeforeFiltered, eventsAfterFiltered, err := applyHistoryVisibilityOnContextEvents(ctx, syncDB, rsAPI, eventsBefore, eventsAfter, device.UserID) + eventsBeforeFiltered, eventsAfterFiltered, err := applyHistoryVisibilityOnContextEvents(ctx, snapshot, rsAPI, eventsBefore, eventsAfter, device.UserID) if err != nil { logrus.WithError(err).Error("unable to apply history visibility filter") return jsonerror.InternalServerError() @@ -152,7 +158,7 @@ func Context( }).Debug("applied history visibility (context eventsBefore/eventsAfter)") // TODO: Get the actual state at the last event returned by SelectContextAfterEvent - state, err := syncDB.CurrentState(ctx, roomID, &stateFilter, nil) + state, err := snapshot.CurrentState(ctx, roomID, &stateFilter, nil) if err != nil { logrus.WithError(err).Error("unable to fetch current room state") return jsonerror.InternalServerError() @@ -173,7 +179,7 @@ func Context( if len(response.State) > filter.Limit { response.State = response.State[len(response.State)-filter.Limit:] } - start, end, err := getStartEnd(ctx, syncDB, eventsBefore, eventsAfter) + start, end, err := getStartEnd(ctx, snapshot, eventsBefore, eventsAfter) if err == nil { response.End = end.String() response.Start = start.String() @@ -188,7 +194,7 @@ func Context( // by combining the events before and after the context event. Returns the filtered events, // and an error, if any. func applyHistoryVisibilityOnContextEvents( - ctx context.Context, syncDB storage.Database, rsAPI roomserver.SyncRoomserverAPI, + ctx context.Context, snapshot storage.DatabaseTransaction, rsAPI roomserver.SyncRoomserverAPI, eventsBefore, eventsAfter []*gomatrixserverlib.HeaderedEvent, userID string, ) (filteredBefore, filteredAfter []*gomatrixserverlib.HeaderedEvent, err error) { @@ -205,7 +211,7 @@ func applyHistoryVisibilityOnContextEvents( } allEvents := append(eventsBefore, eventsAfter...) - filteredEvents, err := internal.ApplyHistoryVisibilityFilter(ctx, syncDB, rsAPI, allEvents, nil, userID, "context") + filteredEvents, err := internal.ApplyHistoryVisibilityFilter(ctx, snapshot, rsAPI, allEvents, nil, userID, "context") if err != nil { return nil, nil, err } @@ -222,15 +228,15 @@ func applyHistoryVisibilityOnContextEvents( return filteredBefore, filteredAfter, nil } -func getStartEnd(ctx context.Context, syncDB storage.Database, startEvents, endEvents []*gomatrixserverlib.HeaderedEvent) (start, end types.TopologyToken, err error) { +func getStartEnd(ctx context.Context, snapshot storage.DatabaseTransaction, startEvents, endEvents []*gomatrixserverlib.HeaderedEvent) (start, end types.TopologyToken, err error) { if len(startEvents) > 0 { - start, err = syncDB.EventPositionInTopology(ctx, startEvents[0].EventID()) + start, err = snapshot.EventPositionInTopology(ctx, startEvents[0].EventID()) if err != nil { return } } if len(endEvents) > 0 { - end, err = syncDB.EventPositionInTopology(ctx, endEvents[0].EventID()) + end, err = snapshot.EventPositionInTopology(ctx, endEvents[0].EventID()) } return } diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index 03614302c..8f3ed3f5b 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -27,6 +27,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/syncapi/internal" @@ -39,6 +40,7 @@ import ( type messagesReq struct { ctx context.Context db storage.Database + snapshot storage.DatabaseTransaction rsAPI api.SyncRoomserverAPI cfg *config.SyncAPI roomID string @@ -70,6 +72,16 @@ func OnIncomingMessagesRequest( ) util.JSONResponse { var err error + // NewDatabaseTransaction is used here instead of NewDatabaseSnapshot as we + // expect to be able to write to the database in response to a /messages + // request that requires backfilling from the roomserver or federation. + snapshot, err := db.NewDatabaseTransaction(req.Context()) + if err != nil { + return jsonerror.InternalServerError() + } + var succeeded bool + defer sqlutil.EndTransactionWithCheck(snapshot, &succeeded, &err) + // check if the user has already forgotten about this room isForgotten, roomExists, err := checkIsRoomForgotten(req.Context(), roomID, device.UserID, rsAPI) if err != nil { @@ -132,7 +144,7 @@ func OnIncomingMessagesRequest( } } else { fromStream = &streamToken - from, err = db.StreamToTopologicalPosition(req.Context(), roomID, streamToken.PDUPosition, backwardOrdering) + from, err = snapshot.StreamToTopologicalPosition(req.Context(), roomID, streamToken.PDUPosition, backwardOrdering) if err != nil { logrus.WithError(err).Errorf("Failed to get topological position for streaming token %v", streamToken) return jsonerror.InternalServerError() @@ -154,7 +166,7 @@ func OnIncomingMessagesRequest( JSON: jsonerror.InvalidArgumentValue("Invalid to parameter: " + err.Error()), } } else { - to, err = db.StreamToTopologicalPosition(req.Context(), roomID, streamToken.PDUPosition, !backwardOrdering) + to, err = snapshot.StreamToTopologicalPosition(req.Context(), roomID, streamToken.PDUPosition, !backwardOrdering) if err != nil { logrus.WithError(err).Errorf("Failed to get topological position for streaming token %v", streamToken) return jsonerror.InternalServerError() @@ -165,7 +177,7 @@ func OnIncomingMessagesRequest( // If "to" isn't provided, it defaults to either the earliest stream // position (if we're going backward) or to the latest one (if we're // going forward). - to, err = setToDefault(req.Context(), db, backwardOrdering, roomID) + to, err = setToDefault(req.Context(), snapshot, backwardOrdering, roomID) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("setToDefault failed") return jsonerror.InternalServerError() @@ -186,6 +198,7 @@ func OnIncomingMessagesRequest( mReq := messagesReq{ ctx: req.Context(), db: db, + snapshot: snapshot, rsAPI: rsAPI, cfg: cfg, roomID: roomID, @@ -217,7 +230,7 @@ func OnIncomingMessagesRequest( Start: start.String(), End: end.String(), } - res.applyLazyLoadMembers(req.Context(), db, roomID, device, filter.LazyLoadMembers, lazyLoadCache) + res.applyLazyLoadMembers(req.Context(), snapshot, roomID, device, filter.LazyLoadMembers, lazyLoadCache) // If we didn't return any events, set the end to an empty string, so it will be omitted // in the response JSON. @@ -229,6 +242,7 @@ func OnIncomingMessagesRequest( } // Respond with the events. + succeeded = true return util.JSONResponse{ Code: http.StatusOK, JSON: res, @@ -239,7 +253,7 @@ func OnIncomingMessagesRequest( // LazyLoadMembers enabled. func (m *messagesResp) applyLazyLoadMembers( ctx context.Context, - db storage.Database, + db storage.DatabaseTransaction, roomID string, device *userapi.Device, lazyLoad bool, @@ -292,7 +306,7 @@ func (r *messagesReq) retrieveEvents() ( end types.TopologyToken, err error, ) { // Retrieve the events from the local database. - streamEvents, err := r.db.GetEventsInTopologicalRange(r.ctx, r.from, r.to, r.roomID, r.filter, r.backwardOrdering) + streamEvents, err := r.snapshot.GetEventsInTopologicalRange(r.ctx, r.from, r.to, r.roomID, r.filter, r.backwardOrdering) if err != nil { err = fmt.Errorf("GetEventsInRange: %w", err) return @@ -348,7 +362,7 @@ func (r *messagesReq) retrieveEvents() ( // Apply room history visibility filter startTime := time.Now() - filteredEvents, err := internal.ApplyHistoryVisibilityFilter(r.ctx, r.db, r.rsAPI, events, nil, r.device.UserID, "messages") + filteredEvents, err := internal.ApplyHistoryVisibilityFilter(r.ctx, r.snapshot, r.rsAPI, events, nil, r.device.UserID, "messages") logrus.WithFields(logrus.Fields{ "duration": time.Since(startTime), "room_id": r.roomID, @@ -366,7 +380,7 @@ func (r *messagesReq) getStartEnd(events []*gomatrixserverlib.HeaderedEvent) (st // else to go. This seems to fix Element iOS from looping on /messages endlessly. end = types.TopologyToken{} } else { - end, err = r.db.EventPositionInTopology( + end, err = r.snapshot.EventPositionInTopology( r.ctx, events[0].EventID(), ) // A stream/topological position is a cursor located between two events. @@ -378,7 +392,7 @@ func (r *messagesReq) getStartEnd(events []*gomatrixserverlib.HeaderedEvent) (st } } else { start = *r.from - end, err = r.db.EventPositionInTopology( + end, err = r.snapshot.EventPositionInTopology( r.ctx, events[len(events)-1].EventID(), ) } @@ -399,7 +413,7 @@ func (r *messagesReq) getStartEnd(events []*gomatrixserverlib.HeaderedEvent) (st func (r *messagesReq) handleEmptyEventsSlice() ( events []*gomatrixserverlib.HeaderedEvent, err error, ) { - backwardExtremities, err := r.db.BackwardExtremitiesForRoom(r.ctx, r.roomID) + backwardExtremities, err := r.snapshot.BackwardExtremitiesForRoom(r.ctx, r.roomID) // Check if we have backward extremities for this room. if len(backwardExtremities) > 0 { @@ -443,7 +457,7 @@ func (r *messagesReq) handleNonEmptyEventsSlice(streamEvents []types.StreamEvent } // Check if the slice contains a backward extremity. - backwardExtremities, err := r.db.BackwardExtremitiesForRoom(r.ctx, r.roomID) + backwardExtremities, err := r.snapshot.BackwardExtremitiesForRoom(r.ctx, r.roomID) if err != nil { return } @@ -463,7 +477,7 @@ func (r *messagesReq) handleNonEmptyEventsSlice(streamEvents []types.StreamEvent } // Append the events ve previously retrieved locally. - events = append(events, r.db.StreamEventsToEvents(nil, streamEvents)...) + events = append(events, r.snapshot.StreamEventsToEvents(nil, streamEvents)...) sort.Sort(eventsByDepth(events)) return @@ -553,7 +567,7 @@ func (r *messagesReq) backfill(roomID string, backwardsExtremities map[string][] // Returns an error if there was an issue with retrieving the latest position // from the database func setToDefault( - ctx context.Context, db storage.Database, backwardOrdering bool, + ctx context.Context, snapshot storage.DatabaseTransaction, backwardOrdering bool, roomID string, ) (to types.TopologyToken, err error) { if backwardOrdering { @@ -561,7 +575,7 @@ func setToDefault( // this is because Database.GetEventsInTopologicalRange is exclusive of the lower-bound. to = types.TopologyToken{} } else { - to, err = db.MaxTopologicalPosition(ctx, roomID) + to, err = snapshot.MaxTopologicalPosition(ctx, roomID) } return diff --git a/syncapi/routing/search.go b/syncapi/routing/search.go index 341efeb14..bac534a2c 100644 --- a/syncapi/routing/search.go +++ b/syncapi/routing/search.go @@ -61,8 +61,14 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts searchReq.SearchCategories.RoomEvents.Filter.Limit = 5 } + snapshot, err := syncDB.NewDatabaseSnapshot(req.Context()) + if err != nil { + return jsonerror.InternalServerError() + } + defer snapshot.Rollback() // nolint:errcheck + // only search rooms the user is actually joined to - joinedRooms, err := syncDB.RoomIDsWithMembership(ctx, device.UserID, "join") + joinedRooms, err := snapshot.RoomIDsWithMembership(ctx, device.UserID, "join") if err != nil { return jsonerror.InternalServerError() } @@ -161,12 +167,12 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts stateForRooms := make(map[string][]gomatrixserverlib.ClientEvent) for _, event := range evs { - eventsBefore, eventsAfter, err := contextEvents(ctx, syncDB, event, roomFilter, searchReq) + eventsBefore, eventsAfter, err := contextEvents(ctx, snapshot, event, roomFilter, searchReq) if err != nil { logrus.WithError(err).Error("failed to get context events") return jsonerror.InternalServerError() } - startToken, endToken, err := getStartEnd(ctx, syncDB, eventsBefore, eventsAfter) + startToken, endToken, err := getStartEnd(ctx, snapshot, eventsBefore, eventsAfter) if err != nil { logrus.WithError(err).Error("failed to get start/end") return jsonerror.InternalServerError() @@ -176,7 +182,7 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts for _, ev := range append(eventsBefore, eventsAfter...) { profile, ok := knownUsersProfiles[event.Sender()] if !ok { - stateEvent, err := syncDB.GetStateEvent(ctx, ev.RoomID(), gomatrixserverlib.MRoomMember, ev.Sender()) + stateEvent, err := snapshot.GetStateEvent(ctx, ev.RoomID(), gomatrixserverlib.MRoomMember, ev.Sender()) if err != nil { logrus.WithError(err).WithField("user_id", event.Sender()).Warn("failed to query userprofile") continue @@ -209,7 +215,7 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts groups[event.RoomID()] = roomGroup if _, ok := stateForRooms[event.RoomID()]; searchReq.SearchCategories.RoomEvents.IncludeState && !ok { stateFilter := gomatrixserverlib.DefaultStateFilter() - state, err := syncDB.CurrentState(ctx, event.RoomID(), &stateFilter, nil) + state, err := snapshot.CurrentState(ctx, event.RoomID(), &stateFilter, nil) if err != nil { logrus.WithError(err).Error("unable to get current state") return jsonerror.InternalServerError() @@ -252,24 +258,24 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts // contextEvents returns the events around a given eventID func contextEvents( ctx context.Context, - syncDB storage.Database, + snapshot storage.DatabaseTransaction, event *gomatrixserverlib.HeaderedEvent, roomFilter *gomatrixserverlib.RoomEventFilter, searchReq SearchRequest, ) ([]*gomatrixserverlib.HeaderedEvent, []*gomatrixserverlib.HeaderedEvent, error) { - id, _, err := syncDB.SelectContextEvent(ctx, event.RoomID(), event.EventID()) + id, _, err := snapshot.SelectContextEvent(ctx, event.RoomID(), event.EventID()) if err != nil { logrus.WithError(err).Error("failed to query context event") return nil, nil, err } roomFilter.Limit = searchReq.SearchCategories.RoomEvents.EventContext.BeforeLimit - eventsBefore, err := syncDB.SelectContextBeforeEvent(ctx, id, event.RoomID(), roomFilter) + eventsBefore, err := snapshot.SelectContextBeforeEvent(ctx, id, event.RoomID(), roomFilter) if err != nil { logrus.WithError(err).Error("failed to query before context event") return nil, nil, err } roomFilter.Limit = searchReq.SearchCategories.RoomEvents.EventContext.AfterLimit - _, eventsAfter, err := syncDB.SelectContextAfterEvent(ctx, id, event.RoomID(), roomFilter) + _, eventsAfter, err := snapshot.SelectContextAfterEvent(ctx, id, event.RoomID(), roomFilter) if err != nil { logrus.WithError(err).Error("failed to query after context event") return nil, nil, err diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index dd03365e9..3732e43fb 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -17,19 +17,17 @@ package storage import ( "context" - "github.com/matrix-org/dendrite/internal/eventutil" - "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/syncapi/storage/shared" "github.com/matrix-org/dendrite/syncapi/types" userapi "github.com/matrix-org/dendrite/userapi/api" ) -type Database interface { - Presence +type DatabaseTransaction interface { SharedUsers - Notifications MaxStreamPositionForPDUs(ctx context.Context) (types.StreamPosition, error) MaxStreamPositionForReceipts(ctx context.Context) (types.StreamPosition, error) @@ -37,6 +35,7 @@ type Database interface { MaxStreamPositionForAccountData(ctx context.Context) (types.StreamPosition, error) MaxStreamPositionForSendToDeviceMessages(ctx context.Context) (types.StreamPosition, error) MaxStreamPositionForNotificationData(ctx context.Context) (types.StreamPosition, error) + MaxStreamPositionForPresence(ctx context.Context) (types.StreamPosition, error) CurrentState(ctx context.Context, roomID string, stateFilterPart *gomatrixserverlib.StateFilter, excludeEventIDs []string) ([]*gomatrixserverlib.HeaderedEvent, error) GetStateDeltasForFullStateSync(ctx context.Context, device *userapi.Device, r types.Range, userID string, stateFilter *gomatrixserverlib.StateFilter) ([]types.StateDelta, []string, error) @@ -44,23 +43,77 @@ type Database interface { RoomIDsWithMembership(ctx context.Context, userID string, membership string) ([]string, error) MembershipCount(ctx context.Context, roomID, membership string, pos types.StreamPosition) (int, error) GetRoomHeroes(ctx context.Context, roomID, userID string, memberships []string) ([]string, error) - RecentEvents(ctx context.Context, roomID string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter, chronologicalOrder bool, onlySyncEvents bool) ([]types.StreamEvent, bool, error) - GetBackwardTopologyPos(ctx context.Context, events []types.StreamEvent) (types.TopologyToken, error) PositionInTopology(ctx context.Context, eventID string) (pos types.StreamPosition, spos types.StreamPosition, err error) - - InviteEventsInRange(ctx context.Context, targetUserID string, r types.Range) (map[string]*gomatrixserverlib.HeaderedEvent, map[string]*gomatrixserverlib.HeaderedEvent, error) + InviteEventsInRange(ctx context.Context, targetUserID string, r types.Range) (map[string]*gomatrixserverlib.HeaderedEvent, map[string]*gomatrixserverlib.HeaderedEvent, types.StreamPosition, error) PeeksInRange(ctx context.Context, userID, deviceID string, r types.Range) (peeks []types.Peek, err error) RoomReceiptsAfter(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) (types.StreamPosition, []types.OutputReceiptEvent, error) - // AllJoinedUsersInRooms returns a map of room ID to a list of all joined user IDs. AllJoinedUsersInRooms(ctx context.Context) (map[string][]string, error) // AllJoinedUsersInRoom returns a map of room ID to a list of all joined user IDs for a given room. AllJoinedUsersInRoom(ctx context.Context, roomIDs []string) (map[string][]string, error) - // AllPeekingDevicesInRooms returns a map of room ID to a list of all peeking devices. AllPeekingDevicesInRooms(ctx context.Context) (map[string][]types.PeekingDevice, error) + // Events lookups a list of event by their event ID. + // Returns a list of events matching the requested IDs found in the database. + // If an event is not found in the database then it will be omitted from the list. + // Returns an error if there was a problem talking with the database. + // Does not include any transaction IDs in the returned events. + Events(ctx context.Context, eventIDs []string) ([]*gomatrixserverlib.HeaderedEvent, error) + // GetStateEvent returns the Matrix state event of a given type for a given room with a given state key + // If no event could be found, returns nil + // If there was an issue during the retrieval, returns an error + GetStateEvent(ctx context.Context, roomID, evType, stateKey string) (*gomatrixserverlib.HeaderedEvent, error) + // GetStateEventsForRoom fetches the state events for a given room. + // Returns an empty slice if no state events could be found for this room. + // Returns an error if there was an issue with the retrieval. + GetStateEventsForRoom(ctx context.Context, roomID string, stateFilterPart *gomatrixserverlib.StateFilter) (stateEvents []*gomatrixserverlib.HeaderedEvent, err error) + // GetAccountDataInRange returns all account data for a given user inserted or + // updated between two given positions + // Returns a map following the format data[roomID] = []dataTypes + // If no data is retrieved, returns an empty map + // If there was an issue with the retrieval, returns an error + GetAccountDataInRange(ctx context.Context, userID string, r types.Range, accountDataFilterPart *gomatrixserverlib.EventFilter) (map[string][]string, types.StreamPosition, error) + // GetEventsInTopologicalRange retrieves all of the events on a given ordering using the given extremities and limit. If backwardsOrdering is true, the most recent event must be first, else last. + GetEventsInTopologicalRange(ctx context.Context, from, to *types.TopologyToken, roomID string, filter *gomatrixserverlib.RoomEventFilter, backwardOrdering bool) (events []types.StreamEvent, err error) + // EventPositionInTopology returns the depth and stream position of the given event. + EventPositionInTopology(ctx context.Context, eventID string) (types.TopologyToken, error) + // BackwardExtremitiesForRoom returns a map of backwards extremity event ID to a list of its prev_events. + BackwardExtremitiesForRoom(ctx context.Context, roomID string) (backwardExtremities map[string][]string, err error) + // MaxTopologicalPosition returns the highest topological position for a given room. + MaxTopologicalPosition(ctx context.Context, roomID string) (types.TopologyToken, error) + // StreamEventsToEvents converts streamEvent to Event. If device is non-nil and + // matches the streamevent.transactionID device then the transaction ID gets + // added to the unsigned section of the output event. + StreamEventsToEvents(device *userapi.Device, in []types.StreamEvent) []*gomatrixserverlib.HeaderedEvent + // SendToDeviceUpdatesForSync returns a list of send-to-device updates. It returns the + // relevant events within the given ranges for the supplied user ID and device ID. + SendToDeviceUpdatesForSync(ctx context.Context, userID, deviceID string, from, to types.StreamPosition) (pos types.StreamPosition, events []types.SendToDeviceEvent, err error) + // GetRoomReceipts gets all receipts for a given roomID + GetRoomReceipts(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) ([]types.OutputReceiptEvent, error) + SelectContextEvent(ctx context.Context, roomID, eventID string) (int, gomatrixserverlib.HeaderedEvent, error) + SelectContextBeforeEvent(ctx context.Context, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) ([]*gomatrixserverlib.HeaderedEvent, error) + SelectContextAfterEvent(ctx context.Context, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) (int, []*gomatrixserverlib.HeaderedEvent, error) + StreamToTopologicalPosition(ctx context.Context, roomID string, streamPos types.StreamPosition, backwardOrdering bool) (types.TopologyToken, error) + IgnoresForUser(ctx context.Context, userID string) (*types.IgnoredUsers, error) + // SelectMembershipForUser returns the membership of the user before and including the given position. If no membership can be found + // returns "leave", the topological position and no error. If an error occurs, other than sql.ErrNoRows, returns that and an empty + // string as the membership. + SelectMembershipForUser(ctx context.Context, roomID, userID string, pos int64) (membership string, topologicalPos int, err error) + // getUserUnreadNotificationCountsForRooms returns the unread notifications for the given rooms + GetUserUnreadNotificationCountsForRooms(ctx context.Context, userID string, roomIDs map[string]string) (map[string]*eventutil.NotificationData, error) + GetPresence(ctx context.Context, userID string) (*types.PresenceInternal, error) + PresenceAfter(ctx context.Context, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (map[string]*types.PresenceInternal, error) +} + +type Database interface { + Presence + Notifications + + NewDatabaseSnapshot(ctx context.Context) (*shared.DatabaseTransaction, error) + NewDatabaseTransaction(ctx context.Context) (*shared.DatabaseTransaction, error) + // Events lookups a list of event by their event ID. // Returns a list of events matching the requested IDs found in the database. // If an event is not found in the database then it will be omitted from the list. @@ -77,20 +130,6 @@ type Database interface { // PurgeRoomState completely purges room state from the sync API. This is done when // receiving an output event that completely resets the state. PurgeRoomState(ctx context.Context, roomID string) error - // GetStateEvent returns the Matrix state event of a given type for a given room with a given state key - // If no event could be found, returns nil - // If there was an issue during the retrieval, returns an error - GetStateEvent(ctx context.Context, roomID, evType, stateKey string) (*gomatrixserverlib.HeaderedEvent, error) - // GetStateEventsForRoom fetches the state events for a given room. - // Returns an empty slice if no state events could be found for this room. - // Returns an error if there was an issue with the retrieval. - GetStateEventsForRoom(ctx context.Context, roomID string, stateFilterPart *gomatrixserverlib.StateFilter) (stateEvents []*gomatrixserverlib.HeaderedEvent, err error) - // GetAccountDataInRange returns all account data for a given user inserted or - // updated between two given positions - // Returns a map following the format data[roomID] = []dataTypes - // If no data is retrieved, returns an empty map - // If there was an issue with the retrieval, returns an error - GetAccountDataInRange(ctx context.Context, userID string, r types.Range, accountDataFilterPart *gomatrixserverlib.EventFilter) (map[string][]string, types.StreamPosition, error) // UpsertAccountData keeps track of new or updated account data, by saving the type // of the new/updated data, and the user ID and room ID the data is related to (empty) // room ID means the data isn't specific to any room) @@ -114,21 +153,6 @@ type Database interface { // DeletePeek deletes all peeks for a given room by a given user // Returns an error if there was a problem communicating with the database. DeletePeeks(ctx context.Context, RoomID, UserID string) (types.StreamPosition, error) - // GetEventsInTopologicalRange retrieves all of the events on a given ordering using the given extremities and limit. If backwardsOrdering is true, the most recent event must be first, else last. - GetEventsInTopologicalRange(ctx context.Context, from, to *types.TopologyToken, roomID string, filter *gomatrixserverlib.RoomEventFilter, backwardOrdering bool) (events []types.StreamEvent, err error) - // EventPositionInTopology returns the depth and stream position of the given event. - EventPositionInTopology(ctx context.Context, eventID string) (types.TopologyToken, error) - // BackwardExtremitiesForRoom returns a map of backwards extremity event ID to a list of its prev_events. - BackwardExtremitiesForRoom(ctx context.Context, roomID string) (backwardExtremities map[string][]string, err error) - // MaxTopologicalPosition returns the highest topological position for a given room. - MaxTopologicalPosition(ctx context.Context, roomID string) (types.TopologyToken, error) - // StreamEventsToEvents converts streamEvent to Event. If device is non-nil and - // matches the streamevent.transactionID device then the transaction ID gets - // added to the unsigned section of the output event. - StreamEventsToEvents(device *userapi.Device, in []types.StreamEvent) []*gomatrixserverlib.HeaderedEvent - // SendToDeviceUpdatesForSync returns a list of send-to-device updates. It returns the - // relevant events within the given ranges for the supplied user ID and device ID. - SendToDeviceUpdatesForSync(ctx context.Context, userID, deviceID string, from, to types.StreamPosition) (pos types.StreamPosition, events []types.SendToDeviceEvent, err error) // StoreNewSendForDeviceMessage stores a new send-to-device event for a user's device. StoreNewSendForDeviceMessage(ctx context.Context, userID, deviceID string, event gomatrixserverlib.SendToDeviceEvent) (types.StreamPosition, error) // CleanSendToDeviceUpdates removes all send-to-device messages BEFORE the specified @@ -146,29 +170,13 @@ type Database interface { RedactEvent(ctx context.Context, redactedEventID string, redactedBecause *gomatrixserverlib.HeaderedEvent) error // StoreReceipt stores new receipt events StoreReceipt(ctx context.Context, roomId, receiptType, userId, eventId string, timestamp gomatrixserverlib.Timestamp) (pos types.StreamPosition, err error) - // GetRoomReceipts gets all receipts for a given roomID - GetRoomReceipts(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) ([]types.OutputReceiptEvent, error) - - SelectContextEvent(ctx context.Context, roomID, eventID string) (int, gomatrixserverlib.HeaderedEvent, error) - SelectContextBeforeEvent(ctx context.Context, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) ([]*gomatrixserverlib.HeaderedEvent, error) - SelectContextAfterEvent(ctx context.Context, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) (int, []*gomatrixserverlib.HeaderedEvent, error) - - StreamToTopologicalPosition(ctx context.Context, roomID string, streamPos types.StreamPosition, backwardOrdering bool) (types.TopologyToken, error) - - IgnoresForUser(ctx context.Context, userID string) (*types.IgnoredUsers, error) UpdateIgnoresForUser(ctx context.Context, userID string, ignores *types.IgnoredUsers) error - // SelectMembershipForUser returns the membership of the user before and including the given position. If no membership can be found - // returns "leave", the topological position and no error. If an error occurs, other than sql.ErrNoRows, returns that and an empty - // string as the membership. - SelectMembershipForUser(ctx context.Context, roomID, userID string, pos int64) (membership string, topologicalPos int, err error) ReIndex(ctx context.Context, limit, afterID int64) (map[int64]gomatrixserverlib.HeaderedEvent, error) } type Presence interface { - UpdatePresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, lastActiveTS gomatrixserverlib.Timestamp, fromSync bool) (types.StreamPosition, error) GetPresence(ctx context.Context, userID string) (*types.PresenceInternal, error) - PresenceAfter(ctx context.Context, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (map[string]*types.PresenceInternal, error) - MaxStreamPositionForPresence(ctx context.Context) (types.StreamPosition, error) + UpdatePresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, lastActiveTS gomatrixserverlib.Timestamp, fromSync bool) (types.StreamPosition, error) } type SharedUsers interface { @@ -179,7 +187,4 @@ type SharedUsers interface { type Notifications interface { // UpsertRoomUnreadNotificationCounts updates the notification statistics about a (user, room) key. UpsertRoomUnreadNotificationCounts(ctx context.Context, userID, roomID string, notificationCount, highlightCount int) (types.StreamPosition, error) - - // getUserUnreadNotificationCountsForRooms returns the unread notifications for the given rooms - GetUserUnreadNotificationCountsForRooms(ctx context.Context, userID string, roomIDs map[string]string) (map[string]*eventutil.NotificationData, error) } diff --git a/syncapi/storage/postgres/invites_table.go b/syncapi/storage/postgres/invites_table.go index f87ccf965..aada70d5e 100644 --- a/syncapi/storage/postgres/invites_table.go +++ b/syncapi/storage/postgres/invites_table.go @@ -55,7 +55,7 @@ const deleteInviteEventSQL = "" + "UPDATE syncapi_invite_events SET deleted=TRUE, id=nextval('syncapi_stream_id') WHERE event_id = $1 AND deleted=FALSE RETURNING id" const selectInviteEventsInRangeSQL = "" + - "SELECT room_id, headered_event_json, deleted FROM syncapi_invite_events" + + "SELECT id, room_id, headered_event_json, deleted FROM syncapi_invite_events" + " WHERE target_user_id = $1 AND id > $2 AND id <= $3" + " ORDER BY id DESC" @@ -121,23 +121,28 @@ func (s *inviteEventsStatements) DeleteInviteEvent( // active invites for the target user ID in the supplied range. func (s *inviteEventsStatements) SelectInviteEventsInRange( ctx context.Context, txn *sql.Tx, targetUserID string, r types.Range, -) (map[string]*gomatrixserverlib.HeaderedEvent, map[string]*gomatrixserverlib.HeaderedEvent, error) { +) (map[string]*gomatrixserverlib.HeaderedEvent, map[string]*gomatrixserverlib.HeaderedEvent, types.StreamPosition, error) { + var lastPos types.StreamPosition stmt := sqlutil.TxStmt(txn, s.selectInviteEventsInRangeStmt) rows, err := stmt.QueryContext(ctx, targetUserID, r.Low(), r.High()) if err != nil { - return nil, nil, err + return nil, nil, lastPos, err } defer internal.CloseAndLogIfError(ctx, rows, "selectInviteEventsInRange: rows.close() failed") result := map[string]*gomatrixserverlib.HeaderedEvent{} retired := map[string]*gomatrixserverlib.HeaderedEvent{} for rows.Next() { var ( + id types.StreamPosition roomID string eventJSON []byte deleted bool ) - if err = rows.Scan(&roomID, &eventJSON, &deleted); err != nil { - return nil, nil, err + if err = rows.Scan(&id, &roomID, &eventJSON, &deleted); err != nil { + return nil, nil, lastPos, err + } + if id > lastPos { + lastPos = id } // if we have seen this room before, it has a higher stream position and hence takes priority @@ -150,7 +155,7 @@ func (s *inviteEventsStatements) SelectInviteEventsInRange( var event *gomatrixserverlib.HeaderedEvent if err := json.Unmarshal(eventJSON, &event); err != nil { - return nil, nil, err + return nil, nil, lastPos, err } if deleted { @@ -159,7 +164,10 @@ func (s *inviteEventsStatements) SelectInviteEventsInRange( result[roomID] = event } } - return result, retired, rows.Err() + if lastPos == 0 { + lastPos = r.To + } + return result, retired, lastPos, rows.Err() } func (s *inviteEventsStatements) SelectMaxInviteID( diff --git a/syncapi/storage/shared/storage_consumer.go b/syncapi/storage/shared/storage_consumer.go new file mode 100644 index 000000000..fb3b295e9 --- /dev/null +++ b/syncapi/storage/shared/storage_consumer.go @@ -0,0 +1,586 @@ +// Copyright 2020 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shared + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + + "github.com/tidwall/gjson" + + userapi "github.com/matrix-org/dendrite/userapi/api" + + "github.com/matrix-org/gomatrixserverlib" + "github.com/sirupsen/logrus" + + "github.com/matrix-org/dendrite/internal/eventutil" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/syncapi/storage/tables" + "github.com/matrix-org/dendrite/syncapi/types" +) + +// Database is a temporary struct until we have made syncserver.go the same for both pq/sqlite +// For now this contains the shared functions +type Database struct { + DB *sql.DB + Writer sqlutil.Writer + Invites tables.Invites + Peeks tables.Peeks + AccountData tables.AccountData + OutputEvents tables.Events + Topology tables.Topology + CurrentRoomState tables.CurrentRoomState + BackwardExtremities tables.BackwardsExtremities + SendToDevice tables.SendToDevice + Filter tables.Filter + Receipts tables.Receipts + Memberships tables.Memberships + NotificationData tables.NotificationData + Ignores tables.Ignores + Presence tables.Presence +} + +func (d *Database) NewDatabaseSnapshot(ctx context.Context) (*DatabaseTransaction, error) { + return d.NewDatabaseTransaction(ctx) + + /* + TODO: Repeatable read is probably the right thing to do here, + but it seems to cause some problems with the invite tests, so + need to investigate that further. + + txn, err := d.DB.BeginTx(ctx, &sql.TxOptions{ + // Set the isolation level so that we see a snapshot of the database. + // In PostgreSQL repeatable read transactions will see a snapshot taken + // at the first query, and since the transaction is read-only it can't + // run into any serialisation errors. + // https://www.postgresql.org/docs/9.5/static/transaction-iso.html#XACT-REPEATABLE-READ + Isolation: sql.LevelRepeatableRead, + ReadOnly: true, + }) + if err != nil { + return nil, err + } + return &DatabaseTransaction{ + Database: d, + txn: txn, + }, nil + */ +} + +func (d *Database) NewDatabaseTransaction(ctx context.Context) (*DatabaseTransaction, error) { + txn, err := d.DB.BeginTx(ctx, nil) + if err != nil { + return nil, err + } + return &DatabaseTransaction{ + Database: d, + txn: txn, + }, nil +} + +func (d *Database) Events(ctx context.Context, eventIDs []string) ([]*gomatrixserverlib.HeaderedEvent, error) { + streamEvents, err := d.OutputEvents.SelectEvents(ctx, nil, eventIDs, nil, false) + if err != nil { + return nil, err + } + + // We don't include a device here as we only include transaction IDs in + // incremental syncs. + return d.StreamEventsToEvents(nil, streamEvents), nil +} + +// AddInviteEvent stores a new invite event for a user. +// If the invite was successfully stored this returns the stream ID it was stored at. +// Returns an error if there was a problem communicating with the database. +func (d *Database) AddInviteEvent( + ctx context.Context, inviteEvent *gomatrixserverlib.HeaderedEvent, +) (sp types.StreamPosition, err error) { + _ = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + sp, err = d.Invites.InsertInviteEvent(ctx, txn, inviteEvent) + return err + }) + return +} + +// RetireInviteEvent removes an old invite event from the database. +// Returns an error if there was a problem communicating with the database. +func (d *Database) RetireInviteEvent( + ctx context.Context, inviteEventID string, +) (sp types.StreamPosition, err error) { + _ = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + sp, err = d.Invites.DeleteInviteEvent(ctx, txn, inviteEventID) + return err + }) + return +} + +// AddPeek tracks the fact that a user has started peeking. +// If the peek was successfully stored this returns the stream ID it was stored at. +// Returns an error if there was a problem communicating with the database. +func (d *Database) AddPeek( + ctx context.Context, roomID, userID, deviceID string, +) (sp types.StreamPosition, err error) { + err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + sp, err = d.Peeks.InsertPeek(ctx, txn, roomID, userID, deviceID) + return err + }) + return +} + +// DeletePeek tracks the fact that a user has stopped peeking from the specified +// device. If the peeks was successfully deleted this returns the stream ID it was +// stored at. Returns an error if there was a problem communicating with the database. +func (d *Database) DeletePeek( + ctx context.Context, roomID, userID, deviceID string, +) (sp types.StreamPosition, err error) { + err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + sp, err = d.Peeks.DeletePeek(ctx, txn, roomID, userID, deviceID) + return err + }) + if err == sql.ErrNoRows { + sp = 0 + err = nil + } + return +} + +// DeletePeeks tracks the fact that a user has stopped peeking from all devices +// If the peeks was successfully deleted this returns the stream ID it was stored at. +// Returns an error if there was a problem communicating with the database. +func (d *Database) DeletePeeks( + ctx context.Context, roomID, userID string, +) (sp types.StreamPosition, err error) { + err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + sp, err = d.Peeks.DeletePeeks(ctx, txn, roomID, userID) + return err + }) + if err == sql.ErrNoRows { + sp = 0 + err = nil + } + return +} + +// UpsertAccountData keeps track of new or updated account data, by saving the type +// of the new/updated data, and the user ID and room ID the data is related to (empty) +// room ID means the data isn't specific to any room) +// If no data with the given type, user ID and room ID exists in the database, +// creates a new row, else update the existing one +// Returns an error if there was an issue with the upsert +func (d *Database) UpsertAccountData( + ctx context.Context, userID, roomID, dataType string, +) (sp types.StreamPosition, err error) { + err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + sp, err = d.AccountData.InsertAccountData(ctx, txn, userID, roomID, dataType) + return err + }) + return +} + +func (d *Database) StreamEventsToEvents(device *userapi.Device, in []types.StreamEvent) []*gomatrixserverlib.HeaderedEvent { + out := make([]*gomatrixserverlib.HeaderedEvent, len(in)) + for i := 0; i < len(in); i++ { + out[i] = in[i].HeaderedEvent + if device != nil && in[i].TransactionID != nil { + if device.UserID == in[i].Sender() && device.SessionID == in[i].TransactionID.SessionID { + err := out[i].SetUnsignedField( + "transaction_id", in[i].TransactionID.TransactionID, + ) + if err != nil { + logrus.WithFields(logrus.Fields{ + "event_id": out[i].EventID(), + }).WithError(err).Warnf("Failed to add transaction ID to event") + } + } + } + } + return out +} + +// handleBackwardExtremities adds this event as a backwards extremity if and only if we do not have all of +// the events listed in the event's 'prev_events'. This function also updates the backwards extremities table +// to account for the fact that the given event is no longer a backwards extremity, but may be marked as such. +// This function should always be called within a sqlutil.Writer for safety in SQLite. +func (d *Database) handleBackwardExtremities(ctx context.Context, txn *sql.Tx, ev *gomatrixserverlib.HeaderedEvent) error { + if err := d.BackwardExtremities.DeleteBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID()); err != nil { + return err + } + + // Check if we have all of the event's previous events. If an event is + // missing, add it to the room's backward extremities. + prevEvents, err := d.OutputEvents.SelectEvents(ctx, txn, ev.PrevEventIDs(), nil, false) + if err != nil { + return err + } + var found bool + for _, eID := range ev.PrevEventIDs() { + found = false + for _, prevEv := range prevEvents { + if eID == prevEv.EventID() { + found = true + } + } + + // If the event is missing, consider it a backward extremity. + if !found { + if err = d.BackwardExtremities.InsertsBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID(), eID); err != nil { + return err + } + } + } + + return nil +} + +func (d *Database) PurgeRoomState( + ctx context.Context, roomID string, +) error { + return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + // If the event is a create event then we'll delete all of the existing + // data for the room. The only reason that a create event would be replayed + // to us in this way is if we're about to receive the entire room state. + if err := d.CurrentRoomState.DeleteRoomStateForRoom(ctx, txn, roomID); err != nil { + return fmt.Errorf("d.CurrentRoomState.DeleteRoomStateForRoom: %w", err) + } + return nil + }) +} + +func (d *Database) WriteEvent( + ctx context.Context, + ev *gomatrixserverlib.HeaderedEvent, + addStateEvents []*gomatrixserverlib.HeaderedEvent, + addStateEventIDs, removeStateEventIDs []string, + transactionID *api.TransactionID, excludeFromSync bool, + historyVisibility gomatrixserverlib.HistoryVisibility, +) (pduPosition types.StreamPosition, returnErr error) { + returnErr = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + var err error + ev.Visibility = historyVisibility + pos, err := d.OutputEvents.InsertEvent( + ctx, txn, ev, addStateEventIDs, removeStateEventIDs, transactionID, excludeFromSync, historyVisibility, + ) + if err != nil { + return fmt.Errorf("d.OutputEvents.InsertEvent: %w", err) + } + pduPosition = pos + var topoPosition types.StreamPosition + if topoPosition, err = d.Topology.InsertEventInTopology(ctx, txn, ev, pos); err != nil { + return fmt.Errorf("d.Topology.InsertEventInTopology: %w", err) + } + + if err = d.handleBackwardExtremities(ctx, txn, ev); err != nil { + return fmt.Errorf("d.handleBackwardExtremities: %w", err) + } + + if len(addStateEvents) == 0 && len(removeStateEventIDs) == 0 { + // Nothing to do, the event may have just been a message event. + return nil + } + for i := range addStateEvents { + addStateEvents[i].Visibility = historyVisibility + } + return d.updateRoomState(ctx, txn, removeStateEventIDs, addStateEvents, pduPosition, topoPosition) + }) + + return pduPosition, returnErr +} + +// This function should always be called within a sqlutil.Writer for safety in SQLite. +func (d *Database) updateRoomState( + ctx context.Context, txn *sql.Tx, + removedEventIDs []string, + addedEvents []*gomatrixserverlib.HeaderedEvent, + pduPosition types.StreamPosition, + topoPosition types.StreamPosition, +) error { + // remove first, then add, as we do not ever delete state, but do replace state which is a remove followed by an add. + for _, eventID := range removedEventIDs { + if err := d.CurrentRoomState.DeleteRoomStateByEventID(ctx, txn, eventID); err != nil { + return fmt.Errorf("d.CurrentRoomState.DeleteRoomStateByEventID: %w", err) + } + } + + for _, event := range addedEvents { + if event.StateKey() == nil { + // ignore non state events + continue + } + var membership *string + if event.Type() == "m.room.member" { + value, err := event.Membership() + if err != nil { + return fmt.Errorf("event.Membership: %w", err) + } + membership = &value + if err = d.Memberships.UpsertMembership(ctx, txn, event, pduPosition, topoPosition); err != nil { + return fmt.Errorf("d.Memberships.UpsertMembership: %w", err) + } + } + + if err := d.CurrentRoomState.UpsertRoomState(ctx, txn, event, membership, pduPosition); err != nil { + return fmt.Errorf("d.CurrentRoomState.UpsertRoomState: %w", err) + } + } + + return nil +} + +func (d *Database) GetFilter( + ctx context.Context, target *gomatrixserverlib.Filter, localpart string, filterID string, +) error { + return d.Filter.SelectFilter(ctx, nil, target, localpart, filterID) +} + +func (d *Database) PutFilter( + ctx context.Context, localpart string, filter *gomatrixserverlib.Filter, +) (string, error) { + var filterID string + var err error + err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + filterID, err = d.Filter.InsertFilter(ctx, txn, filter, localpart) + return err + }) + return filterID, err +} + +func (d *Database) RedactEvent(ctx context.Context, redactedEventID string, redactedBecause *gomatrixserverlib.HeaderedEvent) error { + redactedEvents, err := d.Events(ctx, []string{redactedEventID}) + if err != nil { + return err + } + if len(redactedEvents) == 0 { + logrus.WithField("event_id", redactedEventID).WithField("redaction_event", redactedBecause.EventID()).Warnf("missing redacted event for redaction") + return nil + } + eventToRedact := redactedEvents[0].Unwrap() + redactionEvent := redactedBecause.Unwrap() + if err = eventutil.RedactEvent(redactionEvent, eventToRedact); err != nil { + return err + } + + newEvent := eventToRedact.Headered(redactedBecause.RoomVersion) + err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + return d.OutputEvents.UpdateEventJSON(ctx, txn, newEvent) + }) + return err +} + +// fetchStateEvents converts the set of event IDs into a set of events. It will fetch any which are missing from the database. +// Returns a map of room ID to list of events. +func (d *Database) fetchStateEvents( + ctx context.Context, txn *sql.Tx, + roomIDToEventIDSet map[string]map[string]bool, + eventIDToEvent map[string]types.StreamEvent, +) (map[string][]types.StreamEvent, error) { + stateBetween := make(map[string][]types.StreamEvent) + missingEvents := make(map[string][]string) + for roomID, ids := range roomIDToEventIDSet { + events := stateBetween[roomID] + for id, need := range ids { + if !need { + continue // deleted state + } + e, ok := eventIDToEvent[id] + if ok { + events = append(events, e) + } else { + m := missingEvents[roomID] + m = append(m, id) + missingEvents[roomID] = m + } + } + stateBetween[roomID] = events + } + + if len(missingEvents) > 0 { + // This happens when add_state_ids has an event ID which is not in the provided range. + // We need to explicitly fetch them. + allMissingEventIDs := []string{} + for _, missingEvIDs := range missingEvents { + allMissingEventIDs = append(allMissingEventIDs, missingEvIDs...) + } + evs, err := d.fetchMissingStateEvents(ctx, txn, allMissingEventIDs) + if err != nil { + return nil, err + } + // we know we got them all otherwise an error would've been returned, so just loop the events + for _, ev := range evs { + roomID := ev.RoomID() + stateBetween[roomID] = append(stateBetween[roomID], ev) + } + } + return stateBetween, nil +} + +func (d *Database) fetchMissingStateEvents( + ctx context.Context, txn *sql.Tx, eventIDs []string, +) ([]types.StreamEvent, error) { + // Fetch from the events table first so we pick up the stream ID for the + // event. + events, err := d.OutputEvents.SelectEvents(ctx, txn, eventIDs, nil, false) + if err != nil { + return nil, err + } + + have := map[string]bool{} + for _, event := range events { + have[event.EventID()] = true + } + var missing []string + for _, eventID := range eventIDs { + if !have[eventID] { + missing = append(missing, eventID) + } + } + if len(missing) == 0 { + return events, nil + } + + // If they are missing from the events table then they should be state + // events that we received from outside the main event stream. + // These should be in the room state table. + stateEvents, err := d.CurrentRoomState.SelectEventsWithEventIDs(ctx, txn, missing) + + if err != nil { + return nil, err + } + if len(stateEvents) != len(missing) { + logrus.WithContext(ctx).Warnf("Failed to map all event IDs to events (got %d, wanted %d)", len(stateEvents), len(missing)) + + // TODO: Why is this happening? It's probably the roomserver. Uncomment + // this error again when we work out what it is and fix it, otherwise we + // just end up returning lots of 500s to the client and that breaks + // pretty much everything, rather than just sending what we have. + //return nil, fmt.Errorf("failed to map all event IDs to events: (got %d, wanted %d)", len(stateEvents), len(missing)) + } + events = append(events, stateEvents...) + return events, nil +} + +func (d *Database) StoreNewSendForDeviceMessage( + ctx context.Context, userID, deviceID string, event gomatrixserverlib.SendToDeviceEvent, +) (newPos types.StreamPosition, err error) { + j, err := json.Marshal(event) + if err != nil { + return 0, err + } + // Delegate the database write task to the SendToDeviceWriter. It'll guarantee + // that we don't lock the table for writes in more than one place. + err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + newPos, err = d.SendToDevice.InsertSendToDeviceMessage( + ctx, txn, userID, deviceID, string(j), + ) + return err + }) + if err != nil { + return 0, err + } + return newPos, nil +} + +func (d *Database) CleanSendToDeviceUpdates( + ctx context.Context, + userID, deviceID string, before types.StreamPosition, +) (err error) { + if err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + return d.SendToDevice.DeleteSendToDeviceMessages(ctx, txn, userID, deviceID, before) + }); err != nil { + logrus.WithError(err).Errorf("Failed to clean up old send-to-device messages for user %q device %q", userID, deviceID) + return err + } + return nil +} + +// getMembershipFromEvent returns the value of content.membership iff the event is a state event +// with type 'm.room.member' and state_key of userID. Otherwise, an empty string is returned. +func getMembershipFromEvent(ev *gomatrixserverlib.Event, userID string) (string, string) { + if ev.Type() != "m.room.member" || !ev.StateKeyEquals(userID) { + return "", "" + } + membership, err := ev.Membership() + if err != nil { + return "", "" + } + prevMembership := gjson.GetBytes(ev.Unsigned(), "prev_content.membership").Str + return membership, prevMembership +} + +// StoreReceipt stores user receipts +func (d *Database) StoreReceipt(ctx context.Context, roomId, receiptType, userId, eventId string, timestamp gomatrixserverlib.Timestamp) (pos types.StreamPosition, err error) { + err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + pos, err = d.Receipts.UpsertReceipt(ctx, txn, roomId, receiptType, userId, eventId, timestamp) + return err + }) + return +} + +func (d *Database) UpsertRoomUnreadNotificationCounts(ctx context.Context, userID, roomID string, notificationCount, highlightCount int) (pos types.StreamPosition, err error) { + err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + pos, err = d.NotificationData.UpsertRoomUnreadCounts(ctx, txn, userID, roomID, notificationCount, highlightCount) + return err + }) + return +} + +func (d *Database) SelectContextEvent(ctx context.Context, roomID, eventID string) (int, gomatrixserverlib.HeaderedEvent, error) { + return d.OutputEvents.SelectContextEvent(ctx, nil, roomID, eventID) +} + +func (d *Database) SelectContextBeforeEvent(ctx context.Context, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) ([]*gomatrixserverlib.HeaderedEvent, error) { + return d.OutputEvents.SelectContextBeforeEvent(ctx, nil, id, roomID, filter) +} +func (d *Database) SelectContextAfterEvent(ctx context.Context, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) (int, []*gomatrixserverlib.HeaderedEvent, error) { + return d.OutputEvents.SelectContextAfterEvent(ctx, nil, id, roomID, filter) +} + +func (d *Database) IgnoresForUser(ctx context.Context, userID string) (*types.IgnoredUsers, error) { + return d.Ignores.SelectIgnores(ctx, nil, userID) +} + +func (d *Database) UpdateIgnoresForUser(ctx context.Context, userID string, ignores *types.IgnoredUsers) error { + return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + return d.Ignores.UpsertIgnores(ctx, txn, userID, ignores) + }) +} + +func (d *Database) UpdatePresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, lastActiveTS gomatrixserverlib.Timestamp, fromSync bool) (types.StreamPosition, error) { + var pos types.StreamPosition + var err error + _ = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + pos, err = d.Presence.UpsertPresence(ctx, txn, userID, statusMsg, presence, lastActiveTS, fromSync) + return nil + }) + return pos, err +} + +func (d *Database) GetPresence(ctx context.Context, userID string) (*types.PresenceInternal, error) { + return d.Presence.GetPresenceForUser(ctx, nil, userID) +} + +func (d *Database) SelectMembershipForUser(ctx context.Context, roomID, userID string, pos int64) (membership string, topologicalPos int, err error) { + return d.Memberships.SelectMembershipForUser(ctx, nil, roomID, userID, pos) +} + +func (s *Database) ReIndex(ctx context.Context, limit, afterID int64) (map[int64]gomatrixserverlib.HeaderedEvent, error) { + return s.OutputEvents.ReIndex(ctx, nil, limit, afterID, []string{ + gomatrixserverlib.MRoomName, + gomatrixserverlib.MRoomTopic, + "m.room.message", + }) +} diff --git a/syncapi/storage/shared/storage_sync.go b/syncapi/storage/shared/storage_sync.go new file mode 100644 index 000000000..a19135a69 --- /dev/null +++ b/syncapi/storage/shared/storage_sync.go @@ -0,0 +1,574 @@ +package shared + +import ( + "context" + "database/sql" + "fmt" + + "github.com/matrix-org/dendrite/internal/eventutil" + "github.com/matrix-org/dendrite/syncapi/types" + userapi "github.com/matrix-org/dendrite/userapi/api" + "github.com/matrix-org/gomatrixserverlib" +) + +type DatabaseTransaction struct { + *Database + txn *sql.Tx +} + +func (d *DatabaseTransaction) Commit() error { + if d.txn == nil { + return nil + } + return d.txn.Commit() +} + +func (d *DatabaseTransaction) Rollback() error { + if d.txn == nil { + return nil + } + return d.txn.Rollback() +} + +func (d *DatabaseTransaction) MaxStreamPositionForPDUs(ctx context.Context) (types.StreamPosition, error) { + id, err := d.OutputEvents.SelectMaxEventID(ctx, d.txn) + if err != nil { + return 0, fmt.Errorf("d.OutputEvents.SelectMaxEventID: %w", err) + } + return types.StreamPosition(id), nil +} + +func (d *DatabaseTransaction) MaxStreamPositionForReceipts(ctx context.Context) (types.StreamPosition, error) { + id, err := d.Receipts.SelectMaxReceiptID(ctx, d.txn) + if err != nil { + return 0, fmt.Errorf("d.Receipts.SelectMaxReceiptID: %w", err) + } + return types.StreamPosition(id), nil +} + +func (d *DatabaseTransaction) MaxStreamPositionForInvites(ctx context.Context) (types.StreamPosition, error) { + id, err := d.Invites.SelectMaxInviteID(ctx, d.txn) + if err != nil { + return 0, fmt.Errorf("d.Invites.SelectMaxInviteID: %w", err) + } + return types.StreamPosition(id), nil +} + +func (d *DatabaseTransaction) MaxStreamPositionForSendToDeviceMessages(ctx context.Context) (types.StreamPosition, error) { + id, err := d.SendToDevice.SelectMaxSendToDeviceMessageID(ctx, d.txn) + if err != nil { + return 0, fmt.Errorf("d.SendToDevice.SelectMaxSendToDeviceMessageID: %w", err) + } + return types.StreamPosition(id), nil +} + +func (d *DatabaseTransaction) MaxStreamPositionForAccountData(ctx context.Context) (types.StreamPosition, error) { + id, err := d.AccountData.SelectMaxAccountDataID(ctx, d.txn) + if err != nil { + return 0, fmt.Errorf("d.Invites.SelectMaxAccountDataID: %w", err) + } + return types.StreamPosition(id), nil +} + +func (d *DatabaseTransaction) MaxStreamPositionForNotificationData(ctx context.Context) (types.StreamPosition, error) { + id, err := d.NotificationData.SelectMaxID(ctx, d.txn) + if err != nil { + return 0, fmt.Errorf("d.NotificationData.SelectMaxID: %w", err) + } + return types.StreamPosition(id), nil +} + +func (d *DatabaseTransaction) CurrentState(ctx context.Context, roomID string, stateFilterPart *gomatrixserverlib.StateFilter, excludeEventIDs []string) ([]*gomatrixserverlib.HeaderedEvent, error) { + return d.CurrentRoomState.SelectCurrentState(ctx, d.txn, roomID, stateFilterPart, excludeEventIDs) +} + +func (d *DatabaseTransaction) RoomIDsWithMembership(ctx context.Context, userID string, membership string) ([]string, error) { + return d.CurrentRoomState.SelectRoomIDsWithMembership(ctx, d.txn, userID, membership) +} + +func (d *DatabaseTransaction) MembershipCount(ctx context.Context, roomID, membership string, pos types.StreamPosition) (int, error) { + return d.Memberships.SelectMembershipCount(ctx, d.txn, roomID, membership, pos) +} + +func (d *DatabaseTransaction) GetRoomHeroes(ctx context.Context, roomID, userID string, memberships []string) ([]string, error) { + return d.Memberships.SelectHeroes(ctx, d.txn, roomID, userID, memberships) +} + +func (d *DatabaseTransaction) RecentEvents(ctx context.Context, roomID string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter, chronologicalOrder bool, onlySyncEvents bool) ([]types.StreamEvent, bool, error) { + return d.OutputEvents.SelectRecentEvents(ctx, d.txn, roomID, r, eventFilter, chronologicalOrder, onlySyncEvents) +} + +func (d *DatabaseTransaction) PositionInTopology(ctx context.Context, eventID string) (pos types.StreamPosition, spos types.StreamPosition, err error) { + return d.Topology.SelectPositionInTopology(ctx, d.txn, eventID) +} + +func (d *DatabaseTransaction) InviteEventsInRange(ctx context.Context, targetUserID string, r types.Range) (map[string]*gomatrixserverlib.HeaderedEvent, map[string]*gomatrixserverlib.HeaderedEvent, types.StreamPosition, error) { + return d.Invites.SelectInviteEventsInRange(ctx, d.txn, targetUserID, r) +} + +func (d *DatabaseTransaction) PeeksInRange(ctx context.Context, userID, deviceID string, r types.Range) (peeks []types.Peek, err error) { + return d.Peeks.SelectPeeksInRange(ctx, d.txn, userID, deviceID, r) +} + +func (d *DatabaseTransaction) RoomReceiptsAfter(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) (types.StreamPosition, []types.OutputReceiptEvent, error) { + return d.Receipts.SelectRoomReceiptsAfter(ctx, d.txn, roomIDs, streamPos) +} + +// Events lookups a list of event by their event ID. +// Returns a list of events matching the requested IDs found in the database. +// If an event is not found in the database then it will be omitted from the list. +// Returns an error if there was a problem talking with the database. +// Does not include any transaction IDs in the returned events. +func (d *DatabaseTransaction) Events(ctx context.Context, eventIDs []string) ([]*gomatrixserverlib.HeaderedEvent, error) { + streamEvents, err := d.OutputEvents.SelectEvents(ctx, d.txn, eventIDs, nil, false) + if err != nil { + return nil, err + } + + // We don't include a device here as we only include transaction IDs in + // incremental syncs. + return d.StreamEventsToEvents(nil, streamEvents), nil +} + +func (d *DatabaseTransaction) AllJoinedUsersInRooms(ctx context.Context) (map[string][]string, error) { + return d.CurrentRoomState.SelectJoinedUsers(ctx, d.txn) +} + +func (d *DatabaseTransaction) AllJoinedUsersInRoom(ctx context.Context, roomIDs []string) (map[string][]string, error) { + return d.CurrentRoomState.SelectJoinedUsersInRoom(ctx, d.txn, roomIDs) +} + +func (d *DatabaseTransaction) AllPeekingDevicesInRooms(ctx context.Context) (map[string][]types.PeekingDevice, error) { + return d.Peeks.SelectPeekingDevices(ctx, d.txn) +} + +func (d *DatabaseTransaction) SharedUsers(ctx context.Context, userID string, otherUserIDs []string) ([]string, error) { + return d.CurrentRoomState.SelectSharedUsers(ctx, d.txn, userID, otherUserIDs) +} + +func (d *DatabaseTransaction) GetStateEvent( + ctx context.Context, roomID, evType, stateKey string, +) (*gomatrixserverlib.HeaderedEvent, error) { + return d.CurrentRoomState.SelectStateEvent(ctx, d.txn, roomID, evType, stateKey) +} + +func (d *DatabaseTransaction) GetStateEventsForRoom( + ctx context.Context, roomID string, stateFilter *gomatrixserverlib.StateFilter, +) (stateEvents []*gomatrixserverlib.HeaderedEvent, err error) { + stateEvents, err = d.CurrentRoomState.SelectCurrentState(ctx, d.txn, roomID, stateFilter, nil) + return +} + +// GetAccountDataInRange returns all account data for a given user inserted or +// updated between two given positions +// Returns a map following the format data[roomID] = []dataTypes +// If no data is retrieved, returns an empty map +// If there was an issue with the retrieval, returns an error +func (d *DatabaseTransaction) GetAccountDataInRange( + ctx context.Context, userID string, r types.Range, + accountDataFilterPart *gomatrixserverlib.EventFilter, +) (map[string][]string, types.StreamPosition, error) { + return d.AccountData.SelectAccountDataInRange(ctx, d.txn, userID, r, accountDataFilterPart) +} + +func (d *DatabaseTransaction) GetEventsInTopologicalRange( + ctx context.Context, + from, to *types.TopologyToken, + roomID string, + filter *gomatrixserverlib.RoomEventFilter, + backwardOrdering bool, +) (events []types.StreamEvent, err error) { + var minDepth, maxDepth, maxStreamPosForMaxDepth types.StreamPosition + if backwardOrdering { + // Backward ordering means the 'from' token has a higher depth than the 'to' token + minDepth = to.Depth + maxDepth = from.Depth + // for cases where we have say 5 events with the same depth, the TopologyToken needs to + // know which of the 5 the client has seen. This is done by using the PDU position. + // Events with the same maxDepth but less than this PDU position will be returned. + maxStreamPosForMaxDepth = from.PDUPosition + } else { + // Forward ordering means the 'from' token has a lower depth than the 'to' token. + minDepth = from.Depth + maxDepth = to.Depth + } + + // Select the event IDs from the defined range. + var eIDs []string + eIDs, err = d.Topology.SelectEventIDsInRange( + ctx, d.txn, roomID, minDepth, maxDepth, maxStreamPosForMaxDepth, filter.Limit, !backwardOrdering, + ) + if err != nil { + return + } + + // Retrieve the events' contents using their IDs. + events, err = d.OutputEvents.SelectEvents(ctx, d.txn, eIDs, filter, true) + return +} + +func (d *DatabaseTransaction) BackwardExtremitiesForRoom( + ctx context.Context, roomID string, +) (backwardExtremities map[string][]string, err error) { + return d.BackwardExtremities.SelectBackwardExtremitiesForRoom(ctx, d.txn, roomID) +} + +func (d *DatabaseTransaction) MaxTopologicalPosition( + ctx context.Context, roomID string, +) (types.TopologyToken, error) { + depth, streamPos, err := d.Topology.SelectMaxPositionInTopology(ctx, d.txn, roomID) + if err != nil { + return types.TopologyToken{}, err + } + return types.TopologyToken{Depth: depth, PDUPosition: streamPos}, nil +} + +func (d *DatabaseTransaction) EventPositionInTopology( + ctx context.Context, eventID string, +) (types.TopologyToken, error) { + depth, stream, err := d.Topology.SelectPositionInTopology(ctx, d.txn, eventID) + if err != nil { + return types.TopologyToken{}, err + } + return types.TopologyToken{Depth: depth, PDUPosition: stream}, nil +} + +func (d *DatabaseTransaction) StreamToTopologicalPosition( + ctx context.Context, roomID string, streamPos types.StreamPosition, backwardOrdering bool, +) (types.TopologyToken, error) { + topoPos, err := d.Topology.SelectStreamToTopologicalPosition(ctx, d.txn, roomID, streamPos, backwardOrdering) + switch { + case err == sql.ErrNoRows && backwardOrdering: // no events in range, going backward + return types.TopologyToken{PDUPosition: streamPos}, nil + case err == sql.ErrNoRows && !backwardOrdering: // no events in range, going forward + topoPos, streamPos, err = d.Topology.SelectMaxPositionInTopology(ctx, d.txn, roomID) + if err != nil { + return types.TopologyToken{}, fmt.Errorf("d.Topology.SelectMaxPositionInTopology: %w", err) + } + return types.TopologyToken{Depth: topoPos, PDUPosition: streamPos}, nil + case err != nil: // some other error happened + return types.TopologyToken{}, fmt.Errorf("d.Topology.SelectStreamToTopologicalPosition: %w", err) + default: + return types.TopologyToken{Depth: topoPos, PDUPosition: streamPos}, nil + } +} + +// GetBackwardTopologyPos retrieves the backward topology position, i.e. the position of the +// oldest event in the room's topology. +func (d *DatabaseTransaction) GetBackwardTopologyPos( + ctx context.Context, + events []types.StreamEvent, +) (types.TopologyToken, error) { + zeroToken := types.TopologyToken{} + if len(events) == 0 { + return zeroToken, nil + } + pos, spos, err := d.Topology.SelectPositionInTopology(ctx, d.txn, events[0].EventID()) + if err != nil { + return zeroToken, err + } + tok := types.TopologyToken{Depth: pos, PDUPosition: spos} + tok.Decrement() + return tok, nil +} + +// GetStateDeltas returns the state deltas between fromPos and toPos, +// exclusive of oldPos, inclusive of newPos, for the rooms in which +// the user has new membership events. +// A list of joined room IDs is also returned in case the caller needs it. +func (d *DatabaseTransaction) GetStateDeltas( + ctx context.Context, device *userapi.Device, + r types.Range, userID string, + stateFilter *gomatrixserverlib.StateFilter, +) (deltas []types.StateDelta, joinedRoomsIDs []string, err error) { + // Implement membership change algorithm: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L821 + // - Get membership list changes for this user in this sync response + // - For each room which has membership list changes: + // * Check if the room is 'newly joined' (insufficient to just check for a join event because we allow dupe joins TODO). + // If it is, then we need to send the full room state down (and 'limited' is always true). + // * Check if user is still CURRENTLY invited to the room. If so, add room to 'invited' block. + // * Check if the user is CURRENTLY (TODO) left/banned. If so, add room to 'archived' block. + // - Get all CURRENTLY joined rooms, and add them to 'joined' block. + + // Look up all memberships for the user. We only care about rooms that a + // user has ever interacted with — joined to, kicked/banned from, left. + memberships, err := d.CurrentRoomState.SelectRoomIDsWithAnyMembership(ctx, d.txn, userID) + if err != nil { + if err == sql.ErrNoRows { + return nil, nil, nil + } + return nil, nil, err + } + + allRoomIDs := make([]string, 0, len(memberships)) + joinedRoomIDs := make([]string, 0, len(memberships)) + for roomID, membership := range memberships { + allRoomIDs = append(allRoomIDs, roomID) + if membership == gomatrixserverlib.Join { + joinedRoomIDs = append(joinedRoomIDs, roomID) + } + } + + // get all the state events ever (i.e. for all available rooms) between these two positions + stateNeeded, eventMap, err := d.OutputEvents.SelectStateInRange(ctx, d.txn, r, stateFilter, allRoomIDs) + if err != nil { + if err == sql.ErrNoRows { + return nil, nil, nil + } + return nil, nil, err + } + state, err := d.fetchStateEvents(ctx, d.txn, stateNeeded, eventMap) + if err != nil { + if err == sql.ErrNoRows { + return nil, nil, nil + } + return nil, nil, err + } + + // find out which rooms this user is peeking, if any. + // We do this before joins so any peeks get overwritten + peeks, err := d.Peeks.SelectPeeksInRange(ctx, d.txn, userID, device.ID, r) + if err != nil && err != sql.ErrNoRows { + return nil, nil, err + } + + // add peek blocks + for _, peek := range peeks { + if peek.New { + // send full room state down instead of a delta + var s []types.StreamEvent + s, err = d.currentStateStreamEventsForRoom(ctx, peek.RoomID, stateFilter) + if err != nil { + if err == sql.ErrNoRows { + continue + } + return nil, nil, err + } + state[peek.RoomID] = s + } + if !peek.Deleted { + deltas = append(deltas, types.StateDelta{ + Membership: gomatrixserverlib.Peek, + StateEvents: d.StreamEventsToEvents(device, state[peek.RoomID]), + RoomID: peek.RoomID, + }) + } + } + + // handle newly joined rooms and non-joined rooms + newlyJoinedRooms := make(map[string]bool, len(state)) + for roomID, stateStreamEvents := range state { + for _, ev := range stateStreamEvents { + if membership, prevMembership := getMembershipFromEvent(ev.Event, userID); membership != "" { + if membership == gomatrixserverlib.Join && prevMembership != membership { + // send full room state down instead of a delta + var s []types.StreamEvent + s, err = d.currentStateStreamEventsForRoom(ctx, roomID, stateFilter) + if err != nil { + if err == sql.ErrNoRows { + continue + } + return nil, nil, err + } + state[roomID] = s + newlyJoinedRooms[roomID] = true + continue // we'll add this room in when we do joined rooms + } + + deltas = append(deltas, types.StateDelta{ + Membership: membership, + MembershipPos: ev.StreamPosition, + StateEvents: d.StreamEventsToEvents(device, stateStreamEvents), + RoomID: roomID, + }) + break + } + } + } + + // Add in currently joined rooms + for _, joinedRoomID := range joinedRoomIDs { + deltas = append(deltas, types.StateDelta{ + Membership: gomatrixserverlib.Join, + StateEvents: d.StreamEventsToEvents(device, state[joinedRoomID]), + RoomID: joinedRoomID, + NewlyJoined: newlyJoinedRooms[joinedRoomID], + }) + } + + return deltas, joinedRoomIDs, nil +} + +// GetStateDeltasForFullStateSync is a variant of getStateDeltas used for /sync +// requests with full_state=true. +// Fetches full state for all joined rooms and uses selectStateInRange to get +// updates for other rooms. +func (d *DatabaseTransaction) GetStateDeltasForFullStateSync( + ctx context.Context, device *userapi.Device, + r types.Range, userID string, + stateFilter *gomatrixserverlib.StateFilter, +) ([]types.StateDelta, []string, error) { + // Look up all memberships for the user. We only care about rooms that a + // user has ever interacted with — joined to, kicked/banned from, left. + memberships, err := d.CurrentRoomState.SelectRoomIDsWithAnyMembership(ctx, d.txn, userID) + if err != nil { + if err == sql.ErrNoRows { + return nil, nil, nil + } + return nil, nil, err + } + + allRoomIDs := make([]string, 0, len(memberships)) + joinedRoomIDs := make([]string, 0, len(memberships)) + for roomID, membership := range memberships { + allRoomIDs = append(allRoomIDs, roomID) + if membership == gomatrixserverlib.Join { + joinedRoomIDs = append(joinedRoomIDs, roomID) + } + } + + // Use a reasonable initial capacity + deltas := make(map[string]types.StateDelta) + + peeks, err := d.Peeks.SelectPeeksInRange(ctx, d.txn, userID, device.ID, r) + if err != nil && err != sql.ErrNoRows { + return nil, nil, err + } + + // Add full states for all peeking rooms + for _, peek := range peeks { + if !peek.Deleted { + s, stateErr := d.currentStateStreamEventsForRoom(ctx, peek.RoomID, stateFilter) + if stateErr != nil { + if stateErr == sql.ErrNoRows { + continue + } + return nil, nil, stateErr + } + deltas[peek.RoomID] = types.StateDelta{ + Membership: gomatrixserverlib.Peek, + StateEvents: d.StreamEventsToEvents(device, s), + RoomID: peek.RoomID, + } + } + } + + // Get all the state events ever between these two positions + stateNeeded, eventMap, err := d.OutputEvents.SelectStateInRange(ctx, d.txn, r, stateFilter, allRoomIDs) + if err != nil { + if err == sql.ErrNoRows { + return nil, nil, nil + } + return nil, nil, err + } + state, err := d.fetchStateEvents(ctx, d.txn, stateNeeded, eventMap) + if err != nil { + if err == sql.ErrNoRows { + return nil, nil, nil + } + return nil, nil, err + } + + for roomID, stateStreamEvents := range state { + for _, ev := range stateStreamEvents { + if membership, _ := getMembershipFromEvent(ev.Event, userID); membership != "" { + if membership != gomatrixserverlib.Join { // We've already added full state for all joined rooms above. + deltas[roomID] = types.StateDelta{ + Membership: membership, + MembershipPos: ev.StreamPosition, + StateEvents: d.StreamEventsToEvents(device, stateStreamEvents), + RoomID: roomID, + } + } + + break + } + } + } + + // Add full states for all joined rooms + for _, joinedRoomID := range joinedRoomIDs { + s, stateErr := d.currentStateStreamEventsForRoom(ctx, joinedRoomID, stateFilter) + if stateErr != nil { + if stateErr == sql.ErrNoRows { + continue + } + return nil, nil, stateErr + } + deltas[joinedRoomID] = types.StateDelta{ + Membership: gomatrixserverlib.Join, + StateEvents: d.StreamEventsToEvents(device, s), + RoomID: joinedRoomID, + } + } + + // Create a response array. + result := make([]types.StateDelta, len(deltas)) + i := 0 + for _, delta := range deltas { + result[i] = delta + i++ + } + + return result, joinedRoomIDs, nil +} + +func (d *DatabaseTransaction) currentStateStreamEventsForRoom( + ctx context.Context, roomID string, + stateFilter *gomatrixserverlib.StateFilter, +) ([]types.StreamEvent, error) { + allState, err := d.CurrentRoomState.SelectCurrentState(ctx, d.txn, roomID, stateFilter, nil) + if err != nil { + return nil, err + } + s := make([]types.StreamEvent, len(allState)) + for i := 0; i < len(s); i++ { + s[i] = types.StreamEvent{HeaderedEvent: allState[i], StreamPosition: 0} + } + return s, nil +} + +func (d *DatabaseTransaction) SendToDeviceUpdatesForSync( + ctx context.Context, + userID, deviceID string, + from, to types.StreamPosition, +) (types.StreamPosition, []types.SendToDeviceEvent, error) { + // First of all, get our send-to-device updates for this user. + lastPos, events, err := d.SendToDevice.SelectSendToDeviceMessages(ctx, d.txn, userID, deviceID, from, to) + if err != nil { + return from, nil, fmt.Errorf("d.SendToDevice.SelectSendToDeviceMessages: %w", err) + } + // If there's nothing to do then stop here. + if len(events) == 0 { + return to, nil, nil + } + return lastPos, events, nil +} + +func (d *DatabaseTransaction) GetRoomReceipts(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) ([]types.OutputReceiptEvent, error) { + _, receipts, err := d.Receipts.SelectRoomReceiptsAfter(ctx, d.txn, roomIDs, streamPos) + return receipts, err +} + +func (d *DatabaseTransaction) GetUserUnreadNotificationCountsForRooms(ctx context.Context, userID string, rooms map[string]string) (map[string]*eventutil.NotificationData, error) { + roomIDs := make([]string, 0, len(rooms)) + for roomID, membership := range rooms { + if membership != gomatrixserverlib.Join { + continue + } + roomIDs = append(roomIDs, roomID) + } + return d.NotificationData.SelectUserUnreadCountsForRooms(ctx, d.txn, userID, roomIDs) +} + +func (d *DatabaseTransaction) GetPresence(ctx context.Context, userID string) (*types.PresenceInternal, error) { + return d.Presence.GetPresenceForUser(ctx, d.txn, userID) +} + +func (d *DatabaseTransaction) PresenceAfter(ctx context.Context, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (map[string]*types.PresenceInternal, error) { + return d.Presence.GetPresenceAfter(ctx, d.txn, after, filter) +} + +func (d *DatabaseTransaction) MaxStreamPositionForPresence(ctx context.Context) (types.StreamPosition, error) { + return d.Presence.GetMaxPresenceID(ctx, d.txn) +} diff --git a/syncapi/storage/shared/syncserver.go b/syncapi/storage/shared/syncserver.go deleted file mode 100644 index a05e68804..000000000 --- a/syncapi/storage/shared/syncserver.go +++ /dev/null @@ -1,1103 +0,0 @@ -// Copyright 2020 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package shared - -import ( - "context" - "database/sql" - "encoding/json" - "fmt" - - "github.com/tidwall/gjson" - - userapi "github.com/matrix-org/dendrite/userapi/api" - - "github.com/matrix-org/gomatrixserverlib" - "github.com/sirupsen/logrus" - - "github.com/matrix-org/dendrite/internal/eventutil" - "github.com/matrix-org/dendrite/internal/sqlutil" - "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/syncapi/storage/tables" - "github.com/matrix-org/dendrite/syncapi/types" -) - -// Database is a temporary struct until we have made syncserver.go the same for both pq/sqlite -// For now this contains the shared functions -type Database struct { - DB *sql.DB - Writer sqlutil.Writer - Invites tables.Invites - Peeks tables.Peeks - AccountData tables.AccountData - OutputEvents tables.Events - Topology tables.Topology - CurrentRoomState tables.CurrentRoomState - BackwardExtremities tables.BackwardsExtremities - SendToDevice tables.SendToDevice - Filter tables.Filter - Receipts tables.Receipts - Memberships tables.Memberships - NotificationData tables.NotificationData - Ignores tables.Ignores - Presence tables.Presence -} - -func (d *Database) readOnlySnapshot(ctx context.Context) (*sql.Tx, error) { - return d.DB.BeginTx(ctx, &sql.TxOptions{ - // Set the isolation level so that we see a snapshot of the database. - // In PostgreSQL repeatable read transactions will see a snapshot taken - // at the first query, and since the transaction is read-only it can't - // run into any serialisation errors. - // https://www.postgresql.org/docs/9.5/static/transaction-iso.html#XACT-REPEATABLE-READ - Isolation: sql.LevelRepeatableRead, - ReadOnly: true, - }) -} - -func (d *Database) MaxStreamPositionForPDUs(ctx context.Context) (types.StreamPosition, error) { - id, err := d.OutputEvents.SelectMaxEventID(ctx, nil) - if err != nil { - return 0, fmt.Errorf("d.OutputEvents.SelectMaxEventID: %w", err) - } - return types.StreamPosition(id), nil -} - -func (d *Database) MaxStreamPositionForReceipts(ctx context.Context) (types.StreamPosition, error) { - id, err := d.Receipts.SelectMaxReceiptID(ctx, nil) - if err != nil { - return 0, fmt.Errorf("d.Receipts.SelectMaxReceiptID: %w", err) - } - return types.StreamPosition(id), nil -} - -func (d *Database) MaxStreamPositionForInvites(ctx context.Context) (types.StreamPosition, error) { - id, err := d.Invites.SelectMaxInviteID(ctx, nil) - if err != nil { - return 0, fmt.Errorf("d.Invites.SelectMaxInviteID: %w", err) - } - return types.StreamPosition(id), nil -} - -func (d *Database) MaxStreamPositionForSendToDeviceMessages(ctx context.Context) (types.StreamPosition, error) { - id, err := d.SendToDevice.SelectMaxSendToDeviceMessageID(ctx, nil) - if err != nil { - return 0, fmt.Errorf("d.SendToDevice.SelectMaxSendToDeviceMessageID: %w", err) - } - return types.StreamPosition(id), nil -} - -func (d *Database) MaxStreamPositionForAccountData(ctx context.Context) (types.StreamPosition, error) { - id, err := d.AccountData.SelectMaxAccountDataID(ctx, nil) - if err != nil { - return 0, fmt.Errorf("d.Invites.SelectMaxAccountDataID: %w", err) - } - return types.StreamPosition(id), nil -} - -func (d *Database) MaxStreamPositionForNotificationData(ctx context.Context) (types.StreamPosition, error) { - id, err := d.NotificationData.SelectMaxID(ctx, nil) - if err != nil { - return 0, fmt.Errorf("d.NotificationData.SelectMaxID: %w", err) - } - return types.StreamPosition(id), nil -} - -func (d *Database) CurrentState(ctx context.Context, roomID string, stateFilterPart *gomatrixserverlib.StateFilter, excludeEventIDs []string) ([]*gomatrixserverlib.HeaderedEvent, error) { - return d.CurrentRoomState.SelectCurrentState(ctx, nil, roomID, stateFilterPart, excludeEventIDs) -} - -func (d *Database) RoomIDsWithMembership(ctx context.Context, userID string, membership string) ([]string, error) { - return d.CurrentRoomState.SelectRoomIDsWithMembership(ctx, nil, userID, membership) -} - -func (d *Database) MembershipCount(ctx context.Context, roomID, membership string, pos types.StreamPosition) (int, error) { - return d.Memberships.SelectMembershipCount(ctx, nil, roomID, membership, pos) -} - -func (d *Database) GetRoomHeroes(ctx context.Context, roomID, userID string, memberships []string) ([]string, error) { - return d.Memberships.SelectHeroes(ctx, nil, roomID, userID, memberships) -} - -func (d *Database) RecentEvents(ctx context.Context, roomID string, r types.Range, eventFilter *gomatrixserverlib.RoomEventFilter, chronologicalOrder bool, onlySyncEvents bool) ([]types.StreamEvent, bool, error) { - return d.OutputEvents.SelectRecentEvents(ctx, nil, roomID, r, eventFilter, chronologicalOrder, onlySyncEvents) -} - -func (d *Database) PositionInTopology(ctx context.Context, eventID string) (pos types.StreamPosition, spos types.StreamPosition, err error) { - return d.Topology.SelectPositionInTopology(ctx, nil, eventID) -} - -func (d *Database) InviteEventsInRange(ctx context.Context, targetUserID string, r types.Range) (map[string]*gomatrixserverlib.HeaderedEvent, map[string]*gomatrixserverlib.HeaderedEvent, error) { - return d.Invites.SelectInviteEventsInRange(ctx, nil, targetUserID, r) -} - -func (d *Database) PeeksInRange(ctx context.Context, userID, deviceID string, r types.Range) (peeks []types.Peek, err error) { - return d.Peeks.SelectPeeksInRange(ctx, nil, userID, deviceID, r) -} - -func (d *Database) RoomReceiptsAfter(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) (types.StreamPosition, []types.OutputReceiptEvent, error) { - return d.Receipts.SelectRoomReceiptsAfter(ctx, nil, roomIDs, streamPos) -} - -// Events lookups a list of event by their event ID. -// Returns a list of events matching the requested IDs found in the database. -// If an event is not found in the database then it will be omitted from the list. -// Returns an error if there was a problem talking with the database. -// Does not include any transaction IDs in the returned events. -func (d *Database) Events(ctx context.Context, eventIDs []string) ([]*gomatrixserverlib.HeaderedEvent, error) { - streamEvents, err := d.OutputEvents.SelectEvents(ctx, nil, eventIDs, nil, false) - if err != nil { - return nil, err - } - - // We don't include a device here as we only include transaction IDs in - // incremental syncs. - return d.StreamEventsToEvents(nil, streamEvents), nil -} - -func (d *Database) AllJoinedUsersInRooms(ctx context.Context) (map[string][]string, error) { - return d.CurrentRoomState.SelectJoinedUsers(ctx, nil) -} - -func (d *Database) AllJoinedUsersInRoom(ctx context.Context, roomIDs []string) (map[string][]string, error) { - return d.CurrentRoomState.SelectJoinedUsersInRoom(ctx, nil, roomIDs) -} - -func (d *Database) AllPeekingDevicesInRooms(ctx context.Context) (map[string][]types.PeekingDevice, error) { - return d.Peeks.SelectPeekingDevices(ctx, nil) -} - -func (d *Database) SharedUsers(ctx context.Context, userID string, otherUserIDs []string) ([]string, error) { - return d.CurrentRoomState.SelectSharedUsers(ctx, nil, userID, otherUserIDs) -} - -func (d *Database) GetStateEvent( - ctx context.Context, roomID, evType, stateKey string, -) (*gomatrixserverlib.HeaderedEvent, error) { - return d.CurrentRoomState.SelectStateEvent(ctx, nil, roomID, evType, stateKey) -} - -func (d *Database) GetStateEventsForRoom( - ctx context.Context, roomID string, stateFilter *gomatrixserverlib.StateFilter, -) (stateEvents []*gomatrixserverlib.HeaderedEvent, err error) { - stateEvents, err = d.CurrentRoomState.SelectCurrentState(ctx, nil, roomID, stateFilter, nil) - return -} - -// AddInviteEvent stores a new invite event for a user. -// If the invite was successfully stored this returns the stream ID it was stored at. -// Returns an error if there was a problem communicating with the database. -func (d *Database) AddInviteEvent( - ctx context.Context, inviteEvent *gomatrixserverlib.HeaderedEvent, -) (sp types.StreamPosition, err error) { - _ = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - sp, err = d.Invites.InsertInviteEvent(ctx, txn, inviteEvent) - return err - }) - return -} - -// RetireInviteEvent removes an old invite event from the database. -// Returns an error if there was a problem communicating with the database. -func (d *Database) RetireInviteEvent( - ctx context.Context, inviteEventID string, -) (sp types.StreamPosition, err error) { - _ = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - sp, err = d.Invites.DeleteInviteEvent(ctx, txn, inviteEventID) - return err - }) - return -} - -// AddPeek tracks the fact that a user has started peeking. -// If the peek was successfully stored this returns the stream ID it was stored at. -// Returns an error if there was a problem communicating with the database. -func (d *Database) AddPeek( - ctx context.Context, roomID, userID, deviceID string, -) (sp types.StreamPosition, err error) { - err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - sp, err = d.Peeks.InsertPeek(ctx, txn, roomID, userID, deviceID) - return err - }) - return -} - -// DeletePeek tracks the fact that a user has stopped peeking from the specified -// device. If the peeks was successfully deleted this returns the stream ID it was -// stored at. Returns an error if there was a problem communicating with the database. -func (d *Database) DeletePeek( - ctx context.Context, roomID, userID, deviceID string, -) (sp types.StreamPosition, err error) { - err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - sp, err = d.Peeks.DeletePeek(ctx, txn, roomID, userID, deviceID) - return err - }) - if err == sql.ErrNoRows { - sp = 0 - err = nil - } - return -} - -// DeletePeeks tracks the fact that a user has stopped peeking from all devices -// If the peeks was successfully deleted this returns the stream ID it was stored at. -// Returns an error if there was a problem communicating with the database. -func (d *Database) DeletePeeks( - ctx context.Context, roomID, userID string, -) (sp types.StreamPosition, err error) { - err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - sp, err = d.Peeks.DeletePeeks(ctx, txn, roomID, userID) - return err - }) - if err == sql.ErrNoRows { - sp = 0 - err = nil - } - return -} - -// GetAccountDataInRange returns all account data for a given user inserted or -// updated between two given positions -// Returns a map following the format data[roomID] = []dataTypes -// If no data is retrieved, returns an empty map -// If there was an issue with the retrieval, returns an error -func (d *Database) GetAccountDataInRange( - ctx context.Context, userID string, r types.Range, - accountDataFilterPart *gomatrixserverlib.EventFilter, -) (map[string][]string, types.StreamPosition, error) { - return d.AccountData.SelectAccountDataInRange(ctx, nil, userID, r, accountDataFilterPart) -} - -// UpsertAccountData keeps track of new or updated account data, by saving the type -// of the new/updated data, and the user ID and room ID the data is related to (empty) -// room ID means the data isn't specific to any room) -// If no data with the given type, user ID and room ID exists in the database, -// creates a new row, else update the existing one -// Returns an error if there was an issue with the upsert -func (d *Database) UpsertAccountData( - ctx context.Context, userID, roomID, dataType string, -) (sp types.StreamPosition, err error) { - err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - sp, err = d.AccountData.InsertAccountData(ctx, txn, userID, roomID, dataType) - return err - }) - return -} - -func (d *Database) StreamEventsToEvents(device *userapi.Device, in []types.StreamEvent) []*gomatrixserverlib.HeaderedEvent { - out := make([]*gomatrixserverlib.HeaderedEvent, len(in)) - for i := 0; i < len(in); i++ { - out[i] = in[i].HeaderedEvent - if device != nil && in[i].TransactionID != nil { - if device.UserID == in[i].Sender() && device.SessionID == in[i].TransactionID.SessionID { - err := out[i].SetUnsignedField( - "transaction_id", in[i].TransactionID.TransactionID, - ) - if err != nil { - logrus.WithFields(logrus.Fields{ - "event_id": out[i].EventID(), - }).WithError(err).Warnf("Failed to add transaction ID to event") - } - } - } - } - return out -} - -// handleBackwardExtremities adds this event as a backwards extremity if and only if we do not have all of -// the events listed in the event's 'prev_events'. This function also updates the backwards extremities table -// to account for the fact that the given event is no longer a backwards extremity, but may be marked as such. -// This function should always be called within a sqlutil.Writer for safety in SQLite. -func (d *Database) handleBackwardExtremities(ctx context.Context, txn *sql.Tx, ev *gomatrixserverlib.HeaderedEvent) error { - if err := d.BackwardExtremities.DeleteBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID()); err != nil { - return err - } - - // Check if we have all of the event's previous events. If an event is - // missing, add it to the room's backward extremities. - prevEvents, err := d.OutputEvents.SelectEvents(ctx, txn, ev.PrevEventIDs(), nil, false) - if err != nil { - return err - } - var found bool - for _, eID := range ev.PrevEventIDs() { - found = false - for _, prevEv := range prevEvents { - if eID == prevEv.EventID() { - found = true - } - } - - // If the event is missing, consider it a backward extremity. - if !found { - if err = d.BackwardExtremities.InsertsBackwardExtremity(ctx, txn, ev.RoomID(), ev.EventID(), eID); err != nil { - return err - } - } - } - - return nil -} - -func (d *Database) PurgeRoomState( - ctx context.Context, roomID string, -) error { - return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - // If the event is a create event then we'll delete all of the existing - // data for the room. The only reason that a create event would be replayed - // to us in this way is if we're about to receive the entire room state. - if err := d.CurrentRoomState.DeleteRoomStateForRoom(ctx, txn, roomID); err != nil { - return fmt.Errorf("d.CurrentRoomState.DeleteRoomStateForRoom: %w", err) - } - return nil - }) -} - -func (d *Database) WriteEvent( - ctx context.Context, - ev *gomatrixserverlib.HeaderedEvent, - addStateEvents []*gomatrixserverlib.HeaderedEvent, - addStateEventIDs, removeStateEventIDs []string, - transactionID *api.TransactionID, excludeFromSync bool, - historyVisibility gomatrixserverlib.HistoryVisibility, -) (pduPosition types.StreamPosition, returnErr error) { - returnErr = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - var err error - ev.Visibility = historyVisibility - pos, err := d.OutputEvents.InsertEvent( - ctx, txn, ev, addStateEventIDs, removeStateEventIDs, transactionID, excludeFromSync, historyVisibility, - ) - if err != nil { - return fmt.Errorf("d.OutputEvents.InsertEvent: %w", err) - } - pduPosition = pos - var topoPosition types.StreamPosition - if topoPosition, err = d.Topology.InsertEventInTopology(ctx, txn, ev, pos); err != nil { - return fmt.Errorf("d.Topology.InsertEventInTopology: %w", err) - } - - if err = d.handleBackwardExtremities(ctx, txn, ev); err != nil { - return fmt.Errorf("d.handleBackwardExtremities: %w", err) - } - - if len(addStateEvents) == 0 && len(removeStateEventIDs) == 0 { - // Nothing to do, the event may have just been a message event. - return nil - } - for i := range addStateEvents { - addStateEvents[i].Visibility = historyVisibility - } - return d.updateRoomState(ctx, txn, removeStateEventIDs, addStateEvents, pduPosition, topoPosition) - }) - - return pduPosition, returnErr -} - -// This function should always be called within a sqlutil.Writer for safety in SQLite. -func (d *Database) updateRoomState( - ctx context.Context, txn *sql.Tx, - removedEventIDs []string, - addedEvents []*gomatrixserverlib.HeaderedEvent, - pduPosition types.StreamPosition, - topoPosition types.StreamPosition, -) error { - // remove first, then add, as we do not ever delete state, but do replace state which is a remove followed by an add. - for _, eventID := range removedEventIDs { - if err := d.CurrentRoomState.DeleteRoomStateByEventID(ctx, txn, eventID); err != nil { - return fmt.Errorf("d.CurrentRoomState.DeleteRoomStateByEventID: %w", err) - } - } - - for _, event := range addedEvents { - if event.StateKey() == nil { - // ignore non state events - continue - } - var membership *string - if event.Type() == "m.room.member" { - value, err := event.Membership() - if err != nil { - return fmt.Errorf("event.Membership: %w", err) - } - membership = &value - if err = d.Memberships.UpsertMembership(ctx, txn, event, pduPosition, topoPosition); err != nil { - return fmt.Errorf("d.Memberships.UpsertMembership: %w", err) - } - } - - if err := d.CurrentRoomState.UpsertRoomState(ctx, txn, event, membership, pduPosition); err != nil { - return fmt.Errorf("d.CurrentRoomState.UpsertRoomState: %w", err) - } - } - - return nil -} - -func (d *Database) GetEventsInTopologicalRange( - ctx context.Context, - from, to *types.TopologyToken, - roomID string, - filter *gomatrixserverlib.RoomEventFilter, - backwardOrdering bool, -) (events []types.StreamEvent, err error) { - var minDepth, maxDepth, maxStreamPosForMaxDepth types.StreamPosition - if backwardOrdering { - // Backward ordering means the 'from' token has a higher depth than the 'to' token - minDepth = to.Depth - maxDepth = from.Depth - // for cases where we have say 5 events with the same depth, the TopologyToken needs to - // know which of the 5 the client has seen. This is done by using the PDU position. - // Events with the same maxDepth but less than this PDU position will be returned. - maxStreamPosForMaxDepth = from.PDUPosition - } else { - // Forward ordering means the 'from' token has a lower depth than the 'to' token. - minDepth = from.Depth - maxDepth = to.Depth - } - - // Select the event IDs from the defined range. - var eIDs []string - eIDs, err = d.Topology.SelectEventIDsInRange( - ctx, nil, roomID, minDepth, maxDepth, maxStreamPosForMaxDepth, filter.Limit, !backwardOrdering, - ) - if err != nil { - return - } - - // Retrieve the events' contents using their IDs. - events, err = d.OutputEvents.SelectEvents(ctx, nil, eIDs, filter, true) - return -} - -func (d *Database) BackwardExtremitiesForRoom( - ctx context.Context, roomID string, -) (backwardExtremities map[string][]string, err error) { - return d.BackwardExtremities.SelectBackwardExtremitiesForRoom(ctx, nil, roomID) -} - -func (d *Database) MaxTopologicalPosition( - ctx context.Context, roomID string, -) (types.TopologyToken, error) { - depth, streamPos, err := d.Topology.SelectMaxPositionInTopology(ctx, nil, roomID) - if err != nil { - return types.TopologyToken{}, err - } - return types.TopologyToken{Depth: depth, PDUPosition: streamPos}, nil -} - -func (d *Database) EventPositionInTopology( - ctx context.Context, eventID string, -) (types.TopologyToken, error) { - depth, stream, err := d.Topology.SelectPositionInTopology(ctx, nil, eventID) - if err != nil { - return types.TopologyToken{}, err - } - return types.TopologyToken{Depth: depth, PDUPosition: stream}, nil -} - -func (d *Database) StreamToTopologicalPosition( - ctx context.Context, roomID string, streamPos types.StreamPosition, backwardOrdering bool, -) (types.TopologyToken, error) { - topoPos, err := d.Topology.SelectStreamToTopologicalPosition(ctx, nil, roomID, streamPos, backwardOrdering) - switch { - case err == sql.ErrNoRows && backwardOrdering: // no events in range, going backward - return types.TopologyToken{PDUPosition: streamPos}, nil - case err == sql.ErrNoRows && !backwardOrdering: // no events in range, going forward - topoPos, streamPos, err = d.Topology.SelectMaxPositionInTopology(ctx, nil, roomID) - if err != nil { - return types.TopologyToken{}, fmt.Errorf("d.Topology.SelectMaxPositionInTopology: %w", err) - } - return types.TopologyToken{Depth: topoPos, PDUPosition: streamPos}, nil - case err != nil: // some other error happened - return types.TopologyToken{}, fmt.Errorf("d.Topology.SelectStreamToTopologicalPosition: %w", err) - default: - return types.TopologyToken{Depth: topoPos, PDUPosition: streamPos}, nil - } -} - -func (d *Database) GetFilter( - ctx context.Context, target *gomatrixserverlib.Filter, localpart string, filterID string, -) error { - return d.Filter.SelectFilter(ctx, nil, target, localpart, filterID) -} - -func (d *Database) PutFilter( - ctx context.Context, localpart string, filter *gomatrixserverlib.Filter, -) (string, error) { - var filterID string - var err error - err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - filterID, err = d.Filter.InsertFilter(ctx, txn, filter, localpart) - return err - }) - return filterID, err -} - -func (d *Database) RedactEvent(ctx context.Context, redactedEventID string, redactedBecause *gomatrixserverlib.HeaderedEvent) error { - redactedEvents, err := d.Events(ctx, []string{redactedEventID}) - if err != nil { - return err - } - if len(redactedEvents) == 0 { - logrus.WithField("event_id", redactedEventID).WithField("redaction_event", redactedBecause.EventID()).Warnf("missing redacted event for redaction") - return nil - } - eventToRedact := redactedEvents[0].Unwrap() - redactionEvent := redactedBecause.Unwrap() - if err = eventutil.RedactEvent(redactionEvent, eventToRedact); err != nil { - return err - } - - newEvent := eventToRedact.Headered(redactedBecause.RoomVersion) - err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - return d.OutputEvents.UpdateEventJSON(ctx, txn, newEvent) - }) - return err -} - -// GetBackwardTopologyPos retrieves the backward topology position, i.e. the position of the -// oldest event in the room's topology. -func (d *Database) GetBackwardTopologyPos( - ctx context.Context, - events []types.StreamEvent, -) (types.TopologyToken, error) { - zeroToken := types.TopologyToken{} - if len(events) == 0 { - return zeroToken, nil - } - pos, spos, err := d.Topology.SelectPositionInTopology(ctx, nil, events[0].EventID()) - if err != nil { - return zeroToken, err - } - tok := types.TopologyToken{Depth: pos, PDUPosition: spos} - tok.Decrement() - return tok, nil -} - -// fetchStateEvents converts the set of event IDs into a set of events. It will fetch any which are missing from the database. -// Returns a map of room ID to list of events. -func (d *Database) fetchStateEvents( - ctx context.Context, txn *sql.Tx, - roomIDToEventIDSet map[string]map[string]bool, - eventIDToEvent map[string]types.StreamEvent, -) (map[string][]types.StreamEvent, error) { - stateBetween := make(map[string][]types.StreamEvent) - missingEvents := make(map[string][]string) - for roomID, ids := range roomIDToEventIDSet { - events := stateBetween[roomID] - for id, need := range ids { - if !need { - continue // deleted state - } - e, ok := eventIDToEvent[id] - if ok { - events = append(events, e) - } else { - m := missingEvents[roomID] - m = append(m, id) - missingEvents[roomID] = m - } - } - stateBetween[roomID] = events - } - - if len(missingEvents) > 0 { - // This happens when add_state_ids has an event ID which is not in the provided range. - // We need to explicitly fetch them. - allMissingEventIDs := []string{} - for _, missingEvIDs := range missingEvents { - allMissingEventIDs = append(allMissingEventIDs, missingEvIDs...) - } - evs, err := d.fetchMissingStateEvents(ctx, txn, allMissingEventIDs) - if err != nil { - return nil, err - } - // we know we got them all otherwise an error would've been returned, so just loop the events - for _, ev := range evs { - roomID := ev.RoomID() - stateBetween[roomID] = append(stateBetween[roomID], ev) - } - } - return stateBetween, nil -} - -func (d *Database) fetchMissingStateEvents( - ctx context.Context, txn *sql.Tx, eventIDs []string, -) ([]types.StreamEvent, error) { - // Fetch from the events table first so we pick up the stream ID for the - // event. - events, err := d.OutputEvents.SelectEvents(ctx, txn, eventIDs, nil, false) - if err != nil { - return nil, err - } - - have := map[string]bool{} - for _, event := range events { - have[event.EventID()] = true - } - var missing []string - for _, eventID := range eventIDs { - if !have[eventID] { - missing = append(missing, eventID) - } - } - if len(missing) == 0 { - return events, nil - } - - // If they are missing from the events table then they should be state - // events that we received from outside the main event stream. - // These should be in the room state table. - stateEvents, err := d.CurrentRoomState.SelectEventsWithEventIDs(ctx, txn, missing) - - if err != nil { - return nil, err - } - if len(stateEvents) != len(missing) { - logrus.WithContext(ctx).Warnf("Failed to map all event IDs to events (got %d, wanted %d)", len(stateEvents), len(missing)) - - // TODO: Why is this happening? It's probably the roomserver. Uncomment - // this error again when we work out what it is and fix it, otherwise we - // just end up returning lots of 500s to the client and that breaks - // pretty much everything, rather than just sending what we have. - //return nil, fmt.Errorf("failed to map all event IDs to events: (got %d, wanted %d)", len(stateEvents), len(missing)) - } - events = append(events, stateEvents...) - return events, nil -} - -// GetStateDeltas returns the state deltas between fromPos and toPos, -// exclusive of oldPos, inclusive of newPos, for the rooms in which -// the user has new membership events. -// A list of joined room IDs is also returned in case the caller needs it. -func (d *Database) GetStateDeltas( - ctx context.Context, device *userapi.Device, - r types.Range, userID string, - stateFilter *gomatrixserverlib.StateFilter, -) (deltas []types.StateDelta, joinedRoomsIDs []string, err error) { - // Implement membership change algorithm: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L821 - // - Get membership list changes for this user in this sync response - // - For each room which has membership list changes: - // * Check if the room is 'newly joined' (insufficient to just check for a join event because we allow dupe joins TODO). - // If it is, then we need to send the full room state down (and 'limited' is always true). - // * Check if user is still CURRENTLY invited to the room. If so, add room to 'invited' block. - // * Check if the user is CURRENTLY (TODO) left/banned. If so, add room to 'archived' block. - // - Get all CURRENTLY joined rooms, and add them to 'joined' block. - txn, err := d.readOnlySnapshot(ctx) - if err != nil { - return nil, nil, fmt.Errorf("d.readOnlySnapshot: %w", err) - } - var succeeded bool - defer sqlutil.EndTransactionWithCheck(txn, &succeeded, &err) - - // Look up all memberships for the user. We only care about rooms that a - // user has ever interacted with — joined to, kicked/banned from, left. - memberships, err := d.CurrentRoomState.SelectRoomIDsWithAnyMembership(ctx, txn, userID) - if err != nil { - if err == sql.ErrNoRows { - return nil, nil, nil - } - return nil, nil, err - } - - allRoomIDs := make([]string, 0, len(memberships)) - joinedRoomIDs := make([]string, 0, len(memberships)) - for roomID, membership := range memberships { - allRoomIDs = append(allRoomIDs, roomID) - if membership == gomatrixserverlib.Join { - joinedRoomIDs = append(joinedRoomIDs, roomID) - } - } - - // get all the state events ever (i.e. for all available rooms) between these two positions - stateNeeded, eventMap, err := d.OutputEvents.SelectStateInRange(ctx, txn, r, stateFilter, allRoomIDs) - if err != nil { - if err == sql.ErrNoRows { - return nil, nil, nil - } - return nil, nil, err - } - state, err := d.fetchStateEvents(ctx, txn, stateNeeded, eventMap) - if err != nil { - if err == sql.ErrNoRows { - return nil, nil, nil - } - return nil, nil, err - } - - // find out which rooms this user is peeking, if any. - // We do this before joins so any peeks get overwritten - peeks, err := d.Peeks.SelectPeeksInRange(ctx, txn, userID, device.ID, r) - if err != nil && err != sql.ErrNoRows { - return nil, nil, err - } - - // add peek blocks - for _, peek := range peeks { - if peek.New { - // send full room state down instead of a delta - var s []types.StreamEvent - s, err = d.currentStateStreamEventsForRoom(ctx, txn, peek.RoomID, stateFilter) - if err != nil { - if err == sql.ErrNoRows { - continue - } - return nil, nil, err - } - state[peek.RoomID] = s - } - if !peek.Deleted { - deltas = append(deltas, types.StateDelta{ - Membership: gomatrixserverlib.Peek, - StateEvents: d.StreamEventsToEvents(device, state[peek.RoomID]), - RoomID: peek.RoomID, - }) - } - } - - // handle newly joined rooms and non-joined rooms - newlyJoinedRooms := make(map[string]bool, len(state)) - for roomID, stateStreamEvents := range state { - for _, ev := range stateStreamEvents { - if membership, prevMembership := getMembershipFromEvent(ev.Event, userID); membership != "" { - if membership == gomatrixserverlib.Join && prevMembership != membership { - // send full room state down instead of a delta - var s []types.StreamEvent - s, err = d.currentStateStreamEventsForRoom(ctx, txn, roomID, stateFilter) - if err != nil { - if err == sql.ErrNoRows { - continue - } - return nil, nil, err - } - state[roomID] = s - newlyJoinedRooms[roomID] = true - continue // we'll add this room in when we do joined rooms - } - - deltas = append(deltas, types.StateDelta{ - Membership: membership, - MembershipPos: ev.StreamPosition, - StateEvents: d.StreamEventsToEvents(device, stateStreamEvents), - RoomID: roomID, - }) - break - } - } - } - - // Add in currently joined rooms - for _, joinedRoomID := range joinedRoomIDs { - deltas = append(deltas, types.StateDelta{ - Membership: gomatrixserverlib.Join, - StateEvents: d.StreamEventsToEvents(device, state[joinedRoomID]), - RoomID: joinedRoomID, - NewlyJoined: newlyJoinedRooms[joinedRoomID], - }) - } - - succeeded = true - return deltas, joinedRoomIDs, nil -} - -// GetStateDeltasForFullStateSync is a variant of getStateDeltas used for /sync -// requests with full_state=true. -// Fetches full state for all joined rooms and uses selectStateInRange to get -// updates for other rooms. -func (d *Database) GetStateDeltasForFullStateSync( - ctx context.Context, device *userapi.Device, - r types.Range, userID string, - stateFilter *gomatrixserverlib.StateFilter, -) ([]types.StateDelta, []string, error) { - txn, err := d.readOnlySnapshot(ctx) - if err != nil { - return nil, nil, fmt.Errorf("d.readOnlySnapshot: %w", err) - } - var succeeded bool - defer sqlutil.EndTransactionWithCheck(txn, &succeeded, &err) - - // Look up all memberships for the user. We only care about rooms that a - // user has ever interacted with — joined to, kicked/banned from, left. - memberships, err := d.CurrentRoomState.SelectRoomIDsWithAnyMembership(ctx, txn, userID) - if err != nil { - if err == sql.ErrNoRows { - return nil, nil, nil - } - return nil, nil, err - } - - allRoomIDs := make([]string, 0, len(memberships)) - joinedRoomIDs := make([]string, 0, len(memberships)) - for roomID, membership := range memberships { - allRoomIDs = append(allRoomIDs, roomID) - if membership == gomatrixserverlib.Join { - joinedRoomIDs = append(joinedRoomIDs, roomID) - } - } - - // Use a reasonable initial capacity - deltas := make(map[string]types.StateDelta) - - peeks, err := d.Peeks.SelectPeeksInRange(ctx, txn, userID, device.ID, r) - if err != nil && err != sql.ErrNoRows { - return nil, nil, err - } - - // Add full states for all peeking rooms - for _, peek := range peeks { - if !peek.Deleted { - s, stateErr := d.currentStateStreamEventsForRoom(ctx, txn, peek.RoomID, stateFilter) - if stateErr != nil { - if stateErr == sql.ErrNoRows { - continue - } - return nil, nil, stateErr - } - deltas[peek.RoomID] = types.StateDelta{ - Membership: gomatrixserverlib.Peek, - StateEvents: d.StreamEventsToEvents(device, s), - RoomID: peek.RoomID, - } - } - } - - // Get all the state events ever between these two positions - stateNeeded, eventMap, err := d.OutputEvents.SelectStateInRange(ctx, txn, r, stateFilter, allRoomIDs) - if err != nil { - if err == sql.ErrNoRows { - return nil, nil, nil - } - return nil, nil, err - } - state, err := d.fetchStateEvents(ctx, txn, stateNeeded, eventMap) - if err != nil { - if err == sql.ErrNoRows { - return nil, nil, nil - } - return nil, nil, err - } - - for roomID, stateStreamEvents := range state { - for _, ev := range stateStreamEvents { - if membership, _ := getMembershipFromEvent(ev.Event, userID); membership != "" { - if membership != gomatrixserverlib.Join { // We've already added full state for all joined rooms above. - deltas[roomID] = types.StateDelta{ - Membership: membership, - MembershipPos: ev.StreamPosition, - StateEvents: d.StreamEventsToEvents(device, stateStreamEvents), - RoomID: roomID, - } - } - - break - } - } - } - - // Add full states for all joined rooms - for _, joinedRoomID := range joinedRoomIDs { - s, stateErr := d.currentStateStreamEventsForRoom(ctx, txn, joinedRoomID, stateFilter) - if stateErr != nil { - if stateErr == sql.ErrNoRows { - continue - } - return nil, nil, stateErr - } - deltas[joinedRoomID] = types.StateDelta{ - Membership: gomatrixserverlib.Join, - StateEvents: d.StreamEventsToEvents(device, s), - RoomID: joinedRoomID, - } - } - - // Create a response array. - result := make([]types.StateDelta, len(deltas)) - i := 0 - for _, delta := range deltas { - result[i] = delta - i++ - } - - succeeded = true - return result, joinedRoomIDs, nil -} - -func (d *Database) currentStateStreamEventsForRoom( - ctx context.Context, txn *sql.Tx, roomID string, - stateFilter *gomatrixserverlib.StateFilter, -) ([]types.StreamEvent, error) { - allState, err := d.CurrentRoomState.SelectCurrentState(ctx, txn, roomID, stateFilter, nil) - if err != nil { - return nil, err - } - s := make([]types.StreamEvent, len(allState)) - for i := 0; i < len(s); i++ { - s[i] = types.StreamEvent{HeaderedEvent: allState[i], StreamPosition: 0} - } - return s, nil -} - -func (d *Database) StoreNewSendForDeviceMessage( - ctx context.Context, userID, deviceID string, event gomatrixserverlib.SendToDeviceEvent, -) (newPos types.StreamPosition, err error) { - j, err := json.Marshal(event) - if err != nil { - return 0, err - } - // Delegate the database write task to the SendToDeviceWriter. It'll guarantee - // that we don't lock the table for writes in more than one place. - err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - newPos, err = d.SendToDevice.InsertSendToDeviceMessage( - ctx, txn, userID, deviceID, string(j), - ) - return err - }) - if err != nil { - return 0, err - } - return newPos, nil -} - -func (d *Database) SendToDeviceUpdatesForSync( - ctx context.Context, - userID, deviceID string, - from, to types.StreamPosition, -) (types.StreamPosition, []types.SendToDeviceEvent, error) { - // First of all, get our send-to-device updates for this user. - lastPos, events, err := d.SendToDevice.SelectSendToDeviceMessages(ctx, nil, userID, deviceID, from, to) - if err != nil { - return from, nil, fmt.Errorf("d.SendToDevice.SelectSendToDeviceMessages: %w", err) - } - // If there's nothing to do then stop here. - if len(events) == 0 { - return to, nil, nil - } - return lastPos, events, nil -} - -func (d *Database) CleanSendToDeviceUpdates( - ctx context.Context, - userID, deviceID string, before types.StreamPosition, -) (err error) { - if err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - return d.SendToDevice.DeleteSendToDeviceMessages(ctx, txn, userID, deviceID, before) - }); err != nil { - logrus.WithError(err).Errorf("Failed to clean up old send-to-device messages for user %q device %q", userID, deviceID) - return err - } - return nil -} - -// getMembershipFromEvent returns the value of content.membership iff the event is a state event -// with type 'm.room.member' and state_key of userID. Otherwise, an empty string is returned. -func getMembershipFromEvent(ev *gomatrixserverlib.Event, userID string) (string, string) { - if ev.Type() != "m.room.member" || !ev.StateKeyEquals(userID) { - return "", "" - } - membership, err := ev.Membership() - if err != nil { - return "", "" - } - prevMembership := gjson.GetBytes(ev.Unsigned(), "prev_content.membership").Str - return membership, prevMembership -} - -// StoreReceipt stores user receipts -func (d *Database) StoreReceipt(ctx context.Context, roomId, receiptType, userId, eventId string, timestamp gomatrixserverlib.Timestamp) (pos types.StreamPosition, err error) { - err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - pos, err = d.Receipts.UpsertReceipt(ctx, txn, roomId, receiptType, userId, eventId, timestamp) - return err - }) - return -} - -func (d *Database) GetRoomReceipts(ctx context.Context, roomIDs []string, streamPos types.StreamPosition) ([]types.OutputReceiptEvent, error) { - _, receipts, err := d.Receipts.SelectRoomReceiptsAfter(ctx, nil, roomIDs, streamPos) - return receipts, err -} - -func (d *Database) UpsertRoomUnreadNotificationCounts(ctx context.Context, userID, roomID string, notificationCount, highlightCount int) (pos types.StreamPosition, err error) { - err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - pos, err = d.NotificationData.UpsertRoomUnreadCounts(ctx, txn, userID, roomID, notificationCount, highlightCount) - return err - }) - return -} - -func (d *Database) GetUserUnreadNotificationCountsForRooms(ctx context.Context, userID string, rooms map[string]string) (map[string]*eventutil.NotificationData, error) { - roomIDs := make([]string, 0, len(rooms)) - for roomID, membership := range rooms { - if membership != gomatrixserverlib.Join { - continue - } - roomIDs = append(roomIDs, roomID) - } - return d.NotificationData.SelectUserUnreadCountsForRooms(ctx, nil, userID, roomIDs) -} - -func (d *Database) SelectContextEvent(ctx context.Context, roomID, eventID string) (int, gomatrixserverlib.HeaderedEvent, error) { - return d.OutputEvents.SelectContextEvent(ctx, nil, roomID, eventID) -} - -func (d *Database) SelectContextBeforeEvent(ctx context.Context, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) ([]*gomatrixserverlib.HeaderedEvent, error) { - return d.OutputEvents.SelectContextBeforeEvent(ctx, nil, id, roomID, filter) -} -func (d *Database) SelectContextAfterEvent(ctx context.Context, id int, roomID string, filter *gomatrixserverlib.RoomEventFilter) (int, []*gomatrixserverlib.HeaderedEvent, error) { - return d.OutputEvents.SelectContextAfterEvent(ctx, nil, id, roomID, filter) -} - -func (d *Database) IgnoresForUser(ctx context.Context, userID string) (*types.IgnoredUsers, error) { - return d.Ignores.SelectIgnores(ctx, nil, userID) -} - -func (d *Database) UpdateIgnoresForUser(ctx context.Context, userID string, ignores *types.IgnoredUsers) error { - return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - return d.Ignores.UpsertIgnores(ctx, txn, userID, ignores) - }) -} - -func (d *Database) UpdatePresence(ctx context.Context, userID string, presence types.Presence, statusMsg *string, lastActiveTS gomatrixserverlib.Timestamp, fromSync bool) (types.StreamPosition, error) { - var pos types.StreamPosition - var err error - _ = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - pos, err = d.Presence.UpsertPresence(ctx, txn, userID, statusMsg, presence, lastActiveTS, fromSync) - return nil - }) - return pos, err -} - -func (d *Database) GetPresence(ctx context.Context, userID string) (*types.PresenceInternal, error) { - return d.Presence.GetPresenceForUser(ctx, nil, userID) -} - -func (d *Database) PresenceAfter(ctx context.Context, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (map[string]*types.PresenceInternal, error) { - return d.Presence.GetPresenceAfter(ctx, nil, after, filter) -} - -func (d *Database) MaxStreamPositionForPresence(ctx context.Context) (types.StreamPosition, error) { - return d.Presence.GetMaxPresenceID(ctx, nil) -} - -func (d *Database) SelectMembershipForUser(ctx context.Context, roomID, userID string, pos int64) (membership string, topologicalPos int, err error) { - return d.Memberships.SelectMembershipForUser(ctx, nil, roomID, userID, pos) -} - -func (s *Database) ReIndex(ctx context.Context, limit, afterID int64) (map[int64]gomatrixserverlib.HeaderedEvent, error) { - return s.OutputEvents.ReIndex(ctx, nil, limit, afterID, []string{ - gomatrixserverlib.MRoomName, - gomatrixserverlib.MRoomTopic, - "m.room.message", - }) -} diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go index ba6d8126c..c4019fed2 100644 --- a/syncapi/storage/sqlite3/current_room_state_table.go +++ b/syncapi/storage/sqlite3/current_room_state_table.go @@ -367,7 +367,13 @@ func (s *currentRoomStateStatements) SelectEventsWithEventIDs( for start < len(eventIDs) { n := minOfInts(len(eventIDs)-start, 999) query := strings.Replace(selectEventsWithEventIDsSQL, "($1)", sqlutil.QueryVariadic(n), 1) - rows, err := txn.QueryContext(ctx, query, iEventIDs[start:start+n]...) + var rows *sql.Rows + var err error + if txn == nil { + rows, err = s.db.QueryContext(ctx, query, iEventIDs[start:start+n]...) + } else { + rows, err = txn.QueryContext(ctx, query, iEventIDs[start:start+n]...) + } if err != nil { return nil, err } diff --git a/syncapi/storage/sqlite3/invites_table.go b/syncapi/storage/sqlite3/invites_table.go index 58ab8461e..e2dbcd5c8 100644 --- a/syncapi/storage/sqlite3/invites_table.go +++ b/syncapi/storage/sqlite3/invites_table.go @@ -50,7 +50,7 @@ const deleteInviteEventSQL = "" + "UPDATE syncapi_invite_events SET deleted=true, id=$1 WHERE event_id = $2 AND deleted=false" const selectInviteEventsInRangeSQL = "" + - "SELECT room_id, headered_event_json, deleted FROM syncapi_invite_events" + + "SELECT id, room_id, headered_event_json, deleted FROM syncapi_invite_events" + " WHERE target_user_id = $1 AND id > $2 AND id <= $3" + " ORDER BY id DESC" @@ -132,23 +132,28 @@ func (s *inviteEventsStatements) DeleteInviteEvent( // active invites for the target user ID in the supplied range. func (s *inviteEventsStatements) SelectInviteEventsInRange( ctx context.Context, txn *sql.Tx, targetUserID string, r types.Range, -) (map[string]*gomatrixserverlib.HeaderedEvent, map[string]*gomatrixserverlib.HeaderedEvent, error) { +) (map[string]*gomatrixserverlib.HeaderedEvent, map[string]*gomatrixserverlib.HeaderedEvent, types.StreamPosition, error) { + var lastPos types.StreamPosition stmt := sqlutil.TxStmt(txn, s.selectInviteEventsInRangeStmt) rows, err := stmt.QueryContext(ctx, targetUserID, r.Low(), r.High()) if err != nil { - return nil, nil, err + return nil, nil, lastPos, err } defer internal.CloseAndLogIfError(ctx, rows, "selectInviteEventsInRange: rows.close() failed") result := map[string]*gomatrixserverlib.HeaderedEvent{} retired := map[string]*gomatrixserverlib.HeaderedEvent{} for rows.Next() { var ( + id types.StreamPosition roomID string eventJSON []byte deleted bool ) - if err = rows.Scan(&roomID, &eventJSON, &deleted); err != nil { - return nil, nil, err + if err = rows.Scan(&id, &roomID, &eventJSON, &deleted); err != nil { + return nil, nil, lastPos, err + } + if id > lastPos { + lastPos = id } // if we have seen this room before, it has a higher stream position and hence takes priority @@ -161,15 +166,19 @@ func (s *inviteEventsStatements) SelectInviteEventsInRange( var event *gomatrixserverlib.HeaderedEvent if err := json.Unmarshal(eventJSON, &event); err != nil { - return nil, nil, err + return nil, nil, lastPos, err } + if deleted { retired[roomID] = event } else { result[roomID] = event } } - return result, retired, nil + if lastPos == 0 { + lastPos = r.To + } + return result, retired, lastPos, nil } func (s *inviteEventsStatements) SelectMaxInviteID( diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index a84e2bd16..0879030a6 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -49,6 +49,20 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) return &d, nil } +func (d *SyncServerDatasource) NewDatabaseSnapshot(ctx context.Context) (*shared.DatabaseTransaction, error) { + return &shared.DatabaseTransaction{ + Database: &d.Database, + // not setting a transaction because SQLite doesn't support it + }, nil +} + +func (d *SyncServerDatasource) NewDatabaseTransaction(ctx context.Context) (*shared.DatabaseTransaction, error) { + return &shared.DatabaseTransaction{ + Database: &d.Database, + // not setting a transaction because SQLite doesn't support it + }, nil +} + func (d *SyncServerDatasource) prepare(ctx context.Context) (err error) { if err = d.streamID.Prepare(d.db); err != nil { return err diff --git a/syncapi/storage/storage_test.go b/syncapi/storage/storage_test.go index a62818e9b..5ff185a32 100644 --- a/syncapi/storage/storage_test.go +++ b/syncapi/storage/storage_test.go @@ -60,6 +60,17 @@ func TestWriteEvents(t *testing.T) { }) } +func WithSnapshot(t *testing.T, db storage.Database, f func(snapshot storage.DatabaseTransaction)) { + snapshot, err := db.NewDatabaseSnapshot(ctx) + if err != nil { + t.Fatal(err) + } + f(snapshot) + if err := snapshot.Rollback(); err != nil { + t.Fatal(err) + } +} + // These tests assert basic functionality of RecentEvents for PDUs func TestRecentEventsPDU(t *testing.T) { test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { @@ -79,10 +90,13 @@ func TestRecentEventsPDU(t *testing.T) { // dummy room to make sure SQL queries are filtering on room ID MustWriteEvents(t, db, test.NewRoom(t, alice).Events()) - latest, err := db.MaxStreamPositionForPDUs(ctx) - if err != nil { - t.Fatalf("failed to get MaxStreamPositionForPDUs: %s", err) - } + var latest types.StreamPosition + WithSnapshot(t, db, func(snapshot storage.DatabaseTransaction) { + var err error + if latest, err = snapshot.MaxStreamPositionForPDUs(ctx); err != nil { + t.Fatal("failed to get MaxStreamPositionForPDUs: %w", err) + } + }) testCases := []struct { Name string @@ -140,14 +154,19 @@ func TestRecentEventsPDU(t *testing.T) { tc := testCases[i] t.Run(tc.Name, func(st *testing.T) { var filter gomatrixserverlib.RoomEventFilter + var gotEvents []types.StreamEvent + var limited bool filter.Limit = tc.Limit - gotEvents, limited, err := db.RecentEvents(ctx, r.ID, types.Range{ - From: tc.From, - To: tc.To, - }, &filter, !tc.ReverseOrder, true) - if err != nil { - st.Fatalf("failed to do sync: %s", err) - } + WithSnapshot(t, db, func(snapshot storage.DatabaseTransaction) { + var err error + gotEvents, limited, err = snapshot.RecentEvents(ctx, r.ID, types.Range{ + From: tc.From, + To: tc.To, + }, &filter, !tc.ReverseOrder, true) + if err != nil { + st.Fatalf("failed to do sync: %s", err) + } + }) if limited != tc.WantLimited { st.Errorf("got limited=%v want %v", limited, tc.WantLimited) } @@ -178,22 +197,24 @@ func TestGetEventsInRangeWithTopologyToken(t *testing.T) { events := r.Events() _ = MustWriteEvents(t, db, events) - from, err := db.MaxTopologicalPosition(ctx, r.ID) - if err != nil { - t.Fatalf("failed to get MaxTopologicalPosition: %s", err) - } - t.Logf("max topo pos = %+v", from) - // head towards the beginning of time - to := types.TopologyToken{} + WithSnapshot(t, db, func(snapshot storage.DatabaseTransaction) { + from, err := snapshot.MaxTopologicalPosition(ctx, r.ID) + if err != nil { + t.Fatalf("failed to get MaxTopologicalPosition: %s", err) + } + t.Logf("max topo pos = %+v", from) + // head towards the beginning of time + to := types.TopologyToken{} - // backpaginate 5 messages starting at the latest position. - filter := &gomatrixserverlib.RoomEventFilter{Limit: 5} - paginatedEvents, err := db.GetEventsInTopologicalRange(ctx, &from, &to, r.ID, filter, true) - if err != nil { - t.Fatalf("GetEventsInTopologicalRange returned an error: %s", err) - } - gots := db.StreamEventsToEvents(nil, paginatedEvents) - test.AssertEventsEqual(t, gots, test.Reversed(events[len(events)-5:])) + // backpaginate 5 messages starting at the latest position. + filter := &gomatrixserverlib.RoomEventFilter{Limit: 5} + paginatedEvents, err := snapshot.GetEventsInTopologicalRange(ctx, &from, &to, r.ID, filter, true) + if err != nil { + t.Fatalf("GetEventsInTopologicalRange returned an error: %s", err) + } + gots := snapshot.StreamEventsToEvents(nil, paginatedEvents) + test.AssertEventsEqual(t, gots, test.Reversed(events[len(events)-5:])) + }) }) } @@ -414,13 +435,16 @@ func TestSendToDeviceBehaviour(t *testing.T) { defer closeBase() // At this point there should be no messages. We haven't sent anything // yet. - _, events, err := db.SendToDeviceUpdatesForSync(ctx, alice.ID, deviceID, 0, 100) - if err != nil { - t.Fatal(err) - } - if len(events) != 0 { - t.Fatal("first call should have no updates") - } + + WithSnapshot(t, db, func(snapshot storage.DatabaseTransaction) { + _, events, err := snapshot.SendToDeviceUpdatesForSync(ctx, alice.ID, deviceID, 0, 100) + if err != nil { + t.Fatal(err) + } + if len(events) != 0 { + t.Fatal("first call should have no updates") + } + }) // Try sending a message. streamPos, err := db.StoreNewSendForDeviceMessage(ctx, alice.ID, deviceID, gomatrixserverlib.SendToDeviceEvent{ @@ -432,51 +456,58 @@ func TestSendToDeviceBehaviour(t *testing.T) { t.Fatal(err) } - // At this point we should get exactly one message. We're sending the sync position - // that we were given from the update and the send-to-device update will be updated - // in the database to reflect that this was the sync position we sent the message at. - streamPos, events, err = db.SendToDeviceUpdatesForSync(ctx, alice.ID, deviceID, 0, streamPos) - if err != nil { - t.Fatal(err) - } - if count := len(events); count != 1 { - t.Fatalf("second call should have one update, got %d", count) - } + WithSnapshot(t, db, func(snapshot storage.DatabaseTransaction) { + // At this point we should get exactly one message. We're sending the sync position + // that we were given from the update and the send-to-device update will be updated + // in the database to reflect that this was the sync position we sent the message at. + var events []types.SendToDeviceEvent + streamPos, events, err = snapshot.SendToDeviceUpdatesForSync(ctx, alice.ID, deviceID, 0, streamPos) + if err != nil { + t.Fatal(err) + } + if count := len(events); count != 1 { + t.Fatalf("second call should have one update, got %d", count) + } + + // At this point we should still have one message because we haven't progressed the + // sync position yet. This is equivalent to the client failing to /sync and retrying + // with the same position. + streamPos, events, err = snapshot.SendToDeviceUpdatesForSync(ctx, alice.ID, deviceID, 0, streamPos) + if err != nil { + t.Fatal(err) + } + if len(events) != 1 { + t.Fatal("third call should have one update still") + } + }) - // At this point we should still have one message because we haven't progressed the - // sync position yet. This is equivalent to the client failing to /sync and retrying - // with the same position. - streamPos, events, err = db.SendToDeviceUpdatesForSync(ctx, alice.ID, deviceID, 0, streamPos) - if err != nil { - t.Fatal(err) - } - if len(events) != 1 { - t.Fatal("third call should have one update still") - } err = db.CleanSendToDeviceUpdates(context.Background(), alice.ID, deviceID, streamPos) if err != nil { return } - // At this point we should now have no updates, because we've progressed the sync - // position. Therefore the update from before will not be sent again. - _, events, err = db.SendToDeviceUpdatesForSync(ctx, alice.ID, deviceID, streamPos, streamPos+10) - if err != nil { - t.Fatal(err) - } - if len(events) != 0 { - t.Fatal("fourth call should have no updates") - } + WithSnapshot(t, db, func(snapshot storage.DatabaseTransaction) { + // At this point we should now have no updates, because we've progressed the sync + // position. Therefore the update from before will not be sent again. + var events []types.SendToDeviceEvent + _, events, err = snapshot.SendToDeviceUpdatesForSync(ctx, alice.ID, deviceID, streamPos, streamPos+10) + if err != nil { + t.Fatal(err) + } + if len(events) != 0 { + t.Fatal("fourth call should have no updates") + } - // At this point we should still have no updates, because no new updates have been - // sent. - _, events, err = db.SendToDeviceUpdatesForSync(ctx, alice.ID, deviceID, streamPos, streamPos+10) - if err != nil { - t.Fatal(err) - } - if len(events) != 0 { - t.Fatal("fifth call should have no updates") - } + // At this point we should still have no updates, because no new updates have been + // sent. + _, events, err = snapshot.SendToDeviceUpdatesForSync(ctx, alice.ID, deviceID, streamPos, streamPos+10) + if err != nil { + t.Fatal(err) + } + if len(events) != 0 { + t.Fatal("fifth call should have no updates") + } + }) // Send some more messages and verify the ordering is correct ("in order of arrival") var lastPos types.StreamPosition = 0 @@ -492,18 +523,20 @@ func TestSendToDeviceBehaviour(t *testing.T) { lastPos = streamPos } - _, events, err = db.SendToDeviceUpdatesForSync(ctx, alice.ID, deviceID, 0, lastPos) - if err != nil { - t.Fatalf("unable to get events: %v", err) - } - - for i := 0; i < 10; i++ { - want := json.RawMessage(fmt.Sprintf(`{"count":%d}`, i)) - got := events[i].Content - if !bytes.Equal(got, want) { - t.Fatalf("messages are out of order\nwant: %s\ngot: %s", string(want), string(got)) + WithSnapshot(t, db, func(snapshot storage.DatabaseTransaction) { + _, events, err := snapshot.SendToDeviceUpdatesForSync(ctx, alice.ID, deviceID, 0, lastPos) + if err != nil { + t.Fatalf("unable to get events: %v", err) } - } + + for i := 0; i < 10; i++ { + want := json.RawMessage(fmt.Sprintf(`{"count":%d}`, i)) + got := events[i].Content + if !bytes.Equal(got, want) { + t.Fatalf("messages are out of order\nwant: %s\ngot: %s", string(want), string(got)) + } + } + }) }) } diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go index 89cb537af..2fdc3cfbb 100644 --- a/syncapi/storage/tables/interface.go +++ b/syncapi/storage/tables/interface.go @@ -37,7 +37,7 @@ type Invites interface { DeleteInviteEvent(ctx context.Context, txn *sql.Tx, inviteEventID string) (types.StreamPosition, error) // SelectInviteEventsInRange returns a map of room ID to invite events. If multiple invite/retired invites exist in the given range, return the latest value // for the room. - SelectInviteEventsInRange(ctx context.Context, txn *sql.Tx, targetUserID string, r types.Range) (invites map[string]*gomatrixserverlib.HeaderedEvent, retired map[string]*gomatrixserverlib.HeaderedEvent, err error) + SelectInviteEventsInRange(ctx context.Context, txn *sql.Tx, targetUserID string, r types.Range) (invites map[string]*gomatrixserverlib.HeaderedEvent, retired map[string]*gomatrixserverlib.HeaderedEvent, maxID types.StreamPosition, err error) SelectMaxInviteID(ctx context.Context, txn *sql.Tx) (id int64, err error) } diff --git a/syncapi/streams/stream_accountdata.go b/syncapi/streams/stream_accountdata.go index 0297d5c2f..3f2f7d134 100644 --- a/syncapi/streams/stream_accountdata.go +++ b/syncapi/streams/stream_accountdata.go @@ -5,22 +5,25 @@ import ( "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/types" userapi "github.com/matrix-org/dendrite/userapi/api" ) type AccountDataStreamProvider struct { - StreamProvider + DefaultStreamProvider userAPI userapi.SyncUserAPI } -func (p *AccountDataStreamProvider) Setup() { - p.StreamProvider.Setup() +func (p *AccountDataStreamProvider) Setup( + ctx context.Context, snapshot storage.DatabaseTransaction, +) { + p.DefaultStreamProvider.Setup(ctx, snapshot) p.latestMutex.Lock() defer p.latestMutex.Unlock() - id, err := p.DB.MaxStreamPositionForAccountData(context.Background()) + id, err := snapshot.MaxStreamPositionForAccountData(ctx) if err != nil { panic(err) } @@ -29,13 +32,15 @@ func (p *AccountDataStreamProvider) Setup() { func (p *AccountDataStreamProvider) CompleteSync( ctx context.Context, + snapshot storage.DatabaseTransaction, req *types.SyncRequest, ) types.StreamPosition { - return p.IncrementalSync(ctx, req, 0, p.LatestPosition(ctx)) + return p.IncrementalSync(ctx, snapshot, req, 0, p.LatestPosition(ctx)) } func (p *AccountDataStreamProvider) IncrementalSync( ctx context.Context, + snapshot storage.DatabaseTransaction, req *types.SyncRequest, from, to types.StreamPosition, ) types.StreamPosition { @@ -44,7 +49,7 @@ func (p *AccountDataStreamProvider) IncrementalSync( To: to, } - dataTypes, pos, err := p.DB.GetAccountDataInRange( + dataTypes, pos, err := snapshot.GetAccountDataInRange( ctx, req.Device.UserID, r, &req.Filter.AccountData, ) if err != nil { diff --git a/syncapi/streams/stream_devicelist.go b/syncapi/streams/stream_devicelist.go index 5448ee5bd..7996c2038 100644 --- a/syncapi/streams/stream_devicelist.go +++ b/syncapi/streams/stream_devicelist.go @@ -6,17 +6,19 @@ import ( keyapi "github.com/matrix-org/dendrite/keyserver/api" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/internal" + "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/types" ) type DeviceListStreamProvider struct { - StreamProvider + DefaultStreamProvider rsAPI api.SyncRoomserverAPI keyAPI keyapi.SyncKeyAPI } func (p *DeviceListStreamProvider) CompleteSync( ctx context.Context, + snapshot storage.DatabaseTransaction, req *types.SyncRequest, ) types.StreamPosition { return p.LatestPosition(ctx) @@ -24,11 +26,12 @@ func (p *DeviceListStreamProvider) CompleteSync( func (p *DeviceListStreamProvider) IncrementalSync( ctx context.Context, + snapshot storage.DatabaseTransaction, req *types.SyncRequest, from, to types.StreamPosition, ) types.StreamPosition { var err error - to, _, err = internal.DeviceListCatchup(context.Background(), p.DB, p.keyAPI, p.rsAPI, req.Device.UserID, req.Response, from, to) + to, _, err = internal.DeviceListCatchup(context.Background(), snapshot, p.keyAPI, p.rsAPI, req.Device.UserID, req.Response, from, to) if err != nil { req.Log.WithError(err).Error("internal.DeviceListCatchup failed") return from diff --git a/syncapi/streams/stream_invite.go b/syncapi/streams/stream_invite.go index 925da32f2..17b3b8434 100644 --- a/syncapi/streams/stream_invite.go +++ b/syncapi/streams/stream_invite.go @@ -9,20 +9,23 @@ import ( "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/types" ) type InviteStreamProvider struct { - StreamProvider + DefaultStreamProvider } -func (p *InviteStreamProvider) Setup() { - p.StreamProvider.Setup() +func (p *InviteStreamProvider) Setup( + ctx context.Context, snapshot storage.DatabaseTransaction, +) { + p.DefaultStreamProvider.Setup(ctx, snapshot) p.latestMutex.Lock() defer p.latestMutex.Unlock() - id, err := p.DB.MaxStreamPositionForInvites(context.Background()) + id, err := snapshot.MaxStreamPositionForInvites(ctx) if err != nil { panic(err) } @@ -31,13 +34,15 @@ func (p *InviteStreamProvider) Setup() { func (p *InviteStreamProvider) CompleteSync( ctx context.Context, + snapshot storage.DatabaseTransaction, req *types.SyncRequest, ) types.StreamPosition { - return p.IncrementalSync(ctx, req, 0, p.LatestPosition(ctx)) + return p.IncrementalSync(ctx, snapshot, req, 0, p.LatestPosition(ctx)) } func (p *InviteStreamProvider) IncrementalSync( ctx context.Context, + snapshot storage.DatabaseTransaction, req *types.SyncRequest, from, to types.StreamPosition, ) types.StreamPosition { @@ -46,7 +51,7 @@ func (p *InviteStreamProvider) IncrementalSync( To: to, } - invites, retiredInvites, err := p.DB.InviteEventsInRange( + invites, retiredInvites, maxID, err := snapshot.InviteEventsInRange( ctx, req.Device.UserID, r, ) if err != nil { @@ -86,5 +91,5 @@ func (p *InviteStreamProvider) IncrementalSync( } } - return to + return maxID } diff --git a/syncapi/streams/stream_notificationdata.go b/syncapi/streams/stream_notificationdata.go index 33872734d..5a81fd09a 100644 --- a/syncapi/streams/stream_notificationdata.go +++ b/syncapi/streams/stream_notificationdata.go @@ -3,17 +3,23 @@ package streams import ( "context" + "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/types" ) type NotificationDataStreamProvider struct { - StreamProvider + DefaultStreamProvider } -func (p *NotificationDataStreamProvider) Setup() { - p.StreamProvider.Setup() +func (p *NotificationDataStreamProvider) Setup( + ctx context.Context, snapshot storage.DatabaseTransaction, +) { + p.DefaultStreamProvider.Setup(ctx, snapshot) - id, err := p.DB.MaxStreamPositionForNotificationData(context.Background()) + p.latestMutex.Lock() + defer p.latestMutex.Unlock() + + id, err := snapshot.MaxStreamPositionForNotificationData(ctx) if err != nil { panic(err) } @@ -22,20 +28,22 @@ func (p *NotificationDataStreamProvider) Setup() { func (p *NotificationDataStreamProvider) CompleteSync( ctx context.Context, + snapshot storage.DatabaseTransaction, req *types.SyncRequest, ) types.StreamPosition { - return p.IncrementalSync(ctx, req, 0, p.LatestPosition(ctx)) + return p.IncrementalSync(ctx, snapshot, req, 0, p.LatestPosition(ctx)) } func (p *NotificationDataStreamProvider) IncrementalSync( ctx context.Context, + snapshot storage.DatabaseTransaction, req *types.SyncRequest, from, _ types.StreamPosition, ) types.StreamPosition { // Get the unread notifications for rooms in our join response. // This is to ensure clients always have an unread notification section // and can display the correct numbers. - countsByRoom, err := p.DB.GetUserUnreadNotificationCountsForRooms(ctx, req.Device.UserID, req.Rooms) + countsByRoom, err := snapshot.GetUserUnreadNotificationCountsForRooms(ctx, req.Device.UserID, req.Rooms) if err != nil { req.Log.WithError(err).Error("GetUserUnreadNotificationCountsForRooms failed") return from diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go index 0ab6de886..89c5ba35e 100644 --- a/syncapi/streams/stream_pdu.go +++ b/syncapi/streams/stream_pdu.go @@ -5,7 +5,6 @@ import ( "database/sql" "fmt" "sort" - "sync" "time" "github.com/matrix-org/dendrite/internal/caching" @@ -18,7 +17,6 @@ import ( "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" "github.com/tidwall/gjson" - "go.uber.org/atomic" "github.com/matrix-org/dendrite/syncapi/notifier" ) @@ -33,44 +31,23 @@ const PDU_STREAM_WORKERS = 256 const PDU_STREAM_QUEUESIZE = PDU_STREAM_WORKERS * 8 type PDUStreamProvider struct { - StreamProvider + DefaultStreamProvider - tasks chan func() - workers atomic.Int32 // userID+deviceID -> lazy loading cache lazyLoadCache caching.LazyLoadCache rsAPI roomserverAPI.SyncRoomserverAPI notifier *notifier.Notifier } -func (p *PDUStreamProvider) worker() { - defer p.workers.Dec() - for { - select { - case f := <-p.tasks: - f() - case <-time.After(time.Second * 10): - return - } - } -} - -func (p *PDUStreamProvider) queue(f func()) { - if p.workers.Load() < PDU_STREAM_WORKERS { - p.workers.Inc() - go p.worker() - } - p.tasks <- f -} - -func (p *PDUStreamProvider) Setup() { - p.StreamProvider.Setup() - p.tasks = make(chan func(), PDU_STREAM_QUEUESIZE) +func (p *PDUStreamProvider) Setup( + ctx context.Context, snapshot storage.DatabaseTransaction, +) { + p.DefaultStreamProvider.Setup(ctx, snapshot) p.latestMutex.Lock() defer p.latestMutex.Unlock() - id, err := p.DB.MaxStreamPositionForPDUs(context.Background()) + id, err := snapshot.MaxStreamPositionForPDUs(ctx) if err != nil { panic(err) } @@ -79,6 +56,7 @@ func (p *PDUStreamProvider) Setup() { func (p *PDUStreamProvider) CompleteSync( ctx context.Context, + snapshot storage.DatabaseTransaction, req *types.SyncRequest, ) types.StreamPosition { from := types.StreamPosition(0) @@ -94,7 +72,7 @@ func (p *PDUStreamProvider) CompleteSync( } // Extract room state and recent events for all rooms the user is joined to. - joinedRoomIDs, err := p.DB.RoomIDsWithMembership(ctx, req.Device.UserID, gomatrixserverlib.Join) + joinedRoomIDs, err := snapshot.RoomIDsWithMembership(ctx, req.Device.UserID, gomatrixserverlib.Join) if err != nil { req.Log.WithError(err).Error("p.DB.RoomIDsWithMembership failed") return from @@ -103,7 +81,7 @@ func (p *PDUStreamProvider) CompleteSync( stateFilter := req.Filter.Room.State eventFilter := req.Filter.Room.Timeline - if err = p.addIgnoredUsersToFilter(ctx, req, &eventFilter); err != nil { + if err = p.addIgnoredUsersToFilter(ctx, snapshot, req, &eventFilter); err != nil { req.Log.WithError(err).Error("unable to update event filter with ignored users") } @@ -117,33 +95,20 @@ func (p *PDUStreamProvider) CompleteSync( } // Build up a /sync response. Add joined rooms. - var reqMutex sync.Mutex - var reqWaitGroup sync.WaitGroup - reqWaitGroup.Add(len(joinedRoomIDs)) - for _, room := range joinedRoomIDs { - roomID := room - p.queue(func() { - defer reqWaitGroup.Done() - - jr, jerr := p.getJoinResponseForCompleteSync( - ctx, roomID, r, &stateFilter, &eventFilter, req.WantFullState, req.Device, false, - ) - if jerr != nil { - req.Log.WithError(jerr).Error("p.getJoinResponseForCompleteSync failed") - return - } - - reqMutex.Lock() - defer reqMutex.Unlock() - req.Response.Rooms.Join[roomID] = *jr - req.Rooms[roomID] = gomatrixserverlib.Join - }) + for _, roomID := range joinedRoomIDs { + jr, jerr := p.getJoinResponseForCompleteSync( + ctx, snapshot, roomID, r, &stateFilter, &eventFilter, req.WantFullState, req.Device, false, + ) + if jerr != nil { + req.Log.WithError(jerr).Error("p.getJoinResponseForCompleteSync failed") + continue // return from + } + req.Response.Rooms.Join[roomID] = *jr + req.Rooms[roomID] = gomatrixserverlib.Join } - reqWaitGroup.Wait() - // Add peeked rooms. - peeks, err := p.DB.PeeksInRange(ctx, req.Device.UserID, req.Device.ID, r) + peeks, err := snapshot.PeeksInRange(ctx, req.Device.UserID, req.Device.ID, r) if err != nil { req.Log.WithError(err).Error("p.DB.PeeksInRange failed") return from @@ -152,11 +117,11 @@ func (p *PDUStreamProvider) CompleteSync( if !peek.Deleted { var jr *types.JoinResponse jr, err = p.getJoinResponseForCompleteSync( - ctx, peek.RoomID, r, &stateFilter, &eventFilter, req.WantFullState, req.Device, true, + ctx, snapshot, peek.RoomID, r, &stateFilter, &eventFilter, req.WantFullState, req.Device, true, ) if err != nil { req.Log.WithError(err).Error("p.getJoinResponseForCompleteSync failed") - return from + continue // return from } req.Response.Rooms.Peek[peek.RoomID] = *jr } @@ -167,6 +132,7 @@ func (p *PDUStreamProvider) CompleteSync( func (p *PDUStreamProvider) IncrementalSync( ctx context.Context, + snapshot storage.DatabaseTransaction, req *types.SyncRequest, from, to types.StreamPosition, ) (newPos types.StreamPosition) { @@ -184,12 +150,12 @@ func (p *PDUStreamProvider) IncrementalSync( eventFilter := req.Filter.Room.Timeline if req.WantFullState { - if stateDeltas, syncJoinedRooms, err = p.DB.GetStateDeltasForFullStateSync(ctx, req.Device, r, req.Device.UserID, &stateFilter); err != nil { + if stateDeltas, syncJoinedRooms, err = snapshot.GetStateDeltasForFullStateSync(ctx, req.Device, r, req.Device.UserID, &stateFilter); err != nil { req.Log.WithError(err).Error("p.DB.GetStateDeltasForFullStateSync failed") return } } else { - if stateDeltas, syncJoinedRooms, err = p.DB.GetStateDeltas(ctx, req.Device, r, req.Device.UserID, &stateFilter); err != nil { + if stateDeltas, syncJoinedRooms, err = snapshot.GetStateDeltas(ctx, req.Device, r, req.Device.UserID, &stateFilter); err != nil { req.Log.WithError(err).Error("p.DB.GetStateDeltas failed") return } @@ -203,7 +169,7 @@ func (p *PDUStreamProvider) IncrementalSync( return to } - if err = p.addIgnoredUsersToFilter(ctx, req, &eventFilter); err != nil { + if err = p.addIgnoredUsersToFilter(ctx, snapshot, req, &eventFilter); err != nil { req.Log.WithError(err).Error("unable to update event filter with ignored users") } @@ -222,7 +188,7 @@ func (p *PDUStreamProvider) IncrementalSync( } } var pos types.StreamPosition - if pos, err = p.addRoomDeltaToResponse(ctx, req.Device, newRange, delta, &eventFilter, &stateFilter, req.Response); err != nil { + if pos, err = p.addRoomDeltaToResponse(ctx, snapshot, req.Device, newRange, delta, &eventFilter, &stateFilter, req.Response); err != nil { req.Log.WithError(err).Error("d.addRoomDeltaToResponse failed") return to } @@ -244,6 +210,7 @@ func (p *PDUStreamProvider) IncrementalSync( // nolint:gocyclo func (p *PDUStreamProvider) addRoomDeltaToResponse( ctx context.Context, + snapshot storage.DatabaseTransaction, device *userapi.Device, r types.Range, delta types.StateDelta, @@ -260,7 +227,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( // This is all "okay" assuming history_visibility == "shared" which it is by default. r.To = delta.MembershipPos } - recentStreamEvents, limited, err := p.DB.RecentEvents( + recentStreamEvents, limited, err := snapshot.RecentEvents( ctx, delta.RoomID, r, eventFilter, true, true, ) @@ -270,9 +237,9 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( } return r.From, fmt.Errorf("p.DB.RecentEvents: %w", err) } - recentEvents := p.DB.StreamEventsToEvents(device, recentStreamEvents) + recentEvents := snapshot.StreamEventsToEvents(device, recentStreamEvents) delta.StateEvents = removeDuplicates(delta.StateEvents, recentEvents) // roll back - prevBatch, err := p.DB.GetBackwardTopologyPos(ctx, recentStreamEvents) + prevBatch, err := snapshot.GetBackwardTopologyPos(ctx, recentStreamEvents) if err != nil { return r.From, fmt.Errorf("p.DB.GetBackwardTopologyPos: %w", err) } @@ -291,7 +258,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( latestPosition := r.To updateLatestPosition := func(mostRecentEventID string) { var pos types.StreamPosition - if _, pos, err = p.DB.PositionInTopology(ctx, mostRecentEventID); err == nil { + if _, pos, err = snapshot.PositionInTopology(ctx, mostRecentEventID); err == nil { switch { case r.Backwards && pos < latestPosition: fallthrough @@ -303,7 +270,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( if stateFilter.LazyLoadMembers { delta.StateEvents, err = p.lazyLoadMembers( - ctx, delta.RoomID, true, limited, stateFilter, + ctx, snapshot, delta.RoomID, true, limited, stateFilter, device, recentEvents, delta.StateEvents, ) if err != nil && err != sql.ErrNoRows { @@ -320,7 +287,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( } // Applies the history visibility rules - events, err := applyHistoryVisibilityFilter(ctx, p.DB, p.rsAPI, delta.RoomID, device.UserID, eventFilter.Limit, recentEvents) + events, err := applyHistoryVisibilityFilter(ctx, snapshot, p.rsAPI, delta.RoomID, device.UserID, eventFilter.Limit, recentEvents) if err != nil { logrus.WithError(err).Error("unable to apply history visibility filter") } @@ -336,7 +303,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( case gomatrixserverlib.Join: jr := types.NewJoinResponse() if hasMembershipChange { - p.addRoomSummary(ctx, jr, delta.RoomID, device.UserID, latestPosition) + p.addRoomSummary(ctx, snapshot, jr, delta.RoomID, device.UserID, latestPosition) } jr.Timeline.PrevBatch = &prevBatch jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(events, gomatrixserverlib.FormatSync) @@ -376,7 +343,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( // sure we always return the required events in the timeline. func applyHistoryVisibilityFilter( ctx context.Context, - db storage.Database, + snapshot storage.DatabaseTransaction, rsAPI roomserverAPI.SyncRoomserverAPI, roomID, userID string, limit int, @@ -384,7 +351,7 @@ func applyHistoryVisibilityFilter( ) ([]*gomatrixserverlib.HeaderedEvent, error) { // We need to make sure we always include the latest states events, if they are in the timeline. // We grep at least limit * 2 events, to ensure we really get the needed events. - stateEvents, err := db.CurrentState(ctx, roomID, &gomatrixserverlib.StateFilter{Limit: limit * 2}, nil) + stateEvents, err := snapshot.CurrentState(ctx, roomID, &gomatrixserverlib.StateFilter{Limit: limit * 2}, nil) if err != nil { // Not a fatal error, we can continue without the stateEvents, // they are only needed if there are state events in the timeline. @@ -395,7 +362,7 @@ func applyHistoryVisibilityFilter( alwaysIncludeIDs[ev.EventID()] = struct{}{} } startTime := time.Now() - events, err := internal.ApplyHistoryVisibilityFilter(ctx, db, rsAPI, recentEvents, alwaysIncludeIDs, userID, "sync") + events, err := internal.ApplyHistoryVisibilityFilter(ctx, snapshot, rsAPI, recentEvents, alwaysIncludeIDs, userID, "sync") if err != nil { return nil, err } @@ -408,10 +375,10 @@ func applyHistoryVisibilityFilter( return events, nil } -func (p *PDUStreamProvider) addRoomSummary(ctx context.Context, jr *types.JoinResponse, roomID, userID string, latestPosition types.StreamPosition) { +func (p *PDUStreamProvider) addRoomSummary(ctx context.Context, snapshot storage.DatabaseTransaction, jr *types.JoinResponse, roomID, userID string, latestPosition types.StreamPosition) { // Work out how many members are in the room. - joinedCount, _ := p.DB.MembershipCount(ctx, roomID, gomatrixserverlib.Join, latestPosition) - invitedCount, _ := p.DB.MembershipCount(ctx, roomID, gomatrixserverlib.Invite, latestPosition) + joinedCount, _ := snapshot.MembershipCount(ctx, roomID, gomatrixserverlib.Join, latestPosition) + invitedCount, _ := snapshot.MembershipCount(ctx, roomID, gomatrixserverlib.Invite, latestPosition) jr.Summary.JoinedMemberCount = &joinedCount jr.Summary.InvitedMemberCount = &invitedCount @@ -439,7 +406,7 @@ func (p *PDUStreamProvider) addRoomSummary(ctx context.Context, jr *types.JoinRe } } } - heroes, err := p.DB.GetRoomHeroes(ctx, roomID, userID, []string{"join", "invite"}) + heroes, err := snapshot.GetRoomHeroes(ctx, roomID, userID, []string{"join", "invite"}) if err != nil { return } @@ -449,6 +416,7 @@ func (p *PDUStreamProvider) addRoomSummary(ctx context.Context, jr *types.JoinRe func (p *PDUStreamProvider) getJoinResponseForCompleteSync( ctx context.Context, + snapshot storage.DatabaseTransaction, roomID string, r types.Range, stateFilter *gomatrixserverlib.StateFilter, @@ -460,7 +428,7 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync( jr = types.NewJoinResponse() // TODO: When filters are added, we may need to call this multiple times to get enough events. // See: https://github.com/matrix-org/synapse/blob/v0.19.3/synapse/handlers/sync.py#L316 - recentStreamEvents, limited, err := p.DB.RecentEvents( + recentStreamEvents, limited, err := snapshot.RecentEvents( ctx, roomID, r, eventFilter, true, true, ) if err != nil { @@ -484,7 +452,7 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync( } } - stateEvents, err := p.DB.CurrentState(ctx, roomID, stateFilter, excludingEventIDs) + stateEvents, err := snapshot.CurrentState(ctx, roomID, stateFilter, excludingEventIDs) if err != nil { return } @@ -494,7 +462,7 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync( var prevBatch *types.TopologyToken if len(recentStreamEvents) > 0 { var backwardTopologyPos, backwardStreamPos types.StreamPosition - backwardTopologyPos, backwardStreamPos, err = p.DB.PositionInTopology(ctx, recentStreamEvents[0].EventID()) + backwardTopologyPos, backwardStreamPos, err = snapshot.PositionInTopology(ctx, recentStreamEvents[0].EventID()) if err != nil { return } @@ -505,18 +473,18 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync( prevBatch.Decrement() } - p.addRoomSummary(ctx, jr, roomID, device.UserID, r.From) + p.addRoomSummary(ctx, snapshot, jr, roomID, device.UserID, r.From) // We don't include a device here as we don't need to send down // transaction IDs for complete syncs, but we do it anyway because Sytest demands it for: // "Can sync a room with a message with a transaction id" - which does a complete sync to check. - recentEvents := p.DB.StreamEventsToEvents(device, recentStreamEvents) + recentEvents := snapshot.StreamEventsToEvents(device, recentStreamEvents) stateEvents = removeDuplicates(stateEvents, recentEvents) events := recentEvents // Only apply history visibility checks if the response is for joined rooms if !isPeek { - events, err = applyHistoryVisibilityFilter(ctx, p.DB, p.rsAPI, roomID, device.UserID, eventFilter.Limit, recentEvents) + events, err = applyHistoryVisibilityFilter(ctx, snapshot, p.rsAPI, roomID, device.UserID, eventFilter.Limit, recentEvents) if err != nil { logrus.WithError(err).Error("unable to apply history visibility filter") } @@ -530,7 +498,8 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync( if err != nil { return nil, err } - stateEvents, err = p.lazyLoadMembers(ctx, roomID, + stateEvents, err = p.lazyLoadMembers( + ctx, snapshot, roomID, false, limited, stateFilter, device, recentEvents, stateEvents, ) @@ -549,7 +518,7 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync( } func (p *PDUStreamProvider) lazyLoadMembers( - ctx context.Context, roomID string, + ctx context.Context, snapshot storage.DatabaseTransaction, roomID string, incremental, limited bool, stateFilter *gomatrixserverlib.StateFilter, device *userapi.Device, timelineEvents, stateEvents []*gomatrixserverlib.HeaderedEvent, @@ -598,7 +567,7 @@ func (p *PDUStreamProvider) lazyLoadMembers( filter.Limit = stateFilter.Limit filter.Senders = &wantUsers filter.Types = &[]string{gomatrixserverlib.MRoomMember} - memberships, err := p.DB.GetStateEventsForRoom(ctx, roomID, &filter) + memberships, err := snapshot.GetStateEventsForRoom(ctx, roomID, &filter) if err != nil { return stateEvents, err } @@ -612,8 +581,8 @@ func (p *PDUStreamProvider) lazyLoadMembers( // addIgnoredUsersToFilter adds ignored users to the eventfilter and // the syncreq itself for further use in streams. -func (p *PDUStreamProvider) addIgnoredUsersToFilter(ctx context.Context, req *types.SyncRequest, eventFilter *gomatrixserverlib.RoomEventFilter) error { - ignores, err := p.DB.IgnoresForUser(ctx, req.Device.UserID) +func (p *PDUStreamProvider) addIgnoredUsersToFilter(ctx context.Context, snapshot storage.DatabaseTransaction, req *types.SyncRequest, eventFilter *gomatrixserverlib.RoomEventFilter) error { + ignores, err := snapshot.IgnoresForUser(ctx, req.Device.UserID) if err != nil { if err == sql.ErrNoRows { return nil diff --git a/syncapi/streams/stream_presence.go b/syncapi/streams/stream_presence.go index 15db4d30e..81cea7d5e 100644 --- a/syncapi/streams/stream_presence.go +++ b/syncapi/streams/stream_presence.go @@ -23,20 +23,26 @@ import ( "github.com/tidwall/gjson" "github.com/matrix-org/dendrite/syncapi/notifier" + "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/types" ) type PresenceStreamProvider struct { - StreamProvider + DefaultStreamProvider // cache contains previously sent presence updates to avoid unneeded updates cache sync.Map notifier *notifier.Notifier } -func (p *PresenceStreamProvider) Setup() { - p.StreamProvider.Setup() +func (p *PresenceStreamProvider) Setup( + ctx context.Context, snapshot storage.DatabaseTransaction, +) { + p.DefaultStreamProvider.Setup(ctx, snapshot) - id, err := p.DB.MaxStreamPositionForPresence(context.Background()) + p.latestMutex.Lock() + defer p.latestMutex.Unlock() + + id, err := snapshot.MaxStreamPositionForPresence(ctx) if err != nil { panic(err) } @@ -45,18 +51,20 @@ func (p *PresenceStreamProvider) Setup() { func (p *PresenceStreamProvider) CompleteSync( ctx context.Context, + snapshot storage.DatabaseTransaction, req *types.SyncRequest, ) types.StreamPosition { - return p.IncrementalSync(ctx, req, 0, p.LatestPosition(ctx)) + return p.IncrementalSync(ctx, snapshot, req, 0, p.LatestPosition(ctx)) } func (p *PresenceStreamProvider) IncrementalSync( ctx context.Context, + snapshot storage.DatabaseTransaction, req *types.SyncRequest, from, to types.StreamPosition, ) types.StreamPosition { // We pull out a larger number than the filter asks for, since we're filtering out events later - presences, err := p.DB.PresenceAfter(ctx, from, gomatrixserverlib.EventFilter{Limit: 1000}) + presences, err := snapshot.PresenceAfter(ctx, from, gomatrixserverlib.EventFilter{Limit: 1000}) if err != nil { req.Log.WithError(err).Error("p.DB.PresenceAfter failed") return from @@ -84,7 +92,7 @@ func (p *PresenceStreamProvider) IncrementalSync( } // Bear in mind that this might return nil, but at least populating // a nil means that there's a map entry so we won't repeat this call. - presences[roomUsers[i]], err = p.DB.GetPresence(ctx, roomUsers[i]) + presences[roomUsers[i]], err = snapshot.GetPresence(ctx, roomUsers[i]) if err != nil { req.Log.WithError(err).Error("unable to query presence for user") return from diff --git a/syncapi/streams/stream_receipt.go b/syncapi/streams/stream_receipt.go index f4e84c7d0..8818a5533 100644 --- a/syncapi/streams/stream_receipt.go +++ b/syncapi/streams/stream_receipt.go @@ -4,18 +4,24 @@ import ( "context" "encoding/json" + "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" ) type ReceiptStreamProvider struct { - StreamProvider + DefaultStreamProvider } -func (p *ReceiptStreamProvider) Setup() { - p.StreamProvider.Setup() +func (p *ReceiptStreamProvider) Setup( + ctx context.Context, snapshot storage.DatabaseTransaction, +) { + p.DefaultStreamProvider.Setup(ctx, snapshot) - id, err := p.DB.MaxStreamPositionForReceipts(context.Background()) + p.latestMutex.Lock() + defer p.latestMutex.Unlock() + + id, err := snapshot.MaxStreamPositionForReceipts(ctx) if err != nil { panic(err) } @@ -24,13 +30,15 @@ func (p *ReceiptStreamProvider) Setup() { func (p *ReceiptStreamProvider) CompleteSync( ctx context.Context, + snapshot storage.DatabaseTransaction, req *types.SyncRequest, ) types.StreamPosition { - return p.IncrementalSync(ctx, req, 0, p.LatestPosition(ctx)) + return p.IncrementalSync(ctx, snapshot, req, 0, p.LatestPosition(ctx)) } func (p *ReceiptStreamProvider) IncrementalSync( ctx context.Context, + snapshot storage.DatabaseTransaction, req *types.SyncRequest, from, to types.StreamPosition, ) types.StreamPosition { @@ -41,7 +49,7 @@ func (p *ReceiptStreamProvider) IncrementalSync( } } - lastPos, receipts, err := p.DB.RoomReceiptsAfter(ctx, joinedRooms, from) + lastPos, receipts, err := snapshot.RoomReceiptsAfter(ctx, joinedRooms, from) if err != nil { req.Log.WithError(err).Error("p.DB.RoomReceiptsAfter failed") return from diff --git a/syncapi/streams/stream_sendtodevice.go b/syncapi/streams/stream_sendtodevice.go index 31c6187cb..00b67cc42 100644 --- a/syncapi/streams/stream_sendtodevice.go +++ b/syncapi/streams/stream_sendtodevice.go @@ -3,17 +3,23 @@ package streams import ( "context" + "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/types" ) type SendToDeviceStreamProvider struct { - StreamProvider + DefaultStreamProvider } -func (p *SendToDeviceStreamProvider) Setup() { - p.StreamProvider.Setup() +func (p *SendToDeviceStreamProvider) Setup( + ctx context.Context, snapshot storage.DatabaseTransaction, +) { + p.DefaultStreamProvider.Setup(ctx, snapshot) - id, err := p.DB.MaxStreamPositionForSendToDeviceMessages(context.Background()) + p.latestMutex.Lock() + defer p.latestMutex.Unlock() + + id, err := snapshot.MaxStreamPositionForSendToDeviceMessages(ctx) if err != nil { panic(err) } @@ -22,18 +28,20 @@ func (p *SendToDeviceStreamProvider) Setup() { func (p *SendToDeviceStreamProvider) CompleteSync( ctx context.Context, + snapshot storage.DatabaseTransaction, req *types.SyncRequest, ) types.StreamPosition { - return p.IncrementalSync(ctx, req, 0, p.LatestPosition(ctx)) + return p.IncrementalSync(ctx, snapshot, req, 0, p.LatestPosition(ctx)) } func (p *SendToDeviceStreamProvider) IncrementalSync( ctx context.Context, + snapshot storage.DatabaseTransaction, req *types.SyncRequest, from, to types.StreamPosition, ) types.StreamPosition { // See if we have any new tasks to do for the send-to-device messaging. - lastPos, events, err := p.DB.SendToDeviceUpdatesForSync(req.Context, req.Device.UserID, req.Device.ID, from, to) + lastPos, events, err := snapshot.SendToDeviceUpdatesForSync(req.Context, req.Device.UserID, req.Device.ID, from, to) if err != nil { req.Log.WithError(err).Error("p.DB.SendToDeviceUpdatesForSync failed") return from diff --git a/syncapi/streams/stream_typing.go b/syncapi/streams/stream_typing.go index f781065be..a6f7c7a06 100644 --- a/syncapi/streams/stream_typing.go +++ b/syncapi/streams/stream_typing.go @@ -5,24 +5,27 @@ import ( "encoding/json" "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" ) type TypingStreamProvider struct { - StreamProvider + DefaultStreamProvider EDUCache *caching.EDUCache } func (p *TypingStreamProvider) CompleteSync( ctx context.Context, + snapshot storage.DatabaseTransaction, req *types.SyncRequest, ) types.StreamPosition { - return p.IncrementalSync(ctx, req, 0, p.LatestPosition(ctx)) + return p.IncrementalSync(ctx, snapshot, req, 0, p.LatestPosition(ctx)) } func (p *TypingStreamProvider) IncrementalSync( ctx context.Context, + snapshot storage.DatabaseTransaction, req *types.SyncRequest, from, to types.StreamPosition, ) types.StreamPosition { diff --git a/syncapi/streams/streamprovider.go b/syncapi/streams/streamprovider.go new file mode 100644 index 000000000..8b12e2eba --- /dev/null +++ b/syncapi/streams/streamprovider.go @@ -0,0 +1,28 @@ +package streams + +import ( + "context" + + "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/dendrite/syncapi/types" +) + +type StreamProvider interface { + Setup(ctx context.Context, snapshot storage.DatabaseTransaction) + + // Advance will update the latest position of the stream based on + // an update and will wake callers waiting on StreamNotifyAfter. + Advance(latest types.StreamPosition) + + // CompleteSync will update the response to include all updates as needed + // for a complete sync. It will always return immediately. + CompleteSync(ctx context.Context, snapshot storage.DatabaseTransaction, req *types.SyncRequest) types.StreamPosition + + // IncrementalSync will update the response to include all updates between + // the from and to sync positions. It will always return immediately, + // making no changes if the range contains no updates. + IncrementalSync(ctx context.Context, snapshot storage.DatabaseTransaction, req *types.SyncRequest, from, to types.StreamPosition) types.StreamPosition + + // LatestPosition returns the latest stream position for this stream. + LatestPosition(ctx context.Context) types.StreamPosition +} diff --git a/syncapi/streams/streams.go b/syncapi/streams/streams.go index dbc053bd8..eccbb3a4f 100644 --- a/syncapi/streams/streams.go +++ b/syncapi/streams/streams.go @@ -13,15 +13,15 @@ import ( ) type Streams struct { - PDUStreamProvider types.StreamProvider - TypingStreamProvider types.StreamProvider - ReceiptStreamProvider types.StreamProvider - InviteStreamProvider types.StreamProvider - SendToDeviceStreamProvider types.StreamProvider - AccountDataStreamProvider types.StreamProvider - DeviceListStreamProvider types.StreamProvider - NotificationDataStreamProvider types.StreamProvider - PresenceStreamProvider types.StreamProvider + PDUStreamProvider StreamProvider + TypingStreamProvider StreamProvider + ReceiptStreamProvider StreamProvider + InviteStreamProvider StreamProvider + SendToDeviceStreamProvider StreamProvider + AccountDataStreamProvider StreamProvider + DeviceListStreamProvider StreamProvider + NotificationDataStreamProvider StreamProvider + PresenceStreamProvider StreamProvider } func NewSyncStreamProviders( @@ -31,51 +31,58 @@ func NewSyncStreamProviders( ) *Streams { streams := &Streams{ PDUStreamProvider: &PDUStreamProvider{ - StreamProvider: StreamProvider{DB: d}, - lazyLoadCache: lazyLoadCache, - rsAPI: rsAPI, - notifier: notifier, + DefaultStreamProvider: DefaultStreamProvider{DB: d}, + lazyLoadCache: lazyLoadCache, + rsAPI: rsAPI, + notifier: notifier, }, TypingStreamProvider: &TypingStreamProvider{ - StreamProvider: StreamProvider{DB: d}, - EDUCache: eduCache, + DefaultStreamProvider: DefaultStreamProvider{DB: d}, + EDUCache: eduCache, }, ReceiptStreamProvider: &ReceiptStreamProvider{ - StreamProvider: StreamProvider{DB: d}, + DefaultStreamProvider: DefaultStreamProvider{DB: d}, }, InviteStreamProvider: &InviteStreamProvider{ - StreamProvider: StreamProvider{DB: d}, + DefaultStreamProvider: DefaultStreamProvider{DB: d}, }, SendToDeviceStreamProvider: &SendToDeviceStreamProvider{ - StreamProvider: StreamProvider{DB: d}, + DefaultStreamProvider: DefaultStreamProvider{DB: d}, }, AccountDataStreamProvider: &AccountDataStreamProvider{ - StreamProvider: StreamProvider{DB: d}, - userAPI: userAPI, + DefaultStreamProvider: DefaultStreamProvider{DB: d}, + userAPI: userAPI, }, NotificationDataStreamProvider: &NotificationDataStreamProvider{ - StreamProvider: StreamProvider{DB: d}, + DefaultStreamProvider: DefaultStreamProvider{DB: d}, }, DeviceListStreamProvider: &DeviceListStreamProvider{ - StreamProvider: StreamProvider{DB: d}, - rsAPI: rsAPI, - keyAPI: keyAPI, + DefaultStreamProvider: DefaultStreamProvider{DB: d}, + rsAPI: rsAPI, + keyAPI: keyAPI, }, PresenceStreamProvider: &PresenceStreamProvider{ - StreamProvider: StreamProvider{DB: d}, - notifier: notifier, + DefaultStreamProvider: DefaultStreamProvider{DB: d}, + notifier: notifier, }, } - streams.PDUStreamProvider.Setup() - streams.TypingStreamProvider.Setup() - streams.ReceiptStreamProvider.Setup() - streams.InviteStreamProvider.Setup() - streams.SendToDeviceStreamProvider.Setup() - streams.AccountDataStreamProvider.Setup() - streams.NotificationDataStreamProvider.Setup() - streams.DeviceListStreamProvider.Setup() - streams.PresenceStreamProvider.Setup() + ctx := context.TODO() + snapshot, err := d.NewDatabaseSnapshot(ctx) + if err != nil { + panic(err) + } + defer snapshot.Rollback() // nolint:errcheck + + streams.PDUStreamProvider.Setup(ctx, snapshot) + streams.TypingStreamProvider.Setup(ctx, snapshot) + streams.ReceiptStreamProvider.Setup(ctx, snapshot) + streams.InviteStreamProvider.Setup(ctx, snapshot) + streams.SendToDeviceStreamProvider.Setup(ctx, snapshot) + streams.AccountDataStreamProvider.Setup(ctx, snapshot) + streams.NotificationDataStreamProvider.Setup(ctx, snapshot) + streams.DeviceListStreamProvider.Setup(ctx, snapshot) + streams.PresenceStreamProvider.Setup(ctx, snapshot) return streams } diff --git a/syncapi/streams/template_stream.go b/syncapi/streams/template_stream.go index 15074cc10..f208d84e4 100644 --- a/syncapi/streams/template_stream.go +++ b/syncapi/streams/template_stream.go @@ -8,16 +8,18 @@ import ( "github.com/matrix-org/dendrite/syncapi/types" ) -type StreamProvider struct { +type DefaultStreamProvider struct { DB storage.Database latest types.StreamPosition latestMutex sync.RWMutex } -func (p *StreamProvider) Setup() { +func (p *DefaultStreamProvider) Setup( + ctx context.Context, snapshot storage.DatabaseTransaction, +) { } -func (p *StreamProvider) Advance( +func (p *DefaultStreamProvider) Advance( latest types.StreamPosition, ) { p.latestMutex.Lock() @@ -28,7 +30,7 @@ func (p *StreamProvider) Advance( } } -func (p *StreamProvider) LatestPosition( +func (p *DefaultStreamProvider) LatestPosition( ctx context.Context, ) types.StreamPosition { p.latestMutex.RLock() diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index b2ea105ff..1d0ac1a40 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -305,6 +305,13 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. syncReq.Log.WithField("currentPos", currentPos).Debugln("Responding to sync immediately") } + snapshot, err := rp.db.NewDatabaseSnapshot(req.Context()) + if err != nil { + logrus.WithError(err).Error("Failed to acquire database snapshot for sync request") + return jsonerror.InternalServerError() + } + defer snapshot.Rollback() // nolint:errcheck + if syncReq.Since.IsEmpty() { // Complete sync syncReq.Response.NextBatch = types.StreamingToken{ @@ -312,70 +319,70 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. // might advance while processing other streams, resulting in flakey // tests. DeviceListPosition: rp.streams.DeviceListStreamProvider.CompleteSync( - syncReq.Context, syncReq, + syncReq.Context, snapshot, syncReq, ), PDUPosition: rp.streams.PDUStreamProvider.CompleteSync( - syncReq.Context, syncReq, + syncReq.Context, snapshot, syncReq, ), TypingPosition: rp.streams.TypingStreamProvider.CompleteSync( - syncReq.Context, syncReq, + syncReq.Context, snapshot, syncReq, ), ReceiptPosition: rp.streams.ReceiptStreamProvider.CompleteSync( - syncReq.Context, syncReq, + syncReq.Context, snapshot, syncReq, ), InvitePosition: rp.streams.InviteStreamProvider.CompleteSync( - syncReq.Context, syncReq, + syncReq.Context, snapshot, syncReq, ), SendToDevicePosition: rp.streams.SendToDeviceStreamProvider.CompleteSync( - syncReq.Context, syncReq, + syncReq.Context, snapshot, syncReq, ), AccountDataPosition: rp.streams.AccountDataStreamProvider.CompleteSync( - syncReq.Context, syncReq, + syncReq.Context, snapshot, syncReq, ), NotificationDataPosition: rp.streams.NotificationDataStreamProvider.CompleteSync( - syncReq.Context, syncReq, + syncReq.Context, snapshot, syncReq, ), PresencePosition: rp.streams.PresenceStreamProvider.CompleteSync( - syncReq.Context, syncReq, + syncReq.Context, snapshot, syncReq, ), } } else { // Incremental sync syncReq.Response.NextBatch = types.StreamingToken{ PDUPosition: rp.streams.PDUStreamProvider.IncrementalSync( - syncReq.Context, syncReq, + syncReq.Context, snapshot, syncReq, syncReq.Since.PDUPosition, currentPos.PDUPosition, ), TypingPosition: rp.streams.TypingStreamProvider.IncrementalSync( - syncReq.Context, syncReq, + syncReq.Context, snapshot, syncReq, syncReq.Since.TypingPosition, currentPos.TypingPosition, ), ReceiptPosition: rp.streams.ReceiptStreamProvider.IncrementalSync( - syncReq.Context, syncReq, + syncReq.Context, snapshot, syncReq, syncReq.Since.ReceiptPosition, currentPos.ReceiptPosition, ), InvitePosition: rp.streams.InviteStreamProvider.IncrementalSync( - syncReq.Context, syncReq, + syncReq.Context, snapshot, syncReq, syncReq.Since.InvitePosition, currentPos.InvitePosition, ), SendToDevicePosition: rp.streams.SendToDeviceStreamProvider.IncrementalSync( - syncReq.Context, syncReq, + syncReq.Context, snapshot, syncReq, syncReq.Since.SendToDevicePosition, currentPos.SendToDevicePosition, ), AccountDataPosition: rp.streams.AccountDataStreamProvider.IncrementalSync( - syncReq.Context, syncReq, + syncReq.Context, snapshot, syncReq, syncReq.Since.AccountDataPosition, currentPos.AccountDataPosition, ), NotificationDataPosition: rp.streams.NotificationDataStreamProvider.IncrementalSync( - syncReq.Context, syncReq, + syncReq.Context, snapshot, syncReq, syncReq.Since.NotificationDataPosition, currentPos.NotificationDataPosition, ), DeviceListPosition: rp.streams.DeviceListStreamProvider.IncrementalSync( - syncReq.Context, syncReq, + syncReq.Context, snapshot, syncReq, syncReq.Since.DeviceListPosition, currentPos.DeviceListPosition, ), PresencePosition: rp.streams.PresenceStreamProvider.IncrementalSync( - syncReq.Context, syncReq, + syncReq.Context, snapshot, syncReq, syncReq.Since.PresencePosition, currentPos.PresencePosition, ), } @@ -437,9 +444,15 @@ func (rp *RequestPool) OnIncomingKeyChangeRequest(req *http.Request, device *use util.GetLogger(req.Context()).WithError(err).Error("newSyncRequest failed") return jsonerror.InternalServerError() } - rp.streams.PDUStreamProvider.IncrementalSync(req.Context(), syncReq, fromToken.PDUPosition, toToken.PDUPosition) + snapshot, err := rp.db.NewDatabaseSnapshot(req.Context()) + if err != nil { + logrus.WithError(err).Error("Failed to acquire database snapshot for key change") + return jsonerror.InternalServerError() + } + defer snapshot.Rollback() // nolint:errcheck + rp.streams.PDUStreamProvider.IncrementalSync(req.Context(), snapshot, syncReq, fromToken.PDUPosition, toToken.PDUPosition) _, _, err = internal.DeviceListCatchup( - req.Context(), rp.db, rp.keyAPI, rp.rsAPI, syncReq.Device.UserID, + req.Context(), snapshot, rp.keyAPI, rp.rsAPI, syncReq.Device.UserID, syncReq.Response, fromToken.DeviceListPosition, toToken.DeviceListPosition, ) if err != nil { diff --git a/syncapi/types/provider.go b/syncapi/types/provider.go index a9ea234d0..378cafe99 100644 --- a/syncapi/types/provider.go +++ b/syncapi/types/provider.go @@ -41,23 +41,3 @@ func (r *SyncRequest) IsRoomPresent(roomID string) bool { return false } } - -type StreamProvider interface { - Setup() - - // Advance will update the latest position of the stream based on - // an update and will wake callers waiting on StreamNotifyAfter. - Advance(latest StreamPosition) - - // CompleteSync will update the response to include all updates as needed - // for a complete sync. It will always return immediately. - CompleteSync(ctx context.Context, req *SyncRequest) StreamPosition - - // IncrementalSync will update the response to include all updates between - // the from and to sync positions. It will always return immediately, - // making no changes if the range contains no updates. - IncrementalSync(ctx context.Context, req *SyncRequest, from, to StreamPosition) StreamPosition - - // LatestPosition returns the latest stream position for this stream. - LatestPosition(ctx context.Context) StreamPosition -} From 0e2fb63b4fe832befd266fe7e886832901eac00e Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 30 Sep 2022 13:31:05 +0100 Subject: [PATCH 011/207] Version 0.10.0 (#2753) Changelog and version bump. --- CHANGES.md | 22 ++++++++++++++++++++++ internal/version.go | 4 ++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 657ca1920..eca575cf2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,27 @@ # Changelog +## Dendrite 0.10.0 (2022-09-30) + +### Features + +* High performance full-text searching has been added to Dendrite + * Search must be enabled in the [`search` section of the `sync_api` config](https://github.com/matrix-org/dendrite/blob/6348486a1365c7469a498101f5035a9b6bd16d22/dendrite-sample.monolith.yaml#L279-L290) before it can be used + * The search index is stored on the filesystem rather than the sync API database, so a path to a suitable storage location on disk must be configured +* Sync requests should now complete faster and use considerably less database connections as a result of better transactional isolation +* The notifications code has been refactored to hopefully make notifications more reliable +* A new `/_dendrite/admin/refreshDevices/{userID}` admin endpoint has been added for forcing a refresh of a remote user's device lists without having to modify the database by hand +* A new `/_dendrite/admin/fulltext/reindex` admin endpoint has been added for rebuilding the search index (although this may take some time) + +### Fixes + +* A number of bugs in the device list updater have been fixed, which should help considerably with federated device list synchronisation and E2EE reliability +* A state resolution bug has been fixed which should help to prevent unexpected state resets +* The deprecated `"origin"` field in events will now be correctly ignored in all cases +* Room versions 8 and 9 will now correctly evaluate `"knock"` join rules and membership states +* A database index has been added to speed up finding room memberships in the sync API (contributed by [PiotrKozimor](https://github.com/PiotrKozimor)) +* The client API will now return an `M_UNRECOGNIZED` error for unknown endpoints/methods, which should help with client error handling +* A bug has been fixed when updating push rules which could result in `database is locked` on SQLite + ## Dendrite 0.9.9 (2022-09-22) ### Features diff --git a/internal/version.go b/internal/version.go index f9b101702..0b19757c2 100644 --- a/internal/version.go +++ b/internal/version.go @@ -16,8 +16,8 @@ var build string const ( VersionMajor = 0 - VersionMinor = 9 - VersionPatch = 9 + VersionMinor = 10 + VersionPatch = 0 VersionTag = "" // example: "rc1" ) From aa8ec1acbf3932b0543033bbf735225dac21676a Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 30 Sep 2022 15:39:05 +0100 Subject: [PATCH 012/207] Update 2_domainname.md --- docs/installation/2_domainname.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/installation/2_domainname.md b/docs/installation/2_domainname.md index 7d7fc86bd..e7b3495f7 100644 --- a/docs/installation/2_domainname.md +++ b/docs/installation/2_domainname.md @@ -87,6 +87,12 @@ and contain the following JSON document: For example, this can be done with the following Caddy config: ``` +handle /.well-known/matrix/server { + header Content-Type application/json + header Access-Control-Allow-Origin * + respond `"m.server": "matrix.example.com:8448"` +} + handle /.well-known/matrix/client { header Content-Type application/json header Access-Control-Allow-Origin * From ee40a29e55eb7986e16bb24bd388fb710e43aea9 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 30 Sep 2022 16:07:18 +0100 Subject: [PATCH 013/207] Fix broken `/sync` due to transaction error --- syncapi/consumers/roomserver.go | 5 ++++- syncapi/notifier/notifier.go | 9 +++++++-- syncapi/routing/context.go | 5 ++++- syncapi/routing/search.go | 5 ++++- syncapi/storage/interface.go | 2 ++ syncapi/streams/stream_accountdata.go | 1 + syncapi/streams/stream_devicelist.go | 2 ++ syncapi/streams/stream_invite.go | 1 + syncapi/streams/stream_notificationdata.go | 1 + syncapi/streams/stream_pdu.go | 4 ++++ syncapi/streams/stream_presence.go | 2 ++ syncapi/streams/stream_receipt.go | 1 + syncapi/streams/stream_sendtodevice.go | 1 + syncapi/streams/streams.go | 5 ++++- syncapi/sync/requestpool.go | 9 +++++++-- 15 files changed, 45 insertions(+), 8 deletions(-) diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index e5e8fe293..c7a11dbb4 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -27,6 +27,7 @@ import ( "github.com/tidwall/gjson" "github.com/matrix-org/dendrite/internal/fulltext" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" @@ -454,7 +455,8 @@ func (s *OutputRoomEventConsumer) updateStateEvent(event *gomatrixserverlib.Head if err != nil { return nil, err } - defer snapshot.Rollback() // nolint:errcheck + var succeeded bool + defer sqlutil.EndTransactionWithCheck(snapshot, &succeeded, &err) prevEvent, err := snapshot.GetStateEvent( s.ctx, event.RoomID(), event.Type(), stateKey, @@ -474,6 +476,7 @@ func (s *OutputRoomEventConsumer) updateStateEvent(event *gomatrixserverlib.Head } event.Event, err = event.SetUnsigned(prev) + succeeded = true return event, err } diff --git a/syncapi/notifier/notifier.go b/syncapi/notifier/notifier.go index a8e5bf9ad..db18c6b77 100644 --- a/syncapi/notifier/notifier.go +++ b/syncapi/notifier/notifier.go @@ -19,6 +19,7 @@ import ( "sync" "time" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/types" "github.com/matrix-org/gomatrixserverlib" @@ -323,7 +324,8 @@ func (n *Notifier) Load(ctx context.Context, db storage.Database) error { if err != nil { return err } - defer snapshot.Rollback() // nolint:errcheck + var succeeded bool + defer sqlutil.EndTransactionWithCheck(snapshot, &succeeded, &err) roomToUsers, err := snapshot.AllJoinedUsersInRooms(ctx) if err != nil { @@ -337,6 +339,7 @@ func (n *Notifier) Load(ctx context.Context, db storage.Database) error { } n.setPeekingDevices(roomToPeekingDevices) + succeeded = true return nil } @@ -349,7 +352,8 @@ func (n *Notifier) LoadRooms(ctx context.Context, db storage.Database, roomIDs [ if err != nil { return err } - defer snapshot.Rollback() // nolint:errcheck + var succeeded bool + defer sqlutil.EndTransactionWithCheck(snapshot, &succeeded, &err) roomToUsers, err := snapshot.AllJoinedUsersInRoom(ctx, roomIDs) if err != nil { @@ -357,6 +361,7 @@ func (n *Notifier) LoadRooms(ctx context.Context, db storage.Database, roomIDs [ } n.setUsersJoinedToRooms(roomToUsers) + succeeded = true return nil } diff --git a/syncapi/routing/context.go b/syncapi/routing/context.go index 1ce34b85a..0ed164c7e 100644 --- a/syncapi/routing/context.go +++ b/syncapi/routing/context.go @@ -25,6 +25,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/sqlutil" roomserver "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/internal" "github.com/matrix-org/dendrite/syncapi/storage" @@ -55,7 +56,8 @@ func Context( if err != nil { return jsonerror.InternalServerError() } - defer snapshot.Rollback() // nolint:errcheck + var succeeded bool + defer sqlutil.EndTransactionWithCheck(snapshot, &succeeded, &err) filter, err := parseRoomEventFilter(req) if err != nil { @@ -184,6 +186,7 @@ func Context( response.End = end.String() response.Start = start.String() } + succeeded = true return util.JSONResponse{ Code: http.StatusOK, JSON: response, diff --git a/syncapi/routing/search.go b/syncapi/routing/search.go index bac534a2c..aef355def 100644 --- a/syncapi/routing/search.go +++ b/syncapi/routing/search.go @@ -31,6 +31,7 @@ import ( "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/internal/fulltext" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/userapi/api" ) @@ -65,7 +66,8 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts if err != nil { return jsonerror.InternalServerError() } - defer snapshot.Rollback() // nolint:errcheck + var succeeded bool + defer sqlutil.EndTransactionWithCheck(snapshot, &succeeded, &err) // only search rooms the user is actually joined to joinedRooms, err := snapshot.RoomIDsWithMembership(ctx, device.UserID, "join") @@ -249,6 +251,7 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts logrus.Debugf("Full search request took %v", time.Since(start)) + succeeded = true return util.JSONResponse{ Code: http.StatusOK, JSON: res, diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index 3732e43fb..4a03aca74 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -20,6 +20,7 @@ import ( "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/dendrite/internal/eventutil" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/storage/shared" "github.com/matrix-org/dendrite/syncapi/types" @@ -27,6 +28,7 @@ import ( ) type DatabaseTransaction interface { + sqlutil.Transaction SharedUsers MaxStreamPositionForPDUs(ctx context.Context) (types.StreamPosition, error) diff --git a/syncapi/streams/stream_accountdata.go b/syncapi/streams/stream_accountdata.go index 3f2f7d134..34135d65a 100644 --- a/syncapi/streams/stream_accountdata.go +++ b/syncapi/streams/stream_accountdata.go @@ -54,6 +54,7 @@ func (p *AccountDataStreamProvider) IncrementalSync( ) if err != nil { req.Log.WithError(err).Error("p.DB.GetAccountDataInRange failed") + _ = snapshot.Rollback() return from } diff --git a/syncapi/streams/stream_devicelist.go b/syncapi/streams/stream_devicelist.go index 7996c2038..10ede573a 100644 --- a/syncapi/streams/stream_devicelist.go +++ b/syncapi/streams/stream_devicelist.go @@ -34,11 +34,13 @@ func (p *DeviceListStreamProvider) IncrementalSync( to, _, err = internal.DeviceListCatchup(context.Background(), snapshot, p.keyAPI, p.rsAPI, req.Device.UserID, req.Response, from, to) if err != nil { req.Log.WithError(err).Error("internal.DeviceListCatchup failed") + _ = snapshot.Rollback() return from } err = internal.DeviceOTKCounts(req.Context, p.keyAPI, req.Device.UserID, req.Device.ID, req.Response) if err != nil { req.Log.WithError(err).Error("internal.DeviceListCatchup failed") + _ = snapshot.Rollback() return from } diff --git a/syncapi/streams/stream_invite.go b/syncapi/streams/stream_invite.go index 17b3b8434..b52eaaab1 100644 --- a/syncapi/streams/stream_invite.go +++ b/syncapi/streams/stream_invite.go @@ -56,6 +56,7 @@ func (p *InviteStreamProvider) IncrementalSync( ) if err != nil { req.Log.WithError(err).Error("p.DB.InviteEventsInRange failed") + _ = snapshot.Rollback() return from } diff --git a/syncapi/streams/stream_notificationdata.go b/syncapi/streams/stream_notificationdata.go index 5a81fd09a..e1ee02b21 100644 --- a/syncapi/streams/stream_notificationdata.go +++ b/syncapi/streams/stream_notificationdata.go @@ -46,6 +46,7 @@ func (p *NotificationDataStreamProvider) IncrementalSync( countsByRoom, err := snapshot.GetUserUnreadNotificationCountsForRooms(ctx, req.Device.UserID, req.Rooms) if err != nil { req.Log.WithError(err).Error("GetUserUnreadNotificationCountsForRooms failed") + _ = snapshot.Rollback() return from } diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go index 89c5ba35e..43e2ce8ba 100644 --- a/syncapi/streams/stream_pdu.go +++ b/syncapi/streams/stream_pdu.go @@ -75,6 +75,7 @@ func (p *PDUStreamProvider) CompleteSync( joinedRoomIDs, err := snapshot.RoomIDsWithMembership(ctx, req.Device.UserID, gomatrixserverlib.Join) if err != nil { req.Log.WithError(err).Error("p.DB.RoomIDsWithMembership failed") + _ = snapshot.Rollback() return from } @@ -101,6 +102,7 @@ func (p *PDUStreamProvider) CompleteSync( ) if jerr != nil { req.Log.WithError(jerr).Error("p.getJoinResponseForCompleteSync failed") + _ = snapshot.Rollback() continue // return from } req.Response.Rooms.Join[roomID] = *jr @@ -111,6 +113,7 @@ func (p *PDUStreamProvider) CompleteSync( peeks, err := snapshot.PeeksInRange(ctx, req.Device.UserID, req.Device.ID, r) if err != nil { req.Log.WithError(err).Error("p.DB.PeeksInRange failed") + _ = snapshot.Rollback() return from } for _, peek := range peeks { @@ -121,6 +124,7 @@ func (p *PDUStreamProvider) CompleteSync( ) if err != nil { req.Log.WithError(err).Error("p.getJoinResponseForCompleteSync failed") + _ = snapshot.Rollback() continue // return from } req.Response.Rooms.Peek[peek.RoomID] = *jr diff --git a/syncapi/streams/stream_presence.go b/syncapi/streams/stream_presence.go index 81cea7d5e..d24c85620 100644 --- a/syncapi/streams/stream_presence.go +++ b/syncapi/streams/stream_presence.go @@ -67,6 +67,7 @@ func (p *PresenceStreamProvider) IncrementalSync( presences, err := snapshot.PresenceAfter(ctx, from, gomatrixserverlib.EventFilter{Limit: 1000}) if err != nil { req.Log.WithError(err).Error("p.DB.PresenceAfter failed") + _ = snapshot.Rollback() return from } @@ -95,6 +96,7 @@ func (p *PresenceStreamProvider) IncrementalSync( presences[roomUsers[i]], err = snapshot.GetPresence(ctx, roomUsers[i]) if err != nil { req.Log.WithError(err).Error("unable to query presence for user") + _ = snapshot.Rollback() return from } if len(presences) > req.Filter.Presence.Limit { diff --git a/syncapi/streams/stream_receipt.go b/syncapi/streams/stream_receipt.go index 8818a5533..40e5bd01e 100644 --- a/syncapi/streams/stream_receipt.go +++ b/syncapi/streams/stream_receipt.go @@ -52,6 +52,7 @@ func (p *ReceiptStreamProvider) IncrementalSync( lastPos, receipts, err := snapshot.RoomReceiptsAfter(ctx, joinedRooms, from) if err != nil { req.Log.WithError(err).Error("p.DB.RoomReceiptsAfter failed") + _ = snapshot.Rollback() return from } diff --git a/syncapi/streams/stream_sendtodevice.go b/syncapi/streams/stream_sendtodevice.go index 00b67cc42..3262832a3 100644 --- a/syncapi/streams/stream_sendtodevice.go +++ b/syncapi/streams/stream_sendtodevice.go @@ -44,6 +44,7 @@ func (p *SendToDeviceStreamProvider) IncrementalSync( lastPos, events, err := snapshot.SendToDeviceUpdatesForSync(req.Context, req.Device.UserID, req.Device.ID, from, to) if err != nil { req.Log.WithError(err).Error("p.DB.SendToDeviceUpdatesForSync failed") + _ = snapshot.Rollback() return from } diff --git a/syncapi/streams/streams.go b/syncapi/streams/streams.go index eccbb3a4f..dc8547621 100644 --- a/syncapi/streams/streams.go +++ b/syncapi/streams/streams.go @@ -4,6 +4,7 @@ import ( "context" "github.com/matrix-org/dendrite/internal/caching" + "github.com/matrix-org/dendrite/internal/sqlutil" keyapi "github.com/matrix-org/dendrite/keyserver/api" rsapi "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/syncapi/notifier" @@ -72,7 +73,8 @@ func NewSyncStreamProviders( if err != nil { panic(err) } - defer snapshot.Rollback() // nolint:errcheck + var succeeded bool + defer sqlutil.EndTransactionWithCheck(snapshot, &succeeded, &err) streams.PDUStreamProvider.Setup(ctx, snapshot) streams.TypingStreamProvider.Setup(ctx, snapshot) @@ -84,6 +86,7 @@ func NewSyncStreamProviders( streams.DeviceListStreamProvider.Setup(ctx, snapshot) streams.PresenceStreamProvider.Setup(ctx, snapshot) + succeeded = true return streams } diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index 1d0ac1a40..bef0a5683 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -31,6 +31,7 @@ import ( "github.com/sirupsen/logrus" "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/internal/sqlutil" keyapi "github.com/matrix-org/dendrite/keyserver/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" @@ -310,7 +311,8 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. logrus.WithError(err).Error("Failed to acquire database snapshot for sync request") return jsonerror.InternalServerError() } - defer snapshot.Rollback() // nolint:errcheck + var succeeded bool + defer sqlutil.EndTransactionWithCheck(snapshot, &succeeded, &err) if syncReq.Since.IsEmpty() { // Complete sync @@ -409,6 +411,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. } } + succeeded = true return util.JSONResponse{ Code: http.StatusOK, JSON: syncReq.Response, @@ -449,7 +452,8 @@ func (rp *RequestPool) OnIncomingKeyChangeRequest(req *http.Request, device *use logrus.WithError(err).Error("Failed to acquire database snapshot for key change") return jsonerror.InternalServerError() } - defer snapshot.Rollback() // nolint:errcheck + var succeeded bool + defer sqlutil.EndTransactionWithCheck(snapshot, &succeeded, &err) rp.streams.PDUStreamProvider.IncrementalSync(req.Context(), snapshot, syncReq, fromToken.PDUPosition, toToken.PDUPosition) _, _, err = internal.DeviceListCatchup( req.Context(), snapshot, rp.keyAPI, rp.rsAPI, syncReq.Device.UserID, @@ -459,6 +463,7 @@ func (rp *RequestPool) OnIncomingKeyChangeRequest(req *http.Request, device *use util.GetLogger(req.Context()).WithError(err).Error("Failed to DeviceListCatchup info") return jsonerror.InternalServerError() } + succeeded = true return util.JSONResponse{ Code: 200, JSON: struct { From 60ec9180e6f5b032f147be66aeb0c6713bdfdbe4 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 30 Sep 2022 16:10:44 +0100 Subject: [PATCH 014/207] Update NATS Server to v2.9.2 --- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 0985b986c..c82f76d41 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( github.com/matrix-org/pinecone v0.0.0-20220929155234-2ce51dd4a42c github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.15 - github.com/nats-io/nats-server/v2 v2.9.1-0.20220920152220-52d7b481c4b5 + github.com/nats-io/nats-server/v2 v2.9.2 github.com/nats-io/nats.go v1.17.0 github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 @@ -43,7 +43,7 @@ require ( github.com/uber/jaeger-lib v2.4.1+incompatible github.com/yggdrasil-network/yggdrasil-go v0.4.5-0.20220901155642-4f2abece817c go.uber.org/atomic v1.10.0 - golang.org/x/crypto v0.0.0-20220919173607-35f4265a4bc0 + golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be golang.org/x/image v0.0.0-20220902085622-e7cb96979f69 golang.org/x/mobile v0.0.0-20220722155234-aaac322e2105 golang.org/x/net v0.0.0-20220919232410-f2f64ebce3c1 @@ -91,7 +91,7 @@ require ( github.com/h2non/filetype v1.1.3 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/juju/errors v1.0.0 // indirect - github.com/klauspost/compress v1.15.10 // indirect + github.com/klauspost/compress v1.15.11 // indirect github.com/kr/pretty v0.3.0 // indirect github.com/lucas-clemente/quic-go v0.29.0 // indirect github.com/marten-seemann/qtls-go1-18 v0.1.2 // indirect @@ -120,9 +120,9 @@ require ( go.etcd.io/bbolt v1.3.6 // indirect golang.org/x/exp v0.0.0-20220916125017-b168a2c6b86b // indirect golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect - golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 // indirect + golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec // indirect golang.org/x/text v0.3.8-0.20211004125949-5bd84dd9b33b // indirect - golang.org/x/time v0.0.0-20220920022843-2ce7c2934d45 // indirect + golang.org/x/time v0.0.0-20220922220347-f3bd1da661af // indirect golang.org/x/tools v0.1.12 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/macaroon.v2 v2.1.0 // indirect diff --git a/go.sum b/go.sum index c3db3e09e..a99599cb1 100644 --- a/go.sum +++ b/go.sum @@ -347,8 +347,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/compress v1.15.10 h1:Ai8UzuomSCDw90e1qNMtb15msBXsNpH6gzkkENQNcJo= -github.com/klauspost/compress v1.15.10/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= +github.com/klauspost/compress v1.15.11 h1:Lcadnb3RKGin4FYM/orgq0qde+nc15E5Cbqg4B9Sx9c= +github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -422,8 +422,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nats-io/jwt/v2 v2.3.0 h1:z2mA1a7tIf5ShggOFlR1oBPgd6hGqcDYsISxZByUzdI= github.com/nats-io/jwt/v2 v2.3.0/go.mod h1:0tqz9Hlu6bCBFLWAASKhE5vUA4c24L9KPUUgvwumE/k= -github.com/nats-io/nats-server/v2 v2.9.1-0.20220920152220-52d7b481c4b5 h1:G/YGSXcJ2bUofD8Ts49it4VNezaJLQldI6fZR+wIUts= -github.com/nats-io/nats-server/v2 v2.9.1-0.20220920152220-52d7b481c4b5/go.mod h1:BWKY6217RvhI+FDoOLZ2BH+hOC37xeKRBlQ1Lz7teKI= +github.com/nats-io/nats-server/v2 v2.9.2 h1:XNDgJgOYYaYlquLdbSHI3xssLipfKUOq3EmYIMNCOsE= +github.com/nats-io/nats-server/v2 v2.9.2/go.mod h1:4sq8wvrpbvSzL1n3ZfEYnH4qeUuIl5W990j3kw13rRk= github.com/nats-io/nats.go v1.17.0 h1:1jp5BThsdGlN91hW0k3YEfJbfACjiOYtUiLXG0RL4IE= github.com/nats-io/nats.go v1.17.0/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w= github.com/nats-io/nkeys v0.3.0 h1:cgM5tL53EvYRU+2YLXIK0G2mJtK12Ft9oeooSZMA2G8= @@ -625,8 +625,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220919173607-35f4265a4bc0 h1:a5Yg6ylndHHYJqIPrdq0AhvR6KTvDTAvgBtaidhEevY= -golang.org/x/crypto v0.0.0-20220919173607-35f4265a4bc0/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be h1:fmw3UbQh+nxngCAHrDCCztao/kbYFnWjoqop8dHx05A= +golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -810,8 +810,8 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220730100132-1609e554cd39/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 h1:h+EGohizhe9XlX18rfpa8k8RAc5XyaeamM+0VHRd4lc= -golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec h1:BkDtF2Ih9xZ7le9ndzTA7KJow28VbQW3odyk/8drmuI= +golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220919170432-7a66f970e087 h1:tPwmk4vmvVCMdr98VgL4JH+qZxPL8fqlUOHnyOM8N3w= @@ -829,8 +829,8 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220920022843-2ce7c2934d45 h1:yuLAip3bfURHClMG9VBdzPrQvCWjWiWUTBGV+/fCbUs= -golang.org/x/time v0.0.0-20220920022843-2ce7c2934d45/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220922220347-f3bd1da661af h1:Yx9k8YCG3dvF87UAn2tu2HQLf2dt/eR1bXxpLMWeH+Y= +golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From 3617d5a0ff580c7ca0058490308f7f946df3882f Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Fri, 30 Sep 2022 17:31:41 +0200 Subject: [PATCH 015/207] Remove SendAccountData, since InputAccountData is already doing that --- userapi/internal/api.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/userapi/internal/api.go b/userapi/internal/api.go index 3e761a886..591faffd6 100644 --- a/userapi/internal/api.go +++ b/userapi/internal/api.go @@ -796,11 +796,6 @@ func (a *UserInternalAPI) PerformPushRulesPut( if err := a.InputAccountData(ctx, &userReq, &userRes); err != nil { return err } - if err := a.SyncProducer.SendAccountData(req.UserID, eventutil.AccountData{ - Type: pushRulesAccountDataType, - }); err != nil { - util.GetLogger(ctx).WithError(err).Errorf("syncProducer.SendData failed") - } return nil } From 7d9545ceead42b4761ee9cce169bc8f5858521ed Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 30 Sep 2022 16:34:06 +0100 Subject: [PATCH 016/207] Another `/sync` fix --- syncapi/streams/stream_pdu.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go index 43e2ce8ba..ba90706e8 100644 --- a/syncapi/streams/stream_pdu.go +++ b/syncapi/streams/stream_pdu.go @@ -156,11 +156,13 @@ func (p *PDUStreamProvider) IncrementalSync( if req.WantFullState { if stateDeltas, syncJoinedRooms, err = snapshot.GetStateDeltasForFullStateSync(ctx, req.Device, r, req.Device.UserID, &stateFilter); err != nil { req.Log.WithError(err).Error("p.DB.GetStateDeltasForFullStateSync failed") + _ = snapshot.Rollback() return } } else { if stateDeltas, syncJoinedRooms, err = snapshot.GetStateDeltas(ctx, req.Device, r, req.Device.UserID, &stateFilter); err != nil { req.Log.WithError(err).Error("p.DB.GetStateDeltas failed") + _ = snapshot.Rollback() return } } @@ -175,6 +177,7 @@ func (p *PDUStreamProvider) IncrementalSync( if err = p.addIgnoredUsersToFilter(ctx, snapshot, req, &eventFilter); err != nil { req.Log.WithError(err).Error("unable to update event filter with ignored users") + _ = snapshot.Rollback() } newPos = from @@ -194,7 +197,11 @@ func (p *PDUStreamProvider) IncrementalSync( var pos types.StreamPosition if pos, err = p.addRoomDeltaToResponse(ctx, snapshot, req.Device, newRange, delta, &eventFilter, &stateFilter, req.Response); err != nil { req.Log.WithError(err).Error("d.addRoomDeltaToResponse failed") - return to + _ = snapshot.Rollback() + if err == context.DeadlineExceeded || err == context.Canceled { + return newPos + } + continue // return to } // Reset the position, as it is only for the special case of newly joined rooms if delta.NewlyJoined { From 16048be2366889692bba16089f0246865fbb9c48 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 30 Sep 2022 16:55:10 +0100 Subject: [PATCH 017/207] Handle case when applying history visibility failed --- syncapi/streams/stream_pdu.go | 1 + 1 file changed, 1 insertion(+) diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go index ba90706e8..92e1bccf0 100644 --- a/syncapi/streams/stream_pdu.go +++ b/syncapi/streams/stream_pdu.go @@ -301,6 +301,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( events, err := applyHistoryVisibilityFilter(ctx, snapshot, p.rsAPI, delta.RoomID, device.UserID, eventFilter.Limit, recentEvents) if err != nil { logrus.WithError(err).Error("unable to apply history visibility filter") + _ = snapshot.Rollback() } if len(delta.StateEvents) > 0 { From 0116db79c601d752cc097fc13fe7d22bdb01faae Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 30 Sep 2022 17:07:37 +0100 Subject: [PATCH 018/207] Reset transaction after a failure --- syncapi/storage/interface.go | 1 + syncapi/storage/shared/storage_consumer.go | 2 ++ syncapi/storage/shared/storage_sync.go | 14 +++++++++++++ syncapi/streams/stream_accountdata.go | 2 +- syncapi/streams/stream_devicelist.go | 4 ++-- syncapi/streams/stream_invite.go | 2 +- syncapi/streams/stream_notificationdata.go | 2 +- syncapi/streams/stream_pdu.go | 24 ++++++++++++++-------- syncapi/streams/stream_presence.go | 2 +- syncapi/streams/stream_receipt.go | 2 +- syncapi/streams/stream_sendtodevice.go | 2 +- 11 files changed, 40 insertions(+), 17 deletions(-) diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index 4a03aca74..be75f8ad0 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -29,6 +29,7 @@ import ( type DatabaseTransaction interface { sqlutil.Transaction + Reset() (err error) SharedUsers MaxStreamPositionForPDUs(ctx context.Context) (types.StreamPosition, error) diff --git a/syncapi/storage/shared/storage_consumer.go b/syncapi/storage/shared/storage_consumer.go index fb3b295e9..937ced3a2 100644 --- a/syncapi/storage/shared/storage_consumer.go +++ b/syncapi/storage/shared/storage_consumer.go @@ -77,6 +77,7 @@ func (d *Database) NewDatabaseSnapshot(ctx context.Context) (*DatabaseTransactio } return &DatabaseTransaction{ Database: d, + ctx: ctx, txn: txn, }, nil */ @@ -89,6 +90,7 @@ func (d *Database) NewDatabaseTransaction(ctx context.Context) (*DatabaseTransac } return &DatabaseTransaction{ Database: d, + ctx: ctx, txn: txn, }, nil } diff --git a/syncapi/storage/shared/storage_sync.go b/syncapi/storage/shared/storage_sync.go index a19135a69..6cc83ebc8 100644 --- a/syncapi/storage/shared/storage_sync.go +++ b/syncapi/storage/shared/storage_sync.go @@ -13,6 +13,7 @@ import ( type DatabaseTransaction struct { *Database + ctx context.Context txn *sql.Tx } @@ -30,6 +31,19 @@ func (d *DatabaseTransaction) Rollback() error { return d.txn.Rollback() } +func (d *DatabaseTransaction) Reset() (err error) { + if d.txn == nil { + return nil + } + if err = d.txn.Rollback(); err != nil { + return err + } + if d.txn, err = d.DB.BeginTx(d.ctx, nil); err != nil { + return err + } + return +} + func (d *DatabaseTransaction) MaxStreamPositionForPDUs(ctx context.Context) (types.StreamPosition, error) { id, err := d.OutputEvents.SelectMaxEventID(ctx, d.txn) if err != nil { diff --git a/syncapi/streams/stream_accountdata.go b/syncapi/streams/stream_accountdata.go index 34135d65a..f3e7fbdaa 100644 --- a/syncapi/streams/stream_accountdata.go +++ b/syncapi/streams/stream_accountdata.go @@ -54,7 +54,7 @@ func (p *AccountDataStreamProvider) IncrementalSync( ) if err != nil { req.Log.WithError(err).Error("p.DB.GetAccountDataInRange failed") - _ = snapshot.Rollback() + _ = snapshot.Reset() return from } diff --git a/syncapi/streams/stream_devicelist.go b/syncapi/streams/stream_devicelist.go index 10ede573a..307099b8f 100644 --- a/syncapi/streams/stream_devicelist.go +++ b/syncapi/streams/stream_devicelist.go @@ -34,13 +34,13 @@ func (p *DeviceListStreamProvider) IncrementalSync( to, _, err = internal.DeviceListCatchup(context.Background(), snapshot, p.keyAPI, p.rsAPI, req.Device.UserID, req.Response, from, to) if err != nil { req.Log.WithError(err).Error("internal.DeviceListCatchup failed") - _ = snapshot.Rollback() + _ = snapshot.Reset() return from } err = internal.DeviceOTKCounts(req.Context, p.keyAPI, req.Device.UserID, req.Device.ID, req.Response) if err != nil { req.Log.WithError(err).Error("internal.DeviceListCatchup failed") - _ = snapshot.Rollback() + _ = snapshot.Reset() return from } diff --git a/syncapi/streams/stream_invite.go b/syncapi/streams/stream_invite.go index b52eaaab1..4c889b8f5 100644 --- a/syncapi/streams/stream_invite.go +++ b/syncapi/streams/stream_invite.go @@ -56,7 +56,7 @@ func (p *InviteStreamProvider) IncrementalSync( ) if err != nil { req.Log.WithError(err).Error("p.DB.InviteEventsInRange failed") - _ = snapshot.Rollback() + _ = snapshot.Reset() return from } diff --git a/syncapi/streams/stream_notificationdata.go b/syncapi/streams/stream_notificationdata.go index e1ee02b21..5154dd332 100644 --- a/syncapi/streams/stream_notificationdata.go +++ b/syncapi/streams/stream_notificationdata.go @@ -46,7 +46,7 @@ func (p *NotificationDataStreamProvider) IncrementalSync( countsByRoom, err := snapshot.GetUserUnreadNotificationCountsForRooms(ctx, req.Device.UserID, req.Rooms) if err != nil { req.Log.WithError(err).Error("GetUserUnreadNotificationCountsForRooms failed") - _ = snapshot.Rollback() + _ = snapshot.Reset() return from } diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go index 92e1bccf0..01ddf9ac9 100644 --- a/syncapi/streams/stream_pdu.go +++ b/syncapi/streams/stream_pdu.go @@ -75,7 +75,7 @@ func (p *PDUStreamProvider) CompleteSync( joinedRoomIDs, err := snapshot.RoomIDsWithMembership(ctx, req.Device.UserID, gomatrixserverlib.Join) if err != nil { req.Log.WithError(err).Error("p.DB.RoomIDsWithMembership failed") - _ = snapshot.Rollback() + _ = snapshot.Reset() return from } @@ -102,7 +102,9 @@ func (p *PDUStreamProvider) CompleteSync( ) if jerr != nil { req.Log.WithError(jerr).Error("p.getJoinResponseForCompleteSync failed") - _ = snapshot.Rollback() + if err = snapshot.Reset(); err != nil { + return from + } continue // return from } req.Response.Rooms.Join[roomID] = *jr @@ -113,7 +115,7 @@ func (p *PDUStreamProvider) CompleteSync( peeks, err := snapshot.PeeksInRange(ctx, req.Device.UserID, req.Device.ID, r) if err != nil { req.Log.WithError(err).Error("p.DB.PeeksInRange failed") - _ = snapshot.Rollback() + _ = snapshot.Reset() return from } for _, peek := range peeks { @@ -124,7 +126,9 @@ func (p *PDUStreamProvider) CompleteSync( ) if err != nil { req.Log.WithError(err).Error("p.getJoinResponseForCompleteSync failed") - _ = snapshot.Rollback() + if err = snapshot.Reset(); err != nil { + return from + } continue // return from } req.Response.Rooms.Peek[peek.RoomID] = *jr @@ -156,13 +160,13 @@ func (p *PDUStreamProvider) IncrementalSync( if req.WantFullState { if stateDeltas, syncJoinedRooms, err = snapshot.GetStateDeltasForFullStateSync(ctx, req.Device, r, req.Device.UserID, &stateFilter); err != nil { req.Log.WithError(err).Error("p.DB.GetStateDeltasForFullStateSync failed") - _ = snapshot.Rollback() + _ = snapshot.Reset() return } } else { if stateDeltas, syncJoinedRooms, err = snapshot.GetStateDeltas(ctx, req.Device, r, req.Device.UserID, &stateFilter); err != nil { req.Log.WithError(err).Error("p.DB.GetStateDeltas failed") - _ = snapshot.Rollback() + _ = snapshot.Reset() return } } @@ -177,7 +181,7 @@ func (p *PDUStreamProvider) IncrementalSync( if err = p.addIgnoredUsersToFilter(ctx, snapshot, req, &eventFilter); err != nil { req.Log.WithError(err).Error("unable to update event filter with ignored users") - _ = snapshot.Rollback() + _ = snapshot.Reset() } newPos = from @@ -197,10 +201,12 @@ func (p *PDUStreamProvider) IncrementalSync( var pos types.StreamPosition if pos, err = p.addRoomDeltaToResponse(ctx, snapshot, req.Device, newRange, delta, &eventFilter, &stateFilter, req.Response); err != nil { req.Log.WithError(err).Error("d.addRoomDeltaToResponse failed") - _ = snapshot.Rollback() if err == context.DeadlineExceeded || err == context.Canceled { return newPos } + if err = snapshot.Reset(); err != nil { + return from + } continue // return to } // Reset the position, as it is only for the special case of newly joined rooms @@ -301,7 +307,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( events, err := applyHistoryVisibilityFilter(ctx, snapshot, p.rsAPI, delta.RoomID, device.UserID, eventFilter.Limit, recentEvents) if err != nil { logrus.WithError(err).Error("unable to apply history visibility filter") - _ = snapshot.Rollback() + _ = snapshot.Reset() } if len(delta.StateEvents) > 0 { diff --git a/syncapi/streams/stream_presence.go b/syncapi/streams/stream_presence.go index d24c85620..8a3f01c24 100644 --- a/syncapi/streams/stream_presence.go +++ b/syncapi/streams/stream_presence.go @@ -67,7 +67,7 @@ func (p *PresenceStreamProvider) IncrementalSync( presences, err := snapshot.PresenceAfter(ctx, from, gomatrixserverlib.EventFilter{Limit: 1000}) if err != nil { req.Log.WithError(err).Error("p.DB.PresenceAfter failed") - _ = snapshot.Rollback() + _ = snapshot.Reset() return from } diff --git a/syncapi/streams/stream_receipt.go b/syncapi/streams/stream_receipt.go index 40e5bd01e..79fd65bfe 100644 --- a/syncapi/streams/stream_receipt.go +++ b/syncapi/streams/stream_receipt.go @@ -52,7 +52,7 @@ func (p *ReceiptStreamProvider) IncrementalSync( lastPos, receipts, err := snapshot.RoomReceiptsAfter(ctx, joinedRooms, from) if err != nil { req.Log.WithError(err).Error("p.DB.RoomReceiptsAfter failed") - _ = snapshot.Rollback() + _ = snapshot.Reset() return from } diff --git a/syncapi/streams/stream_sendtodevice.go b/syncapi/streams/stream_sendtodevice.go index 3262832a3..c79efad06 100644 --- a/syncapi/streams/stream_sendtodevice.go +++ b/syncapi/streams/stream_sendtodevice.go @@ -44,7 +44,7 @@ func (p *SendToDeviceStreamProvider) IncrementalSync( lastPos, events, err := snapshot.SendToDeviceUpdatesForSync(req.Context, req.Device.UserID, req.Device.ID, from, to) if err != nil { req.Log.WithError(err).Error("p.DB.SendToDeviceUpdatesForSync failed") - _ = snapshot.Rollback() + _ = snapshot.Reset() return from } From a050503d8d2f606fc909e6c9b04c81b91c1bd984 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 30 Sep 2022 17:15:43 +0100 Subject: [PATCH 019/207] Version 0.10.1 --- CHANGES.md | 11 +++++++++++ internal/version.go | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index eca575cf2..dbe2ccf02 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,16 @@ # Changelog +## Dendrite 0.10.1 (2022-09-30) + +### Features + +* The built-in NATS Server has been updated to version 2.9.2 + +### Fixes + +* A regression introduced in 0.10.0 in `/sync` as a result of transaction errors has been fixed +* Account data updates will no longer send duplicate output events + ## Dendrite 0.10.0 (2022-09-30) ### Features diff --git a/internal/version.go b/internal/version.go index 0b19757c2..d508517be 100644 --- a/internal/version.go +++ b/internal/version.go @@ -17,7 +17,7 @@ var build string const ( VersionMajor = 0 VersionMinor = 10 - VersionPatch = 0 + VersionPatch = 1 VersionTag = "" // example: "rc1" ) From d4710217f8d96ee6889e1e4e7c26defc5456b523 Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Sun, 2 Oct 2022 11:31:40 +0200 Subject: [PATCH 020/207] Use non-HTTPS as default URL, as most people will be running behind a reverse proxy --- cmd/create-account/main.go | 2 +- docs/administration/1_createusers.md | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cmd/create-account/main.go b/cmd/create-account/main.go index a9357f6db..52301415f 100644 --- a/cmd/create-account/main.go +++ b/cmd/create-account/main.go @@ -64,7 +64,7 @@ var ( pwdStdin = flag.Bool("passwordstdin", false, "Reads the password from stdin") isAdmin = flag.Bool("admin", false, "Create an admin account") resetPassword = flag.Bool("reset-password", false, "Deprecated") - serverURL = flag.String("url", "https://localhost:8448", "The URL to connect to.") + serverURL = flag.String("url", "http://localhost:8008", "The URL to connect to.") validUsernameRegex = regexp.MustCompile(`^[0-9a-z_\-=./]+$`) timeout = flag.Duration("timeout", time.Second*30, "Timeout for the http client when connecting to the server") ) diff --git a/docs/administration/1_createusers.md b/docs/administration/1_createusers.md index 3468398ac..94399a04a 100644 --- a/docs/administration/1_createusers.md +++ b/docs/administration/1_createusers.md @@ -1,3 +1,4 @@ + --- title: Creating user accounts parent: Administration @@ -31,11 +32,11 @@ To create a new **admin account**, add the `-admin` flag: ./bin/create-account -config /path/to/dendrite.yaml -username USERNAME -admin ``` -By default `create-account` uses `https://localhost:8448` to connect to Dendrite, this can be overwritten using +By default `create-account` uses `http://localhost:8008` to connect to Dendrite, this can be overwritten using the `-url` flag: ```bash -./bin/create-account -config /path/to/dendrite.yaml -username USERNAME -url http://localhost:8008 +./bin/create-account -config /path/to/dendrite.yaml -username USERNAME -url https://localhost:8448 ``` An example of using `create-account` when running in **Docker**, having found the `CONTAINERNAME` from `docker ps`: From d32f60249d8e248d8f8334b044e016ee384797aa Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 3 Oct 2022 11:38:20 +0100 Subject: [PATCH 021/207] Modify sync transaction behaviour (#2758) This now uses a transaction per stream, so that errors in one stream don't propagate to another, and we therefore no longer need to do hacks to reopen a new transaction after aborting a failed one. --- syncapi/storage/interface.go | 1 - syncapi/storage/shared/storage_sync.go | 13 -- syncapi/streams/stream_accountdata.go | 1 - syncapi/streams/stream_devicelist.go | 2 - syncapi/streams/stream_invite.go | 1 - syncapi/streams/stream_notificationdata.go | 1 - syncapi/streams/stream_pdu.go | 25 +-- syncapi/streams/stream_presence.go | 1 - syncapi/streams/stream_receipt.go | 1 - syncapi/streams/stream_sendtodevice.go | 1 - syncapi/sync/requestpool.go | 199 +++++++++++++++------ 11 files changed, 155 insertions(+), 91 deletions(-) diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index be75f8ad0..4a03aca74 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -29,7 +29,6 @@ import ( type DatabaseTransaction interface { sqlutil.Transaction - Reset() (err error) SharedUsers MaxStreamPositionForPDUs(ctx context.Context) (types.StreamPosition, error) diff --git a/syncapi/storage/shared/storage_sync.go b/syncapi/storage/shared/storage_sync.go index 6cc83ebc8..0e19d97d2 100644 --- a/syncapi/storage/shared/storage_sync.go +++ b/syncapi/storage/shared/storage_sync.go @@ -31,19 +31,6 @@ func (d *DatabaseTransaction) Rollback() error { return d.txn.Rollback() } -func (d *DatabaseTransaction) Reset() (err error) { - if d.txn == nil { - return nil - } - if err = d.txn.Rollback(); err != nil { - return err - } - if d.txn, err = d.DB.BeginTx(d.ctx, nil); err != nil { - return err - } - return -} - func (d *DatabaseTransaction) MaxStreamPositionForPDUs(ctx context.Context) (types.StreamPosition, error) { id, err := d.OutputEvents.SelectMaxEventID(ctx, d.txn) if err != nil { diff --git a/syncapi/streams/stream_accountdata.go b/syncapi/streams/stream_accountdata.go index f3e7fbdaa..3f2f7d134 100644 --- a/syncapi/streams/stream_accountdata.go +++ b/syncapi/streams/stream_accountdata.go @@ -54,7 +54,6 @@ func (p *AccountDataStreamProvider) IncrementalSync( ) if err != nil { req.Log.WithError(err).Error("p.DB.GetAccountDataInRange failed") - _ = snapshot.Reset() return from } diff --git a/syncapi/streams/stream_devicelist.go b/syncapi/streams/stream_devicelist.go index 307099b8f..7996c2038 100644 --- a/syncapi/streams/stream_devicelist.go +++ b/syncapi/streams/stream_devicelist.go @@ -34,13 +34,11 @@ func (p *DeviceListStreamProvider) IncrementalSync( to, _, err = internal.DeviceListCatchup(context.Background(), snapshot, p.keyAPI, p.rsAPI, req.Device.UserID, req.Response, from, to) if err != nil { req.Log.WithError(err).Error("internal.DeviceListCatchup failed") - _ = snapshot.Reset() return from } err = internal.DeviceOTKCounts(req.Context, p.keyAPI, req.Device.UserID, req.Device.ID, req.Response) if err != nil { req.Log.WithError(err).Error("internal.DeviceListCatchup failed") - _ = snapshot.Reset() return from } diff --git a/syncapi/streams/stream_invite.go b/syncapi/streams/stream_invite.go index 4c889b8f5..17b3b8434 100644 --- a/syncapi/streams/stream_invite.go +++ b/syncapi/streams/stream_invite.go @@ -56,7 +56,6 @@ func (p *InviteStreamProvider) IncrementalSync( ) if err != nil { req.Log.WithError(err).Error("p.DB.InviteEventsInRange failed") - _ = snapshot.Reset() return from } diff --git a/syncapi/streams/stream_notificationdata.go b/syncapi/streams/stream_notificationdata.go index 5154dd332..5a81fd09a 100644 --- a/syncapi/streams/stream_notificationdata.go +++ b/syncapi/streams/stream_notificationdata.go @@ -46,7 +46,6 @@ func (p *NotificationDataStreamProvider) IncrementalSync( countsByRoom, err := snapshot.GetUserUnreadNotificationCountsForRooms(ctx, req.Device.UserID, req.Rooms) if err != nil { req.Log.WithError(err).Error("GetUserUnreadNotificationCountsForRooms failed") - _ = snapshot.Reset() return from } diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go index 01ddf9ac9..d252265ff 100644 --- a/syncapi/streams/stream_pdu.go +++ b/syncapi/streams/stream_pdu.go @@ -75,7 +75,6 @@ func (p *PDUStreamProvider) CompleteSync( joinedRoomIDs, err := snapshot.RoomIDsWithMembership(ctx, req.Device.UserID, gomatrixserverlib.Join) if err != nil { req.Log.WithError(err).Error("p.DB.RoomIDsWithMembership failed") - _ = snapshot.Reset() return from } @@ -102,10 +101,10 @@ func (p *PDUStreamProvider) CompleteSync( ) if jerr != nil { req.Log.WithError(jerr).Error("p.getJoinResponseForCompleteSync failed") - if err = snapshot.Reset(); err != nil { + if err == context.DeadlineExceeded || err == context.Canceled || err == sql.ErrTxDone { return from } - continue // return from + continue } req.Response.Rooms.Join[roomID] = *jr req.Rooms[roomID] = gomatrixserverlib.Join @@ -115,7 +114,6 @@ func (p *PDUStreamProvider) CompleteSync( peeks, err := snapshot.PeeksInRange(ctx, req.Device.UserID, req.Device.ID, r) if err != nil { req.Log.WithError(err).Error("p.DB.PeeksInRange failed") - _ = snapshot.Reset() return from } for _, peek := range peeks { @@ -126,10 +124,10 @@ func (p *PDUStreamProvider) CompleteSync( ) if err != nil { req.Log.WithError(err).Error("p.getJoinResponseForCompleteSync failed") - if err = snapshot.Reset(); err != nil { + if err == context.DeadlineExceeded || err == context.Canceled || err == sql.ErrTxDone { return from } - continue // return from + continue } req.Response.Rooms.Peek[peek.RoomID] = *jr } @@ -160,14 +158,12 @@ func (p *PDUStreamProvider) IncrementalSync( if req.WantFullState { if stateDeltas, syncJoinedRooms, err = snapshot.GetStateDeltasForFullStateSync(ctx, req.Device, r, req.Device.UserID, &stateFilter); err != nil { req.Log.WithError(err).Error("p.DB.GetStateDeltasForFullStateSync failed") - _ = snapshot.Reset() - return + return from } } else { if stateDeltas, syncJoinedRooms, err = snapshot.GetStateDeltas(ctx, req.Device, r, req.Device.UserID, &stateFilter); err != nil { req.Log.WithError(err).Error("p.DB.GetStateDeltas failed") - _ = snapshot.Reset() - return + return from } } @@ -181,7 +177,6 @@ func (p *PDUStreamProvider) IncrementalSync( if err = p.addIgnoredUsersToFilter(ctx, snapshot, req, &eventFilter); err != nil { req.Log.WithError(err).Error("unable to update event filter with ignored users") - _ = snapshot.Reset() } newPos = from @@ -201,13 +196,10 @@ func (p *PDUStreamProvider) IncrementalSync( var pos types.StreamPosition if pos, err = p.addRoomDeltaToResponse(ctx, snapshot, req.Device, newRange, delta, &eventFilter, &stateFilter, req.Response); err != nil { req.Log.WithError(err).Error("d.addRoomDeltaToResponse failed") - if err == context.DeadlineExceeded || err == context.Canceled { + if err == context.DeadlineExceeded || err == context.Canceled || err == sql.ErrTxDone { return newPos } - if err = snapshot.Reset(); err != nil { - return from - } - continue // return to + continue } // Reset the position, as it is only for the special case of newly joined rooms if delta.NewlyJoined { @@ -307,7 +299,6 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( events, err := applyHistoryVisibilityFilter(ctx, snapshot, p.rsAPI, delta.RoomID, device.UserID, eventFilter.Limit, recentEvents) if err != nil { logrus.WithError(err).Error("unable to apply history visibility filter") - _ = snapshot.Reset() } if len(delta.StateEvents) > 0 { diff --git a/syncapi/streams/stream_presence.go b/syncapi/streams/stream_presence.go index 8a3f01c24..8b87af452 100644 --- a/syncapi/streams/stream_presence.go +++ b/syncapi/streams/stream_presence.go @@ -67,7 +67,6 @@ func (p *PresenceStreamProvider) IncrementalSync( presences, err := snapshot.PresenceAfter(ctx, from, gomatrixserverlib.EventFilter{Limit: 1000}) if err != nil { req.Log.WithError(err).Error("p.DB.PresenceAfter failed") - _ = snapshot.Reset() return from } diff --git a/syncapi/streams/stream_receipt.go b/syncapi/streams/stream_receipt.go index 79fd65bfe..8818a5533 100644 --- a/syncapi/streams/stream_receipt.go +++ b/syncapi/streams/stream_receipt.go @@ -52,7 +52,6 @@ func (p *ReceiptStreamProvider) IncrementalSync( lastPos, receipts, err := snapshot.RoomReceiptsAfter(ctx, joinedRooms, from) if err != nil { req.Log.WithError(err).Error("p.DB.RoomReceiptsAfter failed") - _ = snapshot.Reset() return from } diff --git a/syncapi/streams/stream_sendtodevice.go b/syncapi/streams/stream_sendtodevice.go index c79efad06..00b67cc42 100644 --- a/syncapi/streams/stream_sendtodevice.go +++ b/syncapi/streams/stream_sendtodevice.go @@ -44,7 +44,6 @@ func (p *SendToDeviceStreamProvider) IncrementalSync( lastPos, events, err := snapshot.SendToDeviceUpdatesForSync(req.Context, req.Device.UserID, req.Device.ID, from, to) if err != nil { req.Log.WithError(err).Error("p.DB.SendToDeviceUpdatesForSync failed") - _ = snapshot.Reset() return from } diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index bef0a5683..a71d32ab8 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -306,13 +306,19 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. syncReq.Log.WithField("currentPos", currentPos).Debugln("Responding to sync immediately") } - snapshot, err := rp.db.NewDatabaseSnapshot(req.Context()) - if err != nil { - logrus.WithError(err).Error("Failed to acquire database snapshot for sync request") - return jsonerror.InternalServerError() + withTransaction := func(from types.StreamPosition, f func(snapshot storage.DatabaseTransaction) types.StreamPosition) types.StreamPosition { + var succeeded bool + snapshot, err := rp.db.NewDatabaseSnapshot(req.Context()) + if err != nil { + logrus.WithError(err).Error("Failed to acquire database snapshot for sync request") + return from + } + defer func() { + succeeded = err == nil + sqlutil.EndTransactionWithCheck(snapshot, &succeeded, &err) + }() + return f(snapshot) } - var succeeded bool - defer sqlutil.EndTransactionWithCheck(snapshot, &succeeded, &err) if syncReq.Since.IsEmpty() { // Complete sync @@ -320,72 +326,162 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. // Get the current DeviceListPosition first, as the currentPosition // might advance while processing other streams, resulting in flakey // tests. - DeviceListPosition: rp.streams.DeviceListStreamProvider.CompleteSync( - syncReq.Context, snapshot, syncReq, + DeviceListPosition: withTransaction( + syncReq.Since.DeviceListPosition, + func(txn storage.DatabaseTransaction) types.StreamPosition { + return rp.streams.DeviceListStreamProvider.CompleteSync( + syncReq.Context, txn, syncReq, + ) + }, ), - PDUPosition: rp.streams.PDUStreamProvider.CompleteSync( - syncReq.Context, snapshot, syncReq, + PDUPosition: withTransaction( + syncReq.Since.PDUPosition, + func(txn storage.DatabaseTransaction) types.StreamPosition { + return rp.streams.PDUStreamProvider.CompleteSync( + syncReq.Context, txn, syncReq, + ) + }, ), - TypingPosition: rp.streams.TypingStreamProvider.CompleteSync( - syncReq.Context, snapshot, syncReq, + TypingPosition: withTransaction( + syncReq.Since.TypingPosition, + func(txn storage.DatabaseTransaction) types.StreamPosition { + return rp.streams.TypingStreamProvider.CompleteSync( + syncReq.Context, txn, syncReq, + ) + }, ), - ReceiptPosition: rp.streams.ReceiptStreamProvider.CompleteSync( - syncReq.Context, snapshot, syncReq, + ReceiptPosition: withTransaction( + syncReq.Since.ReceiptPosition, + func(txn storage.DatabaseTransaction) types.StreamPosition { + return rp.streams.ReceiptStreamProvider.CompleteSync( + syncReq.Context, txn, syncReq, + ) + }, ), - InvitePosition: rp.streams.InviteStreamProvider.CompleteSync( - syncReq.Context, snapshot, syncReq, + InvitePosition: withTransaction( + syncReq.Since.InvitePosition, + func(txn storage.DatabaseTransaction) types.StreamPosition { + return rp.streams.InviteStreamProvider.CompleteSync( + syncReq.Context, txn, syncReq, + ) + }, ), - SendToDevicePosition: rp.streams.SendToDeviceStreamProvider.CompleteSync( - syncReq.Context, snapshot, syncReq, + SendToDevicePosition: withTransaction( + syncReq.Since.SendToDevicePosition, + func(txn storage.DatabaseTransaction) types.StreamPosition { + return rp.streams.SendToDeviceStreamProvider.CompleteSync( + syncReq.Context, txn, syncReq, + ) + }, ), - AccountDataPosition: rp.streams.AccountDataStreamProvider.CompleteSync( - syncReq.Context, snapshot, syncReq, + AccountDataPosition: withTransaction( + syncReq.Since.AccountDataPosition, + func(txn storage.DatabaseTransaction) types.StreamPosition { + return rp.streams.AccountDataStreamProvider.CompleteSync( + syncReq.Context, txn, syncReq, + ) + }, ), - NotificationDataPosition: rp.streams.NotificationDataStreamProvider.CompleteSync( - syncReq.Context, snapshot, syncReq, + NotificationDataPosition: withTransaction( + syncReq.Since.NotificationDataPosition, + func(txn storage.DatabaseTransaction) types.StreamPosition { + return rp.streams.NotificationDataStreamProvider.CompleteSync( + syncReq.Context, txn, syncReq, + ) + }, ), - PresencePosition: rp.streams.PresenceStreamProvider.CompleteSync( - syncReq.Context, snapshot, syncReq, + PresencePosition: withTransaction( + syncReq.Since.PresencePosition, + func(txn storage.DatabaseTransaction) types.StreamPosition { + return rp.streams.PresenceStreamProvider.CompleteSync( + syncReq.Context, txn, syncReq, + ) + }, ), } } else { // Incremental sync syncReq.Response.NextBatch = types.StreamingToken{ - PDUPosition: rp.streams.PDUStreamProvider.IncrementalSync( - syncReq.Context, snapshot, syncReq, - syncReq.Since.PDUPosition, currentPos.PDUPosition, + PDUPosition: withTransaction( + syncReq.Since.PDUPosition, + func(txn storage.DatabaseTransaction) types.StreamPosition { + return rp.streams.PDUStreamProvider.IncrementalSync( + syncReq.Context, txn, syncReq, + syncReq.Since.PDUPosition, currentPos.PDUPosition, + ) + }, ), - TypingPosition: rp.streams.TypingStreamProvider.IncrementalSync( - syncReq.Context, snapshot, syncReq, - syncReq.Since.TypingPosition, currentPos.TypingPosition, + TypingPosition: withTransaction( + syncReq.Since.TypingPosition, + func(txn storage.DatabaseTransaction) types.StreamPosition { + return rp.streams.TypingStreamProvider.IncrementalSync( + syncReq.Context, txn, syncReq, + syncReq.Since.TypingPosition, currentPos.TypingPosition, + ) + }, ), - ReceiptPosition: rp.streams.ReceiptStreamProvider.IncrementalSync( - syncReq.Context, snapshot, syncReq, - syncReq.Since.ReceiptPosition, currentPos.ReceiptPosition, + ReceiptPosition: withTransaction( + syncReq.Since.ReceiptPosition, + func(txn storage.DatabaseTransaction) types.StreamPosition { + return rp.streams.ReceiptStreamProvider.IncrementalSync( + syncReq.Context, txn, syncReq, + syncReq.Since.ReceiptPosition, currentPos.ReceiptPosition, + ) + }, ), - InvitePosition: rp.streams.InviteStreamProvider.IncrementalSync( - syncReq.Context, snapshot, syncReq, - syncReq.Since.InvitePosition, currentPos.InvitePosition, + InvitePosition: withTransaction( + syncReq.Since.InvitePosition, + func(txn storage.DatabaseTransaction) types.StreamPosition { + return rp.streams.InviteStreamProvider.IncrementalSync( + syncReq.Context, txn, syncReq, + syncReq.Since.InvitePosition, currentPos.InvitePosition, + ) + }, ), - SendToDevicePosition: rp.streams.SendToDeviceStreamProvider.IncrementalSync( - syncReq.Context, snapshot, syncReq, - syncReq.Since.SendToDevicePosition, currentPos.SendToDevicePosition, + SendToDevicePosition: withTransaction( + syncReq.Since.SendToDevicePosition, + func(txn storage.DatabaseTransaction) types.StreamPosition { + return rp.streams.SendToDeviceStreamProvider.IncrementalSync( + syncReq.Context, txn, syncReq, + syncReq.Since.SendToDevicePosition, currentPos.SendToDevicePosition, + ) + }, ), - AccountDataPosition: rp.streams.AccountDataStreamProvider.IncrementalSync( - syncReq.Context, snapshot, syncReq, - syncReq.Since.AccountDataPosition, currentPos.AccountDataPosition, + AccountDataPosition: withTransaction( + syncReq.Since.AccountDataPosition, + func(txn storage.DatabaseTransaction) types.StreamPosition { + return rp.streams.AccountDataStreamProvider.IncrementalSync( + syncReq.Context, txn, syncReq, + syncReq.Since.AccountDataPosition, currentPos.AccountDataPosition, + ) + }, ), - NotificationDataPosition: rp.streams.NotificationDataStreamProvider.IncrementalSync( - syncReq.Context, snapshot, syncReq, - syncReq.Since.NotificationDataPosition, currentPos.NotificationDataPosition, + NotificationDataPosition: withTransaction( + syncReq.Since.NotificationDataPosition, + func(txn storage.DatabaseTransaction) types.StreamPosition { + return rp.streams.NotificationDataStreamProvider.IncrementalSync( + syncReq.Context, txn, syncReq, + syncReq.Since.NotificationDataPosition, currentPos.NotificationDataPosition, + ) + }, ), - DeviceListPosition: rp.streams.DeviceListStreamProvider.IncrementalSync( - syncReq.Context, snapshot, syncReq, - syncReq.Since.DeviceListPosition, currentPos.DeviceListPosition, + DeviceListPosition: withTransaction( + syncReq.Since.DeviceListPosition, + func(txn storage.DatabaseTransaction) types.StreamPosition { + return rp.streams.DeviceListStreamProvider.IncrementalSync( + syncReq.Context, txn, syncReq, + syncReq.Since.DeviceListPosition, currentPos.DeviceListPosition, + ) + }, ), - PresencePosition: rp.streams.PresenceStreamProvider.IncrementalSync( - syncReq.Context, snapshot, syncReq, - syncReq.Since.PresencePosition, currentPos.PresencePosition, + PresencePosition: withTransaction( + syncReq.Since.PresencePosition, + func(txn storage.DatabaseTransaction) types.StreamPosition { + return rp.streams.PresenceStreamProvider.IncrementalSync( + syncReq.Context, txn, syncReq, + syncReq.Since.PresencePosition, currentPos.PresencePosition, + ) + }, ), } // it's possible for there to be no updates for this user even though since < current pos, @@ -411,7 +507,6 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. } } - succeeded = true return util.JSONResponse{ Code: http.StatusOK, JSON: syncReq.Response, From b7f4bab358ea6de0932037242f71152bfc27c289 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 3 Oct 2022 11:38:31 +0100 Subject: [PATCH 022/207] Hopefully fix P2P `--config` error (re. #2756) --- cmd/dendrite-demo-pinecone/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go index da63f9a2c..fb988ebe1 100644 --- a/cmd/dendrite-demo-pinecone/main.go +++ b/cmd/dendrite-demo-pinecone/main.go @@ -89,6 +89,7 @@ func main() { if configFlagSet { cfg = setup.ParseFlags(true) sk = cfg.Global.PrivateKey + pk = sk.Public().(ed25519.PublicKey) } else { keyfile := filepath.Join(*instanceDir, *instanceName) + ".pem" if _, err := os.Stat(keyfile); os.IsNotExist(err) { From c1e16fd41e24eb1c139b4e4acbef1a309d7d573b Mon Sep 17 00:00:00 2001 From: Ashley Nelson Date: Mon, 3 Oct 2022 05:57:21 -0500 Subject: [PATCH 023/207] Fix fragility of selectEventsWithEventIDsSQL queries (#2757) This fixes a temporary workaround with the `selectEventsWithEventIDsSQL` queries where fields need to be artificially added to the queries so the row results match the format of the `syncapi_output_room_events` table. I made similar functions that accept row results from the `syncapi_current_room_state` table and convert them into StreamEvents without the fields that are specific to output room events. There is also a unit test in the first commit to ensure the resulting behavior doesn't change from the modified queries and functions. Fixes #601. ### Pull Request Checklist * [x] I have added tests for PR _or_ I have justified why this PR doesn't need tests. * [x] Pull request includes a [sign off](https://github.com/matrix-org/dendrite/blob/main/docs/CONTRIBUTING.md#sign-off) Signed-off-by: `Ashley Nelson ` Co-authored-by: Neil Alexander --- .../postgres/current_room_state_table.go | 38 ++++++-- .../sqlite3/current_room_state_table.go | 38 ++++++-- .../storage/tables/current_room_state_test.go | 88 +++++++++++++++++++ 3 files changed, 150 insertions(+), 14 deletions(-) create mode 100644 syncapi/storage/tables/current_room_state_test.go diff --git a/syncapi/storage/postgres/current_room_state_table.go b/syncapi/storage/postgres/current_room_state_table.go index 4ffd29610..2ccf0be1a 100644 --- a/syncapi/storage/postgres/current_room_state_table.go +++ b/syncapi/storage/postgres/current_room_state_table.go @@ -104,12 +104,7 @@ const selectStateEventSQL = "" + "SELECT headered_event_json FROM syncapi_current_room_state WHERE room_id = $1 AND type = $2 AND state_key = $3" const selectEventsWithEventIDsSQL = "" + - // TODO: The session_id and transaction_id blanks are here because - // the rowsToStreamEvents expects there to be exactly seven columns. We need to - // figure out if these really need to be in the DB, and if so, we need a - // better permanent fix for this. - neilalexander, 2 Jan 2020 - "SELECT event_id, added_at, headered_event_json, 0 AS session_id, false AS exclude_from_sync, '' AS transaction_id, history_visibility" + - " FROM syncapi_current_room_state WHERE event_id = ANY($1)" + "SELECT event_id, added_at, headered_event_json, history_visibility FROM syncapi_current_room_state WHERE event_id = ANY($1)" const selectSharedUsersSQL = "" + "SELECT state_key FROM syncapi_current_room_state WHERE room_id = ANY(" + @@ -365,7 +360,36 @@ func (s *currentRoomStateStatements) SelectEventsWithEventIDs( return nil, err } defer internal.CloseAndLogIfError(ctx, rows, "selectEventsWithEventIDs: rows.close() failed") - return rowsToStreamEvents(rows) + return currentRoomStateRowsToStreamEvents(rows) +} + +func currentRoomStateRowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) { + var events []types.StreamEvent + for rows.Next() { + var ( + eventID string + streamPos types.StreamPosition + eventBytes []byte + historyVisibility gomatrixserverlib.HistoryVisibility + ) + if err := rows.Scan(&eventID, &streamPos, &eventBytes, &historyVisibility); err != nil { + return nil, err + } + // TODO: Handle redacted events + var ev gomatrixserverlib.HeaderedEvent + if err := ev.UnmarshalJSONWithEventID(eventBytes, eventID); err != nil { + return nil, err + } + + ev.Visibility = historyVisibility + + events = append(events, types.StreamEvent{ + HeaderedEvent: &ev, + StreamPosition: streamPos, + }) + } + + return events, nil } func rowsToEvents(rows *sql.Rows) ([]*gomatrixserverlib.HeaderedEvent, error) { diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go index c4019fed2..ff45e786e 100644 --- a/syncapi/storage/sqlite3/current_room_state_table.go +++ b/syncapi/storage/sqlite3/current_room_state_table.go @@ -88,12 +88,7 @@ const selectStateEventSQL = "" + "SELECT headered_event_json FROM syncapi_current_room_state WHERE room_id = $1 AND type = $2 AND state_key = $3" const selectEventsWithEventIDsSQL = "" + - // TODO: The session_id and transaction_id blanks are here because - // the rowsToStreamEvents expects there to be exactly seven columns. We need to - // figure out if these really need to be in the DB, and if so, we need a - // better permanent fix for this. - neilalexander, 2 Jan 2020 - "SELECT event_id, added_at, headered_event_json, 0 AS session_id, false AS exclude_from_sync, '' AS transaction_id, history_visibility" + - " FROM syncapi_current_room_state WHERE event_id IN ($1)" + "SELECT event_id, added_at, headered_event_json, history_visibility FROM syncapi_current_room_state WHERE event_id IN ($1)" const selectSharedUsersSQL = "" + "SELECT state_key FROM syncapi_current_room_state WHERE room_id IN(" + @@ -378,7 +373,7 @@ func (s *currentRoomStateStatements) SelectEventsWithEventIDs( return nil, err } start = start + n - events, err := rowsToStreamEvents(rows) + events, err := currentRoomStateRowsToStreamEvents(rows) internal.CloseAndLogIfError(ctx, rows, "selectEventsWithEventIDs: rows.close() failed") if err != nil { return nil, err @@ -388,6 +383,35 @@ func (s *currentRoomStateStatements) SelectEventsWithEventIDs( return res, nil } +func currentRoomStateRowsToStreamEvents(rows *sql.Rows) ([]types.StreamEvent, error) { + var events []types.StreamEvent + for rows.Next() { + var ( + eventID string + streamPos types.StreamPosition + eventBytes []byte + historyVisibility gomatrixserverlib.HistoryVisibility + ) + if err := rows.Scan(&eventID, &streamPos, &eventBytes, &historyVisibility); err != nil { + return nil, err + } + // TODO: Handle redacted events + var ev gomatrixserverlib.HeaderedEvent + if err := ev.UnmarshalJSONWithEventID(eventBytes, eventID); err != nil { + return nil, err + } + + ev.Visibility = historyVisibility + + events = append(events, types.StreamEvent{ + HeaderedEvent: &ev, + StreamPosition: streamPos, + }) + } + + return events, nil +} + func rowsToEvents(rows *sql.Rows) ([]*gomatrixserverlib.HeaderedEvent, error) { result := []*gomatrixserverlib.HeaderedEvent{} for rows.Next() { diff --git a/syncapi/storage/tables/current_room_state_test.go b/syncapi/storage/tables/current_room_state_test.go new file mode 100644 index 000000000..23287c500 --- /dev/null +++ b/syncapi/storage/tables/current_room_state_test.go @@ -0,0 +1,88 @@ +package tables_test + +import ( + "context" + "database/sql" + "fmt" + "testing" + + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/syncapi/storage/postgres" + "github.com/matrix-org/dendrite/syncapi/storage/sqlite3" + "github.com/matrix-org/dendrite/syncapi/storage/tables" + "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/dendrite/test" +) + +func newCurrentRoomStateTable(t *testing.T, dbType test.DBType) (tables.CurrentRoomState, *sql.DB, func()) { + t.Helper() + connStr, close := test.PrepareDBConnectionString(t, dbType) + db, err := sqlutil.Open(&config.DatabaseOptions{ + ConnectionString: config.DataSource(connStr), + }, sqlutil.NewExclusiveWriter()) + if err != nil { + t.Fatalf("failed to open db: %s", err) + } + + var tab tables.CurrentRoomState + switch dbType { + case test.DBTypePostgres: + tab, err = postgres.NewPostgresCurrentRoomStateTable(db) + case test.DBTypeSQLite: + var stream sqlite3.StreamIDStatements + if err = stream.Prepare(db); err != nil { + t.Fatalf("failed to prepare stream stmts: %s", err) + } + tab, err = sqlite3.NewSqliteCurrentRoomStateTable(db, &stream) + } + if err != nil { + t.Fatalf("failed to make new table: %s", err) + } + return tab, db, close +} + +func TestCurrentRoomStateTable(t *testing.T) { + ctx := context.Background() + alice := test.NewUser(t) + room := test.NewRoom(t, alice) + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + tab, db, close := newCurrentRoomStateTable(t, dbType) + defer close() + events := room.CurrentState() + err := sqlutil.WithTransaction(db, func(txn *sql.Tx) error { + for i, ev := range events { + err := tab.UpsertRoomState(ctx, txn, ev, nil, types.StreamPosition(i)) + if err != nil { + return fmt.Errorf("failed to UpsertRoomState: %w", err) + } + } + wantEventIDs := []string{ + events[0].EventID(), events[1].EventID(), events[2].EventID(), events[3].EventID(), + } + gotEvents, err := tab.SelectEventsWithEventIDs(ctx, txn, wantEventIDs) + if err != nil { + return fmt.Errorf("failed to SelectEventsWithEventIDs: %w", err) + } + if len(gotEvents) != len(wantEventIDs) { + return fmt.Errorf("SelectEventsWithEventIDs\ngot %d, want %d results", len(gotEvents), len(wantEventIDs)) + } + gotEventIDs := make(map[string]struct{}, len(gotEvents)) + for _, event := range gotEvents { + if event.ExcludeFromSync { + return fmt.Errorf("SelectEventsWithEventIDs ExcludeFromSync should be false for current room state event %+v", event) + } + gotEventIDs[event.EventID()] = struct{}{} + } + for _, id := range wantEventIDs { + if _, ok := gotEventIDs[id]; !ok { + return fmt.Errorf("SelectEventsWithEventIDs\nexpected id %q not returned", id) + } + } + return nil + }) + if err != nil { + t.Fatalf("err: %v", err) + } + }) +} From fec3ee384bad7f05e13893a97f65590d2284438b Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 3 Oct 2022 12:59:56 +0100 Subject: [PATCH 024/207] Stop CPU burn in `PerformMarkAsStaleIfNeeded` --- keyserver/internal/internal.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go index 6309066d7..017c29e84 100644 --- a/keyserver/internal/internal.go +++ b/keyserver/internal/internal.go @@ -233,7 +233,7 @@ func (a *KeyInternalAPI) PerformMarkAsStaleIfNeeded(ctx context.Context, req *ap return err } if len(knownDevices) == 0 { - return fmt.Errorf("unknown user %s", req.UserID) + return nil // fmt.Errorf("unknown user %s", req.UserID) } for i := range knownDevices { From 50fa50a3430eed79ddf897d9841fe0b652a32341 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 3 Oct 2022 13:05:58 +0100 Subject: [PATCH 025/207] Update P2P base directories --- cmd/dendrite-demo-pinecone/main.go | 3 +++ cmd/dendrite-demo-yggdrasil/main.go | 3 +++ 2 files changed, 6 insertions(+) diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go index fb988ebe1..be34365b4 100644 --- a/cmd/dendrite-demo-pinecone/main.go +++ b/cmd/dendrite-demo-pinecone/main.go @@ -143,6 +143,9 @@ func main() { cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", filepath.Join(*instanceDir, *instanceName))) cfg.ClientAPI.RegistrationDisabled = false cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true + cfg.MediaAPI.BasePath = config.Path(*instanceDir) + cfg.SyncAPI.Fulltext.Enabled = true + cfg.SyncAPI.Fulltext.IndexPath = config.Path(*instanceDir) if err := cfg.Derive(); err != nil { panic(err) } diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index cd0066679..38c25cdec 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -134,6 +134,9 @@ func main() { cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", filepath.Join(*instanceDir, *instanceName))) cfg.ClientAPI.RegistrationDisabled = false cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true + cfg.MediaAPI.BasePath = config.Path(*instanceDir) + cfg.SyncAPI.Fulltext.Enabled = true + cfg.SyncAPI.Fulltext.IndexPath = config.Path(*instanceDir) if err := cfg.Derive(); err != nil { panic(err) } From 34451d21b8b069e233700dfe4f000f45984be6ea Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 3 Oct 2022 14:35:10 +0100 Subject: [PATCH 026/207] P2P demo tweaks --- build/gobind-pinecone/monolith.go | 38 ++++++++++++++++--------------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go index b0d3fa285..5e9ff8ef3 100644 --- a/build/gobind-pinecone/monolith.go +++ b/build/gobind-pinecone/monolith.go @@ -103,15 +103,15 @@ func (m *DendriteMonolith) SessionCount() int { func (m *DendriteMonolith) RegisterNetworkInterface(name string, index int, mtu int, up bool, broadcast bool, loopback bool, pointToPoint bool, multicast bool, addrs string) { m.PineconeMulticast.RegisterInterface(pineconeMulticast.InterfaceInfo{ - Name: name, - Index: index, - Mtu: mtu, - Up: up, - Broadcast: broadcast, - Loopback: loopback, + Name: name, + Index: index, + Mtu: mtu, + Up: up, + Broadcast: broadcast, + Loopback: loopback, PointToPoint: pointToPoint, - Multicast: multicast, - Addrs: addrs, + Multicast: multicast, + Addrs: addrs, }) } @@ -279,19 +279,21 @@ func (m *DendriteMonolith) Start() { cfg.Global.ServerName = gomatrixserverlib.ServerName(hex.EncodeToString(pk)) cfg.Global.PrivateKey = sk cfg.Global.KeyID = gomatrixserverlib.KeyID(signing.KeyID) - cfg.Global.JetStream.InMemory = true - cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/%s", m.StorageDirectory, prefix)) - cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-account.db", m.StorageDirectory, prefix)) - cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-mediaapi.db", m.StorageDirectory)) - cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-syncapi.db", m.StorageDirectory, prefix)) - cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-roomserver.db", m.StorageDirectory, prefix)) - cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-keyserver.db", m.StorageDirectory, prefix)) - cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-federationsender.db", m.StorageDirectory, prefix)) - cfg.MediaAPI.BasePath = config.Path(fmt.Sprintf("%s/media", m.CacheDirectory)) - cfg.MediaAPI.AbsBasePath = config.Path(fmt.Sprintf("%s/media", m.CacheDirectory)) + cfg.Global.JetStream.InMemory = false + cfg.Global.JetStream.StoragePath = config.Path(filepath.Join(m.CacheDirectory, prefix)) + cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", filepath.Join(m.StorageDirectory, prefix))) + cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", filepath.Join(m.StorageDirectory, prefix))) + cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", filepath.Join(m.StorageDirectory, prefix))) + cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", filepath.Join(m.StorageDirectory, prefix))) + cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", filepath.Join(m.StorageDirectory, prefix))) + cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationsender.db", filepath.Join(m.StorageDirectory, prefix))) + cfg.MediaAPI.BasePath = config.Path(filepath.Join(m.CacheDirectory, "media")) + cfg.MediaAPI.AbsBasePath = config.Path(filepath.Join(m.CacheDirectory, "media")) cfg.MSCs.MSCs = []string{"msc2836", "msc2946"} cfg.ClientAPI.RegistrationDisabled = false cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true + cfg.SyncAPI.Fulltext.Enabled = true + cfg.SyncAPI.Fulltext.IndexPath = config.Path(filepath.Join(m.CacheDirectory, "search")) if err = cfg.Derive(); err != nil { panic(err) } From ba66b5a3b9844f4cb3b3f8f2c814894ebb415695 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 3 Oct 2022 14:43:38 +0100 Subject: [PATCH 027/207] Allow multiple static peers in Pinecone iOS/Android demos --- build/gobind-pinecone/monolith.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go index 5e9ff8ef3..4a96e4bef 100644 --- a/build/gobind-pinecone/monolith.go +++ b/build/gobind-pinecone/monolith.go @@ -126,7 +126,9 @@ func (m *DendriteMonolith) SetMulticastEnabled(enabled bool) { func (m *DendriteMonolith) SetStaticPeer(uri string) { m.PineconeManager.RemovePeers() - m.PineconeManager.AddPeer(strings.TrimSpace(uri)) + for _, uri := range strings.Split(uri, ",") { + m.PineconeManager.AddPeer(strings.TrimSpace(uri)) + } } func (m *DendriteMonolith) DisconnectType(peertype int) { From 34ed316584df916f6959808669c998580f76d88f Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 3 Oct 2022 19:35:26 +0100 Subject: [PATCH 028/207] Fix docs --- docs/administration/1_createusers.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/administration/1_createusers.md b/docs/administration/1_createusers.md index 94399a04a..24eba666d 100644 --- a/docs/administration/1_createusers.md +++ b/docs/administration/1_createusers.md @@ -1,4 +1,3 @@ - --- title: Creating user accounts parent: Administration @@ -15,7 +14,7 @@ User accounts can be created on a Dendrite instance in a number of ways. The `create-account` tool is built in the `bin` folder when building Dendrite with the `build.sh` script. -It uses the `dendrite.yaml` configuration file to connect to a running Dendrite instance and requires +It uses the `dendrite.yaml` configuration file to connect to a running Dendrite instance and requires shared secret registration to be enabled as explained below. An example of using `create-account` to create a **normal account**: @@ -32,7 +31,7 @@ To create a new **admin account**, add the `-admin` flag: ./bin/create-account -config /path/to/dendrite.yaml -username USERNAME -admin ``` -By default `create-account` uses `http://localhost:8008` to connect to Dendrite, this can be overwritten using +By default `create-account` uses `http://localhost:8008` to connect to Dendrite, this can be overwritten using the `-url` flag: ```bash @@ -44,6 +43,7 @@ An example of using `create-account` when running in **Docker**, having found th ```bash docker exec -it CONTAINERNAME /usr/bin/create-account -config /path/to/dendrite.yaml -username USERNAME ``` + ```bash docker exec -it CONTAINERNAME /usr/bin/create-account -config /path/to/dendrite.yaml -username USERNAME -admin ``` From e6c992ba8bcbc59706d6dd55db3b237ebbdc8e7e Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Tue, 4 Oct 2022 11:41:06 +0200 Subject: [PATCH 029/207] Update Dockerfile (#2342) Updates/adds a new multistage (build-kit) Dockerfile. (if accepted, could make `Dockerfile.monolith` and `Dockerfile.polylith` in `build/docker` obsolete) There's no huge difference between the dockerfiles, except this uses a non-root user when running the container, also doesn't copy the working directory to the image when building. Also adds vulnerabilities scans using [Trivy](https://github.com/aquasecurity/trivy) for the created docker images. (untested) Building images is done using ``` docker build . --target image-monolith -t dendrite-monolith docker build . --target image-polylith -t dendrite-polylith ``` As noted in the comments, only adds `dendrite-polylith-multi` to the polylith image and all required binaries to the monolith image. Probably needs some docs updating, if this is accepted. Co-authored-by: Neil Alexander --- .github/workflows/dendrite.yml | 6 +- .github/workflows/docker.yml | 106 +++++++++++++----- Dockerfile | 99 ++++++++++++++++ .../dendritejs-pinecone/jsServer.go | 0 {cmd => build}/dendritejs-pinecone/main.go | 0 .../dendritejs-pinecone/main_noop.go | 0 .../dendritejs-pinecone/main_test.go | 0 build/docker/Dockerfile.demo-pinecone | 25 ----- build/docker/Dockerfile.monolith | 25 ----- build/docker/Dockerfile.polylith | 25 ----- build/docker/README.md | 14 ++- build/docker/crossbuild.sh | 67 +++++++++++ build/docker/images-build.sh | 5 +- 13 files changed, 257 insertions(+), 115 deletions(-) create mode 100644 Dockerfile rename {cmd => build}/dendritejs-pinecone/jsServer.go (100%) rename {cmd => build}/dendritejs-pinecone/main.go (100%) rename {cmd => build}/dendritejs-pinecone/main_noop.go (100%) rename {cmd => build}/dendritejs-pinecone/main_test.go (100%) delete mode 100644 build/docker/Dockerfile.demo-pinecone delete mode 100644 build/docker/Dockerfile.monolith delete mode 100644 build/docker/Dockerfile.polylith create mode 100644 build/docker/crossbuild.sh diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml index be3c7c173..524d36039 100644 --- a/.github/workflows/dendrite.yml +++ b/.github/workflows/dendrite.yml @@ -284,7 +284,7 @@ jobs: API: ${{ matrix.api && 1 }} SYTEST_BRANCH: ${{ github.head_ref }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Run Sytest run: /bootstrap.sh dendrite working-directory: /src @@ -344,8 +344,8 @@ jobs: sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev go get -v github.com/haveyoudebuggedit/gotestfmt/v2/cmd/gotestfmt@latest - - name: Run actions/checkout@v2 for dendrite - uses: actions/checkout@v2 + - name: Run actions/checkout@v3 for dendrite + uses: actions/checkout@v3 with: path: dendrite diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index b4e24e52f..358037c02 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -21,26 +21,32 @@ jobs: monolith: name: Monolith image runs-on: ubuntu-latest + needs: build-flags permissions: contents: read packages: write steps: - name: Checkout - uses: actions/checkout@v2 - - name: Get release tag + uses: actions/checkout@v3 + - name: Get release tag & build flags if: github.event_name == 'release' # Only for GitHub releases - run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + run: | + echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV + BRANCH=$(git symbolic-ref --short HEAD | tr -d \/) + [ ${BRANCH} == "main" ] && BRANCH="" + echo "BRANCH=${BRANCH}" >> $GITHUB_ENV - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Login to Docker Hub - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: username: ${{ env.DOCKER_HUB_USER }} password: ${{ secrets.DOCKER_TOKEN }} - name: Login to GitHub Containers - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ github.repository_owner }} @@ -49,27 +55,41 @@ jobs: - name: Build main monolith image if: github.ref_name == 'main' id: docker_build_monolith - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: cache-from: type=gha cache-to: type=gha,mode=max context: . - file: ./build/docker/Dockerfile.monolith + build-args: FLAGS="-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}" + target: monolith platforms: ${{ env.PLATFORMS }} push: true tags: | ${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }} ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }} + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }} + format: "sarif" + output: "trivy-results.sarif" + + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v1 + with: + sarif_file: "trivy-results.sarif" + - name: Build release monolith image if: github.event_name == 'release' # Only for GitHub releases id: docker_build_monolith_release - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: cache-from: type=gha cache-to: type=gha,mode=max context: . - file: ./build/docker/Dockerfile.monolith + build-args: FLAGS="-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}" + target: monolith platforms: ${{ env.PLATFORMS }} push: true tags: | @@ -81,26 +101,32 @@ jobs: polylith: name: Polylith image runs-on: ubuntu-latest + needs: build-flags permissions: contents: read packages: write steps: - name: Checkout - uses: actions/checkout@v2 - - name: Get release tag + uses: actions/checkout@v3 + - name: Get release tag & build flags if: github.event_name == 'release' # Only for GitHub releases - run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + run: | + echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV + BRANCH=$(git symbolic-ref --short HEAD | tr -d \/) + [ ${BRANCH} == "main" ] && BRANCH="" + echo "BRANCH=${BRANCH}" >> $GITHUB_ENV - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Login to Docker Hub - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: username: ${{ env.DOCKER_HUB_USER }} password: ${{ secrets.DOCKER_TOKEN }} - name: Login to GitHub Containers - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ github.repository_owner }} @@ -109,27 +135,40 @@ jobs: - name: Build main polylith image if: github.ref_name == 'main' id: docker_build_polylith - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: cache-from: type=gha cache-to: type=gha,mode=max context: . - file: ./build/docker/Dockerfile.polylith + build-args: FLAGS="-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}" + target: polylith platforms: ${{ env.PLATFORMS }} push: true tags: | ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }} ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }} + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }} + format: "sarif" + output: "trivy-results.sarif" + + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v1 + with: + sarif_file: "trivy-results.sarif" + - name: Build release polylith image if: github.event_name == 'release' # Only for GitHub releases id: docker_build_polylith_release - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: cache-from: type=gha cache-to: type=gha,mode=max context: . - file: ./build/docker/Dockerfile.polylith + target: polylith platforms: ${{ env.PLATFORMS }} push: true tags: | @@ -141,26 +180,32 @@ jobs: demo-pinecone: name: Pinecone demo image runs-on: ubuntu-latest + needs: build-flags permissions: contents: read packages: write steps: - name: Checkout - uses: actions/checkout@v2 - - name: Get release tag + uses: actions/checkout@v3 + - name: Get release tag & build flags if: github.event_name == 'release' # Only for GitHub releases - run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + run: | + echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV + BRANCH=$(git symbolic-ref --short HEAD | tr -d \/) + [ ${BRANCH} == "main" ] && BRANCH="" + echo "BRANCH=${BRANCH}" >> $GITHUB_ENV - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Login to Docker Hub - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: username: ${{ env.DOCKER_HUB_USER }} password: ${{ secrets.DOCKER_TOKEN }} - name: Login to GitHub Containers - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ github.repository_owner }} @@ -169,12 +214,12 @@ jobs: - name: Build main pinecone demo image if: github.ref_name == 'main' id: docker_build_demo_pinecone - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: cache-from: type=gha cache-to: type=gha,mode=max context: . - file: ./build/docker/Dockerfile.demo-pinecone + target: demo-pinecone platforms: ${{ env.PLATFORMS }} push: true tags: | @@ -184,12 +229,13 @@ jobs: - name: Build release pinecone demo image if: github.event_name == 'release' # Only for GitHub releases id: docker_build_demo_pinecone_release - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: cache-from: type=gha cache-to: type=gha,mode=max context: . - file: ./build/docker/Dockerfile.demo-pinecone + build-args: FLAGS="-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}" + target: demo-pinecone platforms: ${{ env.PLATFORMS }} push: true tags: | diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..bf5831832 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,99 @@ +#syntax=docker/dockerfile:1.2 + +# +# base installs required dependencies and runs go mod download to cache dependencies +# +FROM --platform=${BUILDPLATFORM} docker.io/golang:1.19-alpine AS base +RUN apk --update --no-cache add bash build-base curl + +# +# build creates all needed binaries +# +FROM base AS build +WORKDIR /src +ARG TARGETOS +ARG TARGETARCH +ARG FLAGS +RUN --mount=target=. \ + --mount=type=cache,target=/root/.cache/go-build \ + sh ./build/docker/crossbuild.sh + +# +# The dendrite base image; mainly creates a user and switches to it +# +FROM alpine:latest AS dendrite-base +LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go" +LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite" +LABEL org.opencontainers.image.licenses="Apache-2.0" +LABEL org.opencontainers.image.documentation="https://matrix-org.github.io/dendrite/" +LABEL org.opencontainers.image.vendor="The Matrix.org Foundation C.I.C." +RUN addgroup dendrite && adduser dendrite -G dendrite -u 1337 -D +USER dendrite +WORKDIR /home/dendrite + +# +# Builds the polylith image and only contains the polylith binary +# +FROM dendrite-base AS polylith +LABEL org.opencontainers.image.title="Dendrite (Polylith)" + +COPY --from=build /out/dendrite-polylith-multi /usr/bin/ + +ENTRYPOINT ["/usr/bin/dendrite-polylith-multi"] + +# +# Builds the monolith image and contains all required binaries +# +FROM dendrite-base AS monolith +LABEL org.opencontainers.image.title="Dendrite (Monolith)" + +COPY --from=build /out/create-account /usr/bin/create-account +COPY --from=build /out/generate-config /usr/bin/generate-config +COPY --from=build /out/generate-keys /usr/bin/generate-keys +COPY --from=build /out/dendrite-monolith-server /usr/bin/dendrite-monolith-server + +ENTRYPOINT ["/usr/bin/dendrite-monolith-server"] +EXPOSE 8008 8448 + +# +# Builds the P2P demo image and contains all required binaries +# +FROM dendrite-base AS demo-pinecone +LABEL org.opencontainers.image.title="Dendrite (P2P Demo)" + +COPY --from=build /out/create-account /usr/bin/create-account +COPY --from=build /out/generate-config /usr/bin/generate-config +COPY --from=build /out/generate-keys /usr/bin/generate-keys +COPY --from=build /out/dendrite-demo-pinecone /usr/bin/dendrite-demo-pinecone + +VOLUME /etc/dendrite +WORKDIR /etc/dendrite + +ENTRYPOINT ["/usr/bin/dendrite-demo-pinecone"] +EXPOSE 8008 8448 + +# +# Builds the Complement image, used for integration tests +# +FROM base AS complement +LABEL org.opencontainers.image.title="Dendrite (Complement)" +RUN apk add --no-cache sqlite openssl ca-certificates +COPY --from=build /out/* /usr/bin/ +RUN rm /usr/bin/dendrite-polylith-multi /usr/bin/dendrite-demo* /usr/bin/dendritejs-pinecone + +WORKDIR /dendrite +RUN /usr/bin/generate-keys --private-key matrix_key.pem && \ + mkdir /ca && \ + openssl genrsa -out /ca/ca.key 2048 && \ + openssl req -new -x509 -key /ca/ca.key -days 3650 -subj "/C=GB/ST=London/O=matrix.org/CN=Complement CA" -out /ca/ca.crt + +ENV SERVER_NAME=localhost +ENV API=0 +EXPOSE 8008 8448 + +# At runtime, generate TLS cert based on the CA now mounted at /ca +# At runtime, replace the SERVER_NAME with what we are told +CMD /usr/bin/generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /ca/ca.crt --tls-authority-key /ca/ca.key && \ + /usr/bin/generate-config -server $SERVER_NAME --ci > dendrite.yaml && \ + cp /ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \ + /usr/bin/dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0} \ No newline at end of file diff --git a/cmd/dendritejs-pinecone/jsServer.go b/build/dendritejs-pinecone/jsServer.go similarity index 100% rename from cmd/dendritejs-pinecone/jsServer.go rename to build/dendritejs-pinecone/jsServer.go diff --git a/cmd/dendritejs-pinecone/main.go b/build/dendritejs-pinecone/main.go similarity index 100% rename from cmd/dendritejs-pinecone/main.go rename to build/dendritejs-pinecone/main.go diff --git a/cmd/dendritejs-pinecone/main_noop.go b/build/dendritejs-pinecone/main_noop.go similarity index 100% rename from cmd/dendritejs-pinecone/main_noop.go rename to build/dendritejs-pinecone/main_noop.go diff --git a/cmd/dendritejs-pinecone/main_test.go b/build/dendritejs-pinecone/main_test.go similarity index 100% rename from cmd/dendritejs-pinecone/main_test.go rename to build/dendritejs-pinecone/main_test.go diff --git a/build/docker/Dockerfile.demo-pinecone b/build/docker/Dockerfile.demo-pinecone deleted file mode 100644 index 133c63c53..000000000 --- a/build/docker/Dockerfile.demo-pinecone +++ /dev/null @@ -1,25 +0,0 @@ -FROM docker.io/golang:1.19-alpine AS base - -RUN apk --update --no-cache add bash build-base - -WORKDIR /build - -COPY . /build - -RUN mkdir -p bin -RUN go build -trimpath -o bin/ ./cmd/dendrite-demo-pinecone -RUN go build -trimpath -o bin/ ./cmd/create-account -RUN go build -trimpath -o bin/ ./cmd/generate-keys - -FROM alpine:latest -LABEL org.opencontainers.image.title="Dendrite (Pinecone demo)" -LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go" -LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite" -LABEL org.opencontainers.image.licenses="Apache-2.0" - -COPY --from=base /build/bin/* /usr/bin/ - -VOLUME /etc/dendrite -WORKDIR /etc/dendrite - -ENTRYPOINT ["/usr/bin/dendrite-demo-pinecone"] diff --git a/build/docker/Dockerfile.monolith b/build/docker/Dockerfile.monolith deleted file mode 100644 index 3180e9626..000000000 --- a/build/docker/Dockerfile.monolith +++ /dev/null @@ -1,25 +0,0 @@ -FROM docker.io/golang:1.19-alpine AS base - -RUN apk --update --no-cache add bash build-base - -WORKDIR /build - -COPY . /build - -RUN mkdir -p bin -RUN go build -trimpath -o bin/ ./cmd/dendrite-monolith-server -RUN go build -trimpath -o bin/ ./cmd/create-account -RUN go build -trimpath -o bin/ ./cmd/generate-keys - -FROM alpine:latest -LABEL org.opencontainers.image.title="Dendrite (Monolith)" -LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go" -LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite" -LABEL org.opencontainers.image.licenses="Apache-2.0" - -COPY --from=base /build/bin/* /usr/bin/ - -VOLUME /etc/dendrite -WORKDIR /etc/dendrite - -ENTRYPOINT ["/usr/bin/dendrite-monolith-server"] diff --git a/build/docker/Dockerfile.polylith b/build/docker/Dockerfile.polylith deleted file mode 100644 index 79f8a5f23..000000000 --- a/build/docker/Dockerfile.polylith +++ /dev/null @@ -1,25 +0,0 @@ -FROM docker.io/golang:1.19-alpine AS base - -RUN apk --update --no-cache add bash build-base - -WORKDIR /build - -COPY . /build - -RUN mkdir -p bin -RUN go build -trimpath -o bin/ ./cmd/dendrite-polylith-multi -RUN go build -trimpath -o bin/ ./cmd/create-account -RUN go build -trimpath -o bin/ ./cmd/generate-keys - -FROM alpine:latest -LABEL org.opencontainers.image.title="Dendrite (Polylith)" -LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go" -LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite" -LABEL org.opencontainers.image.licenses="Apache-2.0" - -COPY --from=base /build/bin/* /usr/bin/ - -VOLUME /etc/dendrite -WORKDIR /etc/dendrite - -ENTRYPOINT ["/usr/bin/dendrite-polylith-multi"] diff --git a/build/docker/README.md b/build/docker/README.md index 261519fde..14a9c8594 100644 --- a/build/docker/README.md +++ b/build/docker/README.md @@ -9,11 +9,15 @@ They can be found on Docker Hub: ## Dockerfiles -The `Dockerfile` builds the base image which contains all of the Dendrite -components. The `Dockerfile.component` file takes the given component, as -specified with `--buildarg component=` from the base image and produce -smaller component-specific images, which are substantially smaller and do -not contain the Go toolchain etc. +The `Dockerfile` is a multistage file which can build all three Dendrite +images depending on the supplied `--target`. From the root of the Dendrite +repository, run: + +``` +docker build . --target monolith -t matrixdotorg/dendrite-monolith +docker build . --target polylith -t matrixdotorg/dendrite-monolith +docker build . --target demo-pinecone -t matrixdotorg/dendrite-monolith +``` ## Compose files diff --git a/build/docker/crossbuild.sh b/build/docker/crossbuild.sh new file mode 100644 index 000000000..46e5d7e9b --- /dev/null +++ b/build/docker/crossbuild.sh @@ -0,0 +1,67 @@ +#!/bin/sh +set -e + +# In order to cross-compile with the multi-stage Docker builds, we need to +# ensure that the suitable toolchain for cross-compiling is installed. Since +# the images are Alpine-based, we will use musl. Download and install the +# toolchain inside the build container. + +USERARCH=`go env GOARCH` +GOARCH="$TARGETARCH" +GOOS="linux" + +echo "Target arch: $TARGETARCH" +echo "User arch: $USERARCH" + +if [ "$TARGETARCH" != "$USERARCH" ]; then + if [ "$USERARCH" != "amd64" ]; then + echo "Cross-compiling only supported on amd64" + exit 1 + fi + + echo "Cross compile" + case $GOARCH in + arm64) + curl -s https://more.musl.cc/x86_64-linux-musl/aarch64-linux-musl-cross.tgz | tar xz --strip-components=1 -C /usr + export CC=aarch64-linux-musl-gcc + ;; + + amd64) + curl -s https://more.musl.cc/x86_64-linux-musl/x86_64-linux-musl-cross.tgz | tar xz --strip-components=1 -C /usr + export CC=x86_64-linux-musl-gcc + ;; + + 386) + curl -s https://more.musl.cc/x86_64-linux-musl/i686-linux-musl-cross.tgz | tar xz --strip-components=1 -C /usr + export CC=i686-linux-musl-gcc + ;; + + arm) + curl -s https://more.musl.cc/x86_64-linux-musl/armv7l-linux-musleabihf-cross.tgz | tar xz --strip-components=1 -C /usr + export CC=armv7l-linux-musleabihf-gcc + ;; + + s390x) + curl -s https://more.musl.cc/x86_64-linux-musl/s390x-linux-musl-cross.tgz | tar xz --strip-components=1 -C /usr + export CC=s390x-linux-musl-gcc + ;; + + ppc64le) + curl -s https://more.musl.cc/x86_64-linux-musl/powerpc64le-linux-musl-cross.tgz | tar xz --strip-components=1 -C /usr + export CC=powerpc64le-linux-musl-gcc + ;; + + *) + echo "Unsupported GOARCH=${GOARCH}" + exit 1 + ;; + esac +else + echo "Native compile" +fi + +# Output the go environment just in case it is useful for debugging. +go env + +# Build Dendrite and tools, statically linking them. +CGO_ENABLED=1 go build -v -ldflags="-linkmode external -extldflags -static ${FLAGS}" -trimpath -o /out/ ./cmd/... diff --git a/build/docker/images-build.sh b/build/docker/images-build.sh index c2c140685..1a8326151 100755 --- a/build/docker/images-build.sh +++ b/build/docker/images-build.sh @@ -6,5 +6,6 @@ TAG=${1:-latest} echo "Building tag '${TAG}'" -docker build -t matrixdotorg/dendrite-monolith:${TAG} -f build/docker/Dockerfile.monolith . -docker build -t matrixdotorg/dendrite-polylith:${TAG} -f build/docker/Dockerfile.polylith . \ No newline at end of file +docker build . --target monolith -t matrixdotorg/dendrite-monolith:${TAG} +docker build . --target polylith -t matrixdotorg/dendrite-monolith:${TAG} +docker build . --target demo-pinecone -t matrixdotorg/dendrite-demo-pinecone:${TAG} \ No newline at end of file From ede4632835929bf93d0cb57df58c06efd94fa7e0 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 4 Oct 2022 10:43:58 +0100 Subject: [PATCH 030/207] Fix Docker GHA --- .github/workflows/docker.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 358037c02..c15b6c1a4 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -21,7 +21,6 @@ jobs: monolith: name: Monolith image runs-on: ubuntu-latest - needs: build-flags permissions: contents: read packages: write @@ -101,7 +100,6 @@ jobs: polylith: name: Polylith image runs-on: ubuntu-latest - needs: build-flags permissions: contents: read packages: write @@ -180,7 +178,6 @@ jobs: demo-pinecone: name: Pinecone demo image runs-on: ubuntu-latest - needs: build-flags permissions: contents: read packages: write From 98b73652e00edb8304aa4a3471aa29151f025297 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 4 Oct 2022 11:07:54 +0100 Subject: [PATCH 031/207] Try to populate `-ldflags` in Docker builds --- .github/workflows/docker.yml | 6 +++--- build/docker/crossbuild.sh | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index c15b6c1a4..e0db280fa 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -35,6 +35,7 @@ jobs: BRANCH=$(git symbolic-ref --short HEAD | tr -d \/) [ ${BRANCH} == "main" ] && BRANCH="" echo "BRANCH=${BRANCH}" >> $GITHUB_ENV + echo "FLAGS=\"-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}\"" >> $GITHUB_ENV - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx @@ -59,7 +60,6 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max context: . - build-args: FLAGS="-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}" target: monolith platforms: ${{ env.PLATFORMS }} push: true @@ -114,6 +114,7 @@ jobs: BRANCH=$(git symbolic-ref --short HEAD | tr -d \/) [ ${BRANCH} == "main" ] && BRANCH="" echo "BRANCH=${BRANCH}" >> $GITHUB_ENV + echo "FLAGS=\"-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}\"" >> $GITHUB_ENV - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx @@ -138,7 +139,6 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max context: . - build-args: FLAGS="-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}" target: polylith platforms: ${{ env.PLATFORMS }} push: true @@ -192,6 +192,7 @@ jobs: BRANCH=$(git symbolic-ref --short HEAD | tr -d \/) [ ${BRANCH} == "main" ] && BRANCH="" echo "BRANCH=${BRANCH}" >> $GITHUB_ENV + echo "FLAGS=\"-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}\"" >> $GITHUB_ENV - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx @@ -231,7 +232,6 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max context: . - build-args: FLAGS="-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}" target: demo-pinecone platforms: ${{ env.PLATFORMS }} push: true diff --git a/build/docker/crossbuild.sh b/build/docker/crossbuild.sh index 46e5d7e9b..957893a0a 100644 --- a/build/docker/crossbuild.sh +++ b/build/docker/crossbuild.sh @@ -12,6 +12,7 @@ GOOS="linux" echo "Target arch: $TARGETARCH" echo "User arch: $USERARCH" +echo "Link flags: $FLAGS" if [ "$TARGETARCH" != "$USERARCH" ]; then if [ "$USERARCH" != "amd64" ]; then From 085bf5e28b49948209b464c5b12c02775e6c1d70 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 4 Oct 2022 11:33:05 +0100 Subject: [PATCH 032/207] Revert Docker changes --- .github/workflows/dendrite.yml | 6 +- .github/workflows/docker.yml | 103 +++++------------- Dockerfile | 99 ----------------- build/docker/Dockerfile.demo-pinecone | 25 +++++ build/docker/Dockerfile.monolith | 25 +++++ build/docker/Dockerfile.polylith | 25 +++++ build/docker/README.md | 14 +-- build/docker/crossbuild.sh | 68 ------------ build/docker/images-build.sh | 5 +- .../dendritejs-pinecone/jsServer.go | 0 {build => cmd}/dendritejs-pinecone/main.go | 0 .../dendritejs-pinecone/main_noop.go | 0 .../dendritejs-pinecone/main_test.go | 0 13 files changed, 115 insertions(+), 255 deletions(-) delete mode 100644 Dockerfile create mode 100644 build/docker/Dockerfile.demo-pinecone create mode 100644 build/docker/Dockerfile.monolith create mode 100644 build/docker/Dockerfile.polylith delete mode 100644 build/docker/crossbuild.sh rename {build => cmd}/dendritejs-pinecone/jsServer.go (100%) rename {build => cmd}/dendritejs-pinecone/main.go (100%) rename {build => cmd}/dendritejs-pinecone/main_noop.go (100%) rename {build => cmd}/dendritejs-pinecone/main_test.go (100%) diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml index 524d36039..be3c7c173 100644 --- a/.github/workflows/dendrite.yml +++ b/.github/workflows/dendrite.yml @@ -284,7 +284,7 @@ jobs: API: ${{ matrix.api && 1 }} SYTEST_BRANCH: ${{ github.head_ref }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v2 - name: Run Sytest run: /bootstrap.sh dendrite working-directory: /src @@ -344,8 +344,8 @@ jobs: sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev go get -v github.com/haveyoudebuggedit/gotestfmt/v2/cmd/gotestfmt@latest - - name: Run actions/checkout@v3 for dendrite - uses: actions/checkout@v3 + - name: Run actions/checkout@v2 for dendrite + uses: actions/checkout@v2 with: path: dendrite diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e0db280fa..b4e24e52f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -26,27 +26,21 @@ jobs: packages: write steps: - name: Checkout - uses: actions/checkout@v3 - - name: Get release tag & build flags + uses: actions/checkout@v2 + - name: Get release tag if: github.event_name == 'release' # Only for GitHub releases - run: | - echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV - BRANCH=$(git symbolic-ref --short HEAD | tr -d \/) - [ ${BRANCH} == "main" ] && BRANCH="" - echo "BRANCH=${BRANCH}" >> $GITHUB_ENV - echo "FLAGS=\"-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}\"" >> $GITHUB_ENV + run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v1 - name: Login to Docker Hub - uses: docker/login-action@v2 + uses: docker/login-action@v1 with: username: ${{ env.DOCKER_HUB_USER }} password: ${{ secrets.DOCKER_TOKEN }} - name: Login to GitHub Containers - uses: docker/login-action@v2 + uses: docker/login-action@v1 with: registry: ghcr.io username: ${{ github.repository_owner }} @@ -55,40 +49,27 @@ jobs: - name: Build main monolith image if: github.ref_name == 'main' id: docker_build_monolith - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v2 with: cache-from: type=gha cache-to: type=gha,mode=max context: . - target: monolith + file: ./build/docker/Dockerfile.monolith platforms: ${{ env.PLATFORMS }} push: true tags: | ${{ env.DOCKER_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }} ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }} - - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@master - with: - image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:${{ github.ref_name }} - format: "sarif" - output: "trivy-results.sarif" - - - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v1 - with: - sarif_file: "trivy-results.sarif" - - name: Build release monolith image if: github.event_name == 'release' # Only for GitHub releases id: docker_build_monolith_release - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v2 with: cache-from: type=gha cache-to: type=gha,mode=max context: . - build-args: FLAGS="-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}" - target: monolith + file: ./build/docker/Dockerfile.monolith platforms: ${{ env.PLATFORMS }} push: true tags: | @@ -105,27 +86,21 @@ jobs: packages: write steps: - name: Checkout - uses: actions/checkout@v3 - - name: Get release tag & build flags + uses: actions/checkout@v2 + - name: Get release tag if: github.event_name == 'release' # Only for GitHub releases - run: | - echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV - BRANCH=$(git symbolic-ref --short HEAD | tr -d \/) - [ ${BRANCH} == "main" ] && BRANCH="" - echo "BRANCH=${BRANCH}" >> $GITHUB_ENV - echo "FLAGS=\"-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}\"" >> $GITHUB_ENV + run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v1 - name: Login to Docker Hub - uses: docker/login-action@v2 + uses: docker/login-action@v1 with: username: ${{ env.DOCKER_HUB_USER }} password: ${{ secrets.DOCKER_TOKEN }} - name: Login to GitHub Containers - uses: docker/login-action@v2 + uses: docker/login-action@v1 with: registry: ghcr.io username: ${{ github.repository_owner }} @@ -134,39 +109,27 @@ jobs: - name: Build main polylith image if: github.ref_name == 'main' id: docker_build_polylith - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v2 with: cache-from: type=gha cache-to: type=gha,mode=max context: . - target: polylith + file: ./build/docker/Dockerfile.polylith platforms: ${{ env.PLATFORMS }} push: true tags: | ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }} ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }} - - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@master - with: - image-ref: ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ github.ref_name }} - format: "sarif" - output: "trivy-results.sarif" - - - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v1 - with: - sarif_file: "trivy-results.sarif" - - name: Build release polylith image if: github.event_name == 'release' # Only for GitHub releases id: docker_build_polylith_release - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v2 with: cache-from: type=gha cache-to: type=gha,mode=max context: . - target: polylith + file: ./build/docker/Dockerfile.polylith platforms: ${{ env.PLATFORMS }} push: true tags: | @@ -183,27 +146,21 @@ jobs: packages: write steps: - name: Checkout - uses: actions/checkout@v3 - - name: Get release tag & build flags + uses: actions/checkout@v2 + - name: Get release tag if: github.event_name == 'release' # Only for GitHub releases - run: | - echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - echo "BUILD=$(git rev-parse --short HEAD || "") >> $GITHUB_ENV - BRANCH=$(git symbolic-ref --short HEAD | tr -d \/) - [ ${BRANCH} == "main" ] && BRANCH="" - echo "BRANCH=${BRANCH}" >> $GITHUB_ENV - echo "FLAGS=\"-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}\"" >> $GITHUB_ENV + run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v1 - name: Login to Docker Hub - uses: docker/login-action@v2 + uses: docker/login-action@v1 with: username: ${{ env.DOCKER_HUB_USER }} password: ${{ secrets.DOCKER_TOKEN }} - name: Login to GitHub Containers - uses: docker/login-action@v2 + uses: docker/login-action@v1 with: registry: ghcr.io username: ${{ github.repository_owner }} @@ -212,12 +169,12 @@ jobs: - name: Build main pinecone demo image if: github.ref_name == 'main' id: docker_build_demo_pinecone - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v2 with: cache-from: type=gha cache-to: type=gha,mode=max context: . - target: demo-pinecone + file: ./build/docker/Dockerfile.demo-pinecone platforms: ${{ env.PLATFORMS }} push: true tags: | @@ -227,12 +184,12 @@ jobs: - name: Build release pinecone demo image if: github.event_name == 'release' # Only for GitHub releases id: docker_build_demo_pinecone_release - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v2 with: cache-from: type=gha cache-to: type=gha,mode=max context: . - target: demo-pinecone + file: ./build/docker/Dockerfile.demo-pinecone platforms: ${{ env.PLATFORMS }} push: true tags: | diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index bf5831832..000000000 --- a/Dockerfile +++ /dev/null @@ -1,99 +0,0 @@ -#syntax=docker/dockerfile:1.2 - -# -# base installs required dependencies and runs go mod download to cache dependencies -# -FROM --platform=${BUILDPLATFORM} docker.io/golang:1.19-alpine AS base -RUN apk --update --no-cache add bash build-base curl - -# -# build creates all needed binaries -# -FROM base AS build -WORKDIR /src -ARG TARGETOS -ARG TARGETARCH -ARG FLAGS -RUN --mount=target=. \ - --mount=type=cache,target=/root/.cache/go-build \ - sh ./build/docker/crossbuild.sh - -# -# The dendrite base image; mainly creates a user and switches to it -# -FROM alpine:latest AS dendrite-base -LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go" -LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite" -LABEL org.opencontainers.image.licenses="Apache-2.0" -LABEL org.opencontainers.image.documentation="https://matrix-org.github.io/dendrite/" -LABEL org.opencontainers.image.vendor="The Matrix.org Foundation C.I.C." -RUN addgroup dendrite && adduser dendrite -G dendrite -u 1337 -D -USER dendrite -WORKDIR /home/dendrite - -# -# Builds the polylith image and only contains the polylith binary -# -FROM dendrite-base AS polylith -LABEL org.opencontainers.image.title="Dendrite (Polylith)" - -COPY --from=build /out/dendrite-polylith-multi /usr/bin/ - -ENTRYPOINT ["/usr/bin/dendrite-polylith-multi"] - -# -# Builds the monolith image and contains all required binaries -# -FROM dendrite-base AS monolith -LABEL org.opencontainers.image.title="Dendrite (Monolith)" - -COPY --from=build /out/create-account /usr/bin/create-account -COPY --from=build /out/generate-config /usr/bin/generate-config -COPY --from=build /out/generate-keys /usr/bin/generate-keys -COPY --from=build /out/dendrite-monolith-server /usr/bin/dendrite-monolith-server - -ENTRYPOINT ["/usr/bin/dendrite-monolith-server"] -EXPOSE 8008 8448 - -# -# Builds the P2P demo image and contains all required binaries -# -FROM dendrite-base AS demo-pinecone -LABEL org.opencontainers.image.title="Dendrite (P2P Demo)" - -COPY --from=build /out/create-account /usr/bin/create-account -COPY --from=build /out/generate-config /usr/bin/generate-config -COPY --from=build /out/generate-keys /usr/bin/generate-keys -COPY --from=build /out/dendrite-demo-pinecone /usr/bin/dendrite-demo-pinecone - -VOLUME /etc/dendrite -WORKDIR /etc/dendrite - -ENTRYPOINT ["/usr/bin/dendrite-demo-pinecone"] -EXPOSE 8008 8448 - -# -# Builds the Complement image, used for integration tests -# -FROM base AS complement -LABEL org.opencontainers.image.title="Dendrite (Complement)" -RUN apk add --no-cache sqlite openssl ca-certificates -COPY --from=build /out/* /usr/bin/ -RUN rm /usr/bin/dendrite-polylith-multi /usr/bin/dendrite-demo* /usr/bin/dendritejs-pinecone - -WORKDIR /dendrite -RUN /usr/bin/generate-keys --private-key matrix_key.pem && \ - mkdir /ca && \ - openssl genrsa -out /ca/ca.key 2048 && \ - openssl req -new -x509 -key /ca/ca.key -days 3650 -subj "/C=GB/ST=London/O=matrix.org/CN=Complement CA" -out /ca/ca.crt - -ENV SERVER_NAME=localhost -ENV API=0 -EXPOSE 8008 8448 - -# At runtime, generate TLS cert based on the CA now mounted at /ca -# At runtime, replace the SERVER_NAME with what we are told -CMD /usr/bin/generate-keys --server $SERVER_NAME --tls-cert server.crt --tls-key server.key --tls-authority-cert /ca/ca.crt --tls-authority-key /ca/ca.key && \ - /usr/bin/generate-config -server $SERVER_NAME --ci > dendrite.yaml && \ - cp /ca/ca.crt /usr/local/share/ca-certificates/ && update-ca-certificates && \ - /usr/bin/dendrite-monolith-server --really-enable-open-registration --tls-cert server.crt --tls-key server.key --config dendrite.yaml -api=${API:-0} \ No newline at end of file diff --git a/build/docker/Dockerfile.demo-pinecone b/build/docker/Dockerfile.demo-pinecone new file mode 100644 index 000000000..133c63c53 --- /dev/null +++ b/build/docker/Dockerfile.demo-pinecone @@ -0,0 +1,25 @@ +FROM docker.io/golang:1.19-alpine AS base + +RUN apk --update --no-cache add bash build-base + +WORKDIR /build + +COPY . /build + +RUN mkdir -p bin +RUN go build -trimpath -o bin/ ./cmd/dendrite-demo-pinecone +RUN go build -trimpath -o bin/ ./cmd/create-account +RUN go build -trimpath -o bin/ ./cmd/generate-keys + +FROM alpine:latest +LABEL org.opencontainers.image.title="Dendrite (Pinecone demo)" +LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go" +LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite" +LABEL org.opencontainers.image.licenses="Apache-2.0" + +COPY --from=base /build/bin/* /usr/bin/ + +VOLUME /etc/dendrite +WORKDIR /etc/dendrite + +ENTRYPOINT ["/usr/bin/dendrite-demo-pinecone"] diff --git a/build/docker/Dockerfile.monolith b/build/docker/Dockerfile.monolith new file mode 100644 index 000000000..3180e9626 --- /dev/null +++ b/build/docker/Dockerfile.monolith @@ -0,0 +1,25 @@ +FROM docker.io/golang:1.19-alpine AS base + +RUN apk --update --no-cache add bash build-base + +WORKDIR /build + +COPY . /build + +RUN mkdir -p bin +RUN go build -trimpath -o bin/ ./cmd/dendrite-monolith-server +RUN go build -trimpath -o bin/ ./cmd/create-account +RUN go build -trimpath -o bin/ ./cmd/generate-keys + +FROM alpine:latest +LABEL org.opencontainers.image.title="Dendrite (Monolith)" +LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go" +LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite" +LABEL org.opencontainers.image.licenses="Apache-2.0" + +COPY --from=base /build/bin/* /usr/bin/ + +VOLUME /etc/dendrite +WORKDIR /etc/dendrite + +ENTRYPOINT ["/usr/bin/dendrite-monolith-server"] diff --git a/build/docker/Dockerfile.polylith b/build/docker/Dockerfile.polylith new file mode 100644 index 000000000..79f8a5f23 --- /dev/null +++ b/build/docker/Dockerfile.polylith @@ -0,0 +1,25 @@ +FROM docker.io/golang:1.19-alpine AS base + +RUN apk --update --no-cache add bash build-base + +WORKDIR /build + +COPY . /build + +RUN mkdir -p bin +RUN go build -trimpath -o bin/ ./cmd/dendrite-polylith-multi +RUN go build -trimpath -o bin/ ./cmd/create-account +RUN go build -trimpath -o bin/ ./cmd/generate-keys + +FROM alpine:latest +LABEL org.opencontainers.image.title="Dendrite (Polylith)" +LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go" +LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite" +LABEL org.opencontainers.image.licenses="Apache-2.0" + +COPY --from=base /build/bin/* /usr/bin/ + +VOLUME /etc/dendrite +WORKDIR /etc/dendrite + +ENTRYPOINT ["/usr/bin/dendrite-polylith-multi"] diff --git a/build/docker/README.md b/build/docker/README.md index 14a9c8594..261519fde 100644 --- a/build/docker/README.md +++ b/build/docker/README.md @@ -9,15 +9,11 @@ They can be found on Docker Hub: ## Dockerfiles -The `Dockerfile` is a multistage file which can build all three Dendrite -images depending on the supplied `--target`. From the root of the Dendrite -repository, run: - -``` -docker build . --target monolith -t matrixdotorg/dendrite-monolith -docker build . --target polylith -t matrixdotorg/dendrite-monolith -docker build . --target demo-pinecone -t matrixdotorg/dendrite-monolith -``` +The `Dockerfile` builds the base image which contains all of the Dendrite +components. The `Dockerfile.component` file takes the given component, as +specified with `--buildarg component=` from the base image and produce +smaller component-specific images, which are substantially smaller and do +not contain the Go toolchain etc. ## Compose files diff --git a/build/docker/crossbuild.sh b/build/docker/crossbuild.sh deleted file mode 100644 index 957893a0a..000000000 --- a/build/docker/crossbuild.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/bin/sh -set -e - -# In order to cross-compile with the multi-stage Docker builds, we need to -# ensure that the suitable toolchain for cross-compiling is installed. Since -# the images are Alpine-based, we will use musl. Download and install the -# toolchain inside the build container. - -USERARCH=`go env GOARCH` -GOARCH="$TARGETARCH" -GOOS="linux" - -echo "Target arch: $TARGETARCH" -echo "User arch: $USERARCH" -echo "Link flags: $FLAGS" - -if [ "$TARGETARCH" != "$USERARCH" ]; then - if [ "$USERARCH" != "amd64" ]; then - echo "Cross-compiling only supported on amd64" - exit 1 - fi - - echo "Cross compile" - case $GOARCH in - arm64) - curl -s https://more.musl.cc/x86_64-linux-musl/aarch64-linux-musl-cross.tgz | tar xz --strip-components=1 -C /usr - export CC=aarch64-linux-musl-gcc - ;; - - amd64) - curl -s https://more.musl.cc/x86_64-linux-musl/x86_64-linux-musl-cross.tgz | tar xz --strip-components=1 -C /usr - export CC=x86_64-linux-musl-gcc - ;; - - 386) - curl -s https://more.musl.cc/x86_64-linux-musl/i686-linux-musl-cross.tgz | tar xz --strip-components=1 -C /usr - export CC=i686-linux-musl-gcc - ;; - - arm) - curl -s https://more.musl.cc/x86_64-linux-musl/armv7l-linux-musleabihf-cross.tgz | tar xz --strip-components=1 -C /usr - export CC=armv7l-linux-musleabihf-gcc - ;; - - s390x) - curl -s https://more.musl.cc/x86_64-linux-musl/s390x-linux-musl-cross.tgz | tar xz --strip-components=1 -C /usr - export CC=s390x-linux-musl-gcc - ;; - - ppc64le) - curl -s https://more.musl.cc/x86_64-linux-musl/powerpc64le-linux-musl-cross.tgz | tar xz --strip-components=1 -C /usr - export CC=powerpc64le-linux-musl-gcc - ;; - - *) - echo "Unsupported GOARCH=${GOARCH}" - exit 1 - ;; - esac -else - echo "Native compile" -fi - -# Output the go environment just in case it is useful for debugging. -go env - -# Build Dendrite and tools, statically linking them. -CGO_ENABLED=1 go build -v -ldflags="-linkmode external -extldflags -static ${FLAGS}" -trimpath -o /out/ ./cmd/... diff --git a/build/docker/images-build.sh b/build/docker/images-build.sh index 1a8326151..c2c140685 100755 --- a/build/docker/images-build.sh +++ b/build/docker/images-build.sh @@ -6,6 +6,5 @@ TAG=${1:-latest} echo "Building tag '${TAG}'" -docker build . --target monolith -t matrixdotorg/dendrite-monolith:${TAG} -docker build . --target polylith -t matrixdotorg/dendrite-monolith:${TAG} -docker build . --target demo-pinecone -t matrixdotorg/dendrite-demo-pinecone:${TAG} \ No newline at end of file +docker build -t matrixdotorg/dendrite-monolith:${TAG} -f build/docker/Dockerfile.monolith . +docker build -t matrixdotorg/dendrite-polylith:${TAG} -f build/docker/Dockerfile.polylith . \ No newline at end of file diff --git a/build/dendritejs-pinecone/jsServer.go b/cmd/dendritejs-pinecone/jsServer.go similarity index 100% rename from build/dendritejs-pinecone/jsServer.go rename to cmd/dendritejs-pinecone/jsServer.go diff --git a/build/dendritejs-pinecone/main.go b/cmd/dendritejs-pinecone/main.go similarity index 100% rename from build/dendritejs-pinecone/main.go rename to cmd/dendritejs-pinecone/main.go diff --git a/build/dendritejs-pinecone/main_noop.go b/cmd/dendritejs-pinecone/main_noop.go similarity index 100% rename from build/dendritejs-pinecone/main_noop.go rename to cmd/dendritejs-pinecone/main_noop.go diff --git a/build/dendritejs-pinecone/main_test.go b/cmd/dendritejs-pinecone/main_test.go similarity index 100% rename from build/dendritejs-pinecone/main_test.go rename to cmd/dendritejs-pinecone/main_test.go From a767102f8a74031b5d0612d985c7589dfe0e0821 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 4 Oct 2022 11:34:52 +0100 Subject: [PATCH 033/207] Reduce `max_open_conns` in monolith sample config --- dendrite-sample.monolith.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dendrite-sample.monolith.yaml b/dendrite-sample.monolith.yaml index e41e83d7c..f0fa386d1 100644 --- a/dendrite-sample.monolith.yaml +++ b/dendrite-sample.monolith.yaml @@ -37,7 +37,7 @@ global: # you must configure the "database" block for each component instead. database: connection_string: postgresql://username:password@hostname/dendrite?sslmode=disable - max_open_conns: 100 + max_open_conns: 90 max_idle_conns: 5 conn_max_lifetime: -1 From 3da182212e86daca1d7019efbd424b2bd38b8b3c Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 4 Oct 2022 13:02:41 +0100 Subject: [PATCH 034/207] Track reasons why the process is in a degraded state --- setup/base/base.go | 8 +++++++- setup/jetstream/nats.go | 6 +++--- setup/process/process.go | 35 +++++++++++++++++++++++------------ 3 files changed, 33 insertions(+), 16 deletions(-) diff --git a/setup/base/base.go b/setup/base/base.go index 0636c7b8d..2e3a3a195 100644 --- a/setup/base/base.go +++ b/setup/base/base.go @@ -18,6 +18,7 @@ import ( "context" "crypto/tls" "database/sql" + "encoding/json" "fmt" "io" "net" @@ -467,8 +468,13 @@ func (b *BaseDendrite) SetupAndServeHTTP( w.WriteHeader(200) }) b.DendriteAdminMux.HandleFunc("/monitor/health", func(w http.ResponseWriter, r *http.Request) { - if b.ProcessContext.IsDegraded() { + if isDegraded, reasons := b.ProcessContext.IsDegraded(); isDegraded { w.WriteHeader(503) + _ = json.NewEncoder(w).Encode(struct { + Warnings []string `json:"warnings"` + }{ + Warnings: reasons, + }) return } w.WriteHeader(200) diff --git a/setup/jetstream/nats.go b/setup/jetstream/nats.go index 7409fd6c8..af4eb2949 100644 --- a/setup/jetstream/nats.go +++ b/setup/jetstream/nats.go @@ -169,9 +169,9 @@ func setupNATS(process *process.ProcessContext, cfg *config.JetStream, nc *natsc // We've managed to add the stream in memory. What's on the // disk will be left alone, but our ability to recover from a // future crash will be limited. Yell about it. - sentry.CaptureException(fmt.Errorf("Stream %q is running in-memory; this may be due to data corruption in the JetStream storage directory, investigate as soon as possible", namespaced.Name)) - logrus.Warn("Stream is running in-memory; this may be due to data corruption in the JetStream storage directory, investigate as soon as possible") - process.Degraded() + err := fmt.Errorf("Stream %q is running in-memory; this may be due to data corruption in the JetStream storage directory", namespaced.Name) + sentry.CaptureException(err) + process.Degraded(err) } } } diff --git a/setup/process/process.go b/setup/process/process.go index 06ef60217..b2d2844a8 100644 --- a/setup/process/process.go +++ b/setup/process/process.go @@ -2,19 +2,18 @@ package process import ( "context" - "fmt" "sync" "github.com/getsentry/sentry-go" "github.com/sirupsen/logrus" - "go.uber.org/atomic" ) type ProcessContext struct { - wg *sync.WaitGroup // used to wait for components to shutdown - ctx context.Context // cancelled when Stop is called - shutdown context.CancelFunc // shut down Dendrite - degraded atomic.Bool + mu sync.RWMutex + wg *sync.WaitGroup // used to wait for components to shutdown + ctx context.Context // cancelled when Stop is called + shutdown context.CancelFunc // shut down Dendrite + degraded map[string]struct{} // reasons why the process is degraded } func NewProcessContext() *ProcessContext { @@ -50,13 +49,25 @@ func (b *ProcessContext) WaitForComponentsToFinish() { b.wg.Wait() } -func (b *ProcessContext) Degraded() { - if b.degraded.CompareAndSwap(false, true) { - logrus.Warn("Dendrite is running in a degraded state") - sentry.CaptureException(fmt.Errorf("Process is running in a degraded state")) +func (b *ProcessContext) Degraded(err error) { + b.mu.Lock() + defer b.mu.Unlock() + if _, ok := b.degraded[err.Error()]; !ok { + logrus.WithError(err).Warn("Dendrite has entered a degraded state") + sentry.CaptureException(err) + b.degraded[err.Error()] = struct{}{} } } -func (b *ProcessContext) IsDegraded() bool { - return b.degraded.Load() +func (b *ProcessContext) IsDegraded() (bool, []string) { + b.mu.RLock() + defer b.mu.RUnlock() + if len(b.degraded) == 0 { + return false, nil + } + reasons := make([]string, 0, len(b.degraded)) + for reason := range b.degraded { + reasons = append(reasons, reason) + } + return true, reasons } From ae10aac456e90dc6a3ea56ca4aceb4a4e5aa8e04 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 4 Oct 2022 15:40:04 +0100 Subject: [PATCH 035/207] Don't perform a federated join after invite if we are already joined to the room (#2762) If we are already joined to the room then it shouldn't matter if you were invited or not, so this looks like a bug. --- roomserver/internal/perform/perform_join.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roomserver/internal/perform/perform_join.go b/roomserver/internal/perform/perform_join.go index 43be54beb..167b375b7 100644 --- a/roomserver/internal/perform/perform_join.go +++ b/roomserver/internal/perform/perform_join.go @@ -237,7 +237,7 @@ func (r *Joiner) performJoinRoomByID( // Force a federated join if we're dealing with a pending invite // and we aren't in the room. isInvitePending, inviteSender, _, err := helpers.IsInvitePending(ctx, r.DB, req.RoomIDOrAlias, req.UserID) - if err == nil && isInvitePending { + if err == nil && !serverInRoom && isInvitePending { _, inviterDomain, ierr := gomatrixserverlib.SplitID('@', inviteSender) if ierr != nil { return "", "", fmt.Errorf("gomatrixserverlib.SplitID: %w", err) From 21f88819858dec022dec5919d6db5554605a9b8b Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 4 Oct 2022 16:43:10 +0100 Subject: [PATCH 036/207] Add indexes that optimise `selectStateInRangeSQL` (#2764) This gets rid of some expensive scans on `add_state_ids` and `remove_state_ids`, turning them into much cheaper and faster index scans instead. --- syncapi/storage/postgres/output_room_events_table.go | 2 ++ syncapi/storage/sqlite3/output_room_events_table.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go index cb092150d..b562e6804 100644 --- a/syncapi/storage/postgres/output_room_events_table.go +++ b/syncapi/storage/postgres/output_room_events_table.go @@ -76,6 +76,8 @@ CREATE INDEX IF NOT EXISTS syncapi_output_room_events_type_idx ON syncapi_output CREATE INDEX IF NOT EXISTS syncapi_output_room_events_sender_idx ON syncapi_output_room_events (sender); CREATE INDEX IF NOT EXISTS syncapi_output_room_events_room_id_idx ON syncapi_output_room_events (room_id); CREATE INDEX IF NOT EXISTS syncapi_output_room_events_exclude_from_sync_idx ON syncapi_output_room_events (exclude_from_sync); +CREATE INDEX IF NOT EXISTS syncapi_output_room_events_add_state_ids_idx ON syncapi_output_room_events ((add_state_ids IS NOT NULL)); +CREATE INDEX IF NOT EXISTS syncapi_output_room_events_remove_state_ids_idx ON syncapi_output_room_events ((remove_state_ids IS NOT NULL)); ` const insertEventSQL = "" + diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index 165943027..d6a674b9c 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -55,6 +55,8 @@ CREATE INDEX IF NOT EXISTS syncapi_output_room_events_type_idx ON syncapi_output CREATE INDEX IF NOT EXISTS syncapi_output_room_events_sender_idx ON syncapi_output_room_events (sender); CREATE INDEX IF NOT EXISTS syncapi_output_room_events_room_id_idx ON syncapi_output_room_events (room_id); CREATE INDEX IF NOT EXISTS syncapi_output_room_events_exclude_from_sync_idx ON syncapi_output_room_events (exclude_from_sync); +CREATE INDEX IF NOT EXISTS syncapi_output_room_events_add_state_ids_idx ON syncapi_output_room_events ((add_state_ids IS NOT NULL)); +CREATE INDEX IF NOT EXISTS syncapi_output_room_events_remove_state_ids_idx ON syncapi_output_room_events ((remove_state_ids IS NOT NULL)); ` const insertEventSQL = "" + From e0703522932ab4bf41b23d5229c7a4540b59d830 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Wed, 5 Oct 2022 11:14:33 +0200 Subject: [PATCH 037/207] Side effect import bleve analyzer languages (#2763) ... to actually allow different languages. Fixes #2761 Binary size increases by ~1MB. --- internal/fulltext/bleve.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/internal/fulltext/bleve.go b/internal/fulltext/bleve.go index da8932f5c..7187861dd 100644 --- a/internal/fulltext/bleve.go +++ b/internal/fulltext/bleve.go @@ -21,6 +21,29 @@ import ( "strings" "github.com/blevesearch/bleve/v2" + + // side effect imports to allow all possible languages + _ "github.com/blevesearch/bleve/v2/analysis/lang/ar" + _ "github.com/blevesearch/bleve/v2/analysis/lang/cjk" + _ "github.com/blevesearch/bleve/v2/analysis/lang/ckb" + _ "github.com/blevesearch/bleve/v2/analysis/lang/da" + _ "github.com/blevesearch/bleve/v2/analysis/lang/de" + _ "github.com/blevesearch/bleve/v2/analysis/lang/en" + _ "github.com/blevesearch/bleve/v2/analysis/lang/es" + _ "github.com/blevesearch/bleve/v2/analysis/lang/fa" + _ "github.com/blevesearch/bleve/v2/analysis/lang/fi" + _ "github.com/blevesearch/bleve/v2/analysis/lang/fr" + _ "github.com/blevesearch/bleve/v2/analysis/lang/hi" + _ "github.com/blevesearch/bleve/v2/analysis/lang/hr" + _ "github.com/blevesearch/bleve/v2/analysis/lang/hu" + _ "github.com/blevesearch/bleve/v2/analysis/lang/it" + _ "github.com/blevesearch/bleve/v2/analysis/lang/nl" + _ "github.com/blevesearch/bleve/v2/analysis/lang/no" + _ "github.com/blevesearch/bleve/v2/analysis/lang/pt" + _ "github.com/blevesearch/bleve/v2/analysis/lang/ro" + _ "github.com/blevesearch/bleve/v2/analysis/lang/ru" + _ "github.com/blevesearch/bleve/v2/analysis/lang/sv" + _ "github.com/blevesearch/bleve/v2/analysis/lang/tr" "github.com/blevesearch/bleve/v2/mapping" "github.com/matrix-org/gomatrixserverlib" From ebd137cf6b2fbd767625dc5289b0bef6d1e51971 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 5 Oct 2022 11:07:17 +0100 Subject: [PATCH 038/207] Check PostgreSQL connection count (#2760) This PR queries PostgreSQL for the `max_connections` and `superuser_reserved_connections` settings and then ensures that Dendrite's `max_open_conns` doesn't exceed the allowed value. This is a really common source of configuration problems and can either result in blocking queries or deadlocks, so it seems reasonable that we complain as loudly as possible when it happens. --- internal/sqlutil/sqlutil.go | 39 +++++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/internal/sqlutil/sqlutil.go b/internal/sqlutil/sqlutil.go index 0cdae6d30..789bceeac 100644 --- a/internal/sqlutil/sqlutil.go +++ b/internal/sqlutil/sqlutil.go @@ -2,6 +2,7 @@ package sqlutil import ( "database/sql" + "flag" "fmt" "regexp" @@ -9,6 +10,8 @@ import ( "github.com/sirupsen/logrus" ) +var skipSanityChecks = flag.Bool("skip-db-sanity", false, "Ignore sanity checks on the database connections (NOT RECOMMENDED!)") + // Open opens a database specified by its database driver name and a driver-specific data source name, // usually consisting of at least a database name and connection information. Includes tracing driver // if DENDRITE_TRACE_SQL=1 @@ -37,15 +40,39 @@ func Open(dbProperties *config.DatabaseOptions, writer Writer) (*sql.DB, error) return nil, err } if driverName != "sqlite3" { - logrus.WithFields(logrus.Fields{ - "MaxOpenConns": dbProperties.MaxOpenConns(), - "MaxIdleConns": dbProperties.MaxIdleConns(), - "ConnMaxLifetime": dbProperties.ConnMaxLifetime(), - "dataSourceName": regexp.MustCompile(`://[^@]*@`).ReplaceAllLiteralString(dsn, "://"), - }).Debug("Setting DB connection limits") + logger := logrus.WithFields(logrus.Fields{ + "max_open_conns": dbProperties.MaxOpenConns(), + "max_idle_conns": dbProperties.MaxIdleConns(), + "conn_max_lifetime": dbProperties.ConnMaxLifetime(), + "data_source_name": regexp.MustCompile(`://[^@]*@`).ReplaceAllLiteralString(dsn, "://"), + }) + logger.Debug("Setting DB connection limits") db.SetMaxOpenConns(dbProperties.MaxOpenConns()) db.SetMaxIdleConns(dbProperties.MaxIdleConns()) db.SetConnMaxLifetime(dbProperties.ConnMaxLifetime()) + + if !*skipSanityChecks { + if dbProperties.MaxOpenConns() == 0 { + logrus.Warnf("WARNING: Configuring 'max_open_conns' to be unlimited is not recommended. This can result in bad performance or deadlocks.") + } + + switch driverName { + case "postgres": + // Perform a quick sanity check if possible that we aren't trying to use more database + // connections than PostgreSQL is willing to give us. + var max, reserved int + if err := db.QueryRow("SELECT setting::integer FROM pg_settings WHERE name='max_connections';").Scan(&max); err != nil { + return nil, fmt.Errorf("failed to find maximum connections: %w", err) + } + if err := db.QueryRow("SELECT setting::integer FROM pg_settings WHERE name='superuser_reserved_connections';").Scan(&reserved); err != nil { + return nil, fmt.Errorf("failed to find reserved connections: %w", err) + } + if configured, allowed := dbProperties.MaxOpenConns(), max-reserved; configured > allowed { + logrus.Errorf("ERROR: The configured 'max_open_conns' is greater than the %d non-superuser connections that PostgreSQL is configured to allow. This can result in bad performance or deadlocks. Please pay close attention to your configured database connection counts. If you REALLY know what you are doing and want to override this error, pass the --skip-db-sanity option to Dendrite.", allowed) + return nil, fmt.Errorf("database sanity checks failed") + } + } + } } return db, nil } From 8c0c3441d88a612ca8e9ba4f83c6ff29ca73f5d0 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Wed, 5 Oct 2022 12:12:42 +0200 Subject: [PATCH 039/207] Add `RoomEventType` nats.Header to avoid unneeded unmarshalling (#2765) --- appservice/consumers/roomserver.go | 5 +++++ federationapi/consumers/roomserver.go | 7 +++++++ roomserver/producers/roomevent.go | 13 ++++++------- setup/jetstream/streams.go | 7 ++++--- test/testrig/jetstream.go | 9 ++++----- userapi/consumers/roomserver.go | 7 ++++--- 6 files changed, 30 insertions(+), 18 deletions(-) diff --git a/appservice/consumers/roomserver.go b/appservice/consumers/roomserver.go index d44f32b38..ac68f4bd4 100644 --- a/appservice/consumers/roomserver.go +++ b/appservice/consumers/roomserver.go @@ -101,6 +101,11 @@ func (s *OutputRoomEventConsumer) onMessage( log.WithField("appservice", state.ID).Tracef("Appservice worker received %d message(s) from roomserver", len(msgs)) events := make([]*gomatrixserverlib.HeaderedEvent, 0, len(msgs)) for _, msg := range msgs { + // Only handle events we care about + receivedType := api.OutputType(msg.Header.Get(jetstream.RoomEventType)) + if receivedType != api.OutputTypeNewRoomEvent && receivedType != api.OutputTypeNewInviteEvent { + continue + } // Parse out the event JSON var output api.OutputEvent if err := json.Unmarshal(msg.Data, &output); err != nil { diff --git a/federationapi/consumers/roomserver.go b/federationapi/consumers/roomserver.go index 349b50b05..a42733628 100644 --- a/federationapi/consumers/roomserver.go +++ b/federationapi/consumers/roomserver.go @@ -79,6 +79,13 @@ func (s *OutputRoomEventConsumer) Start() error { // realises that it cannot update the room state using the deltas. func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) bool { msg := msgs[0] // Guaranteed to exist if onMessage is called + receivedType := api.OutputType(msg.Header.Get(jetstream.RoomEventType)) + + // Only handle events we care about + if receivedType != api.OutputTypeNewRoomEvent && receivedType != api.OutputTypeNewInboundPeek { + return true + } + // Parse out the event JSON var output api.OutputEvent if err := json.Unmarshal(msg.Data, &output); err != nil { diff --git a/roomserver/producers/roomevent.go b/roomserver/producers/roomevent.go index 987e6c942..9c4521986 100644 --- a/roomserver/producers/roomevent.go +++ b/roomserver/producers/roomevent.go @@ -17,12 +17,13 @@ package producers import ( "encoding/json" - "github.com/matrix-org/dendrite/roomserver/acls" - "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/setup/jetstream" "github.com/nats-io/nats.go" log "github.com/sirupsen/logrus" "github.com/tidwall/gjson" + + "github.com/matrix-org/dendrite/roomserver/acls" + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/setup/jetstream" ) var keyContentFields = map[string]string{ @@ -40,10 +41,8 @@ type RoomEventProducer struct { func (r *RoomEventProducer) ProduceRoomEvents(roomID string, updates []api.OutputEvent) error { var err error for _, update := range updates { - msg := &nats.Msg{ - Subject: r.Topic, - Header: nats.Header{}, - } + msg := nats.NewMsg(r.Topic) + msg.Header.Set(jetstream.RoomEventType, string(update.Type)) msg.Header.Set(jetstream.RoomID, roomID) msg.Data, err = json.Marshal(update) if err != nil { diff --git a/setup/jetstream/streams.go b/setup/jetstream/streams.go index ee9810dae..590f0cbd9 100644 --- a/setup/jetstream/streams.go +++ b/setup/jetstream/streams.go @@ -9,9 +9,10 @@ import ( ) const ( - UserID = "user_id" - RoomID = "room_id" - EventID = "event_id" + UserID = "user_id" + RoomID = "room_id" + EventID = "event_id" + RoomEventType = "output_room_event_type" ) var ( diff --git a/test/testrig/jetstream.go b/test/testrig/jetstream.go index 74cf95062..b880eea43 100644 --- a/test/testrig/jetstream.go +++ b/test/testrig/jetstream.go @@ -4,10 +4,11 @@ import ( "encoding/json" "testing" + "github.com/nats-io/nats.go" + "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/jetstream" - "github.com/nats-io/nats.go" ) func MustPublishMsgs(t *testing.T, jsctx nats.JetStreamContext, msgs ...*nats.Msg) { @@ -21,10 +22,8 @@ func MustPublishMsgs(t *testing.T, jsctx nats.JetStreamContext, msgs ...*nats.Ms func NewOutputEventMsg(t *testing.T, base *base.BaseDendrite, roomID string, update api.OutputEvent) *nats.Msg { t.Helper() - msg := &nats.Msg{ - Subject: base.Cfg.Global.JetStream.Prefixed(jetstream.OutputRoomEvent), - Header: nats.Header{}, - } + msg := nats.NewMsg(base.Cfg.Global.JetStream.Prefixed(jetstream.OutputRoomEvent)) + msg.Header.Set(jetstream.RoomEventType, string(update.Type)) msg.Header.Set(jetstream.RoomID, roomID) var err error msg.Data, err = json.Marshal(update) diff --git a/userapi/consumers/roomserver.go b/userapi/consumers/roomserver.go index 952de98f7..a12876946 100644 --- a/userapi/consumers/roomserver.go +++ b/userapi/consumers/roomserver.go @@ -72,15 +72,16 @@ func (s *OutputRoomEventConsumer) Start() error { func (s *OutputRoomEventConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) bool { msg := msgs[0] // Guaranteed to exist if onMessage is called + // Only handle events we care about + if rsapi.OutputType(msg.Header.Get(jetstream.RoomEventType)) != rsapi.OutputTypeNewRoomEvent { + return true + } var output rsapi.OutputEvent if err := json.Unmarshal(msg.Data, &output); err != nil { // If the message was invalid, log it and move on to the next message in the stream log.WithError(err).Errorf("roomserver output log: message parse failure") return true } - if output.Type != rsapi.OutputTypeNewRoomEvent { - return true - } event := output.NewRoomEvent.Event if event == nil { log.Errorf("userapi consumer: expected event") From 6f602bb0969fac6d455de1088bf3980f77a4017f Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 5 Oct 2022 11:16:05 +0100 Subject: [PATCH 040/207] Demote `Failed to query device keys for some users` warning to `level=debug` Many of these warnings are due to dead servers and are quite annoying when they fill up the logs. --- keyserver/internal/device_list_update.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/keyserver/internal/device_list_update.go b/keyserver/internal/device_list_update.go index fcfcd092d..8b02f3d6c 100644 --- a/keyserver/internal/device_list_update.go +++ b/keyserver/internal/device_list_update.go @@ -424,7 +424,7 @@ func (u *DeviceListUpdater) processServer(serverName gomatrixserverlib.ServerNam "succeeded": successCount, "failed": len(userIDs) - successCount, "wait_time": waitTime, - }).Warn("Failed to query device keys for some users") + }).Debug("Failed to query device keys for some users") } return waitTime, !allUsersSucceeded } From c85bc3434fc930423e9e27c8bca9b31b5ad7a441 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 5 Oct 2022 12:47:53 +0100 Subject: [PATCH 041/207] Optimise `QuerySharedUsers` so that we can only work on local users (#2766) Otherwise the sync API key change consumer wastes a lot of time trying to wake up the notifiers for non-local users. --- roomserver/api/query.go | 1 + roomserver/internal/query/query.go | 2 +- roomserver/storage/interface.go | 2 +- .../storage/postgres/membership_table.go | 17 +++++++++----- roomserver/storage/shared/storage.go | 4 ++-- .../storage/sqlite3/membership_table.go | 23 +++++++++++-------- roomserver/storage/tables/interface.go | 2 +- .../storage/tables/membership_table_test.go | 2 +- syncapi/consumers/keychange.go | 6 +++-- 9 files changed, 36 insertions(+), 23 deletions(-) diff --git a/roomserver/api/query.go b/roomserver/api/query.go index aa7dc4735..d63c24785 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -278,6 +278,7 @@ type QuerySharedUsersRequest struct { OtherUserIDs []string ExcludeRoomIDs []string IncludeRoomIDs []string + LocalOnly bool } type QuerySharedUsersResponse struct { diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go index b41a92e94..ee8e1cfe7 100644 --- a/roomserver/internal/query/query.go +++ b/roomserver/internal/query/query.go @@ -799,7 +799,7 @@ func (r *Queryer) QuerySharedUsers(ctx context.Context, req *api.QuerySharedUser } roomIDs = roomIDs[:j] - users, err := r.DB.JoinedUsersSetInRooms(ctx, roomIDs, req.OtherUserIDs) + users, err := r.DB.JoinedUsersSetInRooms(ctx, roomIDs, req.OtherUserIDs, req.LocalOnly) if err != nil { return err } diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go index 43e8da7bb..11e175f55 100644 --- a/roomserver/storage/interface.go +++ b/roomserver/storage/interface.go @@ -157,7 +157,7 @@ type Database interface { // If a tuple has the StateKey of '*' and allowWildcards=true then all state events with the EventType should be returned. GetBulkStateContent(ctx context.Context, roomIDs []string, tuples []gomatrixserverlib.StateKeyTuple, allowWildcards bool) ([]tables.StrippedEvent, error) // JoinedUsersSetInRooms returns how many times each of the given users appears across the given rooms. - JoinedUsersSetInRooms(ctx context.Context, roomIDs, userIDs []string) (map[string]int, error) + JoinedUsersSetInRooms(ctx context.Context, roomIDs, userIDs []string, localOnly bool) (map[string]int, error) // GetLocalServerInRoom returns true if we think we're in a given room or false otherwise. GetLocalServerInRoom(ctx context.Context, roomNID types.RoomNID) (bool, error) // GetServerInRoom returns true if we think a server is in a given room or false otherwise. diff --git a/roomserver/storage/postgres/membership_table.go b/roomserver/storage/postgres/membership_table.go index bd3fd5592..0150534e1 100644 --- a/roomserver/storage/postgres/membership_table.go +++ b/roomserver/storage/postgres/membership_table.go @@ -68,14 +68,18 @@ CREATE TABLE IF NOT EXISTS roomserver_membership ( var selectJoinedUsersSetForRoomsAndUserSQL = "" + "SELECT target_nid, COUNT(room_nid) FROM roomserver_membership" + - " WHERE room_nid = ANY($1) AND target_nid = ANY($2) AND" + - " membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) + " and forgotten = false" + + " WHERE (target_local OR $1 = false)" + + " AND room_nid = ANY($2) AND target_nid = ANY($3)" + + " AND membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) + + " AND forgotten = false" + " GROUP BY target_nid" var selectJoinedUsersSetForRoomsSQL = "" + "SELECT target_nid, COUNT(room_nid) FROM roomserver_membership" + - " WHERE room_nid = ANY($1) AND" + - " membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) + " and forgotten = false" + + " WHERE (target_local OR $1 = false) " + + " AND room_nid = ANY($2)" + + " AND membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) + + " AND forgotten = false" + " GROUP BY target_nid" // Insert a row in to membership table so that it can be locked by the @@ -334,6 +338,7 @@ func (s *membershipStatements) SelectJoinedUsersSetForRooms( ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID, userNIDs []types.EventStateKeyNID, + localOnly bool, ) (map[types.EventStateKeyNID]int, error) { var ( rows *sql.Rows @@ -342,9 +347,9 @@ func (s *membershipStatements) SelectJoinedUsersSetForRooms( stmt := sqlutil.TxStmt(txn, s.selectJoinedUsersSetForRoomsStmt) if len(userNIDs) > 0 { stmt = sqlutil.TxStmt(txn, s.selectJoinedUsersSetForRoomsAndUserStmt) - rows, err = stmt.QueryContext(ctx, pq.Array(roomNIDs), pq.Array(userNIDs)) + rows, err = stmt.QueryContext(ctx, localOnly, pq.Array(roomNIDs), pq.Array(userNIDs)) } else { - rows, err = stmt.QueryContext(ctx, pq.Array(roomNIDs)) + rows, err = stmt.QueryContext(ctx, localOnly, pq.Array(roomNIDs)) } if err != nil { diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index 593abbea1..d83a1ff74 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -1280,7 +1280,7 @@ func (d *Database) GetBulkStateContent(ctx context.Context, roomIDs []string, tu } // JoinedUsersSetInRooms returns a map of how many times the given users appear in the specified rooms. -func (d *Database) JoinedUsersSetInRooms(ctx context.Context, roomIDs, userIDs []string) (map[string]int, error) { +func (d *Database) JoinedUsersSetInRooms(ctx context.Context, roomIDs, userIDs []string, localOnly bool) (map[string]int, error) { roomNIDs, err := d.RoomsTable.BulkSelectRoomNIDs(ctx, nil, roomIDs) if err != nil { return nil, err @@ -1295,7 +1295,7 @@ func (d *Database) JoinedUsersSetInRooms(ctx context.Context, roomIDs, userIDs [ userNIDs = append(userNIDs, nid) nidToUserID[nid] = id } - userNIDToCount, err := d.MembershipTable.SelectJoinedUsersSetForRooms(ctx, nil, roomNIDs, userNIDs) + userNIDToCount, err := d.MembershipTable.SelectJoinedUsersSetForRooms(ctx, nil, roomNIDs, userNIDs, localOnly) if err != nil { return nil, err } diff --git a/roomserver/storage/sqlite3/membership_table.go b/roomserver/storage/sqlite3/membership_table.go index f3303eb0e..cd149f0ed 100644 --- a/roomserver/storage/sqlite3/membership_table.go +++ b/roomserver/storage/sqlite3/membership_table.go @@ -44,14 +44,18 @@ const membershipSchema = ` var selectJoinedUsersSetForRoomsAndUserSQL = "" + "SELECT target_nid, COUNT(room_nid) FROM roomserver_membership" + - " WHERE room_nid IN ($1) AND target_nid IN ($2) AND" + - " membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) + " and forgotten = false" + + " WHERE (target_local OR $1 = false)" + + " AND room_nid IN ($2) AND target_nid IN ($3)" + + " AND membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) + + " AND forgotten = false" + " GROUP BY target_nid" var selectJoinedUsersSetForRoomsSQL = "" + "SELECT target_nid, COUNT(room_nid) FROM roomserver_membership" + - " WHERE room_nid IN ($1) AND " + - " membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) + " and forgotten = false" + + " WHERE (target_local OR $1 = false)" + + " AND room_nid IN ($2)" + + " AND membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) + + " AND forgotten = false" + " GROUP BY target_nid" // Insert a row in to membership table so that it can be locked by the @@ -305,8 +309,9 @@ func (s *membershipStatements) SelectRoomsWithMembership( return roomNIDs, nil } -func (s *membershipStatements) SelectJoinedUsersSetForRooms(ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID, userNIDs []types.EventStateKeyNID) (map[types.EventStateKeyNID]int, error) { - params := make([]interface{}, 0, len(roomNIDs)+len(userNIDs)) +func (s *membershipStatements) SelectJoinedUsersSetForRooms(ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID, userNIDs []types.EventStateKeyNID, localOnly bool) (map[types.EventStateKeyNID]int, error) { + params := make([]interface{}, 0, 1+len(roomNIDs)+len(userNIDs)) + params = append(params, localOnly) for _, v := range roomNIDs { params = append(params, v) } @@ -314,10 +319,10 @@ func (s *membershipStatements) SelectJoinedUsersSetForRooms(ctx context.Context, params = append(params, v) } - query := strings.Replace(selectJoinedUsersSetForRoomsSQL, "($1)", sqlutil.QueryVariadic(len(roomNIDs)), 1) + query := strings.Replace(selectJoinedUsersSetForRoomsSQL, "($2)", sqlutil.QueryVariadicOffset(len(roomNIDs), 1), 1) if len(userNIDs) > 0 { - query = strings.Replace(selectJoinedUsersSetForRoomsAndUserSQL, "($1)", sqlutil.QueryVariadic(len(roomNIDs)), 1) - query = strings.Replace(query, "($2)", sqlutil.QueryVariadicOffset(len(userNIDs), len(roomNIDs)), 1) + query = strings.Replace(selectJoinedUsersSetForRoomsAndUserSQL, "($2)", sqlutil.QueryVariadicOffset(len(roomNIDs), 1), 1) + query = strings.Replace(query, "($3)", sqlutil.QueryVariadicOffset(len(userNIDs), len(roomNIDs)+1), 1) } var rows *sql.Rows var err error diff --git a/roomserver/storage/tables/interface.go b/roomserver/storage/tables/interface.go index 68d30f994..d7bcc95ab 100644 --- a/roomserver/storage/tables/interface.go +++ b/roomserver/storage/tables/interface.go @@ -137,7 +137,7 @@ type Membership interface { UpdateMembership(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, senderUserNID types.EventStateKeyNID, membership MembershipState, eventNID types.EventNID, forgotten bool) (bool, error) SelectRoomsWithMembership(ctx context.Context, txn *sql.Tx, userID types.EventStateKeyNID, membershipState MembershipState) ([]types.RoomNID, error) // SelectJoinedUsersSetForRooms returns how many times each of the given users appears across the given rooms. - SelectJoinedUsersSetForRooms(ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID, userNIDs []types.EventStateKeyNID) (map[types.EventStateKeyNID]int, error) + SelectJoinedUsersSetForRooms(ctx context.Context, txn *sql.Tx, roomNIDs []types.RoomNID, userNIDs []types.EventStateKeyNID, localOnly bool) (map[types.EventStateKeyNID]int, error) SelectKnownUsers(ctx context.Context, txn *sql.Tx, userID types.EventStateKeyNID, searchString string, limit int) ([]string, error) UpdateForgetMembership(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, forget bool) error SelectLocalServerInRoom(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID) (bool, error) diff --git a/roomserver/storage/tables/membership_table_test.go b/roomserver/storage/tables/membership_table_test.go index f789ef4ac..c9541d9d2 100644 --- a/roomserver/storage/tables/membership_table_test.go +++ b/roomserver/storage/tables/membership_table_test.go @@ -79,7 +79,7 @@ func TestMembershipTable(t *testing.T) { assert.NoError(t, err) assert.True(t, inRoom) - userJoinedToRooms, err := tab.SelectJoinedUsersSetForRooms(ctx, nil, []types.RoomNID{1}, userNIDs) + userJoinedToRooms, err := tab.SelectJoinedUsersSetForRooms(ctx, nil, []types.RoomNID{1}, userNIDs, false) assert.NoError(t, err) assert.Equal(t, 1, len(userJoinedToRooms)) diff --git a/syncapi/consumers/keychange.go b/syncapi/consumers/keychange.go index dc7d9e207..96ebba7ef 100644 --- a/syncapi/consumers/keychange.go +++ b/syncapi/consumers/keychange.go @@ -111,7 +111,8 @@ func (s *OutputKeyChangeEventConsumer) onDeviceKeyMessage(m api.DeviceMessage, d // work out who we need to notify about the new key var queryRes roomserverAPI.QuerySharedUsersResponse err := s.rsAPI.QuerySharedUsers(s.ctx, &roomserverAPI.QuerySharedUsersRequest{ - UserID: output.UserID, + UserID: output.UserID, + LocalOnly: true, }, &queryRes) if err != nil { logrus.WithError(err).Error("syncapi: failed to QuerySharedUsers for key change event from key server") @@ -135,7 +136,8 @@ func (s *OutputKeyChangeEventConsumer) onCrossSigningMessage(m api.DeviceMessage // work out who we need to notify about the new key var queryRes roomserverAPI.QuerySharedUsersResponse err := s.rsAPI.QuerySharedUsers(s.ctx, &roomserverAPI.QuerySharedUsersRequest{ - UserID: output.UserID, + UserID: output.UserID, + LocalOnly: true, }, &queryRes) if err != nil { logrus.WithError(err).Error("syncapi: failed to QuerySharedUsers for key change event from key server") From 0f777d421c56e386b4c483233277f2b96c4da3a0 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Wed, 5 Oct 2022 14:47:13 +0200 Subject: [PATCH 042/207] Remove empty fields from `/sync` response (#2755) First attempt at removing empty fields from `/sync` responses. Needs https://github.com/matrix-org/sytest/pull/1298 to keep Sytest happy. Co-authored-by: Neil Alexander --- syncapi/internal/keychange_test.go | 33 +++-- syncapi/streams/stream_accountdata.go | 6 +- syncapi/streams/stream_invite.go | 4 +- syncapi/streams/stream_pdu.go | 10 +- syncapi/streams/stream_receipt.go | 5 +- syncapi/streams/stream_typing.go | 9 +- syncapi/types/types.go | 201 ++++++++++++++++++-------- 7 files changed, 178 insertions(+), 90 deletions(-) diff --git a/syncapi/internal/keychange_test.go b/syncapi/internal/keychange_test.go index 3b9c8221c..53f3e5a40 100644 --- a/syncapi/internal/keychange_test.go +++ b/syncapi/internal/keychange_test.go @@ -170,9 +170,12 @@ func joinResponseWithRooms(syncResponse *types.Response, userID string, roomIDs Content: []byte(`{"membership":"join"}`), }, } - - jr := syncResponse.Rooms.Join[roomID] - jr.State.Events = roomEvents + jr, ok := syncResponse.Rooms.Join[roomID] + if !ok { + jr = types.NewJoinResponse() + } + jr.Timeline = &types.Timeline{} + jr.State = &types.ClientEvents{Events: roomEvents} syncResponse.Rooms.Join[roomID] = jr } return syncResponse @@ -191,8 +194,11 @@ func leaveResponseWithRooms(syncResponse *types.Response, userID string, roomIDs }, } - lr := syncResponse.Rooms.Leave[roomID] - lr.Timeline.Events = roomEvents + lr, ok := syncResponse.Rooms.Leave[roomID] + if !ok { + lr = types.NewLeaveResponse() + } + lr.Timeline = &types.Timeline{Events: roomEvents} syncResponse.Rooms.Leave[roomID] = lr } return syncResponse @@ -328,9 +334,13 @@ func TestKeyChangeCatchupNoNewJoinsButMessages(t *testing.T) { }, } - jr := syncResponse.Rooms.Join[roomID] - jr.State.Events = roomStateEvents - jr.Timeline.Events = roomTimelineEvents + jr, ok := syncResponse.Rooms.Join[roomID] + if !ok { + jr = types.NewJoinResponse() + } + + jr.State = &types.ClientEvents{Events: roomStateEvents} + jr.Timeline = &types.Timeline{Events: roomTimelineEvents} syncResponse.Rooms.Join[roomID] = jr rsAPI := &mockRoomserverAPI{ @@ -442,8 +452,11 @@ func TestKeyChangeCatchupChangeAndLeftSameRoom(t *testing.T) { }, } - lr := syncResponse.Rooms.Leave[roomID] - lr.Timeline.Events = roomEvents + lr, ok := syncResponse.Rooms.Leave[roomID] + if !ok { + lr = types.NewLeaveResponse() + } + lr.Timeline = &types.Timeline{Events: roomEvents} syncResponse.Rooms.Leave[roomID] = lr rsAPI := &mockRoomserverAPI{ diff --git a/syncapi/streams/stream_accountdata.go b/syncapi/streams/stream_accountdata.go index 3f2f7d134..3593a6563 100644 --- a/syncapi/streams/stream_accountdata.go +++ b/syncapi/streams/stream_accountdata.go @@ -90,9 +90,9 @@ func (p *AccountDataStreamProvider) IncrementalSync( } } else { if roomData, ok := dataRes.RoomAccountData[roomID][dataType]; ok { - joinData := *types.NewJoinResponse() - if existing, ok := req.Response.Rooms.Join[roomID]; ok { - joinData = existing + joinData, ok := req.Response.Rooms.Join[roomID] + if !ok { + joinData = types.NewJoinResponse() } joinData.AccountData.Events = append( joinData.AccountData.Events, diff --git a/syncapi/streams/stream_invite.go b/syncapi/streams/stream_invite.go index 17b3b8434..7875ffa35 100644 --- a/syncapi/streams/stream_invite.go +++ b/syncapi/streams/stream_invite.go @@ -65,7 +65,7 @@ func (p *InviteStreamProvider) IncrementalSync( continue } ir := types.NewInviteResponse(inviteEvent) - req.Response.Rooms.Invite[roomID] = *ir + req.Response.Rooms.Invite[roomID] = ir } // When doing an initial sync, we don't want to add retired invites, as this @@ -87,7 +87,7 @@ func (p *InviteStreamProvider) IncrementalSync( Type: "m.room.member", Content: gomatrixserverlib.RawJSON(`{"membership":"leave"}`), }) - req.Response.Rooms.Leave[roomID] = *lr + req.Response.Rooms.Leave[roomID] = lr } } diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go index d252265ff..613ac434f 100644 --- a/syncapi/streams/stream_pdu.go +++ b/syncapi/streams/stream_pdu.go @@ -106,7 +106,7 @@ func (p *PDUStreamProvider) CompleteSync( } continue } - req.Response.Rooms.Join[roomID] = *jr + req.Response.Rooms.Join[roomID] = jr req.Rooms[roomID] = gomatrixserverlib.Join } @@ -129,7 +129,7 @@ func (p *PDUStreamProvider) CompleteSync( } continue } - req.Response.Rooms.Peek[peek.RoomID] = *jr + req.Response.Rooms.Peek[peek.RoomID] = jr } } @@ -320,7 +320,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( // didn't "remove" events, return that the response is limited. jr.Timeline.Limited = limited && len(events) == len(recentEvents) jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.StateEvents, gomatrixserverlib.FormatSync) - res.Rooms.Join[delta.RoomID] = *jr + res.Rooms.Join[delta.RoomID] = jr case gomatrixserverlib.Peek: jr := types.NewJoinResponse() @@ -329,7 +329,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = limited jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.StateEvents, gomatrixserverlib.FormatSync) - res.Rooms.Peek[delta.RoomID] = *jr + res.Rooms.Peek[delta.RoomID] = jr case gomatrixserverlib.Leave: fallthrough // transitions to leave are the same as ban @@ -342,7 +342,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( // didn't "remove" events, return that the response is limited. lr.Timeline.Limited = limited && len(events) == len(recentEvents) lr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.StateEvents, gomatrixserverlib.FormatSync) - res.Rooms.Leave[delta.RoomID] = *lr + res.Rooms.Leave[delta.RoomID] = lr } return latestPosition, nil diff --git a/syncapi/streams/stream_receipt.go b/syncapi/streams/stream_receipt.go index 8818a5533..76927cc36 100644 --- a/syncapi/streams/stream_receipt.go +++ b/syncapi/streams/stream_receipt.go @@ -4,9 +4,10 @@ import ( "context" "encoding/json" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/types" - "github.com/matrix-org/gomatrixserverlib" ) type ReceiptStreamProvider struct { @@ -76,7 +77,7 @@ func (p *ReceiptStreamProvider) IncrementalSync( continue } - jr := *types.NewJoinResponse() + jr := types.NewJoinResponse() if existing, ok := req.Response.Rooms.Join[roomID]; ok { jr = existing } diff --git a/syncapi/streams/stream_typing.go b/syncapi/streams/stream_typing.go index a6f7c7a06..84c199b39 100644 --- a/syncapi/streams/stream_typing.go +++ b/syncapi/streams/stream_typing.go @@ -4,10 +4,11 @@ import ( "context" "encoding/json" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/types" - "github.com/matrix-org/gomatrixserverlib" ) type TypingStreamProvider struct { @@ -35,9 +36,9 @@ func (p *TypingStreamProvider) IncrementalSync( continue } - jr := *types.NewJoinResponse() - if existing, ok := req.Response.Rooms.Join[roomID]; ok { - jr = existing + jr, ok := req.Response.Rooms.Join[roomID] + if !ok { + jr = types.NewJoinResponse() } if users, updated := p.EDUCache.GetTypingUsersIfUpdatedAfter( diff --git a/syncapi/types/types.go b/syncapi/types/types.go index 3b85db4a4..b6d340f93 100644 --- a/syncapi/types/types.go +++ b/syncapi/types/types.go @@ -327,29 +327,57 @@ type PrevEventRef struct { PrevSender string `json:"prev_sender"` } +type DeviceLists struct { + Changed []string `json:"changed,omitempty"` + Left []string `json:"left,omitempty"` +} + +type RoomsResponse struct { + Join map[string]*JoinResponse `json:"join,omitempty"` + Peek map[string]*JoinResponse `json:"peek,omitempty"` + Invite map[string]*InviteResponse `json:"invite,omitempty"` + Leave map[string]*LeaveResponse `json:"leave,omitempty"` +} + +type ToDeviceResponse struct { + Events []gomatrixserverlib.SendToDeviceEvent `json:"events,omitempty"` +} + // Response represents a /sync API response. See https://matrix.org/docs/spec/client_server/r0.2.0.html#get-matrix-client-r0-sync type Response struct { - NextBatch StreamingToken `json:"next_batch"` - AccountData struct { - Events []gomatrixserverlib.ClientEvent `json:"events,omitempty"` - } `json:"account_data,omitempty"` - Presence struct { - Events []gomatrixserverlib.ClientEvent `json:"events,omitempty"` - } `json:"presence,omitempty"` - Rooms struct { - Join map[string]JoinResponse `json:"join,omitempty"` - Peek map[string]JoinResponse `json:"peek,omitempty"` - Invite map[string]InviteResponse `json:"invite,omitempty"` - Leave map[string]LeaveResponse `json:"leave,omitempty"` - } `json:"rooms,omitempty"` - ToDevice struct { - Events []gomatrixserverlib.SendToDeviceEvent `json:"events,omitempty"` - } `json:"to_device,omitempty"` - DeviceLists struct { - Changed []string `json:"changed,omitempty"` - Left []string `json:"left,omitempty"` - } `json:"device_lists,omitempty"` - DeviceListsOTKCount map[string]int `json:"device_one_time_keys_count,omitempty"` + NextBatch StreamingToken `json:"next_batch"` + AccountData *ClientEvents `json:"account_data,omitempty"` + Presence *ClientEvents `json:"presence,omitempty"` + Rooms *RoomsResponse `json:"rooms,omitempty"` + ToDevice *ToDeviceResponse `json:"to_device,omitempty"` + DeviceLists *DeviceLists `json:"device_lists,omitempty"` + DeviceListsOTKCount map[string]int `json:"device_one_time_keys_count,omitempty"` +} + +func (r Response) MarshalJSON() ([]byte, error) { + type alias Response + a := alias(r) + if r.AccountData != nil && len(r.AccountData.Events) == 0 { + a.AccountData = nil + } + if r.Presence != nil && len(r.Presence.Events) == 0 { + a.Presence = nil + } + if r.DeviceLists != nil { + if len(r.DeviceLists.Left) == 0 && len(r.DeviceLists.Changed) == 0 { + a.DeviceLists = nil + } + } + if r.Rooms != nil { + if len(r.Rooms.Join) == 0 && len(r.Rooms.Peek) == 0 && + len(r.Rooms.Invite) == 0 && len(r.Rooms.Leave) == 0 { + a.Rooms = nil + } + } + if r.ToDevice != nil && len(r.ToDevice.Events) == 0 { + a.ToDevice = nil + } + return json.Marshal(a) } func (r *Response) HasUpdates() bool { @@ -370,18 +398,21 @@ func NewResponse() *Response { res := Response{} // Pre-initialise the maps. Synapse will return {} even if there are no rooms under a specific section, // so let's do the same thing. Bonus: this means we can't get dreaded 'assignment to entry in nil map' errors. - res.Rooms.Join = map[string]JoinResponse{} - res.Rooms.Peek = map[string]JoinResponse{} - res.Rooms.Invite = map[string]InviteResponse{} - res.Rooms.Leave = map[string]LeaveResponse{} + res.Rooms = &RoomsResponse{ + Join: map[string]*JoinResponse{}, + Peek: map[string]*JoinResponse{}, + Invite: map[string]*InviteResponse{}, + Leave: map[string]*LeaveResponse{}, + } // Also pre-intialise empty slices or else we'll insert 'null' instead of '[]' for the value. // TODO: We really shouldn't have to do all this to coerce encoding/json to Do The Right Thing. We should // really be using our own Marshal/Unmarshal implementations otherwise this may prove to be a CPU bottleneck. // This also applies to NewJoinResponse, NewInviteResponse and NewLeaveResponse. - res.AccountData.Events = []gomatrixserverlib.ClientEvent{} - res.Presence.Events = []gomatrixserverlib.ClientEvent{} - res.ToDevice.Events = []gomatrixserverlib.SendToDeviceEvent{} + res.AccountData = &ClientEvents{} + res.Presence = &ClientEvents{} + res.DeviceLists = &DeviceLists{} + res.ToDevice = &ToDeviceResponse{} res.DeviceListsOTKCount = map[string]int{} return &res @@ -403,38 +434,73 @@ type UnreadNotifications struct { NotificationCount int `json:"notification_count"` } +type ClientEvents struct { + Events []gomatrixserverlib.ClientEvent `json:"events,omitempty"` +} + +type Timeline struct { + Events []gomatrixserverlib.ClientEvent `json:"events"` + Limited bool `json:"limited"` + PrevBatch *TopologyToken `json:"prev_batch,omitempty"` +} + +type Summary struct { + Heroes []string `json:"m.heroes,omitempty"` + JoinedMemberCount *int `json:"m.joined_member_count,omitempty"` + InvitedMemberCount *int `json:"m.invited_member_count,omitempty"` +} + // JoinResponse represents a /sync response for a room which is under the 'join' or 'peek' key. type JoinResponse struct { - Summary struct { - Heroes []string `json:"m.heroes,omitempty"` - JoinedMemberCount *int `json:"m.joined_member_count,omitempty"` - InvitedMemberCount *int `json:"m.invited_member_count,omitempty"` - } `json:"summary"` - State struct { - Events []gomatrixserverlib.ClientEvent `json:"events"` - } `json:"state"` - Timeline struct { - Events []gomatrixserverlib.ClientEvent `json:"events"` - Limited bool `json:"limited"` - PrevBatch *TopologyToken `json:"prev_batch,omitempty"` - } `json:"timeline"` - Ephemeral struct { - Events []gomatrixserverlib.ClientEvent `json:"events"` - } `json:"ephemeral"` - AccountData struct { - Events []gomatrixserverlib.ClientEvent `json:"events"` - } `json:"account_data"` + Summary *Summary `json:"summary,omitempty"` + State *ClientEvents `json:"state,omitempty"` + Timeline *Timeline `json:"timeline,omitempty"` + Ephemeral *ClientEvents `json:"ephemeral,omitempty"` + AccountData *ClientEvents `json:"account_data,omitempty"` *UnreadNotifications `json:"unread_notifications,omitempty"` } +func (jr JoinResponse) MarshalJSON() ([]byte, error) { + type alias JoinResponse + a := alias(jr) + if jr.State != nil && len(jr.State.Events) == 0 { + a.State = nil + } + if jr.Ephemeral != nil && len(jr.Ephemeral.Events) == 0 { + a.Ephemeral = nil + } + if jr.AccountData != nil && len(jr.AccountData.Events) == 0 { + a.AccountData = nil + } + if jr.Timeline != nil && len(jr.Timeline.Events) == 0 { + a.Timeline = nil + } + if jr.Summary != nil { + var nilPtr int + joinedEmpty := jr.Summary.JoinedMemberCount == nil || jr.Summary.JoinedMemberCount == &nilPtr + invitedEmpty := jr.Summary.InvitedMemberCount == nil || jr.Summary.InvitedMemberCount == &nilPtr + if joinedEmpty && invitedEmpty && len(jr.Summary.Heroes) == 0 { + a.Summary = nil + } + + } + if jr.UnreadNotifications != nil && + jr.UnreadNotifications.NotificationCount == 0 && jr.UnreadNotifications.HighlightCount == 0 { + a.UnreadNotifications = nil + } + return json.Marshal(a) +} + // NewJoinResponse creates an empty response with initialised arrays. func NewJoinResponse() *JoinResponse { - res := JoinResponse{} - res.State.Events = []gomatrixserverlib.ClientEvent{} - res.Timeline.Events = []gomatrixserverlib.ClientEvent{} - res.Ephemeral.Events = []gomatrixserverlib.ClientEvent{} - res.AccountData.Events = []gomatrixserverlib.ClientEvent{} - return &res + return &JoinResponse{ + Summary: &Summary{}, + State: &ClientEvents{}, + Timeline: &Timeline{}, + Ephemeral: &ClientEvents{}, + AccountData: &ClientEvents{}, + UnreadNotifications: &UnreadNotifications{}, + } } // InviteResponse represents a /sync response for a room which is under the 'invite' key. @@ -469,21 +535,28 @@ func NewInviteResponse(event *gomatrixserverlib.HeaderedEvent) *InviteResponse { // LeaveResponse represents a /sync response for a room which is under the 'leave' key. type LeaveResponse struct { - State struct { - Events []gomatrixserverlib.ClientEvent `json:"events"` - } `json:"state"` - Timeline struct { - Events []gomatrixserverlib.ClientEvent `json:"events"` - Limited bool `json:"limited"` - PrevBatch *TopologyToken `json:"prev_batch,omitempty"` - } `json:"timeline"` + State *ClientEvents `json:"state,omitempty"` + Timeline *Timeline `json:"timeline,omitempty"` +} + +func (lr LeaveResponse) MarshalJSON() ([]byte, error) { + type alias LeaveResponse + a := alias(lr) + if lr.State != nil && len(lr.State.Events) == 0 { + a.State = nil + } + if lr.Timeline != nil && len(lr.Timeline.Events) == 0 { + a.Timeline = nil + } + return json.Marshal(a) } // NewLeaveResponse creates an empty response with initialised arrays. func NewLeaveResponse() *LeaveResponse { - res := LeaveResponse{} - res.State.Events = []gomatrixserverlib.ClientEvent{} - res.Timeline.Events = []gomatrixserverlib.ClientEvent{} + res := LeaveResponse{ + State: &ClientEvents{}, + Timeline: &Timeline{}, + } return &res } From 9ba3103f8839243ea8529bf1f563afded3e7b591 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 5 Oct 2022 14:54:19 +0100 Subject: [PATCH 043/207] Document database connection limits --- docs/installation/11_optimisation.md | 35 ++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/docs/installation/11_optimisation.md b/docs/installation/11_optimisation.md index f2f67c947..686ec2eb9 100644 --- a/docs/installation/11_optimisation.md +++ b/docs/installation/11_optimisation.md @@ -11,6 +11,41 @@ permalink: /installation/start/optimisation Now that you have Dendrite running, the following tweaks will improve the reliability and performance of your installation. +## PostgreSQL connection limit + +A PostgreSQL database engine is configured to allow only a certain number of connections. +This is typically controlled by the `max_connections` and `superuser_reserved_connections` +configuration items in `postgresql.conf`. Once these limits are violated, **PostgreSQL will +immediately stop accepting new connections** until some of the existing connections are closed. +This is a common source of misconfiguration and requires particular care. + +If your PostgreSQL `max_connections` is set to `100` and `superuser_reserved_connections` is +set to `3` then you have an effective connection limit of 97 database connections. It is +therefore important to ensure that Dendrite doesn't violate that limit, otherwise database +queries will unexpectedly fail and this will cause problems both within Dendrite and for users. + +If you are also running other software that uses the same PostgreSQL database engine, then you +must also take into account that some connections will be already used by your other software +and therefore will not be available to Dendrite. Check the configuration of any other software +using the same database engine for their configured connection limits and adjust your calculations +accordingly. + +Dendrite has a `max_open_conns` configuration item in each `database` block to control how many +connections it will open to the database. + +**If you are using the `global` database pool** then you only need to configure the +`max_open_conns` setting once in the `global` section. + +**If you are defining a `database` config per component** then you will need to ensure that +the **sum total** of all configured `max_open_conns` to a given database server do not exceed +the connection limit. If you configure a total that adds up to more connections than are available +then this will cause database queries to fail. + +You may wish to raise the `max_connections` limit on your PostgreSQL server to accommodate +additional connections, in which case you should also update the `max_open_conns` in your +Dendrite configuration accordingly. However be aware that this is only advisable on particularly +powerful servers that can handle the concurrent load of additional queries running at one time. + ## File descriptor limit Most platforms have a limit on how many file descriptors a single process can open. All From e53dcb25a9a0258b48fd4f7afddd2c3c54d55b7a Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 6 Oct 2022 10:07:13 +0100 Subject: [PATCH 044/207] Tweak logging for federated room joins --- federationapi/internal/perform.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/federationapi/internal/perform.go b/federationapi/internal/perform.go index 84702f4ce..4cdd3a5eb 100644 --- a/federationapi/internal/perform.go +++ b/federationapi/internal/perform.go @@ -259,7 +259,7 @@ func (r *FederationInternalAPI) performJoinUsingServer( if err != nil { return fmt.Errorf("JoinedHostsFromEvents: failed to get joined hosts: %s", err) } - logrus.WithField("hosts", joinedHosts).WithField("room", roomID).Info("Joined federated room with hosts") + logrus.WithField("room", roomID).Infof("Joined federated room with %d hosts", len(joinedHosts)) if _, err = r.db.UpdateRoom(context.Background(), roomID, joinedHosts, nil, true); err != nil { return fmt.Errorf("UpdatedRoom: failed to update room with joined hosts: %s", err) } From 3f82bceb70050c1233b7de6d87ffa5510596d145 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 6 Oct 2022 10:51:06 +0100 Subject: [PATCH 045/207] Don't try to talk to ourselves when finding missing events --- roomserver/internal/input/input_events.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go index 01fd62010..d1b6bc73e 100644 --- a/roomserver/internal/input/input_events.go +++ b/roomserver/internal/input/input_events.go @@ -173,12 +173,15 @@ func (r *Inputer) processRoomEvent( for _, server := range serverRes.ServerNames { servers[server] = struct{}{} } + // Don't try to talk to ourselves. + delete(servers, r.Cfg.Matrix.ServerName) + // Now build up the list of servers. serverRes.ServerNames = serverRes.ServerNames[:0] - if input.Origin != "" { + if input.Origin != "" && input.Origin != r.Cfg.Matrix.ServerName { serverRes.ServerNames = append(serverRes.ServerNames, input.Origin) delete(servers, input.Origin) } - if senderDomain != input.Origin { + if senderDomain != input.Origin && senderDomain != r.Cfg.Matrix.ServerName { serverRes.ServerNames = append(serverRes.ServerNames, senderDomain) delete(servers, senderDomain) } From ec5d1d681d1362f5746c5cb45e93829d6a68aa4d Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Thu, 6 Oct 2022 12:30:24 +0200 Subject: [PATCH 046/207] Always return `one_time_key_counts` on `/keys/upload` (#2769) The OTK count is [required](https://spec.matrix.org/v1.4/client-server-api/#post_matrixclientv3keysupload) in responses to `/keys/upload`, so return those. --- clientapi/routing/keys.go | 4 ++-- keyserver/internal/internal.go | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/clientapi/routing/keys.go b/clientapi/routing/keys.go index b7a76b47e..5c3681382 100644 --- a/clientapi/routing/keys.go +++ b/clientapi/routing/keys.go @@ -19,11 +19,12 @@ import ( "net/http" "time" + "github.com/matrix-org/util" + "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/keyserver/api" userapi "github.com/matrix-org/dendrite/userapi/api" - "github.com/matrix-org/util" ) type uploadKeysRequest struct { @@ -77,7 +78,6 @@ func UploadKeys(req *http.Request, keyAPI api.ClientKeyAPI, device *userapi.Devi } } keyCount := make(map[string]int) - // we only return key counts when the client uploads OTKs if len(uploadRes.OneTimeKeyCounts) > 0 { keyCount = uploadRes.OneTimeKeyCounts[0].KeyCount } diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go index 017c29e84..a0280dff4 100644 --- a/keyserver/internal/internal.go +++ b/keyserver/internal/internal.go @@ -70,6 +70,11 @@ func (a *KeyInternalAPI) PerformUploadKeys(ctx context.Context, req *api.Perform if len(req.OneTimeKeys) > 0 { a.uploadOneTimeKeys(ctx, req, res) } + otks, err := a.DB.OneTimeKeysCount(ctx, req.UserID, req.DeviceID) + if err != nil { + return err + } + res.OneTimeKeyCounts = []api.OneTimeKeysCount{*otks} return nil } From d605d928bce87b381e2f64b8835619d803e67a54 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 6 Oct 2022 11:56:00 +0100 Subject: [PATCH 047/207] Allow specifying old signing keys with the public key and key ID only (#2770) If the private key file is lost, it's often possible to retrieve the public key from another server elsewhere, so we should make it possible to configure it in that way. --- dendrite-sample.monolith.yaml | 7 ++++- dendrite-sample.polylith.yaml | 7 ++++- federationapi/routing/keys.go | 2 +- setup/config/config.go | 48 +++++++++++++++++++++++------------ setup/config/config_global.go | 7 +++-- 5 files changed, 50 insertions(+), 21 deletions(-) diff --git a/dendrite-sample.monolith.yaml b/dendrite-sample.monolith.yaml index f0fa386d1..eadb74a2a 100644 --- a/dendrite-sample.monolith.yaml +++ b/dendrite-sample.monolith.yaml @@ -18,12 +18,17 @@ global: private_key: matrix_key.pem # The paths and expiry timestamps (as a UNIX timestamp in millisecond precision) - # to old signing private keys that were formerly in use on this domain. These + # to old signing keys that were formerly in use on this domain name. These # keys will not be used for federation request or event signing, but will be # provided to any other homeserver that asks when trying to verify old events. old_private_keys: + # If the old private key file is available: # - private_key: old_matrix_key.pem # expired_at: 1601024554498 + # If only the public key (in base64 format) and key ID are known: + # - public_key: mn59Kxfdq9VziYHSBzI7+EDPDcBS2Xl7jeUdiiQcOnM= + # key_id: ed25519:mykeyid + # expired_at: 1601024554498 # How long a remote server can cache our server signing key before requesting it # again. Increasing this number will reduce the number of requests made by other diff --git a/dendrite-sample.polylith.yaml b/dendrite-sample.polylith.yaml index 0ae4cc8fb..aa7e0cc38 100644 --- a/dendrite-sample.polylith.yaml +++ b/dendrite-sample.polylith.yaml @@ -18,12 +18,17 @@ global: private_key: matrix_key.pem # The paths and expiry timestamps (as a UNIX timestamp in millisecond precision) - # to old signing private keys that were formerly in use on this domain. These + # to old signing keys that were formerly in use on this domain name. These # keys will not be used for federation request or event signing, but will be # provided to any other homeserver that asks when trying to verify old events. old_private_keys: + # If the old private key file is available: # - private_key: old_matrix_key.pem # expired_at: 1601024554498 + # If only the public key (in base64 format) and key ID are known: + # - public_key: mn59Kxfdq9VziYHSBzI7+EDPDcBS2Xl7jeUdiiQcOnM= + # key_id: ed25519:mykeyid + # expired_at: 1601024554498 # How long a remote server can cache our server signing key before requesting it # again. Increasing this number will reduce the number of requests made by other diff --git a/federationapi/routing/keys.go b/federationapi/routing/keys.go index b03d4c1d6..8931830f3 100644 --- a/federationapi/routing/keys.go +++ b/federationapi/routing/keys.go @@ -160,7 +160,7 @@ func localKeys(cfg *config.FederationAPI, validUntil time.Time) (*gomatrixserver for _, oldVerifyKey := range cfg.Matrix.OldVerifyKeys { keys.OldVerifyKeys[oldVerifyKey.KeyID] = gomatrixserverlib.OldVerifyKey{ VerifyKey: gomatrixserverlib.VerifyKey{ - Key: gomatrixserverlib.Base64Bytes(oldVerifyKey.PrivateKey.Public().(ed25519.PublicKey)), + Key: oldVerifyKey.PublicKey, }, ExpiredTS: oldVerifyKey.ExpiredAt, } diff --git a/setup/config/config.go b/setup/config/config.go index 5a618d671..e99852ec9 100644 --- a/setup/config/config.go +++ b/setup/config/config.go @@ -231,24 +231,40 @@ func loadConfig( return nil, err } - for i, oldPrivateKey := range c.Global.OldVerifyKeys { - var oldPrivateKeyData []byte + for _, key := range c.Global.OldVerifyKeys { + switch { + case key.PrivateKeyPath != "": + var oldPrivateKeyData []byte + oldPrivateKeyPath := absPath(basePath, key.PrivateKeyPath) + oldPrivateKeyData, err = readFile(oldPrivateKeyPath) + if err != nil { + return nil, fmt.Errorf("failed to read %q: %w", oldPrivateKeyPath, err) + } - oldPrivateKeyPath := absPath(basePath, oldPrivateKey.PrivateKeyPath) - oldPrivateKeyData, err = readFile(oldPrivateKeyPath) - if err != nil { - return nil, err + // NOTSPEC: Ordinarily we should enforce key ID formatting, but since there are + // a number of private keys out there with non-compatible symbols in them due + // to lack of validation in Synapse, we won't enforce that for old verify keys. + keyID, privateKey, perr := readKeyPEM(oldPrivateKeyPath, oldPrivateKeyData, false) + if perr != nil { + return nil, fmt.Errorf("failed to parse %q: %w", oldPrivateKeyPath, perr) + } + + key.KeyID = keyID + key.PrivateKey = privateKey + key.PublicKey = gomatrixserverlib.Base64Bytes(privateKey.Public().(ed25519.PublicKey)) + + case key.KeyID == "": + return nil, fmt.Errorf("'key_id' must be specified if 'public_key' is specified") + + case len(key.PublicKey) == ed25519.PublicKeySize: + continue + + case len(key.PublicKey) > 0: + return nil, fmt.Errorf("the supplied 'public_key' is the wrong length") + + default: + return nil, fmt.Errorf("either specify a 'private_key' path or supply both 'public_key' and 'key_id'") } - - // NOTSPEC: Ordinarily we should enforce key ID formatting, but since there are - // a number of private keys out there with non-compatible symbols in them due - // to lack of validation in Synapse, we won't enforce that for old verify keys. - keyID, privateKey, perr := readKeyPEM(oldPrivateKeyPath, oldPrivateKeyData, false) - if perr != nil { - return nil, perr - } - - c.Global.OldVerifyKeys[i].KeyID, c.Global.OldVerifyKeys[i].PrivateKey = keyID, privateKey } c.MediaAPI.AbsBasePath = Path(absPath(basePath, c.MediaAPI.BasePath)) diff --git a/setup/config/config_global.go b/setup/config/config_global.go index acc608dd7..2efae0d5a 100644 --- a/setup/config/config_global.go +++ b/setup/config/config_global.go @@ -27,7 +27,7 @@ type Global struct { // Information about old private keys that used to be used to sign requests and // events on this domain. They will not be used but will be advertised to other // servers that ask for them to help verify old events. - OldVerifyKeys []OldVerifyKeys `yaml:"old_private_keys"` + OldVerifyKeys []*OldVerifyKeys `yaml:"old_private_keys"` // How long a remote server can cache our server key for before requesting it again. // Increasing this number will reduce the number of requests made by remote servers @@ -127,8 +127,11 @@ type OldVerifyKeys struct { // The private key itself. PrivateKey ed25519.PrivateKey `yaml:"-"` + // The public key, in case only that part is known. + PublicKey gomatrixserverlib.Base64Bytes `yaml:"public_key"` + // The key ID of the private key. - KeyID gomatrixserverlib.KeyID `yaml:"-"` + KeyID gomatrixserverlib.KeyID `yaml:"key_id"` // When the private key was designed as "expired", as a UNIX timestamp // in millisecond precision. From 8c5b16678475112f5a1c13d616931283421aee05 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Thu, 6 Oct 2022 12:57:13 +0200 Subject: [PATCH 048/207] Use the stream positions of the notifier (#2768) Use the stream positions of the notifier, which might have advanced since setting it at the beginning of the loop. This possibly helps in reducing roundtrips to the SyncAPI, just because we didn't fetch the latest data. Also fixes a minor oversight in the receipts stream. --- syncapi/streams/stream_receipt.go | 6 +++--- syncapi/sync/requestpool.go | 18 +++++++++--------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/syncapi/streams/stream_receipt.go b/syncapi/streams/stream_receipt.go index 76927cc36..bba911022 100644 --- a/syncapi/streams/stream_receipt.go +++ b/syncapi/streams/stream_receipt.go @@ -77,9 +77,9 @@ func (p *ReceiptStreamProvider) IncrementalSync( continue } - jr := types.NewJoinResponse() - if existing, ok := req.Response.Rooms.Join[roomID]; ok { - jr = existing + jr, ok := req.Response.Rooms.Join[roomID] + if !ok { + jr = types.NewJoinResponse() } ev := gomatrixserverlib.ClientEvent{ diff --git a/syncapi/sync/requestpool.go b/syncapi/sync/requestpool.go index a71d32ab8..29d92b293 100644 --- a/syncapi/sync/requestpool.go +++ b/syncapi/sync/requestpool.go @@ -407,7 +407,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. func(txn storage.DatabaseTransaction) types.StreamPosition { return rp.streams.PDUStreamProvider.IncrementalSync( syncReq.Context, txn, syncReq, - syncReq.Since.PDUPosition, currentPos.PDUPosition, + syncReq.Since.PDUPosition, rp.Notifier.CurrentPosition().PDUPosition, ) }, ), @@ -416,7 +416,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. func(txn storage.DatabaseTransaction) types.StreamPosition { return rp.streams.TypingStreamProvider.IncrementalSync( syncReq.Context, txn, syncReq, - syncReq.Since.TypingPosition, currentPos.TypingPosition, + syncReq.Since.TypingPosition, rp.Notifier.CurrentPosition().TypingPosition, ) }, ), @@ -425,7 +425,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. func(txn storage.DatabaseTransaction) types.StreamPosition { return rp.streams.ReceiptStreamProvider.IncrementalSync( syncReq.Context, txn, syncReq, - syncReq.Since.ReceiptPosition, currentPos.ReceiptPosition, + syncReq.Since.ReceiptPosition, rp.Notifier.CurrentPosition().ReceiptPosition, ) }, ), @@ -434,7 +434,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. func(txn storage.DatabaseTransaction) types.StreamPosition { return rp.streams.InviteStreamProvider.IncrementalSync( syncReq.Context, txn, syncReq, - syncReq.Since.InvitePosition, currentPos.InvitePosition, + syncReq.Since.InvitePosition, rp.Notifier.CurrentPosition().InvitePosition, ) }, ), @@ -443,7 +443,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. func(txn storage.DatabaseTransaction) types.StreamPosition { return rp.streams.SendToDeviceStreamProvider.IncrementalSync( syncReq.Context, txn, syncReq, - syncReq.Since.SendToDevicePosition, currentPos.SendToDevicePosition, + syncReq.Since.SendToDevicePosition, rp.Notifier.CurrentPosition().SendToDevicePosition, ) }, ), @@ -452,7 +452,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. func(txn storage.DatabaseTransaction) types.StreamPosition { return rp.streams.AccountDataStreamProvider.IncrementalSync( syncReq.Context, txn, syncReq, - syncReq.Since.AccountDataPosition, currentPos.AccountDataPosition, + syncReq.Since.AccountDataPosition, rp.Notifier.CurrentPosition().AccountDataPosition, ) }, ), @@ -461,7 +461,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. func(txn storage.DatabaseTransaction) types.StreamPosition { return rp.streams.NotificationDataStreamProvider.IncrementalSync( syncReq.Context, txn, syncReq, - syncReq.Since.NotificationDataPosition, currentPos.NotificationDataPosition, + syncReq.Since.NotificationDataPosition, rp.Notifier.CurrentPosition().NotificationDataPosition, ) }, ), @@ -470,7 +470,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. func(txn storage.DatabaseTransaction) types.StreamPosition { return rp.streams.DeviceListStreamProvider.IncrementalSync( syncReq.Context, txn, syncReq, - syncReq.Since.DeviceListPosition, currentPos.DeviceListPosition, + syncReq.Since.DeviceListPosition, rp.Notifier.CurrentPosition().DeviceListPosition, ) }, ), @@ -479,7 +479,7 @@ func (rp *RequestPool) OnIncomingSyncRequest(req *http.Request, device *userapi. func(txn storage.DatabaseTransaction) types.StreamPosition { return rp.streams.PresenceStreamProvider.IncrementalSync( syncReq.Context, txn, syncReq, - syncReq.Since.PresencePosition, currentPos.PresencePosition, + syncReq.Since.PresencePosition, rp.Notifier.CurrentPosition().PresencePosition, ) }, ), From 453b50e1d3ce4b4906e050fa8ebe7bcc4c881600 Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Fri, 7 Oct 2022 07:32:27 +0200 Subject: [PATCH 049/207] Update README --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3bb9a2350..dfef11bae 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ $ ./bin/dendrite-monolith-server --tls-cert server.crt --tls-key server.key --co # Create an user account (add -admin for an admin user). # Specify the localpart only, e.g. 'alice' for '@alice:domain.com' -$ ./bin/create-account --config dendrite.yaml --url http://localhost:8008 --username alice +$ ./bin/create-account --config dendrite.yaml --username alice ``` Then point your favourite Matrix client at `http://localhost:8008` or `https://localhost:8448`. @@ -90,7 +90,7 @@ We use a script called Are We Synapse Yet which checks Sytest compliance rates. test rig with around 900 tests. The script works out how many of these tests are passing on Dendrite and it updates with CI. As of August 2022 we're at around 90% CS API coverage and 95% Federation coverage, though check CI for the latest numbers. In practice, this means you can communicate locally and via federation with Synapse -servers such as matrix.org reasonably well, although there are still some missing features (like Search). +servers such as matrix.org reasonably well, although there are still some missing features (like SSO and Third-party ID APIs). We are prioritising features that will benefit single-user homeservers first (e.g Receipts, E2E) rather than features that massive deployments may be interested in (OpenID, Guests, Admin APIs, AS API). @@ -112,6 +112,7 @@ This means Dendrite supports amongst others: - Guests - User Directory - Presence +- Fulltext search ## Contributing From b9d0e9f7ed7ce1f4be72a25c6f5185a6e809f019 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Fri, 7 Oct 2022 10:54:42 +0200 Subject: [PATCH 050/207] Add test for `QueryDeviceMessages` (#2773) Adds tests for `QueryDeviceMessages` and also includes some optimizations to reduce allocations in the DB layer. --- keyserver/internal/internal.go | 8 +- keyserver/internal/internal_test.go | 156 ++++++++++++++++++ .../storage/postgres/device_keys_table.go | 16 +- .../storage/sqlite3/device_keys_table.go | 16 +- 4 files changed, 172 insertions(+), 24 deletions(-) create mode 100644 keyserver/internal/internal_test.go diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go index a0280dff4..06fc4987c 100644 --- a/keyserver/internal/internal.go +++ b/keyserver/internal/internal.go @@ -212,15 +212,13 @@ func (a *KeyInternalAPI) QueryDeviceMessages(ctx context.Context, req *api.Query return nil } maxStreamID := int64(0) + // remove deleted devices + var result []api.DeviceMessage for _, m := range msgs { if m.StreamID > maxStreamID { maxStreamID = m.StreamID } - } - // remove deleted devices - var result []api.DeviceMessage - for _, m := range msgs { - if m.KeyJSON == nil { + if m.KeyJSON == nil || len(m.KeyJSON) == 0 { continue } result = append(result, m) diff --git a/keyserver/internal/internal_test.go b/keyserver/internal/internal_test.go new file mode 100644 index 000000000..8a2c9c5d9 --- /dev/null +++ b/keyserver/internal/internal_test.go @@ -0,0 +1,156 @@ +package internal_test + +import ( + "context" + "reflect" + "testing" + + "github.com/matrix-org/dendrite/keyserver/api" + "github.com/matrix-org/dendrite/keyserver/internal" + "github.com/matrix-org/dendrite/keyserver/storage" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/test" +) + +func mustCreateDatabase(t *testing.T, dbType test.DBType) (storage.Database, func()) { + t.Helper() + connStr, close := test.PrepareDBConnectionString(t, dbType) + db, err := storage.NewDatabase(nil, &config.DatabaseOptions{ + ConnectionString: config.DataSource(connStr), + }) + if err != nil { + t.Fatalf("failed to create new user db: %v", err) + } + return db, close +} + +func Test_QueryDeviceMessages(t *testing.T) { + alice := test.NewUser(t) + type args struct { + req *api.QueryDeviceMessagesRequest + res *api.QueryDeviceMessagesResponse + } + tests := []struct { + name string + args args + wantErr bool + want *api.QueryDeviceMessagesResponse + }{ + { + name: "no existing keys", + args: args{ + req: &api.QueryDeviceMessagesRequest{ + UserID: "@doesNotExist:localhost", + }, + res: &api.QueryDeviceMessagesResponse{}, + }, + want: &api.QueryDeviceMessagesResponse{}, + }, + { + name: "existing user returns devices", + args: args{ + req: &api.QueryDeviceMessagesRequest{ + UserID: alice.ID, + }, + res: &api.QueryDeviceMessagesResponse{}, + }, + want: &api.QueryDeviceMessagesResponse{ + StreamID: 6, + Devices: []api.DeviceMessage{ + { + Type: api.TypeDeviceKeyUpdate, StreamID: 5, DeviceKeys: &api.DeviceKeys{ + DeviceID: "myDevice", + DisplayName: "first device", + UserID: alice.ID, + KeyJSON: []byte("ghi"), + }, + }, + { + Type: api.TypeDeviceKeyUpdate, StreamID: 6, DeviceKeys: &api.DeviceKeys{ + DeviceID: "mySecondDevice", + DisplayName: "second device", + UserID: alice.ID, + KeyJSON: []byte("jkl"), + }, // streamID 6 + }, + }, + }, + }, + } + + deviceMessages := []api.DeviceMessage{ + { // not the user we're looking for + Type: api.TypeDeviceKeyUpdate, DeviceKeys: &api.DeviceKeys{ + UserID: "@doesNotExist:localhost", + }, + // streamID 1 for this user + }, + { // empty keyJSON will be ignored + Type: api.TypeDeviceKeyUpdate, DeviceKeys: &api.DeviceKeys{ + DeviceID: "myDevice", + UserID: alice.ID, + }, // streamID 1 + }, + { + Type: api.TypeDeviceKeyUpdate, DeviceKeys: &api.DeviceKeys{ + DeviceID: "myDevice", + UserID: alice.ID, + KeyJSON: []byte("abc"), + }, // streamID 2 + }, + { + Type: api.TypeDeviceKeyUpdate, DeviceKeys: &api.DeviceKeys{ + DeviceID: "myDevice", + UserID: alice.ID, + KeyJSON: []byte("def"), + }, // streamID 3 + }, + { + Type: api.TypeDeviceKeyUpdate, DeviceKeys: &api.DeviceKeys{ + DeviceID: "myDevice", + UserID: alice.ID, + KeyJSON: []byte(""), + }, // streamID 4 + }, + { + Type: api.TypeDeviceKeyUpdate, DeviceKeys: &api.DeviceKeys{ + DeviceID: "myDevice", + DisplayName: "first device", + UserID: alice.ID, + KeyJSON: []byte("ghi"), + }, // streamID 5 + }, + { + Type: api.TypeDeviceKeyUpdate, DeviceKeys: &api.DeviceKeys{ + DeviceID: "mySecondDevice", + UserID: alice.ID, + KeyJSON: []byte("jkl"), + DisplayName: "second device", + }, // streamID 6 + }, + } + ctx := context.Background() + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + db, closeDB := mustCreateDatabase(t, dbType) + defer closeDB() + if err := db.StoreLocalDeviceKeys(ctx, deviceMessages); err != nil { + t.Fatalf("failed to store local devicesKeys") + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + a := &internal.KeyInternalAPI{ + DB: db, + } + if err := a.QueryDeviceMessages(ctx, tt.args.req, tt.args.res); (err != nil) != tt.wantErr { + t.Errorf("QueryDeviceMessages() error = %v, wantErr %v", err, tt.wantErr) + } + got := tt.args.res + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("QueryDeviceMessages(): got:\n%+v, want:\n%+v", got, tt.want) + } + }) + } + }) +} diff --git a/keyserver/storage/postgres/device_keys_table.go b/keyserver/storage/postgres/device_keys_table.go index ccd20cbd6..2aa11c520 100644 --- a/keyserver/storage/postgres/device_keys_table.go +++ b/keyserver/storage/postgres/device_keys_table.go @@ -20,6 +20,7 @@ import ( "time" "github.com/lib/pq" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/keyserver/api" @@ -204,20 +205,17 @@ func (s *deviceKeysStatements) SelectBatchDeviceKeys(ctx context.Context, userID deviceIDMap[d] = true } var result []api.DeviceMessage + var displayName sql.NullString for rows.Next() { dk := api.DeviceMessage{ - Type: api.TypeDeviceKeyUpdate, - DeviceKeys: &api.DeviceKeys{}, + Type: api.TypeDeviceKeyUpdate, + DeviceKeys: &api.DeviceKeys{ + UserID: userID, + }, } - dk.UserID = userID - var keyJSON string - var streamID int64 - var displayName sql.NullString - if err := rows.Scan(&dk.DeviceID, &keyJSON, &streamID, &displayName); err != nil { + if err := rows.Scan(&dk.DeviceID, &dk.KeyJSON, &dk.StreamID, &displayName); err != nil { return nil, err } - dk.KeyJSON = []byte(keyJSON) - dk.StreamID = streamID if displayName.Valid { dk.DisplayName = displayName.String } diff --git a/keyserver/storage/sqlite3/device_keys_table.go b/keyserver/storage/sqlite3/device_keys_table.go index e77b49b35..73768da5b 100644 --- a/keyserver/storage/sqlite3/device_keys_table.go +++ b/keyserver/storage/sqlite3/device_keys_table.go @@ -137,21 +137,17 @@ func (s *deviceKeysStatements) SelectBatchDeviceKeys(ctx context.Context, userID } defer internal.CloseAndLogIfError(ctx, rows, "selectBatchDeviceKeysStmt: rows.close() failed") var result []api.DeviceMessage + var displayName sql.NullString for rows.Next() { dk := api.DeviceMessage{ - Type: api.TypeDeviceKeyUpdate, - DeviceKeys: &api.DeviceKeys{}, + Type: api.TypeDeviceKeyUpdate, + DeviceKeys: &api.DeviceKeys{ + UserID: userID, + }, } - dk.Type = api.TypeDeviceKeyUpdate - dk.UserID = userID - var keyJSON string - var streamID int64 - var displayName sql.NullString - if err := rows.Scan(&dk.DeviceID, &keyJSON, &streamID, &displayName); err != nil { + if err := rows.Scan(&dk.DeviceID, &dk.KeyJSON, &dk.StreamID, &displayName); err != nil { return nil, err } - dk.KeyJSON = []byte(keyJSON) - dk.StreamID = streamID if displayName.Valid { dk.DisplayName = displayName.String } From 8d8f4689a0c71f79051554e69fb9113592a4b2e4 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 7 Oct 2022 12:21:38 +0100 Subject: [PATCH 051/207] tDatabase transaction tweaks in roomserver --- roomserver/storage/shared/storage.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index d83a1ff74..8859571b7 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -598,7 +598,7 @@ func (d *Database) storeEvent( } // First writer is with a database-provided transaction, so that NIDs are assigned // globally outside of the updater context, to help avoid races. - err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + err = d.Writer.Do(d.DB, txn, func(txn *sql.Tx) error { // TODO: Here we should aim to have two different code paths for new rooms // vs existing ones. @@ -964,9 +964,9 @@ func (d *Database) loadRedactionPair( } if isRedactionEvent { - redactedEvent = d.loadEvent(ctx, info.RedactsEventID) + redactedEvent = d.loadEvent(ctx, txn, info.RedactsEventID) } else { - redactionEvent = d.loadEvent(ctx, info.RedactionEventID) + redactionEvent = d.loadEvent(ctx, txn, info.RedactionEventID) } return redactionEvent, redactedEvent, info.Validated, nil @@ -982,15 +982,15 @@ func (d *Database) applyRedactions(events []types.Event) { } // loadEvent loads a single event or returns nil on any problems/missing event -func (d *Database) loadEvent(ctx context.Context, eventID string) *types.Event { - nids, err := d.EventNIDs(ctx, []string{eventID}) +func (d *Database) loadEvent(ctx context.Context, txn *sql.Tx, eventID string) *types.Event { + nids, err := d.eventNIDs(ctx, txn, []string{eventID}, NoFilter) if err != nil { return nil } if len(nids) == 0 { return nil } - evs, err := d.Events(ctx, []types.EventNID{nids[eventID]}) + evs, err := d.events(ctx, txn, []types.EventNID{nids[eventID]}) if err != nil { return nil } @@ -1358,7 +1358,7 @@ func (d *Database) ForgetRoom(ctx context.Context, userID, roomID string, forget } return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - return d.MembershipTable.UpdateForgetMembership(ctx, nil, roomNIDs[0], stateKeyNID, forget) + return d.MembershipTable.UpdateForgetMembership(ctx, txn, roomNIDs[0], stateKeyNID, forget) }) } From 1b5460a920d271aa03c798709f2f1b3b710e4e0e Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 7 Oct 2022 13:42:35 +0100 Subject: [PATCH 052/207] Ensure we only wake up a given user once (#2775) This ensures that the sync API notifier only wakes up a given user once for a given stream position. --- syncapi/notifier/notifier.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/syncapi/notifier/notifier.go b/syncapi/notifier/notifier.go index db18c6b77..27f7c37ba 100644 --- a/syncapi/notifier/notifier.go +++ b/syncapi/notifier/notifier.go @@ -48,6 +48,7 @@ type Notifier struct { lastCleanUpTime time.Time // This map is reused to prevent allocations and GC pressure in SharedUsers. _sharedUserMap map[string]struct{} + _wakeupUserMap map[string]struct{} } // NewNotifier creates a new notifier set to the given sync position. @@ -61,6 +62,7 @@ func NewNotifier() *Notifier { lock: &sync.RWMutex{}, lastCleanUpTime: time.Now(), _sharedUserMap: map[string]struct{}{}, + _wakeupUserMap: map[string]struct{}{}, } } @@ -408,12 +410,16 @@ func (n *Notifier) setPeekingDevices(roomIDToPeekingDevices map[string][]types.P // specified user IDs, and also the specified peekingDevices func (n *Notifier) _wakeupUsers(userIDs []string, peekingDevices []types.PeekingDevice, newPos types.StreamingToken) { for _, userID := range userIDs { + n._wakeupUserMap[userID] = struct{}{} + } + for userID := range n._wakeupUserMap { for _, stream := range n._fetchUserStreams(userID) { if stream == nil { continue } stream.Broadcast(newPos) // wake up all goroutines Wait()ing on this stream } + delete(n._wakeupUserMap, userID) } for _, peekingDevice := range peekingDevices { From 8e231130e97ce716357bbb4af8f82159dc6e684e Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 7 Oct 2022 14:05:06 +0100 Subject: [PATCH 053/207] Revert "tDatabase transaction tweaks in roomserver" This reverts commit 8d8f4689a0c71f79051554e69fb9113592a4b2e4. --- roomserver/storage/shared/storage.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index 8859571b7..d83a1ff74 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -598,7 +598,7 @@ func (d *Database) storeEvent( } // First writer is with a database-provided transaction, so that NIDs are assigned // globally outside of the updater context, to help avoid races. - err = d.Writer.Do(d.DB, txn, func(txn *sql.Tx) error { + err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { // TODO: Here we should aim to have two different code paths for new rooms // vs existing ones. @@ -964,9 +964,9 @@ func (d *Database) loadRedactionPair( } if isRedactionEvent { - redactedEvent = d.loadEvent(ctx, txn, info.RedactsEventID) + redactedEvent = d.loadEvent(ctx, info.RedactsEventID) } else { - redactionEvent = d.loadEvent(ctx, txn, info.RedactionEventID) + redactionEvent = d.loadEvent(ctx, info.RedactionEventID) } return redactionEvent, redactedEvent, info.Validated, nil @@ -982,15 +982,15 @@ func (d *Database) applyRedactions(events []types.Event) { } // loadEvent loads a single event or returns nil on any problems/missing event -func (d *Database) loadEvent(ctx context.Context, txn *sql.Tx, eventID string) *types.Event { - nids, err := d.eventNIDs(ctx, txn, []string{eventID}, NoFilter) +func (d *Database) loadEvent(ctx context.Context, eventID string) *types.Event { + nids, err := d.EventNIDs(ctx, []string{eventID}) if err != nil { return nil } if len(nids) == 0 { return nil } - evs, err := d.events(ctx, txn, []types.EventNID{nids[eventID]}) + evs, err := d.Events(ctx, []types.EventNID{nids[eventID]}) if err != nil { return nil } @@ -1358,7 +1358,7 @@ func (d *Database) ForgetRoom(ctx context.Context, userID, roomID string, forget } return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - return d.MembershipTable.UpdateForgetMembership(ctx, txn, roomNIDs[0], stateKeyNID, forget) + return d.MembershipTable.UpdateForgetMembership(ctx, nil, roomNIDs[0], stateKeyNID, forget) }) } From 1ca3f3efb512db4c1827eb571588ec31258782a5 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Fri, 7 Oct 2022 16:00:12 +0200 Subject: [PATCH 054/207] Fix issue with DMs shown as normal rooms (#2776) Fixes #2121, test added in https://github.com/matrix-org/complement/pull/494 --- federationapi/api/api.go | 1 + federationapi/internal/perform.go | 19 +++++++++--- roomserver/api/perform.go | 1 + roomserver/internal/helpers/helpers.go | 31 ++++++++++--------- roomserver/internal/perform/perform_join.go | 20 ++++++++++-- roomserver/internal/perform/perform_leave.go | 2 +- roomserver/internal/query/query.go | 2 +- roomserver/storage/interface.go | 5 +-- roomserver/storage/postgres/invite_table.go | 13 ++++---- roomserver/storage/shared/storage.go | 9 +++--- roomserver/storage/sqlite3/invite_table.go | 13 ++++---- roomserver/storage/tables/interface.go | 2 +- .../storage/tables/invite_table_test.go | 11 ++++--- 13 files changed, 82 insertions(+), 47 deletions(-) diff --git a/federationapi/api/api.go b/federationapi/api/api.go index f25538784..362333fc9 100644 --- a/federationapi/api/api.go +++ b/federationapi/api/api.go @@ -159,6 +159,7 @@ type PerformJoinRequest struct { // The sorted list of servers to try. Servers will be tried sequentially, after de-duplication. ServerNames types.ServerNames `json:"server_names"` Content map[string]interface{} `json:"content"` + Unsigned map[string]interface{} `json:"unsigned"` } type PerformJoinResponse struct { diff --git a/federationapi/internal/perform.go b/federationapi/internal/perform.go index 4cdd3a5eb..28ec48d7b 100644 --- a/federationapi/internal/perform.go +++ b/federationapi/internal/perform.go @@ -7,14 +7,15 @@ import ( "fmt" "time" - "github.com/matrix-org/dendrite/federationapi/api" - "github.com/matrix-org/dendrite/federationapi/consumers" - roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/roomserver/version" "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" "github.com/sirupsen/logrus" + + "github.com/matrix-org/dendrite/federationapi/api" + "github.com/matrix-org/dendrite/federationapi/consumers" + roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/roomserver/version" ) // PerformLeaveRequest implements api.FederationInternalAPI @@ -95,6 +96,7 @@ func (r *FederationInternalAPI) PerformJoin( request.Content, serverName, supportedVersions, + request.Unsigned, ); err != nil { logrus.WithError(err).WithFields(logrus.Fields{ "server_name": serverName, @@ -139,6 +141,7 @@ func (r *FederationInternalAPI) performJoinUsingServer( content map[string]interface{}, serverName gomatrixserverlib.ServerName, supportedVersions []gomatrixserverlib.RoomVersion, + unsigned map[string]interface{}, ) error { // Try to perform a make_join using the information supplied in the // request. @@ -267,6 +270,14 @@ func (r *FederationInternalAPI) performJoinUsingServer( // If we successfully performed a send_join above then the other // server now thinks we're a part of the room. Send the newly // returned state to the roomserver to update our local view. + if unsigned != nil { + event, err = event.SetUnsigned(unsigned) + if err != nil { + // non-fatal, log and continue + logrus.WithError(err).Errorf("Failed to set unsigned content") + } + } + if err = roomserverAPI.SendEventWithState( context.Background(), r.rsAPI, diff --git a/roomserver/api/perform.go b/roomserver/api/perform.go index 20931f807..7a362f969 100644 --- a/roomserver/api/perform.go +++ b/roomserver/api/perform.go @@ -80,6 +80,7 @@ type PerformJoinRequest struct { UserID string `json:"user_id"` Content map[string]interface{} `json:"content"` ServerNames []gomatrixserverlib.ServerName `json:"server_names"` + Unsigned map[string]interface{} `json:"unsigned"` } type PerformJoinResponse struct { diff --git a/roomserver/internal/helpers/helpers.go b/roomserver/internal/helpers/helpers.go index cbd1561f7..3b83a0a6d 100644 --- a/roomserver/internal/helpers/helpers.go +++ b/roomserver/internal/helpers/helpers.go @@ -7,6 +7,9 @@ import ( "fmt" "strings" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" + "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/auth" "github.com/matrix-org/dendrite/roomserver/state" @@ -14,8 +17,6 @@ import ( "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" - "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" ) // TODO: temporary package which has helper functions used by both internal/perform packages. @@ -97,35 +98,35 @@ func IsServerCurrentlyInRoom(ctx context.Context, db storage.Database, serverNam func IsInvitePending( ctx context.Context, db storage.Database, roomID, userID string, -) (bool, string, string, error) { +) (bool, string, string, *gomatrixserverlib.Event, error) { // Look up the room NID for the supplied room ID. info, err := db.RoomInfo(ctx, roomID) if err != nil { - return false, "", "", fmt.Errorf("r.DB.RoomInfo: %w", err) + return false, "", "", nil, fmt.Errorf("r.DB.RoomInfo: %w", err) } if info == nil { - return false, "", "", fmt.Errorf("cannot get RoomInfo: unknown room ID %s", roomID) + return false, "", "", nil, fmt.Errorf("cannot get RoomInfo: unknown room ID %s", roomID) } // Look up the state key NID for the supplied user ID. targetUserNIDs, err := db.EventStateKeyNIDs(ctx, []string{userID}) if err != nil { - return false, "", "", fmt.Errorf("r.DB.EventStateKeyNIDs: %w", err) + return false, "", "", nil, fmt.Errorf("r.DB.EventStateKeyNIDs: %w", err) } targetUserNID, targetUserFound := targetUserNIDs[userID] if !targetUserFound { - return false, "", "", fmt.Errorf("missing NID for user %q (%+v)", userID, targetUserNIDs) + return false, "", "", nil, fmt.Errorf("missing NID for user %q (%+v)", userID, targetUserNIDs) } // Let's see if we have an event active for the user in the room. If // we do then it will contain a server name that we can direct the // send_leave to. - senderUserNIDs, eventIDs, err := db.GetInvitesForUser(ctx, info.RoomNID, targetUserNID) + senderUserNIDs, eventIDs, eventJSON, err := db.GetInvitesForUser(ctx, info.RoomNID, targetUserNID) if err != nil { - return false, "", "", fmt.Errorf("r.DB.GetInvitesForUser: %w", err) + return false, "", "", nil, fmt.Errorf("r.DB.GetInvitesForUser: %w", err) } if len(senderUserNIDs) == 0 { - return false, "", "", nil + return false, "", "", nil, nil } userNIDToEventID := make(map[types.EventStateKeyNID]string) for i, nid := range senderUserNIDs { @@ -135,18 +136,20 @@ func IsInvitePending( // Look up the user ID from the NID. senderUsers, err := db.EventStateKeys(ctx, senderUserNIDs) if err != nil { - return false, "", "", fmt.Errorf("r.DB.EventStateKeys: %w", err) + return false, "", "", nil, fmt.Errorf("r.DB.EventStateKeys: %w", err) } if len(senderUsers) == 0 { - return false, "", "", fmt.Errorf("no senderUsers") + return false, "", "", nil, fmt.Errorf("no senderUsers") } senderUser, senderUserFound := senderUsers[senderUserNIDs[0]] if !senderUserFound { - return false, "", "", fmt.Errorf("missing user for NID %d (%+v)", senderUserNIDs[0], senderUsers) + return false, "", "", nil, fmt.Errorf("missing user for NID %d (%+v)", senderUserNIDs[0], senderUsers) } - return true, senderUser, userNIDToEventID[senderUserNIDs[0]], nil + event, err := gomatrixserverlib.NewEventFromTrustedJSON(eventJSON, false, info.RoomVersion) + + return true, senderUser, userNIDToEventID[senderUserNIDs[0]], event, err } // GetMembershipsAtState filters the state events to diff --git a/roomserver/internal/perform/perform_join.go b/roomserver/internal/perform/perform_join.go index 167b375b7..262273ff5 100644 --- a/roomserver/internal/perform/perform_join.go +++ b/roomserver/internal/perform/perform_join.go @@ -22,6 +22,10 @@ import ( "time" "github.com/getsentry/sentry-go" + "github.com/matrix-org/gomatrixserverlib" + "github.com/sirupsen/logrus" + "github.com/tidwall/gjson" + fsAPI "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/roomserver/api" @@ -32,8 +36,6 @@ import ( "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/setup/config" - "github.com/matrix-org/gomatrixserverlib" - "github.com/sirupsen/logrus" ) type Joiner struct { @@ -236,7 +238,7 @@ func (r *Joiner) performJoinRoomByID( // Force a federated join if we're dealing with a pending invite // and we aren't in the room. - isInvitePending, inviteSender, _, err := helpers.IsInvitePending(ctx, r.DB, req.RoomIDOrAlias, req.UserID) + isInvitePending, inviteSender, _, inviteEvent, err := helpers.IsInvitePending(ctx, r.DB, req.RoomIDOrAlias, req.UserID) if err == nil && !serverInRoom && isInvitePending { _, inviterDomain, ierr := gomatrixserverlib.SplitID('@', inviteSender) if ierr != nil { @@ -248,6 +250,17 @@ func (r *Joiner) performJoinRoomByID( if inviterDomain != r.Cfg.Matrix.ServerName { req.ServerNames = append(req.ServerNames, inviterDomain) forceFederatedJoin = true + memberEvent := gjson.Parse(string(inviteEvent.JSON())) + // only set unsigned if we've got a content.membership, which we _should_ + if memberEvent.Get("content.membership").Exists() { + req.Unsigned = map[string]interface{}{ + "prev_sender": memberEvent.Get("sender").Str, + "prev_content": map[string]interface{}{ + "is_direct": memberEvent.Get("content.is_direct").Bool(), + "membership": memberEvent.Get("content.membership").Str, + }, + } + } } } @@ -348,6 +361,7 @@ func (r *Joiner) performFederatedJoinRoomByID( UserID: req.UserID, // the user ID joining the room ServerNames: req.ServerNames, // the server to try joining with Content: req.Content, // the membership event content + Unsigned: req.Unsigned, // the unsigned event content, if any } fedRes := fsAPI.PerformJoinResponse{} r.FSAPI.PerformJoin(ctx, &fedReq, &fedRes) diff --git a/roomserver/internal/perform/perform_leave.go b/roomserver/internal/perform/perform_leave.go index ada3aab06..85b659814 100644 --- a/roomserver/internal/perform/perform_leave.go +++ b/roomserver/internal/perform/perform_leave.go @@ -79,7 +79,7 @@ func (r *Leaver) performLeaveRoomByID( ) ([]api.OutputEvent, error) { // If there's an invite outstanding for the room then respond to // that. - isInvitePending, senderUser, eventID, err := helpers.IsInvitePending(ctx, r.DB, req.RoomID, req.UserID) + isInvitePending, senderUser, eventID, _, err := helpers.IsInvitePending(ctx, r.DB, req.RoomID, req.UserID) if err == nil && isInvitePending { _, senderDomain, serr := gomatrixserverlib.SplitID('@', senderUser) if serr != nil { diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go index ee8e1cfe7..7a424a334 100644 --- a/roomserver/internal/query/query.go +++ b/roomserver/internal/query/query.go @@ -872,7 +872,7 @@ func (r *Queryer) QueryRestrictedJoinAllowed(ctx context.Context, req *api.Query // but we don't specify an authorised via user, since the event auth // will allow the join anyway. var pending bool - if pending, _, _, err = helpers.IsInvitePending(ctx, r.DB, req.RoomID, req.UserID); err != nil { + if pending, _, _, _, err = helpers.IsInvitePending(ctx, r.DB, req.RoomID, req.UserID); err != nil { return fmt.Errorf("helpers.IsInvitePending: %w", err) } else if pending { res.Allowed = true diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go index 11e175f55..ee0624b21 100644 --- a/roomserver/storage/interface.go +++ b/roomserver/storage/interface.go @@ -17,10 +17,11 @@ package storage import ( "context" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/dendrite/roomserver/storage/shared" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" - "github.com/matrix-org/gomatrixserverlib" ) type Database interface { @@ -104,7 +105,7 @@ type Database interface { // Look up the active invites targeting a user in a room and return the // numeric state key IDs for the user IDs who sent them along with the event IDs for the invites. // Returns an error if there was a problem talking to the database. - GetInvitesForUser(ctx context.Context, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID) (senderUserIDs []types.EventStateKeyNID, eventIDs []string, err error) + GetInvitesForUser(ctx context.Context, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID) (senderUserIDs []types.EventStateKeyNID, eventIDs []string, inviteEventJSON []byte, err error) // Save a given room alias with the room ID it refers to. // Returns an error if there was a problem talking to the database. SetRoomAlias(ctx context.Context, alias string, roomID string, creatorUserID string) error diff --git a/roomserver/storage/postgres/invite_table.go b/roomserver/storage/postgres/invite_table.go index 4cddfe2e9..009fd1ac1 100644 --- a/roomserver/storage/postgres/invite_table.go +++ b/roomserver/storage/postgres/invite_table.go @@ -61,7 +61,7 @@ const insertInviteEventSQL = "" + " ON CONFLICT DO NOTHING" const selectInviteActiveForUserInRoomSQL = "" + - "SELECT invite_event_id, sender_nid FROM roomserver_invites" + + "SELECT invite_event_id, sender_nid, invite_event_json FROM roomserver_invites" + " WHERE target_nid = $1 AND room_nid = $2" + " AND NOT retired" @@ -141,25 +141,26 @@ func (s *inviteStatements) UpdateInviteRetired( func (s *inviteStatements) SelectInviteActiveForUserInRoom( ctx context.Context, txn *sql.Tx, targetUserNID types.EventStateKeyNID, roomNID types.RoomNID, -) ([]types.EventStateKeyNID, []string, error) { +) ([]types.EventStateKeyNID, []string, []byte, error) { stmt := sqlutil.TxStmt(txn, s.selectInviteActiveForUserInRoomStmt) rows, err := stmt.QueryContext( ctx, targetUserNID, roomNID, ) if err != nil { - return nil, nil, err + return nil, nil, nil, err } defer internal.CloseAndLogIfError(ctx, rows, "selectInviteActiveForUserInRoom: rows.close() failed") var result []types.EventStateKeyNID var eventIDs []string var inviteEventID string var senderUserNID int64 + var eventJSON []byte for rows.Next() { - if err := rows.Scan(&inviteEventID, &senderUserNID); err != nil { - return nil, nil, err + if err := rows.Scan(&inviteEventID, &senderUserNID, &eventJSON); err != nil { + return nil, nil, nil, err } result = append(result, types.EventStateKeyNID(senderUserNID)) eventIDs = append(eventIDs, inviteEventID) } - return result, eventIDs, rows.Err() + return result, eventIDs, eventJSON, rows.Err() } diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index d83a1ff74..e401f17dc 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -7,13 +7,14 @@ import ( "fmt" "sort" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" + "github.com/tidwall/gjson" + "github.com/matrix-org/dendrite/internal/caching" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" - "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" - "github.com/tidwall/gjson" ) // Ideally, when we have both events we should redact the event JSON and forget about the redaction, but we currently @@ -445,7 +446,7 @@ func (d *Database) GetInvitesForUser( ctx context.Context, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, -) (senderUserIDs []types.EventStateKeyNID, eventIDs []string, err error) { +) (senderUserIDs []types.EventStateKeyNID, eventIDs []string, inviteEventJSON []byte, err error) { return d.InvitesTable.SelectInviteActiveForUserInRoom(ctx, nil, targetUserNID, roomNID) } diff --git a/roomserver/storage/sqlite3/invite_table.go b/roomserver/storage/sqlite3/invite_table.go index e051d63af..ca6e7c511 100644 --- a/roomserver/storage/sqlite3/invite_table.go +++ b/roomserver/storage/sqlite3/invite_table.go @@ -44,7 +44,7 @@ const insertInviteEventSQL = "" + " ON CONFLICT DO NOTHING" const selectInviteActiveForUserInRoomSQL = "" + - "SELECT invite_event_id, sender_nid FROM roomserver_invites" + + "SELECT invite_event_id, sender_nid, invite_event_json FROM roomserver_invites" + " WHERE target_nid = $1 AND room_nid = $2" + " AND NOT retired" @@ -136,25 +136,26 @@ func (s *inviteStatements) UpdateInviteRetired( func (s *inviteStatements) SelectInviteActiveForUserInRoom( ctx context.Context, txn *sql.Tx, targetUserNID types.EventStateKeyNID, roomNID types.RoomNID, -) ([]types.EventStateKeyNID, []string, error) { +) ([]types.EventStateKeyNID, []string, []byte, error) { stmt := sqlutil.TxStmt(txn, s.selectInviteActiveForUserInRoomStmt) rows, err := stmt.QueryContext( ctx, targetUserNID, roomNID, ) if err != nil { - return nil, nil, err + return nil, nil, nil, err } defer internal.CloseAndLogIfError(ctx, rows, "selectInviteActiveForUserInRoom: rows.close() failed") var result []types.EventStateKeyNID var eventIDs []string var eventID string var senderUserNID int64 + var eventJSON []byte for rows.Next() { - if err := rows.Scan(&eventID, &senderUserNID); err != nil { - return nil, nil, err + if err := rows.Scan(&eventID, &senderUserNID, &eventJSON); err != nil { + return nil, nil, nil, err } result = append(result, types.EventStateKeyNID(senderUserNID)) eventIDs = append(eventIDs, eventID) } - return result, eventIDs, nil + return result, eventIDs, eventJSON, nil } diff --git a/roomserver/storage/tables/interface.go b/roomserver/storage/tables/interface.go index d7bcc95ab..8be47855f 100644 --- a/roomserver/storage/tables/interface.go +++ b/roomserver/storage/tables/interface.go @@ -116,7 +116,7 @@ type Invites interface { InsertInviteEvent(ctx context.Context, txn *sql.Tx, inviteEventID string, roomNID types.RoomNID, targetUserNID, senderUserNID types.EventStateKeyNID, inviteEventJSON []byte) (bool, error) UpdateInviteRetired(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID) ([]string, error) // SelectInviteActiveForUserInRoom returns a list of sender state key NIDs and invite event IDs matching those nids. - SelectInviteActiveForUserInRoom(ctx context.Context, txn *sql.Tx, targetUserNID types.EventStateKeyNID, roomNID types.RoomNID) ([]types.EventStateKeyNID, []string, error) + SelectInviteActiveForUserInRoom(ctx context.Context, txn *sql.Tx, targetUserNID types.EventStateKeyNID, roomNID types.RoomNID) ([]types.EventStateKeyNID, []string, []byte, error) } type MembershipState int64 diff --git a/roomserver/storage/tables/invite_table_test.go b/roomserver/storage/tables/invite_table_test.go index 8df3faa2d..e3eedbf14 100644 --- a/roomserver/storage/tables/invite_table_test.go +++ b/roomserver/storage/tables/invite_table_test.go @@ -4,6 +4,9 @@ import ( "context" "testing" + "github.com/matrix-org/util" + "github.com/stretchr/testify/assert" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/postgres" "github.com/matrix-org/dendrite/roomserver/storage/sqlite3" @@ -11,8 +14,6 @@ import ( "github.com/matrix-org/dendrite/roomserver/types" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/test" - "github.com/matrix-org/util" - "github.com/stretchr/testify/assert" ) func mustCreateInviteTable(t *testing.T, dbType test.DBType) (tables.Invites, func()) { @@ -67,7 +68,7 @@ func TestInviteTable(t *testing.T) { assert.NoError(t, err) assert.True(t, newInvite) - stateKeyNIDs, eventIDs, err := tab.SelectInviteActiveForUserInRoom(ctx, nil, targetUserNID, roomNID) + stateKeyNIDs, eventIDs, _, err := tab.SelectInviteActiveForUserInRoom(ctx, nil, targetUserNID, roomNID) assert.NoError(t, err) assert.Equal(t, []string{eventID1, eventID2}, eventIDs) assert.Equal(t, []types.EventStateKeyNID{2, 2}, stateKeyNIDs) @@ -78,13 +79,13 @@ func TestInviteTable(t *testing.T) { assert.Equal(t, []string{eventID1, eventID2}, retiredEventIDs) // This should now be empty - stateKeyNIDs, eventIDs, err = tab.SelectInviteActiveForUserInRoom(ctx, nil, targetUserNID, roomNID) + stateKeyNIDs, eventIDs, _, err = tab.SelectInviteActiveForUserInRoom(ctx, nil, targetUserNID, roomNID) assert.NoError(t, err) assert.Empty(t, eventIDs) assert.Empty(t, stateKeyNIDs) // Non-existent targetUserNID - stateKeyNIDs, eventIDs, err = tab.SelectInviteActiveForUserInRoom(ctx, nil, types.EventStateKeyNID(10), roomNID) + stateKeyNIDs, eventIDs, _, err = tab.SelectInviteActiveForUserInRoom(ctx, nil, types.EventStateKeyNID(10), roomNID) assert.NoError(t, err) assert.Empty(t, stateKeyNIDs) assert.Empty(t, eventIDs) From f1b8df0f49a8a2f2c3c9c4d421b880bd2b008e08 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 7 Oct 2022 15:55:29 +0100 Subject: [PATCH 055/207] Version 0.10.2 (#2778) Changelog and version bump. --- CHANGES.md | 22 ++++++++++++++++++++++ internal/version.go | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index dbe2ccf02..7552e41e5 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,27 @@ # Changelog +## Dendrite 0.10.2 (2022-10-07) + +### Features + +* Dendrite will now fail to start if there is an obvious problem with the configured `max_open_conns` when using PostgreSQL database backends, since this can lead to instability and performance issues + * More information on this is available [in the documentation](https://matrix-org.github.io/dendrite/installation/start/optimisation#postgresql-connection-limit) +* Unnecessary/empty fields will no longer be sent in `/sync` responses +* It is now possible to configure `old_private_keys` from previous Matrix installations on the same domain if only public key is known, to make it easier to expire old keys correctly + * You can configure either just the `private_key` path, or you can supply both the `public_key` and `key_id` + +### Fixes + +* The sync transaction behaviour has been modified further so that errors in one stream should not propagate to other streams unnecessarily +* Rooms should now be classified as DM rooms correctly by passing through `is_direct` and unsigned hints +* A bug which caused marking device lists as stale to consume lots of CPU has been fixed +* Users accepting invites should no longer cause unnecessary federated joins if there are already other local users in the room +* The sync API state range queries have been optimised by adding missing indexes +* It should now be possible to configure non-English languages for full-text search in `search.language` +* The roomserver will no longer attempt to perform federated requests to the local server when trying to fetch missing events +* The `/keys/upload` endpoint will now always return the `one_time_keys_counts`, which may help with E2EE reliability +* The sync API will now retrieve the latest stream position before processing each stream rather than at the beginning of the request, to hopefully reduce the number of round-trips to `/sync` + ## Dendrite 0.10.1 (2022-09-30) ### Features diff --git a/internal/version.go b/internal/version.go index d508517be..56b83f852 100644 --- a/internal/version.go +++ b/internal/version.go @@ -17,7 +17,7 @@ var build string const ( VersionMajor = 0 VersionMinor = 10 - VersionPatch = 1 + VersionPatch = 2 VersionTag = "" // example: "rc1" ) From 980fa55846811eeff89f116c49b38b085143c64e Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 10 Oct 2022 10:39:29 +0100 Subject: [PATCH 056/207] Stronger passwordless account checks (fixes #2780) --- userapi/internal/api.go | 2 ++ userapi/storage/shared/storage.go | 3 +++ 2 files changed, 5 insertions(+) diff --git a/userapi/internal/api.go b/userapi/internal/api.go index 591faffd6..2f7795dfe 100644 --- a/userapi/internal/api.go +++ b/userapi/internal/api.go @@ -838,6 +838,8 @@ func (a *UserInternalAPI) QueryAccountByPassword(ctx context.Context, req *api.Q return nil case bcrypt.ErrMismatchedHashAndPassword: // user exists, but password doesn't match return nil + case bcrypt.ErrHashTooShort: // user exists, but probably a passwordless account + return nil default: res.Exists = true res.Account = acc diff --git a/userapi/storage/shared/storage.go b/userapi/storage/shared/storage.go index 3ff299f1b..09eeedc9f 100644 --- a/userapi/storage/shared/storage.go +++ b/userapi/storage/shared/storage.go @@ -75,6 +75,9 @@ func (d *Database) GetAccountByPassword( if err != nil { return nil, err } + if hash == "" { + return nil, bcrypt.ErrHashTooShort + } if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(plaintextPassword)); err != nil { return nil, err } From 04bab142901b9dc4479b2cf4e5504fbdd9dd3cf1 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 10 Oct 2022 10:45:15 +0100 Subject: [PATCH 057/207] Add regression test for 980fa55846811eeff89f116c49b38b085143c64e --- userapi/userapi_test.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/userapi/userapi_test.go b/userapi/userapi_test.go index 31a69793b..984fe8854 100644 --- a/userapi/userapi_test.go +++ b/userapi/userapi_test.go @@ -151,6 +151,33 @@ func TestQueryProfile(t *testing.T) { }) } +// TestPasswordlessLoginFails ensures that a passwordless account cannot +// be logged into using an arbitrary password (effectively a regression test +// for https://github.com/matrix-org/dendrite/issues/2780). +func TestPasswordlessLoginFails(t *testing.T) { + ctx := context.Background() + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + userAPI, accountDB, close := MustMakeInternalAPI(t, apiTestOpts{}, dbType) + defer close() + _, err := accountDB.CreateAccount(ctx, "auser", "", "", api.AccountTypeAppService) + if err != nil { + t.Fatalf("failed to make account: %s", err) + } + + userReq := &api.QueryAccountByPasswordRequest{ + Localpart: "auser", + PlaintextPassword: "apassword", + } + userRes := &api.QueryAccountByPasswordResponse{} + if err := userAPI.QueryAccountByPassword(ctx, userReq, userRes); err != nil { + t.Fatal(err) + } + if userRes.Exists || userRes.Account != nil { + t.Fatal("QueryAccountByPassword should not return correctly for a passwordless account") + } + }) +} + func TestLoginToken(t *testing.T) { ctx := context.Background() From b32b6d6e8eb5d0a6c22e9215e2fccf4bdd1db8c3 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 10 Oct 2022 11:03:52 +0100 Subject: [PATCH 058/207] Update issue and pull request templates --- .github/ISSUE_TEMPLATE/BUG_REPORT.md | 22 ++++++++++++---------- .github/PULL_REQUEST_TEMPLATE.md | 4 ++-- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/BUG_REPORT.md b/.github/ISSUE_TEMPLATE/BUG_REPORT.md index 206713e04..49c9a37ef 100644 --- a/.github/ISSUE_TEMPLATE/BUG_REPORT.md +++ b/.github/ISSUE_TEMPLATE/BUG_REPORT.md @@ -7,24 +7,27 @@ about: Create a report to help us improve ### Background information -- **Dendrite version or git SHA**: -- **Monolith or Polylith?**: -- **SQLite3 or Postgres?**: -- **Running in Docker?**: +- **Dendrite version or git SHA**: +- **Monolith or Polylith?**: +- **SQLite3 or Postgres?**: +- **Running in Docker?**: - **`go version`**: - **Client used (if applicable)**: - ### Description - - **What** is the problem: - - **Who** is affected: - - **How** is this bug manifesting: - - **When** did this first appear: +- **What** is the problem: +- **Who** is affected: +- **How** is this bug manifesting: +- **When** did this first appear: + * [ ] I have added tests for PR _or_ I have justified why this PR doesn't need tests. -* [ ] Pull request includes a [sign off](https://github.com/matrix-org/dendrite/blob/main/docs/CONTRIBUTING.md#sign-off) +* [ ] Pull request includes a [sign off below using a legally identifiable name](https://matrix-org.github.io/dendrite/development/contributing#sign-off) _or_ I have already signed off privately Signed-off-by: `Your Name ` From 80a0ab6246aa095f428430c38b13861406dd5c78 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 10 Oct 2022 11:09:40 +0100 Subject: [PATCH 059/207] Further tweak to the issue template --- .github/ISSUE_TEMPLATE/BUG_REPORT.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/BUG_REPORT.md b/.github/ISSUE_TEMPLATE/BUG_REPORT.md index 49c9a37ef..f40c56609 100644 --- a/.github/ISSUE_TEMPLATE/BUG_REPORT.md +++ b/.github/ISSUE_TEMPLATE/BUG_REPORT.md @@ -8,9 +8,10 @@ about: Create a report to help us improve All bug reports must provide the following background information Text between ### Background information From fb6cb2dbcbeb7cd7546ca4d126394720d215c310 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 10 Oct 2022 11:14:16 +0100 Subject: [PATCH 060/207] Tweak `GetAccountByPassword` more --- clientapi/auth/password.go | 6 ++++++ userapi/storage/shared/storage.go | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/clientapi/auth/password.go b/clientapi/auth/password.go index bcb4ca97b..890b18183 100644 --- a/clientapi/auth/password.go +++ b/clientapi/auth/password.go @@ -68,6 +68,12 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, JSON: jsonerror.BadJSON("A username must be supplied."), } } + if len(r.Password) == 0 { + return nil, &util.JSONResponse{ + Code: http.StatusUnauthorized, + JSON: jsonerror.BadJSON("A password must be supplied."), + } + } localpart, err := userutil.ParseUsernameParam(username, &t.Config.Matrix.ServerName) if err != nil { return nil, &util.JSONResponse{ diff --git a/userapi/storage/shared/storage.go b/userapi/storage/shared/storage.go index 09eeedc9f..4e28f7b5a 100644 --- a/userapi/storage/shared/storage.go +++ b/userapi/storage/shared/storage.go @@ -75,7 +75,7 @@ func (d *Database) GetAccountByPassword( if err != nil { return nil, err } - if hash == "" { + if len(hash) == 0 && len(plaintextPassword) > 0 { return nil, bcrypt.ErrHashTooShort } if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(plaintextPassword)); err != nil { From 0f09e9d196d3375e38a490881e06668a82fb6c40 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Mon, 10 Oct 2022 12:19:16 +0200 Subject: [PATCH 061/207] Move /event to the SyncAPI (#2782) This allows us to apply history visibility without having to recalculate it in the roomserver. Unblocks https://github.com/matrix-org/complement/pull/495, fix missing part of https://github.com/matrix-org/dendrite/issues/617 --- clientapi/routing/getevent.go | 138 ------------------------- clientapi/routing/routing.go | 9 -- docs/caddy/polylith/Caddyfile | 2 +- docs/hiawatha/polylith-sample.conf | 4 +- docs/nginx/polylith-sample.conf | 4 +- syncapi/internal/history_visibility.go | 8 +- syncapi/routing/getevent.go | 102 ++++++++++++++++++ syncapi/routing/routing.go | 10 ++ 8 files changed, 124 insertions(+), 153 deletions(-) delete mode 100644 clientapi/routing/getevent.go create mode 100644 syncapi/routing/getevent.go diff --git a/clientapi/routing/getevent.go b/clientapi/routing/getevent.go deleted file mode 100644 index 7f5842800..000000000 --- a/clientapi/routing/getevent.go +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright 2019 Alex Chen -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package routing - -import ( - "net/http" - - "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/setup/config" - userapi "github.com/matrix-org/dendrite/userapi/api" - "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" -) - -type getEventRequest struct { - req *http.Request - device *userapi.Device - roomID string - eventID string - cfg *config.ClientAPI - requestedEvent *gomatrixserverlib.Event -} - -// GetEvent implements GET /_matrix/client/r0/rooms/{roomId}/event/{eventId} -// https://matrix.org/docs/spec/client_server/r0.4.0.html#get-matrix-client-r0-rooms-roomid-event-eventid -func GetEvent( - req *http.Request, - device *userapi.Device, - roomID string, - eventID string, - cfg *config.ClientAPI, - rsAPI api.ClientRoomserverAPI, -) util.JSONResponse { - eventsReq := api.QueryEventsByIDRequest{ - EventIDs: []string{eventID}, - } - var eventsResp api.QueryEventsByIDResponse - err := rsAPI.QueryEventsByID(req.Context(), &eventsReq, &eventsResp) - if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("queryAPI.QueryEventsByID failed") - return jsonerror.InternalServerError() - } - - if len(eventsResp.Events) == 0 { - // Event not found locally - return util.JSONResponse{ - Code: http.StatusNotFound, - JSON: jsonerror.NotFound("The event was not found or you do not have permission to read this event"), - } - } - - requestedEvent := eventsResp.Events[0].Event - - r := getEventRequest{ - req: req, - device: device, - roomID: roomID, - eventID: eventID, - cfg: cfg, - requestedEvent: requestedEvent, - } - - stateReq := api.QueryStateAfterEventsRequest{ - RoomID: r.requestedEvent.RoomID(), - PrevEventIDs: r.requestedEvent.PrevEventIDs(), - StateToFetch: []gomatrixserverlib.StateKeyTuple{{ - EventType: gomatrixserverlib.MRoomMember, - StateKey: device.UserID, - }}, - } - var stateResp api.QueryStateAfterEventsResponse - if err := rsAPI.QueryStateAfterEvents(req.Context(), &stateReq, &stateResp); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("queryAPI.QueryStateAfterEvents failed") - return jsonerror.InternalServerError() - } - - if !stateResp.RoomExists { - util.GetLogger(req.Context()).Errorf("Expected to find room for event %s but failed", r.requestedEvent.EventID()) - return jsonerror.InternalServerError() - } - - if !stateResp.PrevEventsExist { - // Missing some events locally; stateResp.StateEvents unavailable. - return util.JSONResponse{ - Code: http.StatusNotFound, - JSON: jsonerror.NotFound("The event was not found or you do not have permission to read this event"), - } - } - - var appService *config.ApplicationService - if device.AppserviceID != "" { - for _, as := range cfg.Derived.ApplicationServices { - if as.ID == device.AppserviceID { - appService = &as - break - } - } - } - - for _, stateEvent := range stateResp.StateEvents { - if appService != nil { - if !appService.IsInterestedInUserID(*stateEvent.StateKey()) { - continue - } - } else if !stateEvent.StateKeyEquals(device.UserID) { - continue - } - membership, err := stateEvent.Membership() - if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("stateEvent.Membership failed") - return jsonerror.InternalServerError() - } - if membership == gomatrixserverlib.Join { - return util.JSONResponse{ - Code: http.StatusOK, - JSON: gomatrixserverlib.ToClientEvent(r.requestedEvent, gomatrixserverlib.FormatAll), - } - } - } - - return util.JSONResponse{ - Code: http.StatusNotFound, - JSON: jsonerror.NotFound("The event was not found or you do not have permission to read this event"), - } -} diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 7d1c434c4..f1fa66ca6 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -367,15 +367,6 @@ func Setup( nil, cfg, rsAPI, transactionsCache) }), ).Methods(http.MethodPut, http.MethodOptions) - v3mux.Handle("/rooms/{roomID}/event/{eventID}", - httputil.MakeAuthAPI("rooms_get_event", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) - if err != nil { - return util.ErrorResponse(err) - } - return GetEvent(req, device, vars["roomID"], vars["eventID"], cfg, rsAPI) - }), - ).Methods(http.MethodGet, http.MethodOptions) v3mux.Handle("/rooms/{roomID}/state", httputil.MakeAuthAPI("room_state", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) diff --git a/docs/caddy/polylith/Caddyfile b/docs/caddy/polylith/Caddyfile index 244e50e7e..906097e4e 100644 --- a/docs/caddy/polylith/Caddyfile +++ b/docs/caddy/polylith/Caddyfile @@ -55,7 +55,7 @@ matrix.example.com { # Change the end of each reverse_proxy line to the correct # address for your various services. @sync_api { - path_regexp /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/messages)$ + path_regexp /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|context/.*?|event/.*?))$ } reverse_proxy @sync_api sync_api:8073 diff --git a/docs/hiawatha/polylith-sample.conf b/docs/hiawatha/polylith-sample.conf index 5ed0cb5ae..036140643 100644 --- a/docs/hiawatha/polylith-sample.conf +++ b/docs/hiawatha/polylith-sample.conf @@ -18,8 +18,10 @@ VirtualHost { # /_matrix/client/.*/user/{userId}/filter/{filterID} # /_matrix/client/.*/keys/changes # /_matrix/client/.*/rooms/{roomId}/messages + # /_matrix/client/.*/rooms/{roomId}/context/{eventID} + # /_matrix/client/.*/rooms/{roomId}/event/{eventID} # to sync_api - ReverseProxy = /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/messages) http://localhost:8073 600 + ReverseProxy = /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|context/.*?|event/.*?))$ http://localhost:8073 600 ReverseProxy = /_matrix/client http://localhost:8071 600 ReverseProxy = /_matrix/federation http://localhost:8072 600 ReverseProxy = /_matrix/key http://localhost:8072 600 diff --git a/docs/nginx/polylith-sample.conf b/docs/nginx/polylith-sample.conf index 274d75658..345d8a6b4 100644 --- a/docs/nginx/polylith-sample.conf +++ b/docs/nginx/polylith-sample.conf @@ -28,8 +28,10 @@ server { # /_matrix/client/.*/user/{userId}/filter/{filterID} # /_matrix/client/.*/keys/changes # /_matrix/client/.*/rooms/{roomId}/messages + # /_matrix/client/.*/rooms/{roomId}/context/{eventID} + # /_matrix/client/.*/rooms/{roomId}/event/{eventID} # to sync_api - location ~ /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/messages)$ { + location ~ /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|context/.*?|event/.*?))$ { proxy_pass http://sync_api:8073; } diff --git a/syncapi/internal/history_visibility.go b/syncapi/internal/history_visibility.go index bbfe19f4c..71d7ddd15 100644 --- a/syncapi/internal/history_visibility.go +++ b/syncapi/internal/history_visibility.go @@ -19,11 +19,13 @@ import ( "math" "time" - "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/gomatrixserverlib" "github.com/prometheus/client_golang/prometheus" + "github.com/sirupsen/logrus" "github.com/tidwall/gjson" + + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/syncapi/storage" ) func init() { @@ -189,7 +191,7 @@ func visibilityForEvents( UserID: userID, }, membershipResp) if err != nil { - return result, err + logrus.WithError(err).Error("visibilityForEvents: failed to fetch membership at event, defaulting to 'leave'") } // Create a map from eventID -> eventVisibility diff --git a/syncapi/routing/getevent.go b/syncapi/routing/getevent.go new file mode 100644 index 000000000..d2cdc1b5f --- /dev/null +++ b/syncapi/routing/getevent.go @@ -0,0 +1,102 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routing + +import ( + "net/http" + + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" + "github.com/sirupsen/logrus" + + "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/syncapi/internal" + "github.com/matrix-org/dendrite/syncapi/storage" + userapi "github.com/matrix-org/dendrite/userapi/api" +) + +// GetEvent implements +// +// GET /_matrix/client/r0/rooms/{roomId}/event/{eventId} +// +// https://spec.matrix.org/v1.4/client-server-api/#get_matrixclientv3roomsroomideventeventid +func GetEvent( + req *http.Request, + device *userapi.Device, + roomID string, + eventID string, + cfg *config.SyncAPI, + syncDB storage.Database, + rsAPI api.SyncRoomserverAPI, +) util.JSONResponse { + ctx := req.Context() + db, err := syncDB.NewDatabaseTransaction(ctx) + logger := util.GetLogger(ctx).WithFields(logrus.Fields{ + "event_id": eventID, + "room_id": roomID, + }) + if err != nil { + logger.WithError(err).Error("GetEvent: syncDB.NewDatabaseTransaction failed") + return jsonerror.InternalServerError() + } + + events, err := db.Events(ctx, []string{eventID}) + if err != nil { + logger.WithError(err).Error("GetEvent: syncDB.Events failed") + return jsonerror.InternalServerError() + } + + // The requested event does not exist in our database + if len(events) == 0 { + logger.Debugf("GetEvent: requested event doesn't exist locally") + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound("The event was not found or you do not have permission to read this event"), + } + } + + // If the request is coming from an appservice, get the user from the request + userID := device.UserID + if asUserID := req.FormValue("user_id"); device.AppserviceID != "" && asUserID != "" { + userID = asUserID + } + + // Apply history visibility to determine if the user is allowed to view the event + events, err = internal.ApplyHistoryVisibilityFilter(ctx, db, rsAPI, events, nil, userID, "event") + if err != nil { + logger.WithError(err).Error("GetEvent: internal.ApplyHistoryVisibilityFilter failed") + return util.JSONResponse{ + Code: http.StatusInternalServerError, + JSON: jsonerror.InternalServerError(), + } + } + + // We only ever expect there to be one event + if len(events) != 1 { + // 0 events -> not allowed to view event; > 1 events -> something that shouldn't happen + logger.WithField("event_count", len(events)).Debug("GetEvent: can't return the requested event") + return util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound("The event was not found or you do not have permission to read this event"), + } + } + + return util.JSONResponse{ + Code: http.StatusOK, + JSON: gomatrixserverlib.HeaderedToClientEvent(events[0], gomatrixserverlib.FormatAll), + } +} diff --git a/syncapi/routing/routing.go b/syncapi/routing/routing.go index 8f84a1341..069dee81f 100644 --- a/syncapi/routing/routing.go +++ b/syncapi/routing/routing.go @@ -60,6 +60,16 @@ func Setup( return OnIncomingMessagesRequest(req, syncDB, vars["roomID"], device, rsAPI, cfg, srp, lazyLoadCache) })).Methods(http.MethodGet, http.MethodOptions) + v3mux.Handle("/rooms/{roomID}/event/{eventID}", + httputil.MakeAuthAPI("rooms_get_event", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return GetEvent(req, device, vars["roomID"], vars["eventID"], cfg, syncDB, rsAPI) + }), + ).Methods(http.MethodGet, http.MethodOptions) + v3mux.Handle("/user/{userId}/filter", httputil.MakeAuthAPI("put_filter", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) From dcc01162874d2634738080614f02cbbeb6bc598f Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Mon, 10 Oct 2022 15:38:00 +0200 Subject: [PATCH 062/207] SyTest List Maintenance --- sytest-blacklist | 1 - sytest-whitelist | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/sytest-blacklist b/sytest-blacklist index 5b2e973a6..634c07cf3 100644 --- a/sytest-blacklist +++ b/sytest-blacklist @@ -47,7 +47,6 @@ Notifications can be viewed with GET /notifications # More flakey -If remote user leaves room we no longer receive device updates Guest users can join guest_access rooms # This will fail in HTTP API mode, so blacklisted for now diff --git a/sytest-whitelist b/sytest-whitelist index 31940b884..9ba9df75d 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -742,3 +742,4 @@ User in private room doesn't appear in user directory User joining then leaving public room appears and dissappears from directory User in remote room doesn't appear in user directory after server left room User in shared private room does appear in user directory until leave +Existing members see new member's presence \ No newline at end of file From 39581af3ba657032fbf66ba3719a1cb334c0519b Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Mon, 10 Oct 2022 15:49:56 +0200 Subject: [PATCH 063/207] CI update --- .github/workflows/dendrite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml index be3c7c173..f8019b3ea 100644 --- a/.github/workflows/dendrite.yml +++ b/.github/workflows/dendrite.yml @@ -342,7 +342,7 @@ jobs: # See https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md specifically GOROOT_1_17_X64 run: | sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev - go get -v github.com/haveyoudebuggedit/gotestfmt/v2/cmd/gotestfmt@latest + go get -v github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest - name: Run actions/checkout@v2 for dendrite uses: actions/checkout@v2 From b000db81ca889b350663493c27eb58654c75e295 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Mon, 10 Oct 2022 17:36:26 +0200 Subject: [PATCH 064/207] Send E2EE related errors to sentry (#2784) Only sends errors if we're not retrying them in NATS. Not sure if those should be scoped/tagged with something like "E2EE". --- federationapi/consumers/keychange.go | 17 ++++++++++++++--- federationapi/consumers/sendtodevice.go | 14 ++++++++++---- federationapi/queue/queue.go | 3 +++ federationapi/routing/send.go | 4 ++++ 4 files changed, 31 insertions(+), 7 deletions(-) diff --git a/federationapi/consumers/keychange.go b/federationapi/consumers/keychange.go index f3314bc98..67dfdc1d3 100644 --- a/federationapi/consumers/keychange.go +++ b/federationapi/consumers/keychange.go @@ -18,6 +18,11 @@ import ( "context" "encoding/json" + "github.com/getsentry/sentry-go" + "github.com/matrix-org/gomatrixserverlib" + "github.com/nats-io/nats.go" + "github.com/sirupsen/logrus" + "github.com/matrix-org/dendrite/federationapi/queue" "github.com/matrix-org/dendrite/federationapi/storage" "github.com/matrix-org/dendrite/federationapi/types" @@ -26,9 +31,6 @@ import ( "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/setup/process" - "github.com/matrix-org/gomatrixserverlib" - "github.com/nats-io/nats.go" - "github.com/sirupsen/logrus" ) // KeyChangeConsumer consumes events that originate in key server. @@ -78,6 +80,7 @@ func (t *KeyChangeConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) boo msg := msgs[0] // Guaranteed to exist if onMessage is called var m api.DeviceMessage if err := json.Unmarshal(msg.Data, &m); err != nil { + sentry.CaptureException(err) logrus.WithError(err).Errorf("failed to read device message from key change topic") return true } @@ -105,6 +108,7 @@ func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) bool { // only send key change events which originated from us _, originServerName, err := gomatrixserverlib.SplitID('@', m.UserID) if err != nil { + sentry.CaptureException(err) logger.WithError(err).Error("Failed to extract domain from key change event") return true } @@ -118,6 +122,7 @@ func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) bool { WantMembership: "join", }, &queryRes) if err != nil { + sentry.CaptureException(err) logger.WithError(err).Error("failed to calculate joined rooms for user") return true } @@ -125,6 +130,7 @@ func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) bool { // send this key change to all servers who share rooms with this user. destinations, err := t.db.GetJoinedHostsForRooms(t.ctx, queryRes.RoomIDs, true) if err != nil { + sentry.CaptureException(err) logger.WithError(err).Error("failed to calculate joined hosts for rooms user is in") return true } @@ -147,6 +153,7 @@ func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) bool { Keys: m.KeyJSON, } if edu.Content, err = json.Marshal(event); err != nil { + sentry.CaptureException(err) logger.WithError(err).Error("failed to marshal EDU JSON") return true } @@ -160,6 +167,7 @@ func (t *KeyChangeConsumer) onCrossSigningMessage(m api.DeviceMessage) bool { output := m.CrossSigningKeyUpdate _, host, err := gomatrixserverlib.SplitID('@', output.UserID) if err != nil { + sentry.CaptureException(err) logrus.WithError(err).Errorf("fedsender key change consumer: user ID parse failure") return true } @@ -176,12 +184,14 @@ func (t *KeyChangeConsumer) onCrossSigningMessage(m api.DeviceMessage) bool { WantMembership: "join", }, &queryRes) if err != nil { + sentry.CaptureException(err) logger.WithError(err).Error("fedsender key change consumer: failed to calculate joined rooms for user") return true } // send this key change to all servers who share rooms with this user. destinations, err := t.db.GetJoinedHostsForRooms(t.ctx, queryRes.RoomIDs, true) if err != nil { + sentry.CaptureException(err) logger.WithError(err).Error("fedsender key change consumer: failed to calculate joined hosts for rooms user is in") return true } @@ -196,6 +206,7 @@ func (t *KeyChangeConsumer) onCrossSigningMessage(m api.DeviceMessage) bool { Origin: string(t.serverName), } if edu.Content, err = json.Marshal(output); err != nil { + sentry.CaptureException(err) logger.WithError(err).Error("fedsender key change consumer: failed to marshal output, dropping") return true } diff --git a/federationapi/consumers/sendtodevice.go b/federationapi/consumers/sendtodevice.go index ffc1d8894..9aec22a3e 100644 --- a/federationapi/consumers/sendtodevice.go +++ b/federationapi/consumers/sendtodevice.go @@ -18,16 +18,18 @@ import ( "context" "encoding/json" + "github.com/getsentry/sentry-go" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" + "github.com/nats-io/nats.go" + log "github.com/sirupsen/logrus" + "github.com/matrix-org/dendrite/federationapi/queue" "github.com/matrix-org/dendrite/federationapi/storage" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/setup/process" syncTypes "github.com/matrix-org/dendrite/syncapi/types" - "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" - "github.com/nats-io/nats.go" - log "github.com/sirupsen/logrus" ) // OutputSendToDeviceConsumer consumes events that originate in the clientapi. @@ -76,6 +78,7 @@ func (t *OutputSendToDeviceConsumer) onMessage(ctx context.Context, msgs []*nats sender := msg.Header.Get("sender") _, originServerName, err := gomatrixserverlib.SplitID('@', sender) if err != nil { + sentry.CaptureException(err) log.WithError(err).WithField("user_id", sender).Error("Failed to extract domain from send-to-device sender") return true } @@ -85,12 +88,14 @@ func (t *OutputSendToDeviceConsumer) onMessage(ctx context.Context, msgs []*nats // Extract the send-to-device event from msg. var ote syncTypes.OutputSendToDeviceEvent if err = json.Unmarshal(msg.Data, &ote); err != nil { + sentry.CaptureException(err) log.WithError(err).Errorf("output log: message parse failed (expected send-to-device)") return true } _, destServerName, err := gomatrixserverlib.SplitID('@', ote.UserID) if err != nil { + sentry.CaptureException(err) log.WithError(err).WithField("user_id", ote.UserID).Error("Failed to extract domain from send-to-device destination") return true } @@ -116,6 +121,7 @@ func (t *OutputSendToDeviceConsumer) onMessage(ctx context.Context, msgs []*nats }, } if edu.Content, err = json.Marshal(tdm); err != nil { + sentry.CaptureException(err) log.WithError(err).Error("failed to marshal EDU JSON") return true } diff --git a/federationapi/queue/queue.go b/federationapi/queue/queue.go index 88664fcf9..8245aa5bd 100644 --- a/federationapi/queue/queue.go +++ b/federationapi/queue/queue.go @@ -21,6 +21,7 @@ import ( "sync" "time" + "github.com/getsentry/sentry-go" "github.com/matrix-org/gomatrixserverlib" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" @@ -307,11 +308,13 @@ func (oqs *OutgoingQueues) SendEDU( ephemeralJSON, err := json.Marshal(e) if err != nil { + sentry.CaptureException(err) return fmt.Errorf("json.Marshal: %w", err) } nid, err := oqs.db.StoreJSON(oqs.process.Context(), string(ephemeralJSON)) if err != nil { + sentry.CaptureException(err) return fmt.Errorf("sendevent: oqs.db.StoreJSON: %w", err) } diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index 060af676d..b3bbaa394 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -22,6 +22,7 @@ import ( "sync" "time" + "github.com/getsentry/sentry-go" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" "github.com/prometheus/client_golang/prometheus" @@ -350,6 +351,7 @@ func (t *txnReq) processEDUs(ctx context.Context) { for deviceID, message := range byUser { // TODO: check that the user and the device actually exist here if err := t.producer.SendToDevice(ctx, directPayload.Sender, userID, deviceID, directPayload.Type, message); err != nil { + sentry.CaptureException(err) util.GetLogger(ctx).WithError(err).WithFields(logrus.Fields{ "sender": directPayload.Sender, "user_id": userID, @@ -360,6 +362,7 @@ func (t *txnReq) processEDUs(ctx context.Context) { } case gomatrixserverlib.MDeviceListUpdate: if err := t.producer.SendDeviceListUpdate(ctx, e.Content, t.Origin); err != nil { + sentry.CaptureException(err) util.GetLogger(ctx).WithError(err).Error("failed to InputDeviceListUpdate") } case gomatrixserverlib.MReceipt: @@ -395,6 +398,7 @@ func (t *txnReq) processEDUs(ctx context.Context) { } case types.MSigningKeyUpdate: if err := t.producer.SendSigningKeyUpdate(ctx, e.Content, t.Origin); err != nil { + sentry.CaptureException(err) logrus.WithError(err).Errorf("Failed to process signing key update") } case gomatrixserverlib.MPresence: From 6bf1912525724baa1a8bf5e0bf51524d7cee59ba Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 10 Oct 2022 16:54:04 +0100 Subject: [PATCH 065/207] Fix joined hosts with `RewritesState` (#2785) This ensures that the joined hosts in the federation API are correct after the state is rewritten. This might fix some races around the time of joining federated rooms. --- federationapi/storage/shared/storage.go | 32 ++++++++++++------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/federationapi/storage/shared/storage.go b/federationapi/storage/shared/storage.go index a00d782f1..9e40f311c 100644 --- a/federationapi/storage/shared/storage.go +++ b/federationapi/storage/shared/storage.go @@ -70,27 +70,27 @@ func (d *Database) UpdateRoom( ) (joinedHosts []types.JoinedHost, err error) { err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { if purgeRoomFirst { - // If the event is a create event then we'll delete all of the existing - // data for the room. The only reason that a create event would be replayed - // to us in this way is if we're about to receive the entire room state. if err = d.FederationJoinedHosts.DeleteJoinedHostsForRoom(ctx, txn, roomID); err != nil { return fmt.Errorf("d.FederationJoinedHosts.DeleteJoinedHosts: %w", err) } - } - - joinedHosts, err = d.FederationJoinedHosts.SelectJoinedHostsWithTx(ctx, txn, roomID) - if err != nil { - return err - } - - for _, add := range addHosts { - err = d.FederationJoinedHosts.InsertJoinedHosts(ctx, txn, roomID, add.MemberEventID, add.ServerName) - if err != nil { + for _, add := range addHosts { + if err = d.FederationJoinedHosts.InsertJoinedHosts(ctx, txn, roomID, add.MemberEventID, add.ServerName); err != nil { + return err + } + joinedHosts = append(joinedHosts, add) + } + } else { + if joinedHosts, err = d.FederationJoinedHosts.SelectJoinedHostsWithTx(ctx, txn, roomID); err != nil { + return err + } + for _, add := range addHosts { + if err = d.FederationJoinedHosts.InsertJoinedHosts(ctx, txn, roomID, add.MemberEventID, add.ServerName); err != nil { + return err + } + } + if err = d.FederationJoinedHosts.DeleteJoinedHosts(ctx, txn, removeHosts); err != nil { return err } - } - if err = d.FederationJoinedHosts.DeleteJoinedHosts(ctx, txn, removeHosts); err != nil { - return err } return nil }) From 9ed8ff6b938e0452d122c68b38cf4fedcd74c0bf Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 11 Oct 2022 10:48:36 +0100 Subject: [PATCH 066/207] Tweak federation `M_NOT_FOUND` errors --- federationapi/routing/events.go | 6 +++++- federationapi/routing/state.go | 15 ++++++++------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/federationapi/routing/events.go b/federationapi/routing/events.go index 23796edfa..6168912bd 100644 --- a/federationapi/routing/events.go +++ b/federationapi/routing/events.go @@ -20,6 +20,7 @@ import ( "net/http" "time" + "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" @@ -95,7 +96,10 @@ func fetchEvent(ctx context.Context, rsAPI api.FederationRoomserverAPI, eventID } if len(eventsResponse.Events) == 0 { - return nil, &util.JSONResponse{Code: http.StatusNotFound, JSON: nil} + return nil, &util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound("Event not found"), + } } return eventsResponse.Events[0].Event, nil diff --git a/federationapi/routing/state.go b/federationapi/routing/state.go index 5377eb88f..1d08d0a82 100644 --- a/federationapi/routing/state.go +++ b/federationapi/routing/state.go @@ -135,23 +135,24 @@ func getState( return nil, nil, &resErr } - if !response.StateKnown { + switch { + case !response.RoomExists: + return nil, nil, &util.JSONResponse{ + Code: http.StatusNotFound, + JSON: jsonerror.NotFound("Room not found"), + } + case !response.StateKnown: return nil, nil, &util.JSONResponse{ Code: http.StatusNotFound, JSON: jsonerror.NotFound("State not known"), } - } - if response.IsRejected { + case response.IsRejected: return nil, nil, &util.JSONResponse{ Code: http.StatusNotFound, JSON: jsonerror.NotFound("Event not found"), } } - if !response.RoomExists { - return nil, nil, &util.JSONResponse{Code: http.StatusNotFound, JSON: nil} - } - return response.StateEvents, response.AuthChainEvents, nil } From 3920b9f9b6155db69822d0dbcd36acb1eaa51c34 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 11 Oct 2022 10:58:34 +0100 Subject: [PATCH 067/207] Tweak `GetStateDeltas` behaviour (#2788) Improves the control flow of `GetStateDeltas` for clarity and possibly also fixes a bug where duplicate state delta entries could be inserted with different memberships instead of being correctly overridden by `join`. --- syncapi/storage/shared/storage_sync.go | 44 ++++++++++++++++++-------- 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/syncapi/storage/shared/storage_sync.go b/syncapi/storage/shared/storage_sync.go index 0e19d97d2..d5b5b3121 100644 --- a/syncapi/storage/shared/storage_sync.go +++ b/syncapi/storage/shared/storage_sync.go @@ -360,34 +360,50 @@ func (d *DatabaseTransaction) GetStateDeltas( newlyJoinedRooms := make(map[string]bool, len(state)) for roomID, stateStreamEvents := range state { for _, ev := range stateStreamEvents { - if membership, prevMembership := getMembershipFromEvent(ev.Event, userID); membership != "" { - if membership == gomatrixserverlib.Join && prevMembership != membership { - // send full room state down instead of a delta + // Look for our membership in the state events and skip over any + // membership events that are not related to us. + membership, prevMembership := getMembershipFromEvent(ev.Event, userID) + if membership == "" { + continue + } + + if membership == gomatrixserverlib.Join { + // If our membership is now join but the previous membership wasn't + // then this is a "join transition", so we'll insert this room. + if prevMembership != membership { + // Get the full room state, as we'll send that down for a newly + // joined room instead of a delta. var s []types.StreamEvent - s, err = d.currentStateStreamEventsForRoom(ctx, roomID, stateFilter) - if err != nil { + if s, err = d.currentStateStreamEventsForRoom(ctx, roomID, stateFilter); err != nil { if err == sql.ErrNoRows { continue } return nil, nil, err } + + // Add the information for this room into the state so that + // it will get added with all of the rest of the joined rooms. state[roomID] = s newlyJoinedRooms[roomID] = true - continue // we'll add this room in when we do joined rooms } - deltas = append(deltas, types.StateDelta{ - Membership: membership, - MembershipPos: ev.StreamPosition, - StateEvents: d.StreamEventsToEvents(device, stateStreamEvents), - RoomID: roomID, - }) - break + // We won't add joined rooms into the delta at this point as they + // are added later on. + continue } + + deltas = append(deltas, types.StateDelta{ + Membership: membership, + MembershipPos: ev.StreamPosition, + StateEvents: d.StreamEventsToEvents(device, stateStreamEvents), + RoomID: roomID, + }) + break } } - // Add in currently joined rooms + // Finally, add in currently joined rooms, including those from the + // join transitions above. for _, joinedRoomID := range joinedRoomIDs { deltas = append(deltas, types.StateDelta{ Membership: gomatrixserverlib.Join, From 0a9aebdf011921680e5b0646bf50d7900423aa69 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 11 Oct 2022 12:27:21 +0100 Subject: [PATCH 068/207] Private read receipts (#2789) Implement behaviours for `m.read.private` receipts. --- clientapi/routing/account_data.go | 44 ++++++++++++++--------------- clientapi/routing/receipt.go | 36 ++++++++++++++++++----- clientapi/routing/routing.go | 2 +- federationapi/consumers/receipts.go | 8 ++++++ internal/eventutil/types.go | 5 ++-- syncapi/streams/stream_receipt.go | 4 +++ 6 files changed, 66 insertions(+), 33 deletions(-) diff --git a/clientapi/routing/account_data.go b/clientapi/routing/account_data.go index b28f0bb1f..4742b1240 100644 --- a/clientapi/routing/account_data.go +++ b/clientapi/routing/account_data.go @@ -154,33 +154,31 @@ func SaveReadMarker( return *resErr } - if r.FullyRead == "" { - return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.BadJSON("Missing m.fully_read mandatory field"), + if r.FullyRead != "" { + data, err := json.Marshal(fullyReadEvent{EventID: r.FullyRead}) + if err != nil { + return jsonerror.InternalServerError() + } + + dataReq := api.InputAccountDataRequest{ + UserID: device.UserID, + DataType: "m.fully_read", + RoomID: roomID, + AccountData: data, + } + dataRes := api.InputAccountDataResponse{} + if err := userAPI.InputAccountData(req.Context(), &dataReq, &dataRes); err != nil { + util.GetLogger(req.Context()).WithError(err).Error("userAPI.InputAccountData failed") + return util.ErrorResponse(err) } } - data, err := json.Marshal(fullyReadEvent{EventID: r.FullyRead}) - if err != nil { - return jsonerror.InternalServerError() - } - - dataReq := api.InputAccountDataRequest{ - UserID: device.UserID, - DataType: "m.fully_read", - RoomID: roomID, - AccountData: data, - } - dataRes := api.InputAccountDataResponse{} - if err := userAPI.InputAccountData(req.Context(), &dataReq, &dataRes); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("userAPI.InputAccountData failed") - return util.ErrorResponse(err) - } - - // Handle the read receipt that may be included in the read marker + // Handle the read receipts that may be included in the read marker. if r.Read != "" { - return SetReceipt(req, syncProducer, device, roomID, "m.read", r.Read) + return SetReceipt(req, userAPI, syncProducer, device, roomID, "m.read", r.Read) + } + if r.ReadPrivate != "" { + return SetReceipt(req, userAPI, syncProducer, device, roomID, "m.read.private", r.ReadPrivate) } return util.JSONResponse{ diff --git a/clientapi/routing/receipt.go b/clientapi/routing/receipt.go index 0f9b1b4ff..99217a780 100644 --- a/clientapi/routing/receipt.go +++ b/clientapi/routing/receipt.go @@ -15,19 +15,22 @@ package routing import ( + "encoding/json" "fmt" "net/http" "time" + "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/dendrite/userapi/api" userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/util" "github.com/sirupsen/logrus" ) -func SetReceipt(req *http.Request, syncProducer *producers.SyncAPIProducer, device *userapi.Device, roomID, receiptType, eventID string) util.JSONResponse { +func SetReceipt(req *http.Request, userAPI api.ClientUserAPI, syncProducer *producers.SyncAPIProducer, device *userapi.Device, roomID, receiptType, eventID string) util.JSONResponse { timestamp := gomatrixserverlib.AsTimestamp(time.Now()) logrus.WithFields(logrus.Fields{ "roomID": roomID, @@ -37,13 +40,32 @@ func SetReceipt(req *http.Request, syncProducer *producers.SyncAPIProducer, devi "timestamp": timestamp, }).Debug("Setting receipt") - // currently only m.read is accepted - if receiptType != "m.read" { - return util.MessageResponse(400, fmt.Sprintf("receipt type must be m.read not '%s'", receiptType)) - } + switch receiptType { + case "m.read", "m.read.private": + if err := syncProducer.SendReceipt(req.Context(), device.UserID, roomID, eventID, receiptType, timestamp); err != nil { + return util.ErrorResponse(err) + } - if err := syncProducer.SendReceipt(req.Context(), device.UserID, roomID, eventID, receiptType, timestamp); err != nil { - return util.ErrorResponse(err) + case "m.fully_read": + data, err := json.Marshal(fullyReadEvent{EventID: eventID}) + if err != nil { + return jsonerror.InternalServerError() + } + + dataReq := api.InputAccountDataRequest{ + UserID: device.UserID, + DataType: "m.fully_read", + RoomID: roomID, + AccountData: data, + } + dataRes := api.InputAccountDataResponse{} + if err := userAPI.InputAccountData(req.Context(), &dataReq, &dataRes); err != nil { + util.GetLogger(req.Context()).WithError(err).Error("userAPI.InputAccountData failed") + return util.ErrorResponse(err) + } + + default: + return util.MessageResponse(400, fmt.Sprintf("Receipt type '%s' not known", receiptType)) } return util.JSONResponse{ diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index f1fa66ca6..e72880ec5 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -1343,7 +1343,7 @@ func Setup( return util.ErrorResponse(err) } - return SetReceipt(req, syncProducer, device, vars["roomId"], vars["receiptType"], vars["eventId"]) + return SetReceipt(req, userAPI, syncProducer, device, vars["roomId"], vars["receiptType"], vars["eventId"]) }), ).Methods(http.MethodPost, http.MethodOptions) v3mux.Handle("/presence/{userId}/status", diff --git a/federationapi/consumers/receipts.go b/federationapi/consumers/receipts.go index 366cb264e..75827cb68 100644 --- a/federationapi/consumers/receipts.go +++ b/federationapi/consumers/receipts.go @@ -81,6 +81,14 @@ func (t *OutputReceiptConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) Type: msg.Header.Get("type"), } + switch receipt.Type { + case "m.read": + // These are allowed to be sent over federation + case "m.read.private", "m.fully_read": + // These must not be sent over federation + return true + } + // only send receipt events which originated from us _, receiptServerName, err := gomatrixserverlib.SplitID('@', receipt.UserID) if err != nil { diff --git a/internal/eventutil/types.go b/internal/eventutil/types.go index afc62d8c2..18175d6a0 100644 --- a/internal/eventutil/types.go +++ b/internal/eventutil/types.go @@ -35,8 +35,9 @@ type AccountData struct { } type ReadMarkerJSON struct { - FullyRead string `json:"m.fully_read"` - Read string `json:"m.read"` + FullyRead string `json:"m.fully_read"` + Read string `json:"m.read"` + ReadPrivate string `json:"m.read.private"` } // NotificationData contains statistics about notifications, sent from diff --git a/syncapi/streams/stream_receipt.go b/syncapi/streams/stream_receipt.go index bba911022..977815078 100644 --- a/syncapi/streams/stream_receipt.go +++ b/syncapi/streams/stream_receipt.go @@ -67,6 +67,10 @@ func (p *ReceiptStreamProvider) IncrementalSync( if _, ok := req.IgnoredUsers.List[receipt.UserID]; ok { continue } + // Don't send private read receipts to other users + if receipt.Type == "m.read.private" && req.Device.UserID != receipt.UserID { + continue + } receiptsByRoom[receipt.RoomID] = append(receiptsByRoom[receipt.RoomID], receipt) } From 3c1474f68f2ac5e564d2bd4bd15f01d2a73f2845 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Tue, 11 Oct 2022 16:04:02 +0200 Subject: [PATCH 069/207] Fix `/get_missing_events` for rooms with `joined`/`invited` history_visibility (#2787) Sytest was using a wrong `history_visibility` for `invited` (https://github.com/matrix-org/sytest/pull/1303), so `invited` was passing for the wrong reason (-> defaulted to `shared`, as `invite` wasn't understood). This change now handles missing events like Synapse, if a server isn't allowed to see the event, it gets a redacted version of it, making the `get_missing_events` tests pass. --- are-we-synapse-yet.list | 2 +- roomserver/auth/auth.go | 30 +++++++------------ roomserver/internal/helpers/helpers.go | 15 +++++----- .../internal/perform/perform_backfill.go | 5 +++- roomserver/internal/query/query.go | 6 ++-- .../storage/postgres/state_snapshot_table.go | 4 ++- sytest-whitelist | 5 ++-- 7 files changed, 33 insertions(+), 34 deletions(-) diff --git a/are-we-synapse-yet.list b/are-we-synapse-yet.list index c776a7400..81c0f8049 100644 --- a/are-we-synapse-yet.list +++ b/are-we-synapse-yet.list @@ -643,7 +643,7 @@ fed Inbound federation redacts events from erased users fme Outbound federation can request missing events fme Inbound federation can return missing events for world_readable visibility fme Inbound federation can return missing events for shared visibility -fme Inbound federation can return missing events for invite visibility +fme Inbound federation can return missing events for invited visibility fme Inbound federation can return missing events for joined visibility fme outliers whose auth_events are in a different room are correctly rejected fbk Outbound federation can backfill events diff --git a/roomserver/auth/auth.go b/roomserver/auth/auth.go index aa1d5bc25..31a856e8e 100644 --- a/roomserver/auth/auth.go +++ b/roomserver/auth/auth.go @@ -13,8 +13,6 @@ package auth import ( - "encoding/json" - "github.com/matrix-org/gomatrixserverlib" ) @@ -30,7 +28,7 @@ func IsServerAllowed( historyVisibility := HistoryVisibilityForRoom(authEvents) // 1. If the history_visibility was set to world_readable, allow. - if historyVisibility == "world_readable" { + if historyVisibility == gomatrixserverlib.HistoryVisibilityWorldReadable { return true } // 2. If the user's membership was join, allow. @@ -39,12 +37,12 @@ func IsServerAllowed( return true } // 3. If history_visibility was set to shared, and the user joined the room at any point after the event was sent, allow. - if historyVisibility == "shared" && serverCurrentlyInRoom { + if historyVisibility == gomatrixserverlib.HistoryVisibilityShared && serverCurrentlyInRoom { return true } // 4. If the user's membership was invite, and the history_visibility was set to invited, allow. invitedUserExists := IsAnyUserOnServerWithMembership(serverName, authEvents, gomatrixserverlib.Invite) - if invitedUserExists && historyVisibility == "invited" { + if invitedUserExists && historyVisibility == gomatrixserverlib.HistoryVisibilityInvited { return true } @@ -52,27 +50,16 @@ func IsServerAllowed( return false } -func HistoryVisibilityForRoom(authEvents []*gomatrixserverlib.Event) string { +func HistoryVisibilityForRoom(authEvents []*gomatrixserverlib.Event) gomatrixserverlib.HistoryVisibility { // https://matrix.org/docs/spec/client_server/r0.6.0#id87 // By default if no history_visibility is set, or if the value is not understood, the visibility is assumed to be shared. - visibility := "shared" - knownStates := []string{"invited", "joined", "shared", "world_readable"} + visibility := gomatrixserverlib.HistoryVisibilityShared for _, ev := range authEvents { if ev.Type() != gomatrixserverlib.MRoomHistoryVisibility { continue } - // TODO: This should be HistoryVisibilityContent to match things like 'MemberContent'. Do this when moving to GMSL - content := struct { - HistoryVisibility string `json:"history_visibility"` - }{} - if err := json.Unmarshal(ev.Content(), &content); err != nil { - break // value is not understood - } - for _, s := range knownStates { - if s == content.HistoryVisibility { - visibility = s - break - } + if vis, err := ev.HistoryVisibility(); err == nil { + visibility = vis } } return visibility @@ -80,6 +67,9 @@ func HistoryVisibilityForRoom(authEvents []*gomatrixserverlib.Event) string { func IsAnyUserOnServerWithMembership(serverName gomatrixserverlib.ServerName, authEvents []*gomatrixserverlib.Event, wantMembership string) bool { for _, ev := range authEvents { + if ev.Type() != gomatrixserverlib.MRoomMember { + continue + } membership, err := ev.Membership() if err != nil || membership != wantMembership { continue diff --git a/roomserver/internal/helpers/helpers.go b/roomserver/internal/helpers/helpers.go index 3b83a0a6d..a6de8ac84 100644 --- a/roomserver/internal/helpers/helpers.go +++ b/roomserver/internal/helpers/helpers.go @@ -324,7 +324,7 @@ func slowGetHistoryVisibilityState( func ScanEventTree( ctx context.Context, db storage.Database, info *types.RoomInfo, front []string, visited map[string]bool, limit int, serverName gomatrixserverlib.ServerName, -) ([]types.EventNID, error) { +) ([]types.EventNID, map[string]struct{}, error) { var resultNIDs []types.EventNID var err error var allowed bool @@ -345,6 +345,7 @@ func ScanEventTree( var checkedServerInRoom bool var isServerInRoom bool + redactEventIDs := make(map[string]struct{}) // Loop through the event IDs to retrieve the requested events and go // through the whole tree (up to the provided limit) using the events' @@ -358,7 +359,7 @@ BFSLoop: // Retrieve the events to process from the database. events, err = db.EventsFromIDs(ctx, front) if err != nil { - return resultNIDs, err + return resultNIDs, redactEventIDs, err } if !checkedServerInRoom && len(events) > 0 { @@ -395,16 +396,16 @@ BFSLoop: ) // drop the error, as we will often error at the DB level if we don't have the prev_event itself. Let's // just return what we have. - return resultNIDs, nil + return resultNIDs, redactEventIDs, nil } // If the event hasn't been seen before and the HS // requesting to retrieve it is allowed to do so, add it to // the list of events to retrieve. - if allowed { - next = append(next, pre) - } else { + next = append(next, pre) + if !allowed { util.GetLogger(ctx).WithField("server", serverName).WithField("event_id", pre).Info("Not allowed to see event") + redactEventIDs[pre] = struct{}{} } } } @@ -413,7 +414,7 @@ BFSLoop: front = next } - return resultNIDs, err + return resultNIDs, redactEventIDs, err } func QueryLatestEventsAndState( diff --git a/roomserver/internal/perform/perform_backfill.go b/roomserver/internal/perform/perform_backfill.go index 69a075733..57e121ea2 100644 --- a/roomserver/internal/perform/perform_backfill.go +++ b/roomserver/internal/perform/perform_backfill.go @@ -78,7 +78,7 @@ func (r *Backfiller) PerformBackfill( } // Scan the event tree for events to send back. - resultNIDs, err := helpers.ScanEventTree(ctx, r.DB, info, front, visited, request.Limit, request.ServerName) + resultNIDs, redactEventIDs, err := helpers.ScanEventTree(ctx, r.DB, info, front, visited, request.Limit, request.ServerName) if err != nil { return err } @@ -95,6 +95,9 @@ func (r *Backfiller) PerformBackfill( } for _, event := range loadedEvents { + if _, ok := redactEventIDs[event.EventID()]; ok { + event.Redact() + } response.Events = append(response.Events, event.Headered(info.RoomVersion)) } diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go index 7a424a334..f41132403 100644 --- a/roomserver/internal/query/query.go +++ b/roomserver/internal/query/query.go @@ -453,7 +453,7 @@ func (r *Queryer) QueryMissingEvents( return fmt.Errorf("missing RoomInfo for room %s", events[0].RoomID()) } - resultNIDs, err := helpers.ScanEventTree(ctx, r.DB, info, front, visited, request.Limit, request.ServerName) + resultNIDs, redactEventIDs, err := helpers.ScanEventTree(ctx, r.DB, info, front, visited, request.Limit, request.ServerName) if err != nil { return err } @@ -470,7 +470,9 @@ func (r *Queryer) QueryMissingEvents( if verr != nil { return verr } - + if _, ok := redactEventIDs[event.EventID()]; ok { + event.Redact() + } response.Events = append(response.Events, event.Headered(roomVersion)) } } diff --git a/roomserver/storage/postgres/state_snapshot_table.go b/roomserver/storage/postgres/state_snapshot_table.go index 99c76befe..a00c026f4 100644 --- a/roomserver/storage/postgres/state_snapshot_table.go +++ b/roomserver/storage/postgres/state_snapshot_table.go @@ -21,10 +21,11 @@ import ( "fmt" "github.com/lib/pq" + "github.com/matrix-org/util" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/roomserver/types" - "github.com/matrix-org/util" ) const stateSnapshotSchema = ` @@ -91,6 +92,7 @@ const bulkSelectStateForHistoryVisibilitySQL = ` WHERE state_snapshot_nid = $1 ) ) + ORDER BY depth ASC ) AS roomserver_events INNER JOIN roomserver_event_state_keys ON roomserver_events.event_state_key_nid = roomserver_event_state_keys.event_state_key_nid diff --git a/sytest-whitelist b/sytest-whitelist index 9ba9df75d..a3218ed70 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -306,7 +306,7 @@ Alternative server names do not cause a routing loop Events whose auth_events are in the wrong room do not mess up the room state Inbound federation can return events Inbound federation can return missing events for world_readable visibility -Inbound federation can return missing events for invite visibility +Inbound federation can return missing events for invited visibility Inbound federation can get public room list PUT /rooms/:room_id/redact/:event_id/:txn_id as power user redacts message PUT /rooms/:room_id/redact/:event_id/:txn_id as original message sender redacts message @@ -742,4 +742,5 @@ User in private room doesn't appear in user directory User joining then leaving public room appears and dissappears from directory User in remote room doesn't appear in user directory after server left room User in shared private room does appear in user directory until leave -Existing members see new member's presence \ No newline at end of file +Existing members see new member's presence +Inbound federation can return missing events for joined visibility \ No newline at end of file From 23a3e04579172de89266e9554428b71172c58495 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 13 Oct 2022 14:50:52 +0100 Subject: [PATCH 070/207] Event relations (#2790) This adds support for tracking `m.relates_to`, as well as adding support for the various `/room/{roomID}/relations/...` endpoints to the CS API. --- docs/caddy/polylith/Caddyfile | 107 ++++++----- docs/hiawatha/polylith-sample.conf | 5 +- docs/nginx/polylith-sample.conf | 5 +- go.mod | 2 +- go.sum | 4 +- syncapi/consumers/roomserver.go | 27 +++ syncapi/routing/relations.go | 124 +++++++++++++ syncapi/routing/routing.go | 43 +++++ syncapi/storage/interface.go | 4 + syncapi/storage/postgres/relations_table.go | 158 +++++++++++++++++ syncapi/storage/postgres/syncserver.go | 5 + syncapi/storage/shared/storage_consumer.go | 35 +++- syncapi/storage/shared/storage_sync.go | 81 +++++++++ syncapi/storage/sqlite3/relations_table.go | 163 +++++++++++++++++ syncapi/storage/sqlite3/stream_id_table.go | 8 + syncapi/storage/sqlite3/syncserver.go | 5 + syncapi/storage/tables/interface.go | 19 ++ syncapi/storage/tables/relations_test.go | 186 ++++++++++++++++++++ syncapi/types/types.go | 13 ++ 19 files changed, 943 insertions(+), 51 deletions(-) create mode 100644 syncapi/routing/relations.go create mode 100644 syncapi/storage/postgres/relations_table.go create mode 100644 syncapi/storage/sqlite3/relations_table.go create mode 100644 syncapi/storage/tables/relations_test.go diff --git a/docs/caddy/polylith/Caddyfile b/docs/caddy/polylith/Caddyfile index 906097e4e..8aeb9317f 100644 --- a/docs/caddy/polylith/Caddyfile +++ b/docs/caddy/polylith/Caddyfile @@ -1,66 +1,85 @@ -# Sample Caddyfile for using Caddy in front of Dendrite. -# -# Customize email address and domain names. -# Optional settings commented out. -# -# BE SURE YOUR DOMAINS ARE POINTED AT YOUR SERVER FIRST. -# Documentation: https://caddyserver.com/docs/ -# -# Bonus tip: If your IP address changes, use Caddy's -# dynamic DNS plugin to update your DNS records to -# point to your new IP automatically: -# https://github.com/mholt/caddy-dynamicdns +# Sample Caddyfile for using Caddy in front of Dendrite + # +# Customize email address and domain names + +# Optional settings commented out + +# + +# BE SURE YOUR DOMAINS ARE POINTED AT YOUR SERVER FIRST + +# Documentation: + +# + +# Bonus tip: If your IP address changes, use Caddy's + +# dynamic DNS plugin to update your DNS records to + +# point to your new IP automatically + +# + +# # Global options block + { - # In case there is a problem with your certificates. - # email example@example.com + # In case there is a problem with your certificates. + # email example@example.com - # Turn off the admin endpoint if you don't need graceful config - # changes and/or are running untrusted code on your machine. - # admin off + # Turn off the admin endpoint if you don't need graceful config + # changes and/or are running untrusted code on your machine. + # admin off - # Enable this if your clients don't send ServerName in TLS handshakes. - # default_sni example.com + # Enable this if your clients don't send ServerName in TLS handshakes. + # default_sni example.com - # Enable debug mode for verbose logging. - # debug + # Enable debug mode for verbose logging. + # debug - # Use Let's Encrypt's staging endpoint for testing. - # acme_ca https://acme-staging-v02.api.letsencrypt.org/directory + # Use Let's Encrypt's staging endpoint for testing. + # acme_ca https://acme-staging-v02.api.letsencrypt.org/directory - # If you're port-forwarding HTTP/HTTPS ports from 80/443 to something - # else, enable these and put the alternate port numbers here. - # http_port 8080 - # https_port 8443 + # If you're port-forwarding HTTP/HTTPS ports from 80/443 to something + # else, enable these and put the alternate port numbers here. + # http_port 8080 + # https_port 8443 } # The server name of your matrix homeserver. This example shows -# "well-known delegation" from the registered domain to a subdomain, + +# "well-known delegation" from the registered domain to a subdomain + # which is only needed if your server_name doesn't match your Matrix + # homeserver URL (i.e. you can show users a vanity domain that looks + # nice and is easy to remember but still have your Matrix server on -# its own subdomain or hosted service). + +# its own subdomain or hosted service) + example.com { - header /.well-known/matrix/* Content-Type application/json - header /.well-known/matrix/* Access-Control-Allow-Origin * - respond /.well-known/matrix/server `{"m.server": "matrix.example.com:443"}` - respond /.well-known/matrix/client `{"m.homeserver": {"base_url": "https://matrix.example.com"}}` + header /.well-known/matrix/*Content-Type application/json + header /.well-known/matrix/* Access-Control-Allow-Origin * + respond /.well-known/matrix/server `{"m.server": "matrix.example.com:443"}` + respond /.well-known/matrix/client `{"m.homeserver": {"base_url": "https://matrix.example.com"}}` } -# The actual domain name whereby your Matrix server is accessed. +# The actual domain name whereby your Matrix server is accessed + matrix.example.com { - # Change the end of each reverse_proxy line to the correct - # address for your various services. - @sync_api { - path_regexp /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|context/.*?|event/.*?))$ - } - reverse_proxy @sync_api sync_api:8073 + # Change the end of each reverse_proxy line to the correct + # address for your various services. + @sync_api { + path_regexp /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|context/.*?|relations/.*?|event/.*?))$ + } + reverse_proxy @sync_api sync_api:8073 - reverse_proxy /_matrix/client* client_api:8071 - reverse_proxy /_matrix/federation* federation_api:8071 - reverse_proxy /_matrix/key* federation_api:8071 - reverse_proxy /_matrix/media* media_api:8071 + reverse_proxy /_matrix/client* client_api:8071 + reverse_proxy /_matrix/federation* federation_api:8071 + reverse_proxy /_matrix/key* federation_api:8071 + reverse_proxy /_matrix/media* media_api:8071 } diff --git a/docs/hiawatha/polylith-sample.conf b/docs/hiawatha/polylith-sample.conf index 036140643..0093fdcf2 100644 --- a/docs/hiawatha/polylith-sample.conf +++ b/docs/hiawatha/polylith-sample.conf @@ -20,8 +20,11 @@ VirtualHost { # /_matrix/client/.*/rooms/{roomId}/messages # /_matrix/client/.*/rooms/{roomId}/context/{eventID} # /_matrix/client/.*/rooms/{roomId}/event/{eventID} + # /_matrix/client/.*/rooms/{roomId}/relations/{eventID} + # /_matrix/client/.*/rooms/{roomId}/relations/{eventID}/{relType} + # /_matrix/client/.*/rooms/{roomId}/relations/{eventID}/{relType}/{eventType} # to sync_api - ReverseProxy = /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|context/.*?|event/.*?))$ http://localhost:8073 600 + ReverseProxy = /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|context/.*?|relations/.*?|event/.*?))$ http://localhost:8073 600 ReverseProxy = /_matrix/client http://localhost:8071 600 ReverseProxy = /_matrix/federation http://localhost:8072 600 ReverseProxy = /_matrix/key http://localhost:8072 600 diff --git a/docs/nginx/polylith-sample.conf b/docs/nginx/polylith-sample.conf index 345d8a6b4..6e81eb5f2 100644 --- a/docs/nginx/polylith-sample.conf +++ b/docs/nginx/polylith-sample.conf @@ -30,8 +30,11 @@ server { # /_matrix/client/.*/rooms/{roomId}/messages # /_matrix/client/.*/rooms/{roomId}/context/{eventID} # /_matrix/client/.*/rooms/{roomId}/event/{eventID} + # /_matrix/client/.*/rooms/{roomId}/relations/{eventID} + # /_matrix/client/.*/rooms/{roomId}/relations/{eventID}/{relType} + # /_matrix/client/.*/rooms/{roomId}/relations/{eventID}/{relType}/{eventType} # to sync_api - location ~ /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|context/.*?|event/.*?))$ { + location ~ /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|context/.*?|relations/.*?|event/.*?))$ { proxy_pass http://sync_api:8073; } diff --git a/go.mod b/go.mod index c82f76d41..eefad89e6 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 - github.com/matrix-org/gomatrixserverlib v0.0.0-20220929190355-91d455cd3621 + github.com/matrix-org/gomatrixserverlib v0.0.0-20221011115330-49fa704b9a64 github.com/matrix-org/pinecone v0.0.0-20220929155234-2ce51dd4a42c github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.15 diff --git a/go.sum b/go.sum index a99599cb1..0d08ac692 100644 --- a/go.sum +++ b/go.sum @@ -384,8 +384,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220929190355-91d455cd3621 h1:a8IaoSPDxevkgXnOUrtIW9AqVNvXBJAG0gtnX687S7g= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220929190355-91d455cd3621/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4= +github.com/matrix-org/gomatrixserverlib v0.0.0-20221011115330-49fa704b9a64 h1:QJmfAPC3P0ZHJzYD/QtbNc5EztKlK1ipRWP5SO/m4jw= +github.com/matrix-org/gomatrixserverlib v0.0.0-20221011115330-49fa704b9a64/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4= github.com/matrix-org/pinecone v0.0.0-20220929155234-2ce51dd4a42c h1:iCHLYwwlPsf4TYFrvhKdhQoAM2lXzcmDZYqwBNWcnVk= github.com/matrix-org/pinecone v0.0.0-20220929155234-2ce51dd4a42c/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index c7a11dbb4..cfbb05327 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -148,6 +148,16 @@ func (s *OutputRoomEventConsumer) onRedactEvent( log.WithError(err).Error("RedactEvent error'd") return err } + + if err = s.db.RedactRelations(ctx, msg.RedactedBecause.RoomID(), msg.RedactedEventID); err != nil { + log.WithFields(log.Fields{ + "room_id": msg.RedactedBecause.RoomID(), + "event_id": msg.RedactedBecause.EventID(), + "redacted_event_id": msg.RedactedEventID, + }).WithError(err).Warn("Failed to redact relations") + return err + } + // fake a room event so we notify clients about the redaction, as if it were // a normal event. return s.onNewRoomEvent(ctx, api.OutputNewRoomEvent{ @@ -271,6 +281,14 @@ func (s *OutputRoomEventConsumer) onNewRoomEvent( return err } + if err = s.db.UpdateRelations(ctx, ev); err != nil { + log.WithFields(log.Fields{ + "event_id": ev.EventID(), + "type": ev.Type(), + }).WithError(err).Warn("Failed to update relations") + return err + } + s.pduStream.Advance(pduPos) s.notifier.OnNewEvent(ev, ev.RoomID(), nil, types.StreamingToken{PDUPosition: pduPos}) @@ -315,6 +333,15 @@ func (s *OutputRoomEventConsumer) onOldRoomEvent( }).WithError(err).Warn("failed to index fulltext element") } + if err = s.db.UpdateRelations(ctx, ev); err != nil { + log.WithFields(log.Fields{ + "room_id": ev.RoomID(), + "event_id": ev.EventID(), + "type": ev.Type(), + }).WithError(err).Warn("Failed to update relations") + return err + } + if pduPos, err = s.notifyJoinedPeeks(ctx, ev, pduPos); err != nil { log.WithError(err).Errorf("Failed to notifyJoinedPeeks for PDU pos %d", pduPos) return err diff --git a/syncapi/routing/relations.go b/syncapi/routing/relations.go new file mode 100644 index 000000000..fee61b0df --- /dev/null +++ b/syncapi/routing/relations.go @@ -0,0 +1,124 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routing + +import ( + "net/http" + "strconv" + + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" + "github.com/sirupsen/logrus" + + "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/syncapi/internal" + "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/dendrite/syncapi/types" + userapi "github.com/matrix-org/dendrite/userapi/api" +) + +type RelationsResponse struct { + Chunk []gomatrixserverlib.ClientEvent `json:"chunk"` + NextBatch string `json:"next_batch,omitempty"` + PrevBatch string `json:"prev_batch,omitempty"` +} + +// nolint:gocyclo +func Relations( + req *http.Request, device *userapi.Device, + syncDB storage.Database, + rsAPI api.SyncRoomserverAPI, + roomID, eventID, relType, eventType string, +) util.JSONResponse { + var err error + var from, to types.StreamPosition + var limit int + dir := req.URL.Query().Get("dir") + if f := req.URL.Query().Get("from"); f != "" { + if from, err = types.NewStreamPositionFromString(f); err != nil { + return util.ErrorResponse(err) + } + } + if t := req.URL.Query().Get("to"); t != "" { + if to, err = types.NewStreamPositionFromString(t); err != nil { + return util.ErrorResponse(err) + } + } + if l := req.URL.Query().Get("limit"); l != "" { + if limit, err = strconv.Atoi(l); err != nil { + return util.ErrorResponse(err) + } + } + if limit == 0 || limit > 50 { + limit = 50 + } + if dir == "" { + dir = "b" + } + if dir != "b" && dir != "f" { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.MissingArgument("Bad or missing dir query parameter (should be either 'b' or 'f')"), + } + } + + snapshot, err := syncDB.NewDatabaseSnapshot(req.Context()) + if err != nil { + logrus.WithError(err).Error("Failed to get snapshot for relations") + return jsonerror.InternalServerError() + } + var succeeded bool + defer sqlutil.EndTransactionWithCheck(snapshot, &succeeded, &err) + + res := &RelationsResponse{ + Chunk: []gomatrixserverlib.ClientEvent{}, + } + var events []types.StreamEvent + events, res.PrevBatch, res.NextBatch, err = snapshot.RelationsFor( + req.Context(), roomID, eventID, relType, eventType, from, to, dir == "b", limit, + ) + if err != nil { + return util.ErrorResponse(err) + } + + headeredEvents := make([]*gomatrixserverlib.HeaderedEvent, 0, len(events)) + for _, event := range events { + headeredEvents = append(headeredEvents, event.HeaderedEvent) + } + + // Apply history visibility to the result events. + filteredEvents, err := internal.ApplyHistoryVisibilityFilter(req.Context(), snapshot, rsAPI, headeredEvents, nil, device.UserID, "relations") + if err != nil { + return util.ErrorResponse(err) + } + + // Convert the events into client events, and optionally filter based on the event + // type if it was specified. + res.Chunk = make([]gomatrixserverlib.ClientEvent, 0, len(filteredEvents)) + for _, event := range filteredEvents { + res.Chunk = append( + res.Chunk, + gomatrixserverlib.ToClientEvent(event.Event, gomatrixserverlib.FormatAll), + ) + } + + succeeded = true + return util.JSONResponse{ + Code: http.StatusOK, + JSON: res, + } +} diff --git a/syncapi/routing/routing.go b/syncapi/routing/routing.go index 069dee81f..71fa93c1e 100644 --- a/syncapi/routing/routing.go +++ b/syncapi/routing/routing.go @@ -45,6 +45,7 @@ func Setup( lazyLoadCache caching.LazyLoadCache, fts *fulltext.Search, ) { + v1unstablemux := csMux.PathPrefix("/{apiversion:(?:v1|unstable)}/").Subrouter() v3mux := csMux.PathPrefix("/{apiversion:(?:r0|v3)}/").Subrouter() // TODO: Add AS support for all handlers below. @@ -110,6 +111,48 @@ func Setup( }), ).Methods(http.MethodGet, http.MethodOptions) + v1unstablemux.Handle("/rooms/{roomId}/relations/{eventId}", + httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + + return Relations( + req, device, syncDB, rsAPI, + vars["roomId"], vars["eventId"], "", "", + ) + }), + ).Methods(http.MethodGet, http.MethodOptions) + + v1unstablemux.Handle("/rooms/{roomId}/relations/{eventId}/{relType}", + httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + + return Relations( + req, device, syncDB, rsAPI, + vars["roomId"], vars["eventId"], vars["relType"], "", + ) + }), + ).Methods(http.MethodGet, http.MethodOptions) + + v1unstablemux.Handle("/rooms/{roomId}/relations/{eventId}/{relType}/{eventType}", + httputil.MakeAuthAPI(gomatrixserverlib.Join, userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + + return Relations( + req, device, syncDB, rsAPI, + vars["roomId"], vars["eventId"], vars["relType"], vars["eventType"], + ) + }), + ).Methods(http.MethodGet, http.MethodOptions) + v3mux.Handle("/search", httputil.MakeAuthAPI("search", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { if !cfg.Fulltext.Enabled { diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index 4a03aca74..02d45f801 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -38,6 +38,7 @@ type DatabaseTransaction interface { MaxStreamPositionForSendToDeviceMessages(ctx context.Context) (types.StreamPosition, error) MaxStreamPositionForNotificationData(ctx context.Context) (types.StreamPosition, error) MaxStreamPositionForPresence(ctx context.Context) (types.StreamPosition, error) + MaxStreamPositionForRelations(ctx context.Context) (types.StreamPosition, error) CurrentState(ctx context.Context, roomID string, stateFilterPart *gomatrixserverlib.StateFilter, excludeEventIDs []string) ([]*gomatrixserverlib.HeaderedEvent, error) GetStateDeltasForFullStateSync(ctx context.Context, device *userapi.Device, r types.Range, userID string, stateFilter *gomatrixserverlib.StateFilter) ([]types.StateDelta, []string, error) @@ -107,6 +108,7 @@ type DatabaseTransaction interface { GetUserUnreadNotificationCountsForRooms(ctx context.Context, userID string, roomIDs map[string]string) (map[string]*eventutil.NotificationData, error) GetPresence(ctx context.Context, userID string) (*types.PresenceInternal, error) PresenceAfter(ctx context.Context, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (map[string]*types.PresenceInternal, error) + RelationsFor(ctx context.Context, roomID, eventID, relType, eventType string, from, to types.StreamPosition, backwards bool, limit int) (events []types.StreamEvent, prevBatch, nextBatch string, err error) } type Database interface { @@ -174,6 +176,8 @@ type Database interface { StoreReceipt(ctx context.Context, roomId, receiptType, userId, eventId string, timestamp gomatrixserverlib.Timestamp) (pos types.StreamPosition, err error) UpdateIgnoresForUser(ctx context.Context, userID string, ignores *types.IgnoredUsers) error ReIndex(ctx context.Context, limit, afterID int64) (map[int64]gomatrixserverlib.HeaderedEvent, error) + UpdateRelations(ctx context.Context, event *gomatrixserverlib.HeaderedEvent) error + RedactRelations(ctx context.Context, roomID, redactedEventID string) error } type Presence interface { diff --git a/syncapi/storage/postgres/relations_table.go b/syncapi/storage/postgres/relations_table.go new file mode 100644 index 000000000..5a76e9c33 --- /dev/null +++ b/syncapi/storage/postgres/relations_table.go @@ -0,0 +1,158 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package postgres + +import ( + "context" + "database/sql" + + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/syncapi/storage/tables" + "github.com/matrix-org/dendrite/syncapi/types" +) + +const relationsSchema = ` +CREATE SEQUENCE IF NOT EXISTS syncapi_relation_id; + +CREATE TABLE IF NOT EXISTS syncapi_relations ( + id BIGINT PRIMARY KEY DEFAULT nextval('syncapi_relation_id'), + room_id TEXT NOT NULL, + event_id TEXT NOT NULL, + child_event_id TEXT NOT NULL, + child_event_type TEXT NOT NULL, + rel_type TEXT NOT NULL, + CONSTRAINT syncapi_relations_unique UNIQUE (room_id, event_id, child_event_id, rel_type) +); +` + +const insertRelationSQL = "" + + "INSERT INTO syncapi_relations (" + + " room_id, event_id, child_event_id, child_event_type, rel_type" + + ") VALUES ($1, $2, $3, $4, $5) " + + " ON CONFLICT DO NOTHING" + +const deleteRelationSQL = "" + + "DELETE FROM syncapi_relations WHERE room_id = $1 AND child_event_id = $2" + +const selectRelationsInRangeAscSQL = "" + + "SELECT id, child_event_id, rel_type FROM syncapi_relations" + + " WHERE room_id = $1 AND event_id = $2" + + " AND ( $3 = '' OR rel_type = $3 )" + + " AND ( $4 = '' OR child_event_type = $4 )" + + " AND id > $5 AND id <= $6" + + " ORDER BY id ASC LIMIT $7" + +const selectRelationsInRangeDescSQL = "" + + "SELECT id, child_event_id, rel_type FROM syncapi_relations" + + " WHERE room_id = $1 AND event_id = $2" + + " AND ( $3 = '' OR rel_type = $3 )" + + " AND ( $4 = '' OR child_event_type = $4 )" + + " AND id >= $5 AND id < $6" + + " ORDER BY id DESC LIMIT $7" + +const selectMaxRelationIDSQL = "" + + "SELECT COALESCE(MAX(id), 0) FROM syncapi_relations" + +type relationsStatements struct { + insertRelationStmt *sql.Stmt + selectRelationsInRangeAscStmt *sql.Stmt + selectRelationsInRangeDescStmt *sql.Stmt + deleteRelationStmt *sql.Stmt + selectMaxRelationIDStmt *sql.Stmt +} + +func NewPostgresRelationsTable(db *sql.DB) (tables.Relations, error) { + s := &relationsStatements{} + _, err := db.Exec(relationsSchema) + if err != nil { + return nil, err + } + return s, sqlutil.StatementList{ + {&s.insertRelationStmt, insertRelationSQL}, + {&s.selectRelationsInRangeAscStmt, selectRelationsInRangeAscSQL}, + {&s.selectRelationsInRangeDescStmt, selectRelationsInRangeDescSQL}, + {&s.deleteRelationStmt, deleteRelationSQL}, + {&s.selectMaxRelationIDStmt, selectMaxRelationIDSQL}, + }.Prepare(db) +} + +func (s *relationsStatements) InsertRelation( + ctx context.Context, txn *sql.Tx, roomID, eventID, childEventID, childEventType, relType string, +) (err error) { + _, err = sqlutil.TxStmt(txn, s.insertRelationStmt).ExecContext( + ctx, roomID, eventID, childEventID, childEventType, relType, + ) + return +} + +func (s *relationsStatements) DeleteRelation( + ctx context.Context, txn *sql.Tx, roomID, childEventID string, +) error { + stmt := sqlutil.TxStmt(txn, s.deleteRelationStmt) + _, err := stmt.ExecContext( + ctx, roomID, childEventID, + ) + return err +} + +// SelectRelationsInRange returns a map rel_type -> []child_event_id +func (s *relationsStatements) SelectRelationsInRange( + ctx context.Context, txn *sql.Tx, roomID, eventID, relType, eventType string, + r types.Range, limit int, +) (map[string][]types.RelationEntry, types.StreamPosition, error) { + var lastPos types.StreamPosition + var stmt *sql.Stmt + if r.Backwards { + stmt = sqlutil.TxStmt(txn, s.selectRelationsInRangeDescStmt) + } else { + stmt = sqlutil.TxStmt(txn, s.selectRelationsInRangeAscStmt) + } + rows, err := stmt.QueryContext(ctx, roomID, eventID, relType, eventType, r.Low(), r.High(), limit) + if err != nil { + return nil, lastPos, err + } + defer internal.CloseAndLogIfError(ctx, rows, "selectRelationsInRange: rows.close() failed") + result := map[string][]types.RelationEntry{} + var ( + id types.StreamPosition + childEventID string + relationType string + ) + for rows.Next() { + if err = rows.Scan(&id, &childEventID, &relationType); err != nil { + return nil, lastPos, err + } + if id > lastPos { + lastPos = id + } + result[relationType] = append(result[relationType], types.RelationEntry{ + Position: id, + EventID: childEventID, + }) + } + if lastPos == 0 { + lastPos = r.To + } + return result, lastPos, rows.Err() +} + +func (s *relationsStatements) SelectMaxRelationID( + ctx context.Context, txn *sql.Tx, +) (id int64, err error) { + stmt := sqlutil.TxStmt(txn, s.selectMaxRelationIDStmt) + err = stmt.QueryRowContext(ctx).Scan(&id) + return +} diff --git a/syncapi/storage/postgres/syncserver.go b/syncapi/storage/postgres/syncserver.go index 979ff6647..850d24a07 100644 --- a/syncapi/storage/postgres/syncserver.go +++ b/syncapi/storage/postgres/syncserver.go @@ -98,6 +98,10 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) if err != nil { return nil, err } + relations, err := NewPostgresRelationsTable(d.db) + if err != nil { + return nil, err + } // apply migrations which need multiple tables m := sqlutil.NewMigrator(d.db) @@ -129,6 +133,7 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions) NotificationData: notificationData, Ignores: ignores, Presence: presence, + Relations: relations, } return &d, nil } diff --git a/syncapi/storage/shared/storage_consumer.go b/syncapi/storage/shared/storage_consumer.go index 937ced3a2..bf12203db 100644 --- a/syncapi/storage/shared/storage_consumer.go +++ b/syncapi/storage/shared/storage_consumer.go @@ -53,6 +53,7 @@ type Database struct { NotificationData tables.NotificationData Ignores tables.Ignores Presence tables.Presence + Relations tables.Relations } func (d *Database) NewDatabaseSnapshot(ctx context.Context) (*DatabaseTransaction, error) { @@ -579,10 +580,40 @@ func (d *Database) SelectMembershipForUser(ctx context.Context, roomID, userID s return d.Memberships.SelectMembershipForUser(ctx, nil, roomID, userID, pos) } -func (s *Database) ReIndex(ctx context.Context, limit, afterID int64) (map[int64]gomatrixserverlib.HeaderedEvent, error) { - return s.OutputEvents.ReIndex(ctx, nil, limit, afterID, []string{ +func (d *Database) ReIndex(ctx context.Context, limit, afterID int64) (map[int64]gomatrixserverlib.HeaderedEvent, error) { + return d.OutputEvents.ReIndex(ctx, nil, limit, afterID, []string{ gomatrixserverlib.MRoomName, gomatrixserverlib.MRoomTopic, "m.room.message", }) } + +func (d *Database) UpdateRelations(ctx context.Context, event *gomatrixserverlib.HeaderedEvent) error { + var content gomatrixserverlib.RelationContent + if err := json.Unmarshal(event.Content(), &content); err != nil { + return fmt.Errorf("json.Unmarshal: %w", err) + } + switch { + case content.Relations == nil: + return nil + case content.Relations.EventID == "": + return nil + case content.Relations.RelationType == "": + return nil + case event.Type() == gomatrixserverlib.MRoomRedaction: + return nil + default: + return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + return d.Relations.InsertRelation( + ctx, txn, event.RoomID(), content.Relations.EventID, + event.EventID(), event.Type(), content.Relations.RelationType, + ) + }) + } +} + +func (d *Database) RedactRelations(ctx context.Context, roomID, redactedEventID string) error { + return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + return d.Relations.DeleteRelation(ctx, txn, roomID, redactedEventID) + }) +} diff --git a/syncapi/storage/shared/storage_sync.go b/syncapi/storage/shared/storage_sync.go index d5b5b3121..cb61c1c26 100644 --- a/syncapi/storage/shared/storage_sync.go +++ b/syncapi/storage/shared/storage_sync.go @@ -589,3 +589,84 @@ func (d *DatabaseTransaction) PresenceAfter(ctx context.Context, after types.Str func (d *DatabaseTransaction) MaxStreamPositionForPresence(ctx context.Context) (types.StreamPosition, error) { return d.Presence.GetMaxPresenceID(ctx, d.txn) } + +func (d *DatabaseTransaction) MaxStreamPositionForRelations(ctx context.Context) (types.StreamPosition, error) { + id, err := d.Relations.SelectMaxRelationID(ctx, d.txn) + return types.StreamPosition(id), err +} + +func (d *DatabaseTransaction) RelationsFor(ctx context.Context, roomID, eventID, relType, eventType string, from, to types.StreamPosition, backwards bool, limit int) ( + events []types.StreamEvent, prevBatch, nextBatch string, err error, +) { + r := types.Range{ + From: from, + To: to, + Backwards: backwards, + } + + if r.Backwards && r.From == 0 { + // If we're working backwards (dir=b) and there's no ?from= specified then + // we will automatically want to work backwards from the current position, + // so find out what that is. + if r.From, err = d.MaxStreamPositionForRelations(ctx); err != nil { + return nil, "", "", fmt.Errorf("d.MaxStreamPositionForRelations: %w", err) + } + // The result normally isn't inclusive of the event *at* the ?from= + // position, so add 1 here so that we include the most recent relation. + r.From++ + } else if !r.Backwards && r.To == 0 { + // If we're working forwards (dir=f) and there's no ?to= specified then + // we will automatically want to work forwards towards the current position, + // so find out what that is. + if r.To, err = d.MaxStreamPositionForRelations(ctx); err != nil { + return nil, "", "", fmt.Errorf("d.MaxStreamPositionForRelations: %w", err) + } + } + + // First look up any relations from the database. We add one to the limit here + // so that we can tell if we're overflowing, as we will only set the "next_batch" + // in the response if we are. + relations, _, err := d.Relations.SelectRelationsInRange(ctx, d.txn, roomID, eventID, relType, eventType, r, limit+1) + if err != nil { + return nil, "", "", fmt.Errorf("d.Relations.SelectRelationsInRange: %w", err) + } + + // If we specified a relation type then just get those results, otherwise collate + // them from all of the returned relation types. + entries := []types.RelationEntry{} + if relType != "" { + entries = relations[relType] + } else { + for _, e := range relations { + entries = append(entries, e...) + } + } + + // If there were no entries returned, there were no relations, so stop at this point. + if len(entries) == 0 { + return nil, "", "", nil + } + + // Otherwise, let's try and work out what sensible prev_batch and next_batch values + // could be. We've requested an extra event by adding one to the limit already so + // that we can determine whether or not to provide a "next_batch", so trim off that + // event off the end if needs be. + if len(entries) > limit { + entries = entries[:len(entries)-1] + nextBatch = fmt.Sprintf("%d", entries[len(entries)-1].Position) + } + // TODO: set prevBatch? doesn't seem to affect the tests... + + // Extract all of the event IDs from the relation entries so that we can pull the + // events out of the database. Then go and fetch the events. + eventIDs := make([]string, 0, len(entries)) + for _, entry := range entries { + eventIDs = append(eventIDs, entry.EventID) + } + events, err = d.OutputEvents.SelectEvents(ctx, d.txn, eventIDs, nil, true) + if err != nil { + return nil, "", "", fmt.Errorf("d.OutputEvents.SelectEvents: %w", err) + } + + return events, prevBatch, nextBatch, nil +} diff --git a/syncapi/storage/sqlite3/relations_table.go b/syncapi/storage/sqlite3/relations_table.go new file mode 100644 index 000000000..7cbb5408f --- /dev/null +++ b/syncapi/storage/sqlite3/relations_table.go @@ -0,0 +1,163 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlite3 + +import ( + "context" + "database/sql" + + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/syncapi/storage/tables" + "github.com/matrix-org/dendrite/syncapi/types" +) + +const relationsSchema = ` +CREATE TABLE IF NOT EXISTS syncapi_relations ( + id BIGINT PRIMARY KEY, + room_id TEXT NOT NULL, + event_id TEXT NOT NULL, + child_event_id TEXT NOT NULL, + child_event_type TEXT NOT NULL, + rel_type TEXT NOT NULL, + UNIQUE (room_id, event_id, child_event_id, rel_type) +); +` + +const insertRelationSQL = "" + + "INSERT INTO syncapi_relations (" + + " id, room_id, event_id, child_event_id, child_event_type, rel_type" + + ") VALUES ($1, $2, $3, $4, $5, $6) " + + " ON CONFLICT DO NOTHING" + +const deleteRelationSQL = "" + + "DELETE FROM syncapi_relations WHERE room_id = $1 AND child_event_id = $2" + +const selectRelationsInRangeAscSQL = "" + + "SELECT id, child_event_id, rel_type FROM syncapi_relations" + + " WHERE room_id = $1 AND event_id = $2" + + " AND ( $3 = '' OR rel_type = $3 )" + + " AND ( $4 = '' OR child_event_type = $4 )" + + " AND id > $5 AND id <= $6" + + " ORDER BY id ASC LIMIT $7" + +const selectRelationsInRangeDescSQL = "" + + "SELECT id, child_event_id, rel_type FROM syncapi_relations" + + " WHERE room_id = $1 AND event_id = $2" + + " AND ( $3 = '' OR rel_type = $3 )" + + " AND ( $4 = '' OR child_event_type = $4 )" + + " AND id >= $5 AND id < $6" + + " ORDER BY id DESC LIMIT $7" + +const selectMaxRelationIDSQL = "" + + "SELECT COALESCE(MAX(id), 0) FROM syncapi_relations" + +type relationsStatements struct { + streamIDStatements *StreamIDStatements + insertRelationStmt *sql.Stmt + selectRelationsInRangeAscStmt *sql.Stmt + selectRelationsInRangeDescStmt *sql.Stmt + deleteRelationStmt *sql.Stmt + selectMaxRelationIDStmt *sql.Stmt +} + +func NewSqliteRelationsTable(db *sql.DB, streamID *StreamIDStatements) (tables.Relations, error) { + s := &relationsStatements{ + streamIDStatements: streamID, + } + _, err := db.Exec(relationsSchema) + if err != nil { + return nil, err + } + return s, sqlutil.StatementList{ + {&s.insertRelationStmt, insertRelationSQL}, + {&s.selectRelationsInRangeAscStmt, selectRelationsInRangeAscSQL}, + {&s.selectRelationsInRangeDescStmt, selectRelationsInRangeDescSQL}, + {&s.deleteRelationStmt, deleteRelationSQL}, + {&s.selectMaxRelationIDStmt, selectMaxRelationIDSQL}, + }.Prepare(db) +} + +func (s *relationsStatements) InsertRelation( + ctx context.Context, txn *sql.Tx, roomID, eventID, childEventID, childEventType, relType string, +) (err error) { + var streamPos types.StreamPosition + if streamPos, err = s.streamIDStatements.nextRelationID(ctx, txn); err != nil { + return + } + _, err = sqlutil.TxStmt(txn, s.insertRelationStmt).ExecContext( + ctx, streamPos, roomID, eventID, childEventID, childEventType, relType, + ) + return +} + +func (s *relationsStatements) DeleteRelation( + ctx context.Context, txn *sql.Tx, roomID, childEventID string, +) error { + stmt := sqlutil.TxStmt(txn, s.deleteRelationStmt) + _, err := stmt.ExecContext( + ctx, roomID, childEventID, + ) + return err +} + +// SelectRelationsInRange returns a map rel_type -> []child_event_id +func (s *relationsStatements) SelectRelationsInRange( + ctx context.Context, txn *sql.Tx, roomID, eventID, relType, eventType string, + r types.Range, limit int, +) (map[string][]types.RelationEntry, types.StreamPosition, error) { + var lastPos types.StreamPosition + var stmt *sql.Stmt + if r.Backwards { + stmt = sqlutil.TxStmt(txn, s.selectRelationsInRangeDescStmt) + } else { + stmt = sqlutil.TxStmt(txn, s.selectRelationsInRangeAscStmt) + } + rows, err := stmt.QueryContext(ctx, roomID, eventID, relType, eventType, r.Low(), r.High(), limit) + if err != nil { + return nil, lastPos, err + } + defer internal.CloseAndLogIfError(ctx, rows, "selectRelationsInRange: rows.close() failed") + result := map[string][]types.RelationEntry{} + var ( + id types.StreamPosition + childEventID string + relationType string + ) + for rows.Next() { + if err = rows.Scan(&id, &childEventID, &relationType); err != nil { + return nil, lastPos, err + } + if id > lastPos { + lastPos = id + } + result[relationType] = append(result[relationType], types.RelationEntry{ + Position: id, + EventID: childEventID, + }) + } + if lastPos == 0 { + lastPos = r.To + } + return result, lastPos, rows.Err() +} + +func (s *relationsStatements) SelectMaxRelationID( + ctx context.Context, txn *sql.Tx, +) (id int64, err error) { + stmt := sqlutil.TxStmt(txn, s.selectMaxRelationIDStmt) + err = stmt.QueryRowContext(ctx).Scan(&id) + return +} diff --git a/syncapi/storage/sqlite3/stream_id_table.go b/syncapi/storage/sqlite3/stream_id_table.go index 1160a437e..a4bba508e 100644 --- a/syncapi/storage/sqlite3/stream_id_table.go +++ b/syncapi/storage/sqlite3/stream_id_table.go @@ -28,6 +28,8 @@ INSERT INTO syncapi_stream_id (stream_name, stream_id) VALUES ("presence", 0) ON CONFLICT DO NOTHING; INSERT INTO syncapi_stream_id (stream_name, stream_id) VALUES ("notification", 0) ON CONFLICT DO NOTHING; +INSERT INTO syncapi_stream_id (stream_name, stream_id) VALUES ("relation", 0) + ON CONFLICT DO NOTHING; ` const increaseStreamIDStmt = "" + @@ -86,3 +88,9 @@ func (s *StreamIDStatements) nextNotificationID(ctx context.Context, txn *sql.Tx err = increaseStmt.QueryRowContext(ctx, "notification").Scan(&pos) return } + +func (s *StreamIDStatements) nextRelationID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error) { + increaseStmt := sqlutil.TxStmt(txn, s.increaseStreamIDStmt) + err = increaseStmt.QueryRowContext(ctx, "relation").Scan(&pos) + return +} diff --git a/syncapi/storage/sqlite3/syncserver.go b/syncapi/storage/sqlite3/syncserver.go index 0879030a6..510546909 100644 --- a/syncapi/storage/sqlite3/syncserver.go +++ b/syncapi/storage/sqlite3/syncserver.go @@ -123,6 +123,10 @@ func (d *SyncServerDatasource) prepare(ctx context.Context) (err error) { if err != nil { return err } + relations, err := NewSqliteRelationsTable(d.db, &d.streamID) + if err != nil { + return err + } // apply migrations which need multiple tables m := sqlutil.NewMigrator(d.db) @@ -153,6 +157,7 @@ func (d *SyncServerDatasource) prepare(ctx context.Context) (err error) { NotificationData: notificationData, Ignores: ignores, Presence: presence, + Relations: relations, } return nil } diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go index 2fdc3cfbb..e48c050dd 100644 --- a/syncapi/storage/tables/interface.go +++ b/syncapi/storage/tables/interface.go @@ -206,3 +206,22 @@ type Presence interface { GetMaxPresenceID(ctx context.Context, txn *sql.Tx) (pos types.StreamPosition, err error) GetPresenceAfter(ctx context.Context, txn *sql.Tx, after types.StreamPosition, filter gomatrixserverlib.EventFilter) (presences map[string]*types.PresenceInternal, err error) } + +type Relations interface { + // Inserts a relation which refers from the child event ID to the event ID in the given room. + // If the relation already exists then this function will do nothing and return no error. + InsertRelation(ctx context.Context, txn *sql.Tx, roomID, eventID, childEventID, childEventType, relType string) (err error) + // Deletes a relation which already exists as the result of an event redaction. If the relation + // does not exist then this function will do nothing and return no error. + DeleteRelation(ctx context.Context, txn *sql.Tx, roomID, childEventID string) error + // SelectRelationsInRange will return relations grouped by relation type within the given range. + // The map is relType -> []entry. If a relType parameter is specified then the results will only + // contain relations of that type, otherwise if "" is specified then all relations in the range + // will be returned, inclusive of the "to" position but excluding the "from" position. The stream + // position returned is the maximum position of the returned results. + SelectRelationsInRange(ctx context.Context, txn *sql.Tx, roomID, eventID, relType, eventType string, r types.Range, limit int) (map[string][]types.RelationEntry, types.StreamPosition, error) + // SelectMaxRelationID returns the maximum ID of all relations, used to determine what the boundaries + // should be if there are no boundaries supplied (i.e. we want to work backwards but don't have a + // "from" or want to work forwards and don't have a "to"). + SelectMaxRelationID(ctx context.Context, txn *sql.Tx) (id int64, err error) +} diff --git a/syncapi/storage/tables/relations_test.go b/syncapi/storage/tables/relations_test.go new file mode 100644 index 000000000..46270e36d --- /dev/null +++ b/syncapi/storage/tables/relations_test.go @@ -0,0 +1,186 @@ +package tables_test + +import ( + "context" + "database/sql" + "testing" + + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/syncapi/storage/postgres" + "github.com/matrix-org/dendrite/syncapi/storage/sqlite3" + "github.com/matrix-org/dendrite/syncapi/storage/tables" + "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/dendrite/test" +) + +func newRelationsTable(t *testing.T, dbType test.DBType) (tables.Relations, *sql.DB, func()) { + t.Helper() + connStr, close := test.PrepareDBConnectionString(t, dbType) + db, err := sqlutil.Open(&config.DatabaseOptions{ + ConnectionString: config.DataSource(connStr), + }, sqlutil.NewExclusiveWriter()) + if err != nil { + t.Fatalf("failed to open db: %s", err) + } + + var tab tables.Relations + switch dbType { + case test.DBTypePostgres: + tab, err = postgres.NewPostgresRelationsTable(db) + case test.DBTypeSQLite: + var stream sqlite3.StreamIDStatements + if err = stream.Prepare(db); err != nil { + t.Fatalf("failed to prepare stream stmts: %s", err) + } + tab, err = sqlite3.NewSqliteRelationsTable(db, &stream) + } + if err != nil { + t.Fatalf("failed to make new table: %s", err) + } + return tab, db, close +} + +func compareRelationsToExpected(t *testing.T, tab tables.Relations, r types.Range, expected []types.RelationEntry) { + ctx := context.Background() + relations, _, err := tab.SelectRelationsInRange(ctx, nil, roomID, "a", "", "", r, 50) + if err != nil { + t.Fatal(err) + } + if len(relations[relType]) != len(expected) { + t.Fatalf("incorrect number of values returned for range %v (got %d, want %d)", r, len(relations[relType]), len(expected)) + } + for i := 0; i < len(relations[relType]); i++ { + got := relations[relType][i] + want := expected[i] + if got != want { + t.Fatalf("range %v position %d should have been %q but got %q", r, i, got, want) + } + } +} + +const roomID = "!roomid:server" +const childType = "m.room.something" +const relType = "m.reaction" + +func TestRelationsTable(t *testing.T) { + ctx := context.Background() + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + tab, _, close := newRelationsTable(t, dbType) + defer close() + + // Insert some relations + for _, child := range []string{"b", "c", "d"} { + if err := tab.InsertRelation(ctx, nil, roomID, "a", child, childType, relType); err != nil { + t.Fatal(err) + } + } + + // Check the max position, we've inserted three things so it + // should be 3 + if max, err := tab.SelectMaxRelationID(ctx, nil); err != nil { + t.Fatal(err) + } else if max != 3 { + t.Fatalf("max position should have been 3 but got %d", max) + } + + // Query some ranges for "a" + for r, expected := range map[types.Range][]types.RelationEntry{ + {From: 0, To: 10, Backwards: false}: { + {Position: 1, EventID: "b"}, + {Position: 2, EventID: "c"}, + {Position: 3, EventID: "d"}, + }, + {From: 1, To: 2, Backwards: false}: { + {Position: 2, EventID: "c"}, + }, + {From: 1, To: 3, Backwards: false}: { + {Position: 2, EventID: "c"}, + {Position: 3, EventID: "d"}, + }, + {From: 10, To: 0, Backwards: true}: { + {Position: 3, EventID: "d"}, + {Position: 2, EventID: "c"}, + {Position: 1, EventID: "b"}, + }, + {From: 3, To: 1, Backwards: true}: { + {Position: 2, EventID: "c"}, + {Position: 1, EventID: "b"}, + }, + } { + compareRelationsToExpected(t, tab, r, expected) + } + + // Now delete one of the relations + if err := tab.DeleteRelation(ctx, nil, roomID, "c"); err != nil { + t.Fatal(err) + } + + // Query some more ranges for "a" + for r, expected := range map[types.Range][]types.RelationEntry{ + {From: 0, To: 10, Backwards: false}: { + {Position: 1, EventID: "b"}, + {Position: 3, EventID: "d"}, + }, + {From: 1, To: 2, Backwards: false}: {}, + {From: 1, To: 3, Backwards: false}: { + {Position: 3, EventID: "d"}, + }, + {From: 10, To: 0, Backwards: true}: { + {Position: 3, EventID: "d"}, + {Position: 1, EventID: "b"}, + }, + {From: 3, To: 1, Backwards: true}: { + {Position: 1, EventID: "b"}, + }, + } { + compareRelationsToExpected(t, tab, r, expected) + } + + // Insert some new relations + for _, child := range []string{"e", "f", "g", "h"} { + if err := tab.InsertRelation(ctx, nil, roomID, "a", child, childType, relType); err != nil { + t.Fatal(err) + } + } + + // Check the max position, we've inserted four things so it + // should now be 7 + if max, err := tab.SelectMaxRelationID(ctx, nil); err != nil { + t.Fatal(err) + } else if max != 7 { + t.Fatalf("max position should have been 3 but got %d", max) + } + + // Query last set of ranges for "a" + for r, expected := range map[types.Range][]types.RelationEntry{ + {From: 0, To: 10, Backwards: false}: { + {Position: 1, EventID: "b"}, + {Position: 3, EventID: "d"}, + {Position: 4, EventID: "e"}, + {Position: 5, EventID: "f"}, + {Position: 6, EventID: "g"}, + {Position: 7, EventID: "h"}, + }, + {From: 1, To: 2, Backwards: false}: {}, + {From: 1, To: 3, Backwards: false}: { + {Position: 3, EventID: "d"}, + }, + {From: 10, To: 0, Backwards: true}: { + {Position: 7, EventID: "h"}, + {Position: 6, EventID: "g"}, + {Position: 5, EventID: "f"}, + {Position: 4, EventID: "e"}, + {Position: 3, EventID: "d"}, + {Position: 1, EventID: "b"}, + }, + {From: 6, To: 3, Backwards: true}: { + {Position: 5, EventID: "f"}, + {Position: 4, EventID: "e"}, + {Position: 3, EventID: "d"}, + }, + } { + compareRelationsToExpected(t, tab, r, expected) + } + }) +} diff --git a/syncapi/types/types.go b/syncapi/types/types.go index b6d340f93..60a74a285 100644 --- a/syncapi/types/types.go +++ b/syncapi/types/types.go @@ -47,6 +47,14 @@ type StateDelta struct { // StreamPosition represents the offset in the sync stream a client is at. type StreamPosition int64 +func NewStreamPositionFromString(s string) (StreamPosition, error) { + n, err := strconv.Atoi(s) + if err != nil { + return 0, err + } + return StreamPosition(n), nil +} + // StreamEvent is the same as gomatrixserverlib.Event but also has the PDU stream position for this event. type StreamEvent struct { *gomatrixserverlib.HeaderedEvent @@ -599,3 +607,8 @@ type OutputSendToDeviceEvent struct { type IgnoredUsers struct { List map[string]interface{} `json:"ignored_users"` } + +type RelationEntry struct { + Position StreamPosition + EventID string +} From dcedd1b6bf1e890ff425bdf1fcd8a2e0850778b5 Mon Sep 17 00:00:00 2001 From: devonh Date: Thu, 13 Oct 2022 14:38:13 +0000 Subject: [PATCH 071/207] Federation backoff fixes and tests (#2792) This fixes some edge cases where federation queue backoffs and blacklisting weren't behaving as expected. It also adds new tests for the federation queues to ensure their behaviour continues to work correctly. --- federationapi/queue/destinationqueue.go | 10 + federationapi/queue/queue.go | 1 + federationapi/queue/queue_test.go | 422 ++++++++++++++++++++++++ federationapi/statistics/statistics.go | 8 +- go.mod | 2 +- 5 files changed, 441 insertions(+), 2 deletions(-) create mode 100644 federationapi/queue/queue_test.go diff --git a/federationapi/queue/destinationqueue.go b/federationapi/queue/destinationqueue.go index 5cb8cae1f..4ae554ef3 100644 --- a/federationapi/queue/destinationqueue.go +++ b/federationapi/queue/destinationqueue.go @@ -75,6 +75,7 @@ func (oq *destinationQueue) sendEvent(event *gomatrixserverlib.HeaderedEvent, re logrus.Errorf("attempt to send nil PDU with destination %q", oq.destination) return } + // Create a database entry that associates the given PDU NID with // this destination queue. We'll then be able to retrieve the PDU // later. @@ -108,6 +109,8 @@ func (oq *destinationQueue) sendEvent(event *gomatrixserverlib.HeaderedEvent, re case oq.notify <- struct{}{}: default: } + } else { + oq.overflowed.Store(true) } } @@ -153,6 +156,8 @@ func (oq *destinationQueue) sendEDU(event *gomatrixserverlib.EDU, receipt *share case oq.notify <- struct{}{}: default: } + } else { + oq.overflowed.Store(true) } } @@ -335,6 +340,11 @@ func (oq *destinationQueue) backgroundSend() { // We failed to send the transaction. Mark it as a failure. oq.statistics.Failure() + // Queue up another attempt since the transaction failed. + select { + case oq.notify <- struct{}{}: + default: + } } else if transaction { // If we successfully sent the transaction then clear out // the pending events and EDUs, and wipe our transaction ID. diff --git a/federationapi/queue/queue.go b/federationapi/queue/queue.go index 8245aa5bd..5d352eca6 100644 --- a/federationapi/queue/queue.go +++ b/federationapi/queue/queue.go @@ -332,6 +332,7 @@ func (oqs *OutgoingQueues) RetryServer(srv gomatrixserverlib.ServerName) { if oqs.disabled { return } + oqs.statistics.ForServer(srv).RemoveBlacklist() if queue := oqs.getQueue(srv); queue != nil { queue.wakeQueueIfNeeded() } diff --git a/federationapi/queue/queue_test.go b/federationapi/queue/queue_test.go new file mode 100644 index 000000000..8e4a675f4 --- /dev/null +++ b/federationapi/queue/queue_test.go @@ -0,0 +1,422 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package queue + +import ( + "context" + "encoding/json" + "fmt" + "sync" + "testing" + "time" + + "go.uber.org/atomic" + "gotest.tools/v3/poll" + + "github.com/matrix-org/dendrite/federationapi/api" + "github.com/matrix-org/dendrite/federationapi/statistics" + "github.com/matrix-org/dendrite/federationapi/storage" + "github.com/matrix-org/dendrite/federationapi/storage/shared" + rsapi "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/setup/process" + "github.com/matrix-org/dendrite/test" + "github.com/matrix-org/gomatrixserverlib" + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" +) + +var dbMutex sync.Mutex + +type fakeDatabase struct { + storage.Database + pendingPDUServers map[gomatrixserverlib.ServerName]struct{} + pendingEDUServers map[gomatrixserverlib.ServerName]struct{} + blacklistedServers map[gomatrixserverlib.ServerName]struct{} + pendingPDUs map[*shared.Receipt]*gomatrixserverlib.HeaderedEvent + pendingEDUs map[*shared.Receipt]*gomatrixserverlib.EDU + associatedPDUs map[gomatrixserverlib.ServerName]map[*shared.Receipt]struct{} + associatedEDUs map[gomatrixserverlib.ServerName]map[*shared.Receipt]struct{} +} + +func (d *fakeDatabase) StoreJSON(ctx context.Context, js string) (*shared.Receipt, error) { + dbMutex.Lock() + defer dbMutex.Unlock() + + var event gomatrixserverlib.HeaderedEvent + if err := json.Unmarshal([]byte(js), &event); err == nil { + receipt := &shared.Receipt{} + d.pendingPDUs[receipt] = &event + return receipt, nil + } + + var edu gomatrixserverlib.EDU + if err := json.Unmarshal([]byte(js), &edu); err == nil { + receipt := &shared.Receipt{} + d.pendingEDUs[receipt] = &edu + return receipt, nil + } + + return nil, errors.New("Failed to determine type of json to store") +} + +func (d *fakeDatabase) GetPendingPDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, limit int) (pdus map[*shared.Receipt]*gomatrixserverlib.HeaderedEvent, err error) { + dbMutex.Lock() + defer dbMutex.Unlock() + + pdus = make(map[*shared.Receipt]*gomatrixserverlib.HeaderedEvent) + if receipts, ok := d.associatedPDUs[serverName]; ok { + for receipt := range receipts { + if event, ok := d.pendingPDUs[receipt]; ok { + pdus[receipt] = event + } + } + } + return pdus, nil +} + +func (d *fakeDatabase) GetPendingEDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, limit int) (edus map[*shared.Receipt]*gomatrixserverlib.EDU, err error) { + dbMutex.Lock() + defer dbMutex.Unlock() + + edus = make(map[*shared.Receipt]*gomatrixserverlib.EDU) + if receipts, ok := d.associatedEDUs[serverName]; ok { + for receipt := range receipts { + if event, ok := d.pendingEDUs[receipt]; ok { + edus[receipt] = event + } + } + } + return edus, nil +} + +func (d *fakeDatabase) AssociatePDUWithDestination(ctx context.Context, transactionID gomatrixserverlib.TransactionID, serverName gomatrixserverlib.ServerName, receipt *shared.Receipt) error { + dbMutex.Lock() + defer dbMutex.Unlock() + + if _, ok := d.pendingPDUs[receipt]; ok { + if _, ok := d.associatedPDUs[serverName]; !ok { + d.associatedPDUs[serverName] = make(map[*shared.Receipt]struct{}) + } + d.associatedPDUs[serverName][receipt] = struct{}{} + return nil + } else { + return errors.New("PDU doesn't exist") + } +} + +func (d *fakeDatabase) AssociateEDUWithDestination(ctx context.Context, serverName gomatrixserverlib.ServerName, receipt *shared.Receipt, eduType string, expireEDUTypes map[string]time.Duration) error { + dbMutex.Lock() + defer dbMutex.Unlock() + + if _, ok := d.pendingEDUs[receipt]; ok { + if _, ok := d.associatedEDUs[serverName]; !ok { + d.associatedEDUs[serverName] = make(map[*shared.Receipt]struct{}) + } + d.associatedEDUs[serverName][receipt] = struct{}{} + return nil + } else { + return errors.New("EDU doesn't exist") + } +} + +func (d *fakeDatabase) CleanPDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, receipts []*shared.Receipt) error { + dbMutex.Lock() + defer dbMutex.Unlock() + + if pdus, ok := d.associatedPDUs[serverName]; ok { + for _, receipt := range receipts { + delete(pdus, receipt) + } + } + + return nil +} + +func (d *fakeDatabase) CleanEDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, receipts []*shared.Receipt) error { + dbMutex.Lock() + defer dbMutex.Unlock() + + if edus, ok := d.associatedEDUs[serverName]; ok { + for _, receipt := range receipts { + delete(edus, receipt) + } + } + + return nil +} + +func (d *fakeDatabase) GetPendingPDUCount(ctx context.Context, serverName gomatrixserverlib.ServerName) (int64, error) { + dbMutex.Lock() + defer dbMutex.Unlock() + + var count int64 + if pdus, ok := d.associatedPDUs[serverName]; ok { + count = int64(len(pdus)) + } + return count, nil +} + +func (d *fakeDatabase) GetPendingEDUCount(ctx context.Context, serverName gomatrixserverlib.ServerName) (int64, error) { + dbMutex.Lock() + defer dbMutex.Unlock() + + var count int64 + if edus, ok := d.associatedEDUs[serverName]; ok { + count = int64(len(edus)) + } + return count, nil +} + +func (d *fakeDatabase) GetPendingPDUServerNames(ctx context.Context) ([]gomatrixserverlib.ServerName, error) { + dbMutex.Lock() + defer dbMutex.Unlock() + + servers := []gomatrixserverlib.ServerName{} + for server := range d.pendingPDUServers { + servers = append(servers, server) + } + return servers, nil +} + +func (d *fakeDatabase) GetPendingEDUServerNames(ctx context.Context) ([]gomatrixserverlib.ServerName, error) { + dbMutex.Lock() + defer dbMutex.Unlock() + + servers := []gomatrixserverlib.ServerName{} + for server := range d.pendingEDUServers { + servers = append(servers, server) + } + return servers, nil +} + +func (d *fakeDatabase) AddServerToBlacklist(serverName gomatrixserverlib.ServerName) error { + dbMutex.Lock() + defer dbMutex.Unlock() + + d.blacklistedServers[serverName] = struct{}{} + return nil +} + +func (d *fakeDatabase) RemoveServerFromBlacklist(serverName gomatrixserverlib.ServerName) error { + dbMutex.Lock() + defer dbMutex.Unlock() + + delete(d.blacklistedServers, serverName) + return nil +} + +func (d *fakeDatabase) RemoveAllServersFromBlacklist() error { + dbMutex.Lock() + defer dbMutex.Unlock() + + d.blacklistedServers = make(map[gomatrixserverlib.ServerName]struct{}) + return nil +} + +func (d *fakeDatabase) IsServerBlacklisted(serverName gomatrixserverlib.ServerName) (bool, error) { + dbMutex.Lock() + defer dbMutex.Unlock() + + isBlacklisted := false + if _, ok := d.blacklistedServers[serverName]; ok { + isBlacklisted = true + } + + return isBlacklisted, nil +} + +type stubFederationRoomServerAPI struct { + rsapi.FederationRoomserverAPI +} + +func (r *stubFederationRoomServerAPI) QueryServerBannedFromRoom(ctx context.Context, req *rsapi.QueryServerBannedFromRoomRequest, res *rsapi.QueryServerBannedFromRoomResponse) error { + res.Banned = false + return nil +} + +type stubFederationClient struct { + api.FederationClient + shouldTxSucceed bool + txCount atomic.Uint32 +} + +func (f *stubFederationClient) SendTransaction(ctx context.Context, t gomatrixserverlib.Transaction) (res gomatrixserverlib.RespSend, err error) { + var result error + if !f.shouldTxSucceed { + result = fmt.Errorf("transaction failed") + } + + f.txCount.Add(1) + return gomatrixserverlib.RespSend{}, result +} + +func createDatabase() storage.Database { + return &fakeDatabase{ + pendingPDUServers: make(map[gomatrixserverlib.ServerName]struct{}), + pendingEDUServers: make(map[gomatrixserverlib.ServerName]struct{}), + blacklistedServers: make(map[gomatrixserverlib.ServerName]struct{}), + pendingPDUs: make(map[*shared.Receipt]*gomatrixserverlib.HeaderedEvent), + pendingEDUs: make(map[*shared.Receipt]*gomatrixserverlib.EDU), + associatedPDUs: make(map[gomatrixserverlib.ServerName]map[*shared.Receipt]struct{}), + } +} + +func mustCreateEvent(t *testing.T) *gomatrixserverlib.HeaderedEvent { + t.Helper() + content := `{"type":"m.room.message"}` + ev, err := gomatrixserverlib.NewEventFromTrustedJSON([]byte(content), false, gomatrixserverlib.RoomVersionV10) + if err != nil { + t.Fatalf("failed to create event: %v", err) + } + return ev.Headered(gomatrixserverlib.RoomVersionV10) +} + +func testSetup(failuresUntilBlacklist uint32, shouldTxSucceed bool) (storage.Database, *stubFederationClient, *OutgoingQueues) { + db := createDatabase() + + fc := &stubFederationClient{ + shouldTxSucceed: shouldTxSucceed, + txCount: *atomic.NewUint32(0), + } + rs := &stubFederationRoomServerAPI{} + stats := &statistics.Statistics{ + DB: db, + FailuresUntilBlacklist: failuresUntilBlacklist, + } + signingInfo := &SigningInfo{ + KeyID: "ed25519:auto", + PrivateKey: test.PrivateKeyA, + ServerName: "localhost", + } + queues := NewOutgoingQueues(db, process.NewProcessContext(), false, "localhost", fc, rs, stats, signingInfo) + + return db, fc, queues +} + +func TestSendTransactionOnSuccessRemovedFromDB(t *testing.T) { + ctx := context.Background() + failuresUntilBlacklist := uint32(16) + destination := gomatrixserverlib.ServerName("remotehost") + db, fc, queues := testSetup(failuresUntilBlacklist, true) + + ev := mustCreateEvent(t) + err := queues.SendEvent(ev, "localhost", []gomatrixserverlib.ServerName{destination}) + assert.NoError(t, err) + + check := func(log poll.LogT) poll.Result { + if fc.txCount.Load() >= 1 { + data, err := db.GetPendingPDUs(ctx, destination, 100) + assert.NoError(t, err) + if len(data) == 0 { + return poll.Success() + } + return poll.Continue("waiting for event to be removed from database") + } + return poll.Continue("waiting for more send attempts before checking database") + } + poll.WaitOn(t, check, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond)) +} + +func TestSendTransactionOnFailStoredInDB(t *testing.T) { + ctx := context.Background() + failuresUntilBlacklist := uint32(16) + destination := gomatrixserverlib.ServerName("remotehost") + db, fc, queues := testSetup(failuresUntilBlacklist, false) + + ev := mustCreateEvent(t) + err := queues.SendEvent(ev, "localhost", []gomatrixserverlib.ServerName{destination}) + assert.NoError(t, err) + + check := func(log poll.LogT) poll.Result { + // Wait for 2 backoff attempts to ensure there was adequate time to attempt sending + if fc.txCount.Load() >= 2 { + data, err := db.GetPendingPDUs(ctx, destination, 100) + assert.NoError(t, err) + if len(data) == 1 { + return poll.Success() + } + return poll.Continue("waiting for event to be added to database") + } + return poll.Continue("waiting for more send attempts before checking database") + } + poll.WaitOn(t, check, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond)) +} + +func TestSendTransactionMultipleFailuresBlacklisted(t *testing.T) { + ctx := context.Background() + failuresUntilBlacklist := uint32(2) + destination := gomatrixserverlib.ServerName("remotehost") + db, fc, queues := testSetup(failuresUntilBlacklist, false) + + ev := mustCreateEvent(t) + err := queues.SendEvent(ev, "localhost", []gomatrixserverlib.ServerName{destination}) + assert.NoError(t, err) + + check := func(log poll.LogT) poll.Result { + if fc.txCount.Load() >= failuresUntilBlacklist { + data, err := db.GetPendingPDUs(ctx, destination, 100) + assert.NoError(t, err) + if len(data) == 1 { + if val, _ := db.IsServerBlacklisted(destination); val { + return poll.Success() + } + return poll.Continue("waiting for server to be blacklisted") + } + return poll.Continue("waiting for event to be added to database") + } + return poll.Continue("waiting for more send attempts before checking database") + } + poll.WaitOn(t, check, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond)) +} + +func TestRetryServerSendsSuccessfully(t *testing.T) { + ctx := context.Background() + failuresUntilBlacklist := uint32(1) + destination := gomatrixserverlib.ServerName("remotehost") + db, fc, queues := testSetup(failuresUntilBlacklist, false) + + ev := mustCreateEvent(t) + err := queues.SendEvent(ev, "localhost", []gomatrixserverlib.ServerName{destination}) + assert.NoError(t, err) + + checkBlacklisted := func(log poll.LogT) poll.Result { + if fc.txCount.Load() >= failuresUntilBlacklist { + data, err := db.GetPendingPDUs(ctx, destination, 100) + assert.NoError(t, err) + if len(data) == 1 { + if val, _ := db.IsServerBlacklisted(destination); val { + return poll.Success() + } + return poll.Continue("waiting for server to be blacklisted") + } + return poll.Continue("waiting for event to be added to database") + } + return poll.Continue("waiting for more send attempts before checking database") + } + poll.WaitOn(t, checkBlacklisted, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond)) + + fc.shouldTxSucceed = true + db.RemoveServerFromBlacklist(destination) + queues.RetryServer(destination) + checkRetry := func(log poll.LogT) poll.Result { + data, err := db.GetPendingPDUs(ctx, destination, 100) + assert.NoError(t, err) + if len(data) == 0 { + return poll.Success() + } + return poll.Continue("waiting for event to be removed from database") + } + poll.WaitOn(t, checkRetry, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond)) +} diff --git a/federationapi/statistics/statistics.go b/federationapi/statistics/statistics.go index db6d5c735..61a965791 100644 --- a/federationapi/statistics/statistics.go +++ b/federationapi/statistics/statistics.go @@ -95,8 +95,8 @@ func (s *ServerStatistics) cancel() { // we will unblacklist it. func (s *ServerStatistics) Success() { s.cancel() - s.successCounter.Inc() s.backoffCount.Store(0) + s.successCounter.Inc() if s.statistics.DB != nil { if err := s.statistics.DB.RemoveServerFromBlacklist(s.serverName); err != nil { logrus.WithError(err).Errorf("Failed to remove %q from blacklist", s.serverName) @@ -174,6 +174,12 @@ func (s *ServerStatistics) Blacklisted() bool { return s.blacklisted.Load() } +// RemoveBlacklist removes the blacklisted status from the server. +func (s *ServerStatistics) RemoveBlacklist() { + s.cancel() + s.backoffCount.Store(0) +} + // SuccessCount returns the number of successful requests. This is // usually useful in constructing transaction IDs. func (s *ServerStatistics) SuccessCount() uint32 { diff --git a/go.mod b/go.mod index eefad89e6..eeae9608f 100644 --- a/go.mod +++ b/go.mod @@ -50,6 +50,7 @@ require ( golang.org/x/term v0.0.0-20220919170432-7a66f970e087 gopkg.in/h2non/bimg.v1 v1.1.9 gopkg.in/yaml.v2 v2.4.0 + gotest.tools/v3 v3.0.3 nhooyr.io/websocket v1.8.7 ) @@ -128,7 +129,6 @@ require ( gopkg.in/macaroon.v2 v2.1.0 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - gotest.tools/v3 v3.0.3 // indirect ) go 1.18 From f3be4b31850add1a33c932aa3fa7b0bb740554f2 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 13 Oct 2022 16:06:50 +0100 Subject: [PATCH 072/207] Revert "Federation backoff fixes and tests (#2792)" This reverts commit dcedd1b6bf1e890ff425bdf1fcd8a2e0850778b5. --- federationapi/queue/destinationqueue.go | 10 - federationapi/queue/queue.go | 1 - federationapi/queue/queue_test.go | 422 ------------------------ federationapi/statistics/statistics.go | 8 +- go.mod | 2 +- 5 files changed, 2 insertions(+), 441 deletions(-) delete mode 100644 federationapi/queue/queue_test.go diff --git a/federationapi/queue/destinationqueue.go b/federationapi/queue/destinationqueue.go index 4ae554ef3..5cb8cae1f 100644 --- a/federationapi/queue/destinationqueue.go +++ b/federationapi/queue/destinationqueue.go @@ -75,7 +75,6 @@ func (oq *destinationQueue) sendEvent(event *gomatrixserverlib.HeaderedEvent, re logrus.Errorf("attempt to send nil PDU with destination %q", oq.destination) return } - // Create a database entry that associates the given PDU NID with // this destination queue. We'll then be able to retrieve the PDU // later. @@ -109,8 +108,6 @@ func (oq *destinationQueue) sendEvent(event *gomatrixserverlib.HeaderedEvent, re case oq.notify <- struct{}{}: default: } - } else { - oq.overflowed.Store(true) } } @@ -156,8 +153,6 @@ func (oq *destinationQueue) sendEDU(event *gomatrixserverlib.EDU, receipt *share case oq.notify <- struct{}{}: default: } - } else { - oq.overflowed.Store(true) } } @@ -340,11 +335,6 @@ func (oq *destinationQueue) backgroundSend() { // We failed to send the transaction. Mark it as a failure. oq.statistics.Failure() - // Queue up another attempt since the transaction failed. - select { - case oq.notify <- struct{}{}: - default: - } } else if transaction { // If we successfully sent the transaction then clear out // the pending events and EDUs, and wipe our transaction ID. diff --git a/federationapi/queue/queue.go b/federationapi/queue/queue.go index 5d352eca6..8245aa5bd 100644 --- a/federationapi/queue/queue.go +++ b/federationapi/queue/queue.go @@ -332,7 +332,6 @@ func (oqs *OutgoingQueues) RetryServer(srv gomatrixserverlib.ServerName) { if oqs.disabled { return } - oqs.statistics.ForServer(srv).RemoveBlacklist() if queue := oqs.getQueue(srv); queue != nil { queue.wakeQueueIfNeeded() } diff --git a/federationapi/queue/queue_test.go b/federationapi/queue/queue_test.go deleted file mode 100644 index 8e4a675f4..000000000 --- a/federationapi/queue/queue_test.go +++ /dev/null @@ -1,422 +0,0 @@ -// Copyright 2022 The Matrix.org Foundation C.I.C. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package queue - -import ( - "context" - "encoding/json" - "fmt" - "sync" - "testing" - "time" - - "go.uber.org/atomic" - "gotest.tools/v3/poll" - - "github.com/matrix-org/dendrite/federationapi/api" - "github.com/matrix-org/dendrite/federationapi/statistics" - "github.com/matrix-org/dendrite/federationapi/storage" - "github.com/matrix-org/dendrite/federationapi/storage/shared" - rsapi "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/setup/process" - "github.com/matrix-org/dendrite/test" - "github.com/matrix-org/gomatrixserverlib" - "github.com/pkg/errors" - "github.com/stretchr/testify/assert" -) - -var dbMutex sync.Mutex - -type fakeDatabase struct { - storage.Database - pendingPDUServers map[gomatrixserverlib.ServerName]struct{} - pendingEDUServers map[gomatrixserverlib.ServerName]struct{} - blacklistedServers map[gomatrixserverlib.ServerName]struct{} - pendingPDUs map[*shared.Receipt]*gomatrixserverlib.HeaderedEvent - pendingEDUs map[*shared.Receipt]*gomatrixserverlib.EDU - associatedPDUs map[gomatrixserverlib.ServerName]map[*shared.Receipt]struct{} - associatedEDUs map[gomatrixserverlib.ServerName]map[*shared.Receipt]struct{} -} - -func (d *fakeDatabase) StoreJSON(ctx context.Context, js string) (*shared.Receipt, error) { - dbMutex.Lock() - defer dbMutex.Unlock() - - var event gomatrixserverlib.HeaderedEvent - if err := json.Unmarshal([]byte(js), &event); err == nil { - receipt := &shared.Receipt{} - d.pendingPDUs[receipt] = &event - return receipt, nil - } - - var edu gomatrixserverlib.EDU - if err := json.Unmarshal([]byte(js), &edu); err == nil { - receipt := &shared.Receipt{} - d.pendingEDUs[receipt] = &edu - return receipt, nil - } - - return nil, errors.New("Failed to determine type of json to store") -} - -func (d *fakeDatabase) GetPendingPDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, limit int) (pdus map[*shared.Receipt]*gomatrixserverlib.HeaderedEvent, err error) { - dbMutex.Lock() - defer dbMutex.Unlock() - - pdus = make(map[*shared.Receipt]*gomatrixserverlib.HeaderedEvent) - if receipts, ok := d.associatedPDUs[serverName]; ok { - for receipt := range receipts { - if event, ok := d.pendingPDUs[receipt]; ok { - pdus[receipt] = event - } - } - } - return pdus, nil -} - -func (d *fakeDatabase) GetPendingEDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, limit int) (edus map[*shared.Receipt]*gomatrixserverlib.EDU, err error) { - dbMutex.Lock() - defer dbMutex.Unlock() - - edus = make(map[*shared.Receipt]*gomatrixserverlib.EDU) - if receipts, ok := d.associatedEDUs[serverName]; ok { - for receipt := range receipts { - if event, ok := d.pendingEDUs[receipt]; ok { - edus[receipt] = event - } - } - } - return edus, nil -} - -func (d *fakeDatabase) AssociatePDUWithDestination(ctx context.Context, transactionID gomatrixserverlib.TransactionID, serverName gomatrixserverlib.ServerName, receipt *shared.Receipt) error { - dbMutex.Lock() - defer dbMutex.Unlock() - - if _, ok := d.pendingPDUs[receipt]; ok { - if _, ok := d.associatedPDUs[serverName]; !ok { - d.associatedPDUs[serverName] = make(map[*shared.Receipt]struct{}) - } - d.associatedPDUs[serverName][receipt] = struct{}{} - return nil - } else { - return errors.New("PDU doesn't exist") - } -} - -func (d *fakeDatabase) AssociateEDUWithDestination(ctx context.Context, serverName gomatrixserverlib.ServerName, receipt *shared.Receipt, eduType string, expireEDUTypes map[string]time.Duration) error { - dbMutex.Lock() - defer dbMutex.Unlock() - - if _, ok := d.pendingEDUs[receipt]; ok { - if _, ok := d.associatedEDUs[serverName]; !ok { - d.associatedEDUs[serverName] = make(map[*shared.Receipt]struct{}) - } - d.associatedEDUs[serverName][receipt] = struct{}{} - return nil - } else { - return errors.New("EDU doesn't exist") - } -} - -func (d *fakeDatabase) CleanPDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, receipts []*shared.Receipt) error { - dbMutex.Lock() - defer dbMutex.Unlock() - - if pdus, ok := d.associatedPDUs[serverName]; ok { - for _, receipt := range receipts { - delete(pdus, receipt) - } - } - - return nil -} - -func (d *fakeDatabase) CleanEDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, receipts []*shared.Receipt) error { - dbMutex.Lock() - defer dbMutex.Unlock() - - if edus, ok := d.associatedEDUs[serverName]; ok { - for _, receipt := range receipts { - delete(edus, receipt) - } - } - - return nil -} - -func (d *fakeDatabase) GetPendingPDUCount(ctx context.Context, serverName gomatrixserverlib.ServerName) (int64, error) { - dbMutex.Lock() - defer dbMutex.Unlock() - - var count int64 - if pdus, ok := d.associatedPDUs[serverName]; ok { - count = int64(len(pdus)) - } - return count, nil -} - -func (d *fakeDatabase) GetPendingEDUCount(ctx context.Context, serverName gomatrixserverlib.ServerName) (int64, error) { - dbMutex.Lock() - defer dbMutex.Unlock() - - var count int64 - if edus, ok := d.associatedEDUs[serverName]; ok { - count = int64(len(edus)) - } - return count, nil -} - -func (d *fakeDatabase) GetPendingPDUServerNames(ctx context.Context) ([]gomatrixserverlib.ServerName, error) { - dbMutex.Lock() - defer dbMutex.Unlock() - - servers := []gomatrixserverlib.ServerName{} - for server := range d.pendingPDUServers { - servers = append(servers, server) - } - return servers, nil -} - -func (d *fakeDatabase) GetPendingEDUServerNames(ctx context.Context) ([]gomatrixserverlib.ServerName, error) { - dbMutex.Lock() - defer dbMutex.Unlock() - - servers := []gomatrixserverlib.ServerName{} - for server := range d.pendingEDUServers { - servers = append(servers, server) - } - return servers, nil -} - -func (d *fakeDatabase) AddServerToBlacklist(serverName gomatrixserverlib.ServerName) error { - dbMutex.Lock() - defer dbMutex.Unlock() - - d.blacklistedServers[serverName] = struct{}{} - return nil -} - -func (d *fakeDatabase) RemoveServerFromBlacklist(serverName gomatrixserverlib.ServerName) error { - dbMutex.Lock() - defer dbMutex.Unlock() - - delete(d.blacklistedServers, serverName) - return nil -} - -func (d *fakeDatabase) RemoveAllServersFromBlacklist() error { - dbMutex.Lock() - defer dbMutex.Unlock() - - d.blacklistedServers = make(map[gomatrixserverlib.ServerName]struct{}) - return nil -} - -func (d *fakeDatabase) IsServerBlacklisted(serverName gomatrixserverlib.ServerName) (bool, error) { - dbMutex.Lock() - defer dbMutex.Unlock() - - isBlacklisted := false - if _, ok := d.blacklistedServers[serverName]; ok { - isBlacklisted = true - } - - return isBlacklisted, nil -} - -type stubFederationRoomServerAPI struct { - rsapi.FederationRoomserverAPI -} - -func (r *stubFederationRoomServerAPI) QueryServerBannedFromRoom(ctx context.Context, req *rsapi.QueryServerBannedFromRoomRequest, res *rsapi.QueryServerBannedFromRoomResponse) error { - res.Banned = false - return nil -} - -type stubFederationClient struct { - api.FederationClient - shouldTxSucceed bool - txCount atomic.Uint32 -} - -func (f *stubFederationClient) SendTransaction(ctx context.Context, t gomatrixserverlib.Transaction) (res gomatrixserverlib.RespSend, err error) { - var result error - if !f.shouldTxSucceed { - result = fmt.Errorf("transaction failed") - } - - f.txCount.Add(1) - return gomatrixserverlib.RespSend{}, result -} - -func createDatabase() storage.Database { - return &fakeDatabase{ - pendingPDUServers: make(map[gomatrixserverlib.ServerName]struct{}), - pendingEDUServers: make(map[gomatrixserverlib.ServerName]struct{}), - blacklistedServers: make(map[gomatrixserverlib.ServerName]struct{}), - pendingPDUs: make(map[*shared.Receipt]*gomatrixserverlib.HeaderedEvent), - pendingEDUs: make(map[*shared.Receipt]*gomatrixserverlib.EDU), - associatedPDUs: make(map[gomatrixserverlib.ServerName]map[*shared.Receipt]struct{}), - } -} - -func mustCreateEvent(t *testing.T) *gomatrixserverlib.HeaderedEvent { - t.Helper() - content := `{"type":"m.room.message"}` - ev, err := gomatrixserverlib.NewEventFromTrustedJSON([]byte(content), false, gomatrixserverlib.RoomVersionV10) - if err != nil { - t.Fatalf("failed to create event: %v", err) - } - return ev.Headered(gomatrixserverlib.RoomVersionV10) -} - -func testSetup(failuresUntilBlacklist uint32, shouldTxSucceed bool) (storage.Database, *stubFederationClient, *OutgoingQueues) { - db := createDatabase() - - fc := &stubFederationClient{ - shouldTxSucceed: shouldTxSucceed, - txCount: *atomic.NewUint32(0), - } - rs := &stubFederationRoomServerAPI{} - stats := &statistics.Statistics{ - DB: db, - FailuresUntilBlacklist: failuresUntilBlacklist, - } - signingInfo := &SigningInfo{ - KeyID: "ed25519:auto", - PrivateKey: test.PrivateKeyA, - ServerName: "localhost", - } - queues := NewOutgoingQueues(db, process.NewProcessContext(), false, "localhost", fc, rs, stats, signingInfo) - - return db, fc, queues -} - -func TestSendTransactionOnSuccessRemovedFromDB(t *testing.T) { - ctx := context.Background() - failuresUntilBlacklist := uint32(16) - destination := gomatrixserverlib.ServerName("remotehost") - db, fc, queues := testSetup(failuresUntilBlacklist, true) - - ev := mustCreateEvent(t) - err := queues.SendEvent(ev, "localhost", []gomatrixserverlib.ServerName{destination}) - assert.NoError(t, err) - - check := func(log poll.LogT) poll.Result { - if fc.txCount.Load() >= 1 { - data, err := db.GetPendingPDUs(ctx, destination, 100) - assert.NoError(t, err) - if len(data) == 0 { - return poll.Success() - } - return poll.Continue("waiting for event to be removed from database") - } - return poll.Continue("waiting for more send attempts before checking database") - } - poll.WaitOn(t, check, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond)) -} - -func TestSendTransactionOnFailStoredInDB(t *testing.T) { - ctx := context.Background() - failuresUntilBlacklist := uint32(16) - destination := gomatrixserverlib.ServerName("remotehost") - db, fc, queues := testSetup(failuresUntilBlacklist, false) - - ev := mustCreateEvent(t) - err := queues.SendEvent(ev, "localhost", []gomatrixserverlib.ServerName{destination}) - assert.NoError(t, err) - - check := func(log poll.LogT) poll.Result { - // Wait for 2 backoff attempts to ensure there was adequate time to attempt sending - if fc.txCount.Load() >= 2 { - data, err := db.GetPendingPDUs(ctx, destination, 100) - assert.NoError(t, err) - if len(data) == 1 { - return poll.Success() - } - return poll.Continue("waiting for event to be added to database") - } - return poll.Continue("waiting for more send attempts before checking database") - } - poll.WaitOn(t, check, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond)) -} - -func TestSendTransactionMultipleFailuresBlacklisted(t *testing.T) { - ctx := context.Background() - failuresUntilBlacklist := uint32(2) - destination := gomatrixserverlib.ServerName("remotehost") - db, fc, queues := testSetup(failuresUntilBlacklist, false) - - ev := mustCreateEvent(t) - err := queues.SendEvent(ev, "localhost", []gomatrixserverlib.ServerName{destination}) - assert.NoError(t, err) - - check := func(log poll.LogT) poll.Result { - if fc.txCount.Load() >= failuresUntilBlacklist { - data, err := db.GetPendingPDUs(ctx, destination, 100) - assert.NoError(t, err) - if len(data) == 1 { - if val, _ := db.IsServerBlacklisted(destination); val { - return poll.Success() - } - return poll.Continue("waiting for server to be blacklisted") - } - return poll.Continue("waiting for event to be added to database") - } - return poll.Continue("waiting for more send attempts before checking database") - } - poll.WaitOn(t, check, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond)) -} - -func TestRetryServerSendsSuccessfully(t *testing.T) { - ctx := context.Background() - failuresUntilBlacklist := uint32(1) - destination := gomatrixserverlib.ServerName("remotehost") - db, fc, queues := testSetup(failuresUntilBlacklist, false) - - ev := mustCreateEvent(t) - err := queues.SendEvent(ev, "localhost", []gomatrixserverlib.ServerName{destination}) - assert.NoError(t, err) - - checkBlacklisted := func(log poll.LogT) poll.Result { - if fc.txCount.Load() >= failuresUntilBlacklist { - data, err := db.GetPendingPDUs(ctx, destination, 100) - assert.NoError(t, err) - if len(data) == 1 { - if val, _ := db.IsServerBlacklisted(destination); val { - return poll.Success() - } - return poll.Continue("waiting for server to be blacklisted") - } - return poll.Continue("waiting for event to be added to database") - } - return poll.Continue("waiting for more send attempts before checking database") - } - poll.WaitOn(t, checkBlacklisted, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond)) - - fc.shouldTxSucceed = true - db.RemoveServerFromBlacklist(destination) - queues.RetryServer(destination) - checkRetry := func(log poll.LogT) poll.Result { - data, err := db.GetPendingPDUs(ctx, destination, 100) - assert.NoError(t, err) - if len(data) == 0 { - return poll.Success() - } - return poll.Continue("waiting for event to be removed from database") - } - poll.WaitOn(t, checkRetry, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond)) -} diff --git a/federationapi/statistics/statistics.go b/federationapi/statistics/statistics.go index 61a965791..db6d5c735 100644 --- a/federationapi/statistics/statistics.go +++ b/federationapi/statistics/statistics.go @@ -95,8 +95,8 @@ func (s *ServerStatistics) cancel() { // we will unblacklist it. func (s *ServerStatistics) Success() { s.cancel() - s.backoffCount.Store(0) s.successCounter.Inc() + s.backoffCount.Store(0) if s.statistics.DB != nil { if err := s.statistics.DB.RemoveServerFromBlacklist(s.serverName); err != nil { logrus.WithError(err).Errorf("Failed to remove %q from blacklist", s.serverName) @@ -174,12 +174,6 @@ func (s *ServerStatistics) Blacklisted() bool { return s.blacklisted.Load() } -// RemoveBlacklist removes the blacklisted status from the server. -func (s *ServerStatistics) RemoveBlacklist() { - s.cancel() - s.backoffCount.Store(0) -} - // SuccessCount returns the number of successful requests. This is // usually useful in constructing transaction IDs. func (s *ServerStatistics) SuccessCount() uint32 { diff --git a/go.mod b/go.mod index eeae9608f..eefad89e6 100644 --- a/go.mod +++ b/go.mod @@ -50,7 +50,6 @@ require ( golang.org/x/term v0.0.0-20220919170432-7a66f970e087 gopkg.in/h2non/bimg.v1 v1.1.9 gopkg.in/yaml.v2 v2.4.0 - gotest.tools/v3 v3.0.3 nhooyr.io/websocket v1.8.7 ) @@ -129,6 +128,7 @@ require ( gopkg.in/macaroon.v2 v2.1.0 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + gotest.tools/v3 v3.0.3 // indirect ) go 1.18 From 088ad1dd21a06ad3c4ae2db2a73936ed3e0d809e Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Fri, 14 Oct 2022 09:14:54 +0200 Subject: [PATCH 073/207] Fix `outliers whose auth_events are in a different room are correctly rejected` (#2791) Fixes `outliers whose auth_events are in a different room are correctly rejected`, by validating that auth events are all from the same room and not using rejected events for event auth. --- go.mod | 2 +- go.sum | 4 +- roomserver/internal/helpers/auth.go | 20 +++++- roomserver/internal/input/input_events.go | 31 ++++++--- .../internal/input/input_events_test.go | 63 +++++++++++++++++++ sytest-whitelist | 3 +- test/event.go | 7 +++ test/room.go | 9 ++- 8 files changed, 124 insertions(+), 15 deletions(-) create mode 100644 roomserver/internal/input/input_events_test.go diff --git a/go.mod b/go.mod index eefad89e6..23d5655d8 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 - github.com/matrix-org/gomatrixserverlib v0.0.0-20221011115330-49fa704b9a64 + github.com/matrix-org/gomatrixserverlib v0.0.0-20221014061925-a132619fa241 github.com/matrix-org/pinecone v0.0.0-20220929155234-2ce51dd4a42c github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.15 diff --git a/go.sum b/go.sum index 0d08ac692..a1069da37 100644 --- a/go.sum +++ b/go.sum @@ -384,8 +384,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20221011115330-49fa704b9a64 h1:QJmfAPC3P0ZHJzYD/QtbNc5EztKlK1ipRWP5SO/m4jw= -github.com/matrix-org/gomatrixserverlib v0.0.0-20221011115330-49fa704b9a64/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4= +github.com/matrix-org/gomatrixserverlib v0.0.0-20221014061925-a132619fa241 h1:e5o68MWeU7wjTvvNKmVo655oCYesoNRoPeBb1Xfz54g= +github.com/matrix-org/gomatrixserverlib v0.0.0-20221014061925-a132619fa241/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4= github.com/matrix-org/pinecone v0.0.0-20220929155234-2ce51dd4a42c h1:iCHLYwwlPsf4TYFrvhKdhQoAM2lXzcmDZYqwBNWcnVk= github.com/matrix-org/pinecone v0.0.0-20220929155234-2ce51dd4a42c/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= diff --git a/roomserver/internal/helpers/auth.go b/roomserver/internal/helpers/auth.go index 935a045df..03d8bca0b 100644 --- a/roomserver/internal/helpers/auth.go +++ b/roomserver/internal/helpers/auth.go @@ -19,10 +19,11 @@ import ( "fmt" "sort" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/dendrite/roomserver/state" "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/dendrite/roomserver/types" - "github.com/matrix-org/gomatrixserverlib" ) // CheckForSoftFail returns true if the event should be soft-failed @@ -129,6 +130,12 @@ type authEvents struct { stateKeyNIDMap map[string]types.EventStateKeyNID state stateEntryMap events EventMap + valid bool +} + +// Valid verifies that all auth events are from the same room. +func (ae *authEvents) Valid() bool { + return ae.valid } // Create implements gomatrixserverlib.AuthEventProvider @@ -197,6 +204,7 @@ func loadAuthEvents( needed gomatrixserverlib.StateNeeded, state []types.StateEntry, ) (result authEvents, err error) { + result.valid = true // Look up the numeric IDs for the state keys needed for auth. var neededStateKeys []string neededStateKeys = append(neededStateKeys, needed.Member...) @@ -218,6 +226,16 @@ func loadAuthEvents( if result.events, err = db.Events(ctx, eventNIDs); err != nil { return } + roomID := "" + for _, ev := range result.events { + if roomID == "" { + roomID = ev.RoomID() + } + if ev.RoomID() != roomID { + result.valid = false + break + } + } return } diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go index d1b6bc73e..60160e8e5 100644 --- a/roomserver/internal/input/input_events.go +++ b/roomserver/internal/input/input_events.go @@ -19,9 +19,16 @@ package input import ( "context" "database/sql" + "errors" "fmt" "time" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" + "github.com/opentracing/opentracing-go" + "github.com/prometheus/client_golang/prometheus" + "github.com/sirupsen/logrus" + fedapi "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/eventutil" @@ -31,11 +38,6 @@ import ( "github.com/matrix-org/dendrite/roomserver/internal/helpers" "github.com/matrix-org/dendrite/roomserver/state" "github.com/matrix-org/dendrite/roomserver/types" - "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" - "github.com/opentracing/opentracing-go" - "github.com/prometheus/client_golang/prometheus" - "github.com/sirupsen/logrus" ) // TODO: Does this value make sense? @@ -196,7 +198,7 @@ func (r *Inputer) processRoomEvent( isRejected := false authEvents := gomatrixserverlib.NewAuthEvents(nil) knownEvents := map[string]*types.Event{} - if err = r.fetchAuthEvents(ctx, logger, headered, &authEvents, knownEvents, serverRes.ServerNames); err != nil { + if err = r.fetchAuthEvents(ctx, logger, roomInfo, headered, &authEvents, knownEvents, serverRes.ServerNames); err != nil { return fmt.Errorf("r.fetchAuthEvents: %w", err) } @@ -336,7 +338,7 @@ func (r *Inputer) processRoomEvent( // doesn't have any associated state to store and we don't need to // notify anyone about it. if input.Kind == api.KindOutlier { - logger.Debug("Stored outlier") + logger.WithField("rejected", isRejected).Debug("Stored outlier") hooks.Run(hooks.KindNewEventPersisted, headered) return nil } @@ -536,6 +538,7 @@ func (r *Inputer) processStateBefore( func (r *Inputer) fetchAuthEvents( ctx context.Context, logger *logrus.Entry, + roomInfo *types.RoomInfo, event *gomatrixserverlib.HeaderedEvent, auth *gomatrixserverlib.AuthEvents, known map[string]*types.Event, @@ -557,9 +560,19 @@ func (r *Inputer) fetchAuthEvents( continue } ev := authEvents[0] + + isRejected := false + if roomInfo != nil { + isRejected, err = r.DB.IsEventRejected(ctx, roomInfo.RoomNID, ev.EventID()) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return fmt.Errorf("r.DB.IsEventRejected failed: %w", err) + } + } known[authEventID] = &ev // don't take the pointer of the iterated event - if err = auth.AddEvent(ev.Event); err != nil { - return fmt.Errorf("auth.AddEvent: %w", err) + if !isRejected { + if err = auth.AddEvent(ev.Event); err != nil { + return fmt.Errorf("auth.AddEvent: %w", err) + } } } diff --git a/roomserver/internal/input/input_events_test.go b/roomserver/internal/input/input_events_test.go new file mode 100644 index 000000000..818e7715c --- /dev/null +++ b/roomserver/internal/input/input_events_test.go @@ -0,0 +1,63 @@ +package input + +import ( + "testing" + + "github.com/matrix-org/gomatrixserverlib" + + "github.com/matrix-org/dendrite/test" +) + +func Test_EventAuth(t *testing.T) { + alice := test.NewUser(t) + bob := test.NewUser(t) + + // create two rooms, so we can craft "illegal" auth events + room1 := test.NewRoom(t, alice) + room2 := test.NewRoom(t, alice, test.RoomPreset(test.PresetPublicChat)) + + authEventIDs := make([]string, 0, 4) + authEvents := []*gomatrixserverlib.Event{} + + // Add the legal auth events from room2 + for _, x := range room2.Events() { + if x.Type() == gomatrixserverlib.MRoomCreate { + authEventIDs = append(authEventIDs, x.EventID()) + authEvents = append(authEvents, x.Event) + } + if x.Type() == gomatrixserverlib.MRoomPowerLevels { + authEventIDs = append(authEventIDs, x.EventID()) + authEvents = append(authEvents, x.Event) + } + if x.Type() == gomatrixserverlib.MRoomJoinRules { + authEventIDs = append(authEventIDs, x.EventID()) + authEvents = append(authEvents, x.Event) + } + } + + // Add the illegal auth event from room1 (rooms are different) + for _, x := range room1.Events() { + if x.Type() == gomatrixserverlib.MRoomMember { + authEventIDs = append(authEventIDs, x.EventID()) + authEvents = append(authEvents, x.Event) + } + } + + // Craft the illegal join event, with auth events from different rooms + ev := room2.CreateEvent(t, bob, "m.room.member", map[string]interface{}{ + "membership": "join", + }, test.WithStateKey(bob.ID), test.WithAuthIDs(authEventIDs)) + + // Add the auth events to the allower + allower := gomatrixserverlib.NewAuthEvents(nil) + for _, a := range authEvents { + if err := allower.AddEvent(a); err != nil { + t.Fatalf("allower.AddEvent failed: %v", err) + } + } + + // Finally check that the event is NOT allowed + if err := gomatrixserverlib.Allowed(ev.Event, &allower); err == nil { + t.Fatalf("event should not be allowed, but it was") + } +} diff --git a/sytest-whitelist b/sytest-whitelist index a3218ed70..2bd8b9403 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -743,4 +743,5 @@ User joining then leaving public room appears and dissappears from directory User in remote room doesn't appear in user directory after server left room User in shared private room does appear in user directory until leave Existing members see new member's presence -Inbound federation can return missing events for joined visibility \ No newline at end of file +Inbound federation can return missing events for joined visibility +outliers whose auth_events are in a different room are correctly rejected \ No newline at end of file diff --git a/test/event.go b/test/event.go index 73fc656bd..0c7bf4355 100644 --- a/test/event.go +++ b/test/event.go @@ -30,6 +30,7 @@ type eventMods struct { unsigned interface{} keyID gomatrixserverlib.KeyID privKey ed25519.PrivateKey + authEvents []string } type eventModifier func(e *eventMods) @@ -52,6 +53,12 @@ func WithUnsigned(unsigned interface{}) eventModifier { } } +func WithAuthIDs(evs []string) eventModifier { + return func(e *eventMods) { + e.authEvents = evs + } +} + func WithKeyID(keyID gomatrixserverlib.KeyID) eventModifier { return func(e *eventMods) { e.keyID = keyID diff --git a/test/room.go b/test/room.go index 94eb51bbe..4328bf84f 100644 --- a/test/room.go +++ b/test/room.go @@ -21,8 +21,9 @@ import ( "testing" "time" - "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/gomatrixserverlib" + + "github.com/matrix-org/dendrite/internal/eventutil" ) type Preset int @@ -174,11 +175,17 @@ func (r *Room) CreateEvent(t *testing.T, creator *User, eventType string, conten if err != nil { t.Fatalf("CreateEvent[%s]: failed to StateNeededForEventBuilder: %s", eventType, err) } + refs, err := eventsNeeded.AuthEventReferences(&r.authEvents) if err != nil { t.Fatalf("CreateEvent[%s]: failed to AuthEventReferences: %s", eventType, err) } builder.AuthEvents = refs + + if len(mod.authEvents) > 0 { + builder.AuthEvents = mod.authEvents + } + ev, err := builder.Build( mod.originServerTS, mod.origin, mod.keyID, mod.privKey, r.Version, From fb44e33909660b5e37f2d422baf4ffa7ddc30b0a Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Fri, 14 Oct 2022 10:37:04 +0200 Subject: [PATCH 074/207] Relax test a bit --- userapi/storage/storage_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/userapi/storage/storage_test.go b/userapi/storage/storage_test.go index ca7c1bfd2..1538a8138 100644 --- a/userapi/storage/storage_test.go +++ b/userapi/storage/storage_test.go @@ -192,19 +192,18 @@ func Test_Devices(t *testing.T) { newName := "new display name" err = db.UpdateDevice(ctx, localpart, deviceWithID.ID, &newName) assert.NoError(t, err, "unable to update device displayname") + updatedAfterTimestamp := time.Now().Unix() err = db.UpdateDeviceLastSeen(ctx, localpart, deviceWithID.ID, "127.0.0.1", "Element Web") assert.NoError(t, err, "unable to update device last seen") deviceWithID.DisplayName = newName deviceWithID.LastSeenIP = "127.0.0.1" - deviceWithID.LastSeenTS = int64(gomatrixserverlib.AsTimestamp(time.Now().Truncate(time.Second))) gotDevice, err = db.GetDeviceByID(ctx, localpart, deviceWithID.ID) assert.NoError(t, err, "unable to get device by id") assert.Equal(t, 2, len(devices)) assert.Equal(t, deviceWithID.DisplayName, gotDevice.DisplayName) assert.Equal(t, deviceWithID.LastSeenIP, gotDevice.LastSeenIP) - truncatedTime := gomatrixserverlib.Timestamp(gotDevice.LastSeenTS).Time().Truncate(time.Second) - assert.Equal(t, gomatrixserverlib.Timestamp(deviceWithID.LastSeenTS), gomatrixserverlib.AsTimestamp(truncatedTime)) + assert.Greater(t, gotDevice.LastSeenTS, updatedAfterTimestamp) // create one more device and remove the devices step by step newDeviceID := util.RandomString(16) From a8bc558a606266b2dc1d1c19cf0c052f3b733679 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Fri, 14 Oct 2022 10:38:12 +0200 Subject: [PATCH 075/207] Always add `UnreadNotifications` to joined room reponses (#2793) Fixes a minor bug, where we failed to add `UnreadNotifications` to the join response, if it wasn't in `GetUserUnreadNotificationCountsForRooms`. --- syncapi/streams/stream_notificationdata.go | 3 ++- syncapi/types/types.go | 8 +++++--- sytest-whitelist | 4 +++- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/syncapi/streams/stream_notificationdata.go b/syncapi/streams/stream_notificationdata.go index 5a81fd09a..66ee0ded9 100644 --- a/syncapi/streams/stream_notificationdata.go +++ b/syncapi/streams/stream_notificationdata.go @@ -3,6 +3,7 @@ package streams import ( "context" + "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/syncapi/storage" "github.com/matrix-org/dendrite/syncapi/types" ) @@ -53,7 +54,7 @@ func (p *NotificationDataStreamProvider) IncrementalSync( for roomID, jr := range req.Response.Rooms.Join { counts := countsByRoom[roomID] if counts == nil { - continue + counts = &eventutil.NotificationData{} } jr.UnreadNotifications = &types.UnreadNotifications{ HighlightCount: counts.UnreadHighlightCount, diff --git a/syncapi/types/types.go b/syncapi/types/types.go index 60a74a285..57ce7b6ff 100644 --- a/syncapi/types/types.go +++ b/syncapi/types/types.go @@ -492,9 +492,11 @@ func (jr JoinResponse) MarshalJSON() ([]byte, error) { } } - if jr.UnreadNotifications != nil && - jr.UnreadNotifications.NotificationCount == 0 && jr.UnreadNotifications.HighlightCount == 0 { - a.UnreadNotifications = nil + if jr.UnreadNotifications != nil { + // if everything else is nil, also remove UnreadNotifications + if a.State == nil && a.Ephemeral == nil && a.AccountData == nil && a.Timeline == nil && a.Summary == nil { + a.UnreadNotifications = nil + } } return json.Marshal(a) } diff --git a/sytest-whitelist b/sytest-whitelist index 2bd8b9403..93d447d28 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -744,4 +744,6 @@ User in remote room doesn't appear in user directory after server left room User in shared private room does appear in user directory until leave Existing members see new member's presence Inbound federation can return missing events for joined visibility -outliers whose auth_events are in a different room are correctly rejected \ No newline at end of file +outliers whose auth_events are in a different room are correctly rejected +Messages that notify from another user increment notification_count +Messages that highlight from another user increment unread highlight count \ No newline at end of file From 82d1d434c559285963c3b59e3df04b75337a6a78 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 14 Oct 2022 10:10:25 +0100 Subject: [PATCH 076/207] Update to NATS Server v2.9.3 and nats.go v1.18.0 --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 23d5655d8..943760a00 100644 --- a/go.mod +++ b/go.mod @@ -26,8 +26,8 @@ require ( github.com/matrix-org/pinecone v0.0.0-20220929155234-2ce51dd4a42c github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.15 - github.com/nats-io/nats-server/v2 v2.9.2 - github.com/nats-io/nats.go v1.17.0 + github.com/nats-io/nats-server/v2 v2.9.3 + github.com/nats-io/nats.go v1.18.0 github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 github.com/ngrok/sqlmw v0.0.0-20220520173518-97c9c04efc79 diff --git a/go.sum b/go.sum index a1069da37..a8252ab77 100644 --- a/go.sum +++ b/go.sum @@ -422,10 +422,10 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nats-io/jwt/v2 v2.3.0 h1:z2mA1a7tIf5ShggOFlR1oBPgd6hGqcDYsISxZByUzdI= github.com/nats-io/jwt/v2 v2.3.0/go.mod h1:0tqz9Hlu6bCBFLWAASKhE5vUA4c24L9KPUUgvwumE/k= -github.com/nats-io/nats-server/v2 v2.9.2 h1:XNDgJgOYYaYlquLdbSHI3xssLipfKUOq3EmYIMNCOsE= -github.com/nats-io/nats-server/v2 v2.9.2/go.mod h1:4sq8wvrpbvSzL1n3ZfEYnH4qeUuIl5W990j3kw13rRk= -github.com/nats-io/nats.go v1.17.0 h1:1jp5BThsdGlN91hW0k3YEfJbfACjiOYtUiLXG0RL4IE= -github.com/nats-io/nats.go v1.17.0/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w= +github.com/nats-io/nats-server/v2 v2.9.3 h1:HrfzA7G9LNetKkm1z+jU/e9kuAe+E6uaBuuq9EB5sQQ= +github.com/nats-io/nats-server/v2 v2.9.3/go.mod h1:4sq8wvrpbvSzL1n3ZfEYnH4qeUuIl5W990j3kw13rRk= +github.com/nats-io/nats.go v1.18.0 h1:o480Ao6kuSSFyJO75rGTXCEPj7LGkY84C1Ye+Uhm4c0= +github.com/nats-io/nats.go v1.18.0/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w= github.com/nats-io/nkeys v0.3.0 h1:cgM5tL53EvYRU+2YLXIK0G2mJtK12Ft9oeooSZMA2G8= github.com/nats-io/nkeys v0.3.0/go.mod h1:gvUNGjVcM2IPr5rCsRsC6Wb3Hr2CQAm08dsxtV6A5y4= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= From f76969831ed2d0165b83b15ddda2560a774063c0 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 14 Oct 2022 10:59:32 +0100 Subject: [PATCH 077/207] Update direct dependencies (#2794) This updates a number of Dendrite's dependencies. --- go.mod | 51 +++++++++++------------ go.sum | 128 +++++++++++++++++++++++++++++++-------------------------- 2 files changed, 94 insertions(+), 85 deletions(-) diff --git a/go.mod b/go.mod index 943760a00..7bfd9b45f 100644 --- a/go.mod +++ b/go.mod @@ -8,10 +8,10 @@ require ( github.com/Masterminds/semver/v3 v3.1.1 github.com/blevesearch/bleve/v2 v2.3.4 github.com/codeclysm/extract v2.2.0+incompatible - github.com/dgraph-io/ristretto v0.1.1-0.20220403145359-8e850b710d6d - github.com/docker/docker v20.10.18+incompatible + github.com/dgraph-io/ristretto v0.1.1 + github.com/docker/docker v20.10.19+incompatible github.com/docker/go-connections v0.4.0 - github.com/getsentry/sentry-go v0.13.0 + github.com/getsentry/sentry-go v0.14.0 github.com/gologme/log v1.3.0 github.com/google/go-cmp v0.5.9 github.com/google/uuid v1.3.0 @@ -21,9 +21,9 @@ require ( github.com/lib/pq v1.10.7 github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 - github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 + github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 github.com/matrix-org/gomatrixserverlib v0.0.0-20221014061925-a132619fa241 - github.com/matrix-org/pinecone v0.0.0-20220929155234-2ce51dd4a42c + github.com/matrix-org/pinecone v0.0.0-20221007145426-3adc85477dd3 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.15 github.com/nats-io/nats-server/v2 v2.9.3 @@ -43,10 +43,10 @@ require ( github.com/uber/jaeger-lib v2.4.1+incompatible github.com/yggdrasil-network/yggdrasil-go v0.4.5-0.20220901155642-4f2abece817c go.uber.org/atomic v1.10.0 - golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be + golang.org/x/crypto v0.0.0-20221012134737-56aed061732a golang.org/x/image v0.0.0-20220902085622-e7cb96979f69 - golang.org/x/mobile v0.0.0-20220722155234-aaac322e2105 - golang.org/x/net v0.0.0-20220919232410-f2f64ebce3c1 + golang.org/x/mobile v0.0.0-20221012134814-c746ac228303 + golang.org/x/net v0.0.0-20221014081412-f15817d10f9b golang.org/x/term v0.0.0-20220919170432-7a66f970e087 gopkg.in/h2non/bimg.v1 v1.1.9 gopkg.in/yaml.v2 v2.4.0 @@ -54,14 +54,13 @@ require ( ) require ( - github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect - github.com/Microsoft/go-winio v0.5.2 // indirect + github.com/Microsoft/go-winio v0.6.0 // indirect github.com/RoaringBitmap/roaring v1.2.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.3.3 // indirect github.com/blevesearch/bleve_index_api v1.0.3 // indirect - github.com/blevesearch/geo v0.1.14 // indirect + github.com/blevesearch/geo v0.1.15 // indirect github.com/blevesearch/go-porterstemmer v1.0.3 // indirect github.com/blevesearch/gtreap v0.1.1 // indirect github.com/blevesearch/mmap-go v1.0.4 // indirect @@ -69,7 +68,7 @@ require ( github.com/blevesearch/segment v0.9.0 // indirect github.com/blevesearch/snowballstem v0.9.0 // indirect github.com/blevesearch/upsidedown_store_api v1.0.1 // indirect - github.com/blevesearch/vellum v1.0.8 // indirect + github.com/blevesearch/vellum v1.0.9 // indirect github.com/blevesearch/zapx/v11 v11.3.5 // indirect github.com/blevesearch/zapx/v12 v12.3.5 // indirect github.com/blevesearch/zapx/v13 v13.3.5 // indirect @@ -80,7 +79,7 @@ require ( github.com/docker/distribution v2.8.1+incompatible // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.0 // indirect - github.com/fsnotify/fsnotify v1.5.4 // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect @@ -92,13 +91,13 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/juju/errors v1.0.0 // indirect github.com/klauspost/compress v1.15.11 // indirect - github.com/kr/pretty v0.3.0 // indirect - github.com/lucas-clemente/quic-go v0.29.0 // indirect - github.com/marten-seemann/qtls-go1-18 v0.1.2 // indirect - github.com/marten-seemann/qtls-go1-19 v0.1.0 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/lucas-clemente/quic-go v0.29.2 // indirect + github.com/marten-seemann/qtls-go1-18 v0.1.3 // indirect + github.com/marten-seemann/qtls-go1-19 v0.1.1 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect github.com/minio/highwayhash v1.0.2 // indirect - github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 // indirect + github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/morikuni/aec v1.0.0 // indirect @@ -108,27 +107,27 @@ require ( github.com/nats-io/nuid v1.0.1 // indirect github.com/nxadm/tail v1.4.8 // indirect github.com/onsi/ginkgo v1.16.5 // indirect - github.com/onsi/gomega v1.17.0 // indirect + github.com/onsi/gomega v1.22.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect + github.com/opencontainers/image-spec v1.1.0-rc2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect github.com/tidwall/match v1.1.1 // indirect - github.com/tidwall/pretty v1.2.0 // indirect + github.com/tidwall/pretty v1.2.1 // indirect go.etcd.io/bbolt v1.3.6 // indirect - golang.org/x/exp v0.0.0-20220916125017-b168a2c6b86b // indirect + golang.org/x/exp v0.0.0-20221012211006-4de253d81b95 // indirect golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect - golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec // indirect - golang.org/x/text v0.3.8-0.20211004125949-5bd84dd9b33b // indirect + golang.org/x/sys v0.0.0-20221013171732-95e765b1cc43 // indirect + golang.org/x/text v0.3.8 // indirect golang.org/x/time v0.0.0-20220922220347-f3bd1da661af // indirect golang.org/x/tools v0.1.12 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/macaroon.v2 v2.1.0 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - gotest.tools/v3 v3.0.3 // indirect + gotest.tools/v3 v3.4.0 // indirect ) go 1.18 diff --git a/go.sum b/go.sum index a8252ab77..ed76397cc 100644 --- a/go.sum +++ b/go.sum @@ -42,7 +42,6 @@ github.com/Arceliar/ironwood v0.0.0-20220903132624-ee60c16bcfcf h1:kjPkmDHUTWUma github.com/Arceliar/ironwood v0.0.0-20220903132624-ee60c16bcfcf/go.mod h1:RP72rucOFm5udrnEzTmIWLRVGQiV/fSUAQXJ0RST/nk= github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979 h1:WndgpSW13S32VLQ3ugUxx2EnnWmgba1kCqPkd4Gk1yQ= github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI= -github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -55,8 +54,8 @@ github.com/MFAshby/stdemuxerhook v1.0.0 h1:1XFGzakrsHMv76AeanPDL26NOgwjPl/OUxbGh github.com/MFAshby/stdemuxerhook v1.0.0/go.mod h1:nLMI9FUf9Hz98n+yAXsTMUR4RZQy28uCTLG1Fzvj/uY= github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= -github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= -github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= +github.com/Microsoft/go-winio v0.6.0 h1:slsWYD/zyx7lCXoZVlvQrj0hPTM1HI4+v1sIda2yDvg= +github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE= github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w= github.com/RoaringBitmap/roaring v0.9.4/go.mod h1:icnadbWcNyfEHlYdr+tDlOTih1Bf/h+rzPpv4sbomAA= github.com/RoaringBitmap/roaring v1.2.1 h1:58/LJlg/81wfEHd5L9qsHduznOIhyv4qb1yWcSvVq9A= @@ -95,8 +94,8 @@ github.com/blevesearch/bleve/v2 v2.3.4/go.mod h1:Ot0zYum8XQRfPcwhae8bZmNyYubynso github.com/blevesearch/bleve_index_api v1.0.3 h1:DDSWaPXOZZJ2BB73ZTWjKxydAugjwywcqU+91AAqcAg= github.com/blevesearch/bleve_index_api v1.0.3/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4= github.com/blevesearch/geo v0.1.13/go.mod h1:cRIvqCdk3cgMhGeHNNe6yPzb+w56otxbfo1FBJfR2Pc= -github.com/blevesearch/geo v0.1.14 h1:TTDpJN6l9ck/cUYbXSn4aCElNls0Whe44rcQKsB7EfU= -github.com/blevesearch/geo v0.1.14/go.mod h1:cRIvqCdk3cgMhGeHNNe6yPzb+w56otxbfo1FBJfR2Pc= +github.com/blevesearch/geo v0.1.15 h1:0NybEduqE5fduFRYiUKF0uqybAIFKXYjkBdXKYn7oA4= +github.com/blevesearch/geo v0.1.15/go.mod h1:cRIvqCdk3cgMhGeHNNe6yPzb+w56otxbfo1FBJfR2Pc= github.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:9eJDeqxJ3E7WnLebQUlPD7ZjSce7AnDb9vjGmMCbD0A= github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo= github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M= @@ -115,8 +114,9 @@ github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs= github.com/blevesearch/upsidedown_store_api v1.0.1 h1:1SYRwyoFLwG3sj0ed89RLtM15amfX2pXlYbFOnF8zNU= github.com/blevesearch/upsidedown_store_api v1.0.1/go.mod h1:MQDVGpHZrpe3Uy26zJBf/a8h0FZY6xJbthIMm8myH2Q= -github.com/blevesearch/vellum v1.0.8 h1:iMGh4lfxza4BnWO/UJTMPlI3HsK9YawjPv+TteVa9ck= github.com/blevesearch/vellum v1.0.8/go.mod h1:+cpRi/tqq49xUYSQN2P7A5zNSNrS+MscLeeaZ3J46UA= +github.com/blevesearch/vellum v1.0.9 h1:PL+NWVk3dDGPCV0hoDu9XLLJgqU4E5s/dOeEJByQ2uQ= +github.com/blevesearch/vellum v1.0.9/go.mod h1:ul1oT0FhSMDIExNjIxHqJoGpVrBpKCdgDQNxfqgJt7k= github.com/blevesearch/zapx/v11 v11.3.5 h1:eBQWQ7huA+mzm0sAGnZDwgGGli7S45EO+N+ObFWssbI= github.com/blevesearch/zapx/v11 v11.3.5/go.mod h1:5UdIa/HRMdeRCiLQOyFESsnqBGiip7vQmYReA9toevU= github.com/blevesearch/zapx/v12 v12.3.5 h1:5pX2hU+R1aZihT7ac1dNWh1n4wqkIM9pZzWp0ANED9s= @@ -157,14 +157,14 @@ github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgraph-io/ristretto v0.1.1-0.20220403145359-8e850b710d6d h1:Wrc3UKTS+cffkOx0xRGFC+ZesNuTfn0ThvEC72N0krk= -github.com/dgraph-io/ristretto v0.1.1-0.20220403145359-8e850b710d6d/go.mod h1:RAy2GVV4sTWVlNMavv3xhLsk18rxhfhDnombTe6EF5c= +github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= +github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v20.10.18+incompatible h1:SN84VYXTBNGn92T/QwIRPlum9zfemfitN7pbsp26WSc= -github.com/docker/docker v20.10.18+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v20.10.19+incompatible h1:lzEmjivyNHFHMNAFLXORMBXyGIhw/UP4DvJwvyKYq64= +github.com/docker/docker v20.10.19+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -184,20 +184,21 @@ github.com/frankban/quicktest v1.0.0/go.mod h1:R98jIehRai+d1/3Hv2//jOVCTJhW1VBav github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= -github.com/getsentry/sentry-go v0.13.0 h1:20dgTiUSfxRB/EhMPtxcL9ZEbM1ZdR+W/7f7NWD+xWo= -github.com/getsentry/sentry-go v0.13.0/go.mod h1:EOsfu5ZdvKPfeHYV6pTVQnsjfp30+XA7//UooKNumH0= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/getsentry/sentry-go v0.14.0 h1:rlOBkuFZRKKdUnKO+0U3JclRDQKlRu5vVQtkWSQvC70= +github.com/getsentry/sentry-go v0.14.0/go.mod h1:RZPJKSw+adu8PBNygiri/A98FqVr2HtRckJk9XVxJ9I= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= -github.com/gin-gonic/gin v1.7.7 h1:3DoBmSbJbZAWqXJC3SLjAPfutPJJRN1U5pALB7EeTTs= +github.com/gin-gonic/gin v1.8.1 h1:4+fr/el88TOO3ewCmQr8cx/CtZ/umlIRIs5M4NTNjf8= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= -github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -210,12 +211,12 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= -github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= +github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho= github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= -github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE= +github.com/go-playground/validator/v10 v10.11.1 h1:prmOlTVv+YjZjmRmNSF3VmspqJIxJWXmqUsHwfTRRkQ= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= @@ -225,6 +226,7 @@ github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8= github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= +github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -353,59 +355,60 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lucas-clemente/quic-go v0.28.1/go.mod h1:oGz5DKK41cJt5+773+BSO9BXDsREY4HLf7+0odGAPO0= -github.com/lucas-clemente/quic-go v0.29.0 h1:Vw0mGTfmWqGzh4jx/kMymsIkFK6rErFVmg+t9RLrnZE= -github.com/lucas-clemente/quic-go v0.29.0/go.mod h1:CTcNfLYJS2UuRNB+zcNlgvkjBhxX6Hm3WUxxAQx2mgE= +github.com/lucas-clemente/quic-go v0.29.2 h1:O8Mt0O6LpvEW+wfC40vZdcw0DngwYzoxq5xULZNzSI8= +github.com/lucas-clemente/quic-go v0.29.2/go.mod h1:g6/h9YMmLuU54tL1gW25uIi3VlBp3uv+sBihplIuskE= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc= github.com/marten-seemann/qtls-go1-16 v0.1.5/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk= github.com/marten-seemann/qtls-go1-17 v0.1.2/go.mod h1:C2ekUKcDdz9SDWxec1N/MvcXBpaX9l3Nx67XaR84L5s= -github.com/marten-seemann/qtls-go1-18 v0.1.2 h1:JH6jmzbduz0ITVQ7ShevK10Av5+jBEKAHMntXmIV7kM= github.com/marten-seemann/qtls-go1-18 v0.1.2/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4= +github.com/marten-seemann/qtls-go1-18 v0.1.3 h1:R4H2Ks8P6pAtUagjFty2p7BVHn3XiwDAl7TTQf5h7TI= +github.com/marten-seemann/qtls-go1-18 v0.1.3/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4= github.com/marten-seemann/qtls-go1-19 v0.1.0-beta.1/go.mod h1:5HTDWtVudo/WFsHKRNuOhWlbdjrfs5JHrYb0wIJqGpI= -github.com/marten-seemann/qtls-go1-19 v0.1.0 h1:rLFKD/9mp/uq1SYGYuVZhm83wkmU95pK5df3GufyYYU= -github.com/marten-seemann/qtls-go1-19 v0.1.0/go.mod h1:5HTDWtVudo/WFsHKRNuOhWlbdjrfs5JHrYb0wIJqGpI= +github.com/marten-seemann/qtls-go1-19 v0.1.1 h1:mnbxeq3oEyQxQXwI4ReCgW9DPoPR94sNlqWoDZnjRIE= +github.com/marten-seemann/qtls-go1-19 v0.1.1/go.mod h1:5HTDWtVudo/WFsHKRNuOhWlbdjrfs5JHrYb0wIJqGpI= github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e h1:DP5RC0Z3XdyBEW5dKt8YPeN6vZbm6OzVaGVp7f1BQRM= github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e/go.mod h1:NgPCr+UavRGH6n5jmdX8DuqFZ4JiCWIJoZiuhTRLSUg= github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw2QV3YD/fRrzEDPNGgTlJlvXY0EHHnT87wF3OA= github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= -github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4= -github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= +github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8r4Fzarl4+Y3K5hjothkVW5z7T1dUM11U= +github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= github.com/matrix-org/gomatrixserverlib v0.0.0-20221014061925-a132619fa241 h1:e5o68MWeU7wjTvvNKmVo655oCYesoNRoPeBb1Xfz54g= github.com/matrix-org/gomatrixserverlib v0.0.0-20221014061925-a132619fa241/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4= -github.com/matrix-org/pinecone v0.0.0-20220929155234-2ce51dd4a42c h1:iCHLYwwlPsf4TYFrvhKdhQoAM2lXzcmDZYqwBNWcnVk= -github.com/matrix-org/pinecone v0.0.0-20220929155234-2ce51dd4a42c/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k= +github.com/matrix-org/pinecone v0.0.0-20221007145426-3adc85477dd3 h1:lzkSQvBv8TuqKJCPoVwOVvEnARTlua5rrNy/Qw2Vxeo= +github.com/matrix-org/pinecone v0.0.0-20221007145426-3adc85477dd3/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= +github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/mattomatic/dijkstra v0.0.0-20130617153013-6f6d134eb237/go.mod h1:UOnLAUmVG5paym8pD3C4B9BQylUDC2vXFJJpT7JrlEA= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/matttproud/golang_protobuf_extensions v1.0.2 h1:hAHbPm5IJGijwng3PWk09JkG9WeqChjprR5s9bBZ+OM= +github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 h1:yH0SvLzcbZxcJXho2yh7CqdENGMQe73Cw3woZBpPli0= -github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A= +github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae h1:O4SWKdcHVCvYqyDV+9CJA1fcDN2L11Bule0iFy3YlAI= +github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -454,20 +457,23 @@ github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= -github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE= -github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.22.1 h1:pY8O4lBfsHKZHM/6nrxkhVPUznOlIu3quZcKP/M20KI= +github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 h1:rc3tiVYb5z54aKaDfakKn0dDjIyPpTtszkjuMzyt7ec= -github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034= +github.com/opencontainers/image-spec v1.1.0-rc2/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= +github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml/v2 v2.0.5 h1:ipoSadvV8oGUjnUbMub59IDPPwfxF694nG/jwbMiyQg= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -503,8 +509,8 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= @@ -533,7 +539,6 @@ github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5k github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= @@ -565,8 +570,9 @@ github.com/tidwall/gjson v1.14.3 h1:9jvXn7olKEHU1S9vwoMGliaT8jq1vJ7IH/n9zD9Dnlw= github.com/tidwall/gjson v1.14.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= -github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= @@ -577,8 +583,8 @@ github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6 github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= +github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= @@ -625,8 +631,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be h1:fmw3UbQh+nxngCAHrDCCztao/kbYFnWjoqop8dHx05A= -golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20221012134737-56aed061732a h1:NmSIgad6KjE6VvHciPZuNRTKxGhlPfD6OA87W/PLkqg= +golang.org/x/crypto v0.0.0-20221012134737-56aed061732a/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -641,8 +647,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20220916125017-b168a2c6b86b h1:SCE/18RnFsLrjydh/R/s5EVvHoZprqEQUuoxK8q2Pc4= -golang.org/x/exp v0.0.0-20220916125017-b168a2c6b86b/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= +golang.org/x/exp v0.0.0-20221012211006-4de253d81b95 h1:sBdrWpxhGDdTAYNqbgBLAR+ULAPPhfgncLr1X0lyWtg= +golang.org/x/exp v0.0.0-20221012211006-4de253d81b95/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -661,8 +667,9 @@ golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPI golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mobile v0.0.0-20220722155234-aaac322e2105 h1:3vUV5x5+3LfQbgk7paCM6INOaJG9xXQbn79xoNkwfIk= golang.org/x/mobile v0.0.0-20220722155234-aaac322e2105/go.mod h1:pe2sM7Uk+2Su1y7u/6Z8KJ24D7lepUjFZbhFOrmDfuQ= +golang.org/x/mobile v0.0.0-20221012134814-c746ac228303 h1:K4fp1rDuJBz0FCPAWzIJwnzwNEM7S6yobdZzMrZ/Zws= +golang.org/x/mobile v0.0.0-20221012134814-c746ac228303/go.mod h1:M32cGdzp91A8Ex9qQtyZinr19EYxzkFqDjW2oyHzTDQ= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= @@ -717,8 +724,8 @@ golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220728211354-c7608f3a8462/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.0.0-20220919232410-f2f64ebce3c1 h1:TWZxd/th7FbRSMret2MVQdlI8uT49QEtwZdvJrxjEHU= -golang.org/x/net v0.0.0-20220919232410-f2f64ebce3c1/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20221014081412-f15817d10f9b h1:tvrvnPFcdzp294diPnrdZZZ8XUt2Tyj7svb7X52iDuU= +golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -741,6 +748,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -763,7 +771,6 @@ golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -787,11 +794,11 @@ golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -810,8 +817,10 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220730100132-1609e554cd39/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec h1:BkDtF2Ih9xZ7le9ndzTA7KJow28VbQW3odyk/8drmuI= -golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20221013171732-95e765b1cc43 h1:OK7RB6t2WQX54srQQYSXMW8dF5C6/8+oA/s5QBmmto4= +golang.org/x/sys v0.0.0-20221013171732-95e765b1cc43/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220919170432-7a66f970e087 h1:tPwmk4vmvVCMdr98VgL4JH+qZxPL8fqlUOHnyOM8N3w= @@ -823,8 +832,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8-0.20211004125949-5bd84dd9b33b h1:NXqSWXSRUSCaFuvitrWtU169I3876zRTalMRbfd6LL0= -golang.org/x/text v0.3.8-0.20211004125949-5bd84dd9b33b/go.mod h1:EFNZuWvGYxIRUEX+K8UmCFwYmZjqcrnq15ZuVldZkZ0= +golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -879,6 +888,7 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.8-0.20211022200916-316ba0b74098/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= @@ -1013,8 +1023,8 @@ gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= -gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= -gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= +gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= +gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g= grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From eac5678449f9134769344a6d7b3761e79eed1328 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 14 Oct 2022 14:53:53 +0100 Subject: [PATCH 078/207] Update dependency now that it is fixed --- go.mod | 2 +- go.sum | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 7bfd9b45f..1303d4004 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/google/uuid v1.3.0 github.com/gorilla/mux v1.8.0 github.com/gorilla/websocket v1.5.0 - github.com/kardianos/minwinsvc v1.0.0 + github.com/kardianos/minwinsvc v1.0.2 github.com/lib/pq v1.10.7 github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 diff --git a/go.sum b/go.sum index ed76397cc..54055f2fd 100644 --- a/go.sum +++ b/go.sum @@ -343,8 +343,8 @@ github.com/juju/errors v1.0.0/go.mod h1:B5x9thDqx0wIMH3+aLIMP9HjItInYWObRovoCFM5 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= -github.com/kardianos/minwinsvc v1.0.0 h1:+JfAi8IBJna0jY2dJGZqi7o15z13JelFIklJCAENALA= -github.com/kardianos/minwinsvc v1.0.0/go.mod h1:Bgd0oc+D0Qo3bBytmNtyRKVlp85dAloLKhfxanPFFRc= +github.com/kardianos/minwinsvc v1.0.2 h1:JmZKFJQrmTGa/WiW+vkJXKmfzdjabuEW4Tirj5lLdR0= +github.com/kardianos/minwinsvc v1.0.2/go.mod h1:LUZNYhNmxujx2tR7FbdxqYJ9XDDoCd3MQcl1o//FWl4= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= @@ -790,7 +790,6 @@ golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= From cd8f7e125172629984f60546446e7c9d261b26c0 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 14 Oct 2022 15:14:29 +0100 Subject: [PATCH 079/207] Set inactivity threshold on durable consumers in the roomserver input API (#2795) This prevents us from holding onto durable consumers indefinitely for rooms that have long since turned inactive, since they do have a bit of a processing overhead in the NATS Server. If we clear up a consumer and then a room becomes active again, the consumer gets recreated as needed. The threshold is set to 24 hours for now, we can tweak it later if needs be. --- roomserver/internal/input/input.go | 34 +++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/roomserver/internal/input/input.go b/roomserver/internal/input/input.go index c47793f0a..f5099ca11 100644 --- a/roomserver/internal/input/input.go +++ b/roomserver/internal/input/input.go @@ -89,6 +89,13 @@ type Inputer struct { Queryer *query.Queryer } +// If a room consumer is inactive for a while then we will allow NATS +// to clean it up. This stops us from holding onto durable consumers +// indefinitely for rooms that might no longer be active, since they do +// have an interest overhead in the NATS Server. If the room becomes +// active again then we'll recreate the consumer anyway. +const inactiveThreshold = time.Hour * 24 + type worker struct { phony.Inbox sync.Mutex @@ -125,11 +132,12 @@ func (r *Inputer) startWorkerForRoom(roomID string) { if _, err := w.r.JetStream.AddConsumer( r.Cfg.Matrix.JetStream.Prefixed(jetstream.InputRoomEvent), &nats.ConsumerConfig{ - Durable: consumer, - AckPolicy: nats.AckAllPolicy, - DeliverPolicy: nats.DeliverAllPolicy, - FilterSubject: subject, - AckWait: MaximumMissingProcessingTime + (time.Second * 10), + Durable: consumer, + AckPolicy: nats.AckAllPolicy, + DeliverPolicy: nats.DeliverAllPolicy, + FilterSubject: subject, + AckWait: MaximumMissingProcessingTime + (time.Second * 10), + InactiveThreshold: inactiveThreshold, }, ); err != nil { logrus.WithError(err).Errorf("Failed to create consumer for room %q", w.roomID) @@ -145,6 +153,7 @@ func (r *Inputer) startWorkerForRoom(roomID string) { nats.DeliverAll(), nats.AckWait(MaximumMissingProcessingTime+(time.Second*10)), nats.Bind(r.InputRoomEventTopic, consumer), + nats.InactiveThreshold(inactiveThreshold), ) if err != nil { logrus.WithError(err).Errorf("Failed to subscribe to stream for room %q", w.roomID) @@ -180,6 +189,21 @@ func (r *Inputer) Start() error { nats.ReplayInstant(), nats.BindStream(r.InputRoomEventTopic), ) + + // Make sure that the room consumers have the right config. + stream := r.Cfg.Matrix.JetStream.Prefixed(jetstream.InputRoomEvent) + for consumer := range r.JetStream.Consumers(stream) { + switch { + case consumer.Config.Durable == "": + continue // Ignore ephemeral consumers + case consumer.Config.InactiveThreshold != inactiveThreshold: + consumer.Config.InactiveThreshold = inactiveThreshold + if _, cerr := r.JetStream.UpdateConsumer(stream, &consumer.Config); cerr != nil { + logrus.WithError(cerr).Warnf("Failed to update inactive threshold on consumer %q", consumer.Name) + } + } + } + return err } From 81dbad39a3007a67b92f7abd669170a545a1f597 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 14 Oct 2022 15:53:58 +0100 Subject: [PATCH 080/207] Dendrite 0.10.3 commit c6e18c18e93b54c006c6b4d0044aa53a0735a906 Author: Neil Alexander Date: Fri Oct 14 15:42:58 2022 +0100 Changelog and version bump --- CHANGES.md | 17 +++++++++++++++++ internal/version.go | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 7552e41e5..eea2c3c7c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,22 @@ # Changelog +## Dendrite 0.10.3 (2022-10-14) + +### Features + +* Event relations are now tracked and support for the `/room/{roomID}/relations/...` client API endpoints have been added +* Support has been added for private read receipts +* The built-in NATS Server has been updated to version 2.9.3 + +### Fixes + +* The `unread_notifications` are now always populated in joined room responses +* The `/get_missing_events` federation API endpoint should now work correctly for rooms with `joined` and `invited` visibility settings, returning redacted events for events that other servers are not allowed to see +* The `/event` client API endpoint now applies history visibility correctly +* Read markers should now be updated much more reliably +* A rare bug in the sync API which could cause some `join` memberships to be incorrectly overwritten by other memberships when working out which rooms to populate has been fixed +* The federation API now correctly updates the joined hosts table during a state rewrite + ## Dendrite 0.10.2 (2022-10-07) ### Features diff --git a/internal/version.go b/internal/version.go index 56b83f852..c888748a8 100644 --- a/internal/version.go +++ b/internal/version.go @@ -17,7 +17,7 @@ var build string const ( VersionMajor = 0 VersionMinor = 10 - VersionPatch = 2 + VersionPatch = 3 VersionTag = "" // example: "rc1" ) From 83c9dde219440e0d730d63d2d65fc9eaaea64762 Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Mon, 17 Oct 2022 07:27:11 +0200 Subject: [PATCH 081/207] Return error if we fail to read the response body --- cmd/create-account/main.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/create-account/main.go b/cmd/create-account/main.go index 52301415f..c8e239f29 100644 --- a/cmd/create-account/main.go +++ b/cmd/create-account/main.go @@ -179,7 +179,10 @@ func sharedSecretRegister(sharedSecret, serverURL, localpart, password string, a body, _ = io.ReadAll(regResp.Body) return "", fmt.Errorf(gjson.GetBytes(body, "error").Str) } - r, _ := io.ReadAll(regResp.Body) + r, err := io.ReadAll(regResp.Body) + if err != nil { + return "", fmt.Errorf("failed to read response body (HTTP %d): %w", regResp.StatusCode, err) + } return gjson.GetBytes(r, "access_token").Str, nil } From d72d4f8d5d0016a8dcbf77aba92671f3469eb630 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 17 Oct 2022 10:38:22 +0100 Subject: [PATCH 082/207] Set `org.matrix.msc2285.stable` in `/versions` --- clientapi/routing/routing.go | 1 + 1 file changed, 1 insertion(+) diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index e72880ec5..ec5ca899e 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -70,6 +70,7 @@ func Setup( unstableFeatures := map[string]bool{ "org.matrix.e2e_cross_signing": true, + "org.matrix.msc2285.stable": true, } for _, msc := range cfg.MSCs.MSCs { unstableFeatures["org.matrix."+msc] = true From 07bfb791ca616bd3a4aa96691b74c96146d59d90 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Mon, 17 Oct 2022 14:48:35 +0200 Subject: [PATCH 083/207] Scope transactions to endpoints (#2799) To avoid returning results from e.g. `/redact` on `/sendToDevice` requests. Takes the raw URL path and uses `filepath.Dir` to remove the `txnID` (file) from it. Co-authored-by: Neil Alexander --- clientapi/routing/redaction.go | 9 ++--- clientapi/routing/sendevent.go | 4 +-- clientapi/routing/sendtodevice.go | 7 ++-- clientapi/routing/server_notices.go | 7 ++-- internal/transactions/transactions.go | 16 +++++---- internal/transactions/transactions_test.go | 42 ++++++++++++++++++---- 6 files changed, 59 insertions(+), 26 deletions(-) diff --git a/clientapi/routing/redaction.go b/clientapi/routing/redaction.go index 27f0ba5d0..a0f3b1152 100644 --- a/clientapi/routing/redaction.go +++ b/clientapi/routing/redaction.go @@ -19,6 +19,9 @@ import ( "net/http" "time" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" + "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/internal/eventutil" @@ -26,8 +29,6 @@ import ( roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" userapi "github.com/matrix-org/dendrite/userapi/api" - "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" ) type redactionContent struct { @@ -51,7 +52,7 @@ func SendRedaction( if txnID != nil { // Try to fetch response from transactionsCache - if res, ok := txnCache.FetchTransaction(device.AccessToken, *txnID); ok { + if res, ok := txnCache.FetchTransaction(device.AccessToken, *txnID, req.URL); ok { return *res } } @@ -144,7 +145,7 @@ func SendRedaction( // Add response to transactionsCache if txnID != nil { - txnCache.AddTransaction(device.AccessToken, *txnID, &res) + txnCache.AddTransaction(device.AccessToken, *txnID, req.URL, &res) } return res diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go index 85f1053f3..114e9088d 100644 --- a/clientapi/routing/sendevent.go +++ b/clientapi/routing/sendevent.go @@ -86,7 +86,7 @@ func SendEvent( if txnID != nil { // Try to fetch response from transactionsCache - if res, ok := txnCache.FetchTransaction(device.AccessToken, *txnID); ok { + if res, ok := txnCache.FetchTransaction(device.AccessToken, *txnID, req.URL); ok { return *res } } @@ -206,7 +206,7 @@ func SendEvent( } // Add response to transactionsCache if txnID != nil { - txnCache.AddTransaction(device.AccessToken, *txnID, &res) + txnCache.AddTransaction(device.AccessToken, *txnID, req.URL, &res) } // Take a note of how long it took to generate the event vs submit diff --git a/clientapi/routing/sendtodevice.go b/clientapi/routing/sendtodevice.go index 4a5f08883..0c0227937 100644 --- a/clientapi/routing/sendtodevice.go +++ b/clientapi/routing/sendtodevice.go @@ -16,12 +16,13 @@ import ( "encoding/json" "net/http" + "github.com/matrix-org/util" + "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/producers" "github.com/matrix-org/dendrite/internal/transactions" userapi "github.com/matrix-org/dendrite/userapi/api" - "github.com/matrix-org/util" ) // SendToDevice handles PUT /_matrix/client/r0/sendToDevice/{eventType}/{txnId} @@ -33,7 +34,7 @@ func SendToDevice( eventType string, txnID *string, ) util.JSONResponse { if txnID != nil { - if res, ok := txnCache.FetchTransaction(device.AccessToken, *txnID); ok { + if res, ok := txnCache.FetchTransaction(device.AccessToken, *txnID, req.URL); ok { return *res } } @@ -63,7 +64,7 @@ func SendToDevice( } if txnID != nil { - txnCache.AddTransaction(device.AccessToken, *txnID, &res) + txnCache.AddTransaction(device.AccessToken, *txnID, req.URL, &res) } return res diff --git a/clientapi/routing/server_notices.go b/clientapi/routing/server_notices.go index 9edeed2f7..7729eddd8 100644 --- a/clientapi/routing/server_notices.go +++ b/clientapi/routing/server_notices.go @@ -21,7 +21,6 @@ import ( "net/http" "time" - "github.com/matrix-org/dendrite/roomserver/version" "github.com/matrix-org/gomatrix" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib/tokens" @@ -29,6 +28,8 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" + "github.com/matrix-org/dendrite/roomserver/version" + appserviceAPI "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" @@ -73,7 +74,7 @@ func SendServerNotice( if txnID != nil { // Try to fetch response from transactionsCache - if res, ok := txnCache.FetchTransaction(device.AccessToken, *txnID); ok { + if res, ok := txnCache.FetchTransaction(device.AccessToken, *txnID, req.URL); ok { return *res } } @@ -251,7 +252,7 @@ func SendServerNotice( } // Add response to transactionsCache if txnID != nil { - txnCache.AddTransaction(device.AccessToken, *txnID, &res) + txnCache.AddTransaction(device.AccessToken, *txnID, req.URL, &res) } // Take a note of how long it took to generate the event vs submit diff --git a/internal/transactions/transactions.go b/internal/transactions/transactions.go index d2eb0f27f..7ff6f5044 100644 --- a/internal/transactions/transactions.go +++ b/internal/transactions/transactions.go @@ -13,6 +13,8 @@ package transactions import ( + "net/url" + "path/filepath" "sync" "time" @@ -29,6 +31,7 @@ type txnsMap map[CacheKey]*util.JSONResponse type CacheKey struct { AccessToken string TxnID string + Endpoint string } // Cache represents a temporary store for response entries. @@ -57,14 +60,14 @@ func NewWithCleanupPeriod(cleanupPeriod time.Duration) *Cache { return &t } -// FetchTransaction looks up an entry for the (accessToken, txnID) tuple in Cache. +// FetchTransaction looks up an entry for the (accessToken, txnID, req.URL) tuple in Cache. // Looks in both the txnMaps. // Returns (JSON response, true) if txnID is found, else the returned bool is false. -func (t *Cache) FetchTransaction(accessToken, txnID string) (*util.JSONResponse, bool) { +func (t *Cache) FetchTransaction(accessToken, txnID string, u *url.URL) (*util.JSONResponse, bool) { t.RLock() defer t.RUnlock() for _, txns := range t.txnsMaps { - res, ok := txns[CacheKey{accessToken, txnID}] + res, ok := txns[CacheKey{accessToken, txnID, filepath.Dir(u.Path)}] if ok { return res, true } @@ -72,13 +75,12 @@ func (t *Cache) FetchTransaction(accessToken, txnID string) (*util.JSONResponse, return nil, false } -// AddTransaction adds an entry for the (accessToken, txnID) tuple in Cache. +// AddTransaction adds an entry for the (accessToken, txnID, req.URL) tuple in Cache. // Adds to the front txnMap. -func (t *Cache) AddTransaction(accessToken, txnID string, res *util.JSONResponse) { +func (t *Cache) AddTransaction(accessToken, txnID string, u *url.URL, res *util.JSONResponse) { t.Lock() defer t.Unlock() - - t.txnsMaps[0][CacheKey{accessToken, txnID}] = res + t.txnsMaps[0][CacheKey{accessToken, txnID, filepath.Dir(u.Path)}] = res } // cacheCleanService is responsible for cleaning up entries after cleanupPeriod. diff --git a/internal/transactions/transactions_test.go b/internal/transactions/transactions_test.go index aa837f76c..c552550ac 100644 --- a/internal/transactions/transactions_test.go +++ b/internal/transactions/transactions_test.go @@ -14,6 +14,9 @@ package transactions import ( "net/http" + "net/url" + "path/filepath" + "reflect" "strconv" "testing" @@ -24,6 +27,16 @@ type fakeType struct { ID string `json:"ID"` } +func TestCompare(t *testing.T) { + u1, _ := url.Parse("/send/1?accessToken=123") + u2, _ := url.Parse("/send/1") + c1 := CacheKey{"1", "2", filepath.Dir(u1.Path)} + c2 := CacheKey{"1", "2", filepath.Dir(u2.Path)} + if !reflect.DeepEqual(c1, c2) { + t.Fatalf("Cache keys differ: %+v <> %+v", c1, c2) + } +} + var ( fakeAccessToken = "aRandomAccessToken" fakeAccessToken2 = "anotherRandomAccessToken" @@ -34,23 +47,28 @@ var ( fakeResponse2 = &util.JSONResponse{ Code: http.StatusOK, JSON: fakeType{ID: "1"}, } + fakeResponse3 = &util.JSONResponse{ + Code: http.StatusOK, JSON: fakeType{ID: "2"}, + } ) // TestCache creates a New Cache and tests AddTransaction & FetchTransaction func TestCache(t *testing.T) { fakeTxnCache := New() - fakeTxnCache.AddTransaction(fakeAccessToken, fakeTxnID, fakeResponse) + u, _ := url.Parse("") + fakeTxnCache.AddTransaction(fakeAccessToken, fakeTxnID, u, fakeResponse) // Add entries for noise. for i := 1; i <= 100; i++ { fakeTxnCache.AddTransaction( fakeAccessToken, fakeTxnID+strconv.Itoa(i), + u, &util.JSONResponse{Code: http.StatusOK, JSON: fakeType{ID: strconv.Itoa(i)}}, ) } - testResponse, ok := fakeTxnCache.FetchTransaction(fakeAccessToken, fakeTxnID) + testResponse, ok := fakeTxnCache.FetchTransaction(fakeAccessToken, fakeTxnID, u) if !ok { t.Error("Failed to retrieve entry for txnID: ", fakeTxnID) } else if testResponse.JSON != fakeResponse.JSON { @@ -59,20 +77,30 @@ func TestCache(t *testing.T) { } // TestCacheScope ensures transactions with the same transaction ID are not shared -// across multiple access tokens. +// across multiple access tokens and endpoints. func TestCacheScope(t *testing.T) { cache := New() - cache.AddTransaction(fakeAccessToken, fakeTxnID, fakeResponse) - cache.AddTransaction(fakeAccessToken2, fakeTxnID, fakeResponse2) + sendEndpoint, _ := url.Parse("/send/1?accessToken=test") + sendToDeviceEndpoint, _ := url.Parse("/sendToDevice/1") + cache.AddTransaction(fakeAccessToken, fakeTxnID, sendEndpoint, fakeResponse) + cache.AddTransaction(fakeAccessToken2, fakeTxnID, sendEndpoint, fakeResponse2) + cache.AddTransaction(fakeAccessToken2, fakeTxnID, sendToDeviceEndpoint, fakeResponse3) - if res, ok := cache.FetchTransaction(fakeAccessToken, fakeTxnID); !ok { + if res, ok := cache.FetchTransaction(fakeAccessToken, fakeTxnID, sendEndpoint); !ok { t.Errorf("failed to retrieve entry for (%s, %s)", fakeAccessToken, fakeTxnID) } else if res.JSON != fakeResponse.JSON { t.Errorf("Wrong cache entry for (%s, %s). Expected: %v; got: %v", fakeAccessToken, fakeTxnID, fakeResponse.JSON, res.JSON) } - if res, ok := cache.FetchTransaction(fakeAccessToken2, fakeTxnID); !ok { + if res, ok := cache.FetchTransaction(fakeAccessToken2, fakeTxnID, sendEndpoint); !ok { t.Errorf("failed to retrieve entry for (%s, %s)", fakeAccessToken, fakeTxnID) } else if res.JSON != fakeResponse2.JSON { t.Errorf("Wrong cache entry for (%s, %s). Expected: %v; got: %v", fakeAccessToken, fakeTxnID, fakeResponse2.JSON, res.JSON) } + + // Ensure the txnID is not shared across endpoints + if res, ok := cache.FetchTransaction(fakeAccessToken2, fakeTxnID, sendToDeviceEndpoint); !ok { + t.Errorf("failed to retrieve entry for (%s, %s)", fakeAccessToken, fakeTxnID) + } else if res.JSON != fakeResponse3.JSON { + t.Errorf("Wrong cache entry for (%s, %s). Expected: %v; got: %v", fakeAccessToken, fakeTxnID, fakeResponse2.JSON, res.JSON) + } } From 9c189b1b80f9c338ac3cfa71cdaca60016de45f7 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 18 Oct 2022 09:51:31 +0100 Subject: [PATCH 084/207] Try to make `AddEvent` less expensive (update to matrix-org/gomatrixserverlib@a72a83f) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 1303d4004..911d36c1c 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 - github.com/matrix-org/gomatrixserverlib v0.0.0-20221014061925-a132619fa241 + github.com/matrix-org/gomatrixserverlib v0.0.0-20221018085104-a72a83f0e19a github.com/matrix-org/pinecone v0.0.0-20221007145426-3adc85477dd3 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.15 diff --git a/go.sum b/go.sum index 54055f2fd..a141fc9b4 100644 --- a/go.sum +++ b/go.sum @@ -387,8 +387,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8r4Fzarl4+Y3K5hjothkVW5z7T1dUM11U= github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20221014061925-a132619fa241 h1:e5o68MWeU7wjTvvNKmVo655oCYesoNRoPeBb1Xfz54g= -github.com/matrix-org/gomatrixserverlib v0.0.0-20221014061925-a132619fa241/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4= +github.com/matrix-org/gomatrixserverlib v0.0.0-20221018085104-a72a83f0e19a h1:bQKHk3AWlgm7XhzPhuU3Iw3pUptW5l1DR/1y0o7zCKQ= +github.com/matrix-org/gomatrixserverlib v0.0.0-20221018085104-a72a83f0e19a/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4= github.com/matrix-org/pinecone v0.0.0-20221007145426-3adc85477dd3 h1:lzkSQvBv8TuqKJCPoVwOVvEnARTlua5rrNy/Qw2Vxeo= github.com/matrix-org/pinecone v0.0.0-20221007145426-3adc85477dd3/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= From 3aa92efaa3e814ad0596fc5fc174a2e43124dcf5 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 18 Oct 2022 15:59:08 +0100 Subject: [PATCH 085/207] Namespace user API tables (#2806) This migrates all the various user API tables, indices and sequences to be `userapi_`-namespaced, rather than the mess they are all now. --- userapi/consumers/roomserver_test.go | 9 +- .../storage/postgres/account_data_table.go | 8 +- userapi/storage/postgres/accounts_table.go | 14 +-- .../deltas/20200929203058_is_active.go | 4 +- .../deltas/20201001204705_last_seen_ts_ip.go | 12 +- .../2022021013023800_add_account_type.go | 10 +- .../deltas/2022101711000000_rename_tables.go | 102 ++++++++++++++++ userapi/storage/postgres/devices_table.go | 28 ++--- userapi/storage/postgres/key_backup_table.go | 18 +-- .../postgres/key_backup_version_table.go | 20 ++-- userapi/storage/postgres/logintoken_table.go | 12 +- userapi/storage/postgres/openid_table.go | 6 +- userapi/storage/postgres/profile_table.go | 12 +- userapi/storage/postgres/stats_table.go | 16 +-- userapi/storage/postgres/storage.go | 11 ++ userapi/storage/postgres/threepid_table.go | 12 +- userapi/storage/sqlite3/account_data_table.go | 8 +- userapi/storage/sqlite3/accounts_table.go | 14 +-- .../deltas/20200929203058_is_active.go | 20 ++-- .../deltas/20201001204705_last_seen_ts_ip.go | 20 ++-- .../2022021012490600_add_account_type.go | 16 +-- .../deltas/2022101711000000_rename_tables.go | 109 ++++++++++++++++++ userapi/storage/sqlite3/devices_table.go | 24 ++-- userapi/storage/sqlite3/key_backup_table.go | 18 +-- .../sqlite3/key_backup_version_table.go | 16 +-- userapi/storage/sqlite3/logintoken_table.go | 12 +- userapi/storage/sqlite3/openid_table.go | 6 +- userapi/storage/sqlite3/profile_table.go | 12 +- userapi/storage/sqlite3/stats_table.go | 16 +-- userapi/storage/sqlite3/storage.go | 11 ++ userapi/storage/sqlite3/threepid_table.go | 12 +- userapi/storage/storage_test.go | 9 +- userapi/storage/tables/stats_table_test.go | 4 +- userapi/userapi_test.go | 14 ++- 34 files changed, 441 insertions(+), 194 deletions(-) create mode 100644 userapi/storage/postgres/deltas/2022101711000000_rename_tables.go create mode 100644 userapi/storage/sqlite3/deltas/2022101711000000_rename_tables.go diff --git a/userapi/consumers/roomserver_test.go b/userapi/consumers/roomserver_test.go index 3bbeb439a..e4587670f 100644 --- a/userapi/consumers/roomserver_test.go +++ b/userapi/consumers/roomserver_test.go @@ -10,19 +10,24 @@ import ( "github.com/matrix-org/dendrite/internal/pushrules" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/test" + "github.com/matrix-org/dendrite/test/testrig" "github.com/matrix-org/dendrite/userapi/storage" ) func mustCreateDatabase(t *testing.T, dbType test.DBType) (storage.Database, func()) { + base, baseclose := testrig.CreateBaseDendrite(t, dbType) t.Helper() connStr, close := test.PrepareDBConnectionString(t, dbType) - db, err := storage.NewUserAPIDatabase(nil, &config.DatabaseOptions{ + db, err := storage.NewUserAPIDatabase(base, &config.DatabaseOptions{ ConnectionString: config.DataSource(connStr), }, "", 4, 0, 0, "") if err != nil { t.Fatalf("failed to create new user db: %v", err) } - return db, close + return db, func() { + close() + baseclose() + } } func mustCreateEvent(t *testing.T, content string) *gomatrixserverlib.HeaderedEvent { diff --git a/userapi/storage/postgres/account_data_table.go b/userapi/storage/postgres/account_data_table.go index 67113367b..0b6a3af6d 100644 --- a/userapi/storage/postgres/account_data_table.go +++ b/userapi/storage/postgres/account_data_table.go @@ -26,7 +26,7 @@ import ( const accountDataSchema = ` -- Stores data about accounts data. -CREATE TABLE IF NOT EXISTS account_data ( +CREATE TABLE IF NOT EXISTS userapi_account_datas ( -- The Matrix user ID localpart for this account localpart TEXT NOT NULL, -- The room ID for this data (empty string if not specific to a room) @@ -41,15 +41,15 @@ CREATE TABLE IF NOT EXISTS account_data ( ` const insertAccountDataSQL = ` - INSERT INTO account_data(localpart, room_id, type, content) VALUES($1, $2, $3, $4) + INSERT INTO userapi_account_datas(localpart, room_id, type, content) VALUES($1, $2, $3, $4) ON CONFLICT (localpart, room_id, type) DO UPDATE SET content = EXCLUDED.content ` const selectAccountDataSQL = "" + - "SELECT room_id, type, content FROM account_data WHERE localpart = $1" + "SELECT room_id, type, content FROM userapi_account_datas WHERE localpart = $1" const selectAccountDataByTypeSQL = "" + - "SELECT content FROM account_data WHERE localpart = $1 AND room_id = $2 AND type = $3" + "SELECT content FROM userapi_account_datas WHERE localpart = $1 AND room_id = $2 AND type = $3" type accountDataStatements struct { insertAccountDataStmt *sql.Stmt diff --git a/userapi/storage/postgres/accounts_table.go b/userapi/storage/postgres/accounts_table.go index 33fb6dd42..7c309eb4f 100644 --- a/userapi/storage/postgres/accounts_table.go +++ b/userapi/storage/postgres/accounts_table.go @@ -32,7 +32,7 @@ import ( const accountsSchema = ` -- Stores data about accounts. -CREATE TABLE IF NOT EXISTS account_accounts ( +CREATE TABLE IF NOT EXISTS userapi_accounts ( -- The Matrix user ID localpart for this account localpart TEXT NOT NULL PRIMARY KEY, -- When this account was first created, as a unix timestamp (ms resolution). @@ -51,22 +51,22 @@ CREATE TABLE IF NOT EXISTS account_accounts ( ` const insertAccountSQL = "" + - "INSERT INTO account_accounts(localpart, created_ts, password_hash, appservice_id, account_type) VALUES ($1, $2, $3, $4, $5)" + "INSERT INTO userapi_accounts(localpart, created_ts, password_hash, appservice_id, account_type) VALUES ($1, $2, $3, $4, $5)" const updatePasswordSQL = "" + - "UPDATE account_accounts SET password_hash = $1 WHERE localpart = $2" + "UPDATE userapi_accounts SET password_hash = $1 WHERE localpart = $2" const deactivateAccountSQL = "" + - "UPDATE account_accounts SET is_deactivated = TRUE WHERE localpart = $1" + "UPDATE userapi_accounts SET is_deactivated = TRUE WHERE localpart = $1" const selectAccountByLocalpartSQL = "" + - "SELECT localpart, appservice_id, account_type FROM account_accounts WHERE localpart = $1" + "SELECT localpart, appservice_id, account_type FROM userapi_accounts WHERE localpart = $1" const selectPasswordHashSQL = "" + - "SELECT password_hash FROM account_accounts WHERE localpart = $1 AND is_deactivated = FALSE" + "SELECT password_hash FROM userapi_accounts WHERE localpart = $1 AND is_deactivated = FALSE" const selectNewNumericLocalpartSQL = "" + - "SELECT COALESCE(MAX(localpart::bigint), 0) FROM account_accounts WHERE localpart ~ '^[0-9]{1,}$'" + "SELECT COALESCE(MAX(localpart::bigint), 0) FROM userapi_accounts WHERE localpart ~ '^[0-9]{1,}$'" type accountsStatements struct { insertAccountStmt *sql.Stmt diff --git a/userapi/storage/postgres/deltas/20200929203058_is_active.go b/userapi/storage/postgres/deltas/20200929203058_is_active.go index 24f87e073..2c5cc2f58 100644 --- a/userapi/storage/postgres/deltas/20200929203058_is_active.go +++ b/userapi/storage/postgres/deltas/20200929203058_is_active.go @@ -7,7 +7,7 @@ import ( ) func UpIsActive(ctx context.Context, tx *sql.Tx) error { - _, err := tx.ExecContext(ctx, "ALTER TABLE account_accounts ADD COLUMN IF NOT EXISTS is_deactivated BOOLEAN DEFAULT FALSE;") + _, err := tx.ExecContext(ctx, "ALTER TABLE userapi_accounts ADD COLUMN IF NOT EXISTS is_deactivated BOOLEAN DEFAULT FALSE;") if err != nil { return fmt.Errorf("failed to execute upgrade: %w", err) } @@ -15,7 +15,7 @@ func UpIsActive(ctx context.Context, tx *sql.Tx) error { } func DownIsActive(ctx context.Context, tx *sql.Tx) error { - _, err := tx.ExecContext(ctx, "ALTER TABLE account_accounts DROP COLUMN is_deactivated;") + _, err := tx.ExecContext(ctx, "ALTER TABLE userapi_accounts DROP COLUMN is_deactivated;") if err != nil { return fmt.Errorf("failed to execute downgrade: %w", err) } diff --git a/userapi/storage/postgres/deltas/20201001204705_last_seen_ts_ip.go b/userapi/storage/postgres/deltas/20201001204705_last_seen_ts_ip.go index edd3353f0..40e237027 100644 --- a/userapi/storage/postgres/deltas/20201001204705_last_seen_ts_ip.go +++ b/userapi/storage/postgres/deltas/20201001204705_last_seen_ts_ip.go @@ -8,9 +8,9 @@ import ( func UpLastSeenTSIP(ctx context.Context, tx *sql.Tx) error { _, err := tx.ExecContext(ctx, ` -ALTER TABLE device_devices ADD COLUMN IF NOT EXISTS last_seen_ts BIGINT NOT NULL DEFAULT EXTRACT(EPOCH FROM CURRENT_TIMESTAMP)*1000; -ALTER TABLE device_devices ADD COLUMN IF NOT EXISTS ip TEXT; -ALTER TABLE device_devices ADD COLUMN IF NOT EXISTS user_agent TEXT;`) +ALTER TABLE userapi_devices ADD COLUMN IF NOT EXISTS last_seen_ts BIGINT NOT NULL DEFAULT EXTRACT(EPOCH FROM CURRENT_TIMESTAMP)*1000; +ALTER TABLE userapi_devices ADD COLUMN IF NOT EXISTS ip TEXT; +ALTER TABLE userapi_devices ADD COLUMN IF NOT EXISTS user_agent TEXT;`) if err != nil { return fmt.Errorf("failed to execute upgrade: %w", err) } @@ -19,9 +19,9 @@ ALTER TABLE device_devices ADD COLUMN IF NOT EXISTS user_agent TEXT;`) func DownLastSeenTSIP(ctx context.Context, tx *sql.Tx) error { _, err := tx.ExecContext(ctx, ` - ALTER TABLE device_devices DROP COLUMN last_seen_ts; - ALTER TABLE device_devices DROP COLUMN ip; - ALTER TABLE device_devices DROP COLUMN user_agent;`) + ALTER TABLE userapi_devices DROP COLUMN last_seen_ts; + ALTER TABLE userapi_devices DROP COLUMN ip; + ALTER TABLE userapi_devices DROP COLUMN user_agent;`) if err != nil { return fmt.Errorf("failed to execute downgrade: %w", err) } diff --git a/userapi/storage/postgres/deltas/2022021013023800_add_account_type.go b/userapi/storage/postgres/deltas/2022021013023800_add_account_type.go index eb7c3a958..164847e51 100644 --- a/userapi/storage/postgres/deltas/2022021013023800_add_account_type.go +++ b/userapi/storage/postgres/deltas/2022021013023800_add_account_type.go @@ -9,10 +9,10 @@ import ( func UpAddAccountType(ctx context.Context, tx *sql.Tx) error { // initially set every account to useraccount, change appservice and guest accounts afterwards // (user = 1, guest = 2, admin = 3, appservice = 4) - _, err := tx.ExecContext(ctx, `ALTER TABLE account_accounts ADD COLUMN IF NOT EXISTS account_type SMALLINT NOT NULL DEFAULT 1; -UPDATE account_accounts SET account_type = 4 WHERE appservice_id <> ''; -UPDATE account_accounts SET account_type = 2 WHERE localpart ~ '^[0-9]+$'; -ALTER TABLE account_accounts ALTER COLUMN account_type DROP DEFAULT;`, + _, err := tx.ExecContext(ctx, `ALTER TABLE userapi_accounts ADD COLUMN IF NOT EXISTS account_type SMALLINT NOT NULL DEFAULT 1; +UPDATE userapi_accounts SET account_type = 4 WHERE appservice_id <> ''; +UPDATE userapi_accounts SET account_type = 2 WHERE localpart ~ '^[0-9]+$'; +ALTER TABLE userapi_accounts ALTER COLUMN account_type DROP DEFAULT;`, ) if err != nil { return fmt.Errorf("failed to execute upgrade: %w", err) @@ -21,7 +21,7 @@ ALTER TABLE account_accounts ALTER COLUMN account_type DROP DEFAULT;`, } func DownAddAccountType(ctx context.Context, tx *sql.Tx) error { - _, err := tx.ExecContext(ctx, "ALTER TABLE account_accounts DROP COLUMN account_type;") + _, err := tx.ExecContext(ctx, "ALTER TABLE userapi_accounts DROP COLUMN account_type;") if err != nil { return fmt.Errorf("failed to execute downgrade: %w", err) } diff --git a/userapi/storage/postgres/deltas/2022101711000000_rename_tables.go b/userapi/storage/postgres/deltas/2022101711000000_rename_tables.go new file mode 100644 index 000000000..1d73d0af4 --- /dev/null +++ b/userapi/storage/postgres/deltas/2022101711000000_rename_tables.go @@ -0,0 +1,102 @@ +package deltas + +import ( + "context" + "database/sql" + "fmt" + + "github.com/lib/pq" +) + +var renameTableMappings = map[string]string{ + "account_accounts": "userapi_accounts", + "account_data": "userapi_account_datas", + "device_devices": "userapi_devices", + "account_e2e_room_keys": "userapi_key_backups", + "account_e2e_room_keys_versions": "userapi_key_backup_versions", + "login_tokens": "userapi_login_tokens", + "open_id_tokens": "userapi_openid_tokens", + "account_profiles": "userapi_profiles", + "account_threepid": "userapi_threepids", +} + +var renameSequenceMappings = map[string]string{ + "device_session_id_seq": "userapi_device_session_id_seq", + "account_e2e_room_keys_versions_seq": "userapi_key_backup_versions_seq", +} + +var renameIndicesMappings = map[string]string{ + "device_localpart_id_idx": "userapi_device_localpart_id_idx", + "e2e_room_keys_idx": "userapi_key_backups_idx", + "e2e_room_keys_versions_idx": "userapi_key_backups_versions_idx", + "account_e2e_room_keys_versions_idx": "userapi_key_backup_versions_idx", + "login_tokens_expiration_idx": "userapi_login_tokens_expiration_idx", + "account_threepid_localpart": "userapi_threepid_idx", +} + +// I know what you're thinking: you're wondering "why doesn't this use $1 +// and pass variadic parameters to ExecContext?" — the answer is because +// PostgreSQL doesn't expect the table name to be specified as a substituted +// argument in that way so it results in a syntax error in the query. + +func UpRenameTables(ctx context.Context, tx *sql.Tx) error { + for old, new := range renameTableMappings { + q := fmt.Sprintf( + "ALTER TABLE IF EXISTS %s RENAME TO %s;", + pq.QuoteIdentifier(old), pq.QuoteIdentifier(new), + ) + if _, err := tx.ExecContext(ctx, q); err != nil { + return fmt.Errorf("rename table %q to %q error: %w", old, new, err) + } + } + for old, new := range renameSequenceMappings { + q := fmt.Sprintf( + "ALTER SEQUENCE IF EXISTS %s RENAME TO %s;", + pq.QuoteIdentifier(old), pq.QuoteIdentifier(new), + ) + if _, err := tx.ExecContext(ctx, q); err != nil { + return fmt.Errorf("rename table %q to %q error: %w", old, new, err) + } + } + for old, new := range renameIndicesMappings { + q := fmt.Sprintf( + "ALTER INDEX IF EXISTS %s RENAME TO %s;", + pq.QuoteIdentifier(old), pq.QuoteIdentifier(new), + ) + if _, err := tx.ExecContext(ctx, q); err != nil { + return fmt.Errorf("rename table %q to %q error: %w", old, new, err) + } + } + return nil +} + +func DownRenameTables(ctx context.Context, tx *sql.Tx) error { + for old, new := range renameTableMappings { + q := fmt.Sprintf( + "ALTER TABLE IF EXISTS %s RENAME TO %s;", + pq.QuoteIdentifier(new), pq.QuoteIdentifier(old), + ) + if _, err := tx.ExecContext(ctx, q); err != nil { + return fmt.Errorf("rename table %q to %q error: %w", new, old, err) + } + } + for old, new := range renameSequenceMappings { + q := fmt.Sprintf( + "ALTER SEQUENCE IF EXISTS %s RENAME TO %s;", + pq.QuoteIdentifier(new), pq.QuoteIdentifier(old), + ) + if _, err := tx.ExecContext(ctx, q); err != nil { + return fmt.Errorf("rename table %q to %q error: %w", new, old, err) + } + } + for old, new := range renameIndicesMappings { + q := fmt.Sprintf( + "ALTER INDEX IF EXISTS %s RENAME TO %s;", + pq.QuoteIdentifier(new), pq.QuoteIdentifier(old), + ) + if _, err := tx.ExecContext(ctx, q); err != nil { + return fmt.Errorf("rename table %q to %q error: %w", new, old, err) + } + } + return nil +} diff --git a/userapi/storage/postgres/devices_table.go b/userapi/storage/postgres/devices_table.go index f65681aae..8b7fbd6cf 100644 --- a/userapi/storage/postgres/devices_table.go +++ b/userapi/storage/postgres/devices_table.go @@ -31,10 +31,10 @@ import ( const devicesSchema = ` -- This sequence is used for automatic allocation of session_id. -CREATE SEQUENCE IF NOT EXISTS device_session_id_seq START 1; +CREATE SEQUENCE IF NOT EXISTS userapi_device_session_id_seq START 1; -- Stores data about devices. -CREATE TABLE IF NOT EXISTS device_devices ( +CREATE TABLE IF NOT EXISTS userapi_devices ( -- The access token granted to this device. This has to be the primary key -- so we can distinguish which device is making a given request. access_token TEXT NOT NULL PRIMARY KEY, @@ -42,7 +42,7 @@ CREATE TABLE IF NOT EXISTS device_devices ( -- This can be used as a secure substitution of the access token in situations -- where data is associated with access tokens (e.g. transaction storage), -- so we don't have to store users' access tokens everywhere. - session_id BIGINT NOT NULL DEFAULT nextval('device_session_id_seq'), + session_id BIGINT NOT NULL DEFAULT nextval('userapi_device_session_id_seq'), -- The device identifier. This only needs to uniquely identify a device for a given user, not globally. -- access_tokens will be clobbered based on the device ID for a user. device_id TEXT NOT NULL, @@ -65,39 +65,39 @@ CREATE TABLE IF NOT EXISTS device_devices ( ); -- Device IDs must be unique for a given user. -CREATE UNIQUE INDEX IF NOT EXISTS device_localpart_id_idx ON device_devices(localpart, device_id); +CREATE UNIQUE INDEX IF NOT EXISTS userapi_device_localpart_id_idx ON userapi_devices(localpart, device_id); ` const insertDeviceSQL = "" + - "INSERT INTO device_devices(device_id, localpart, access_token, created_ts, display_name, last_seen_ts, ip, user_agent) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)" + + "INSERT INTO userapi_devices(device_id, localpart, access_token, created_ts, display_name, last_seen_ts, ip, user_agent) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)" + " RETURNING session_id" const selectDeviceByTokenSQL = "" + - "SELECT session_id, device_id, localpart FROM device_devices WHERE access_token = $1" + "SELECT session_id, device_id, localpart FROM userapi_devices WHERE access_token = $1" const selectDeviceByIDSQL = "" + - "SELECT display_name, last_seen_ts, ip FROM device_devices WHERE localpart = $1 and device_id = $2" + "SELECT display_name, last_seen_ts, ip FROM userapi_devices WHERE localpart = $1 and device_id = $2" const selectDevicesByLocalpartSQL = "" + - "SELECT device_id, display_name, last_seen_ts, ip, user_agent FROM device_devices WHERE localpart = $1 AND device_id != $2 ORDER BY last_seen_ts DESC" + "SELECT device_id, display_name, last_seen_ts, ip, user_agent FROM userapi_devices WHERE localpart = $1 AND device_id != $2 ORDER BY last_seen_ts DESC" const updateDeviceNameSQL = "" + - "UPDATE device_devices SET display_name = $1 WHERE localpart = $2 AND device_id = $3" + "UPDATE userapi_devices SET display_name = $1 WHERE localpart = $2 AND device_id = $3" const deleteDeviceSQL = "" + - "DELETE FROM device_devices WHERE device_id = $1 AND localpart = $2" + "DELETE FROM userapi_devices WHERE device_id = $1 AND localpart = $2" const deleteDevicesByLocalpartSQL = "" + - "DELETE FROM device_devices WHERE localpart = $1 AND device_id != $2" + "DELETE FROM userapi_devices WHERE localpart = $1 AND device_id != $2" const deleteDevicesSQL = "" + - "DELETE FROM device_devices WHERE localpart = $1 AND device_id = ANY($2)" + "DELETE FROM userapi_devices WHERE localpart = $1 AND device_id = ANY($2)" const selectDevicesByIDSQL = "" + - "SELECT device_id, localpart, display_name, last_seen_ts FROM device_devices WHERE device_id = ANY($1) ORDER BY last_seen_ts DESC" + "SELECT device_id, localpart, display_name, last_seen_ts FROM userapi_devices WHERE device_id = ANY($1) ORDER BY last_seen_ts DESC" const updateDeviceLastSeen = "" + - "UPDATE device_devices SET last_seen_ts = $1, ip = $2, user_agent = $3 WHERE localpart = $4 AND device_id = $5" + "UPDATE userapi_devices SET last_seen_ts = $1, ip = $2, user_agent = $3 WHERE localpart = $4 AND device_id = $5" type devicesStatements struct { insertDeviceStmt *sql.Stmt diff --git a/userapi/storage/postgres/key_backup_table.go b/userapi/storage/postgres/key_backup_table.go index ac0e80617..7b58f7bae 100644 --- a/userapi/storage/postgres/key_backup_table.go +++ b/userapi/storage/postgres/key_backup_table.go @@ -26,7 +26,7 @@ import ( ) const keyBackupTableSchema = ` -CREATE TABLE IF NOT EXISTS account_e2e_room_keys ( +CREATE TABLE IF NOT EXISTS userapi_key_backups ( user_id TEXT NOT NULL, room_id TEXT NOT NULL, session_id TEXT NOT NULL, @@ -37,31 +37,31 @@ CREATE TABLE IF NOT EXISTS account_e2e_room_keys ( is_verified BOOLEAN NOT NULL, session_data TEXT NOT NULL ); -CREATE UNIQUE INDEX IF NOT EXISTS e2e_room_keys_idx ON account_e2e_room_keys(user_id, room_id, session_id, version); -CREATE INDEX IF NOT EXISTS e2e_room_keys_versions_idx ON account_e2e_room_keys(user_id, version); +CREATE UNIQUE INDEX IF NOT EXISTS userapi_key_backups_idx ON userapi_key_backups(user_id, room_id, session_id, version); +CREATE INDEX IF NOT EXISTS userapi_key_backups_versions_idx ON userapi_key_backups(user_id, version); ` const insertBackupKeySQL = "" + - "INSERT INTO account_e2e_room_keys(user_id, room_id, session_id, version, first_message_index, forwarded_count, is_verified, session_data) " + + "INSERT INTO userapi_key_backups(user_id, room_id, session_id, version, first_message_index, forwarded_count, is_verified, session_data) " + "VALUES ($1, $2, $3, $4, $5, $6, $7, $8)" const updateBackupKeySQL = "" + - "UPDATE account_e2e_room_keys SET first_message_index=$1, forwarded_count=$2, is_verified=$3, session_data=$4 " + + "UPDATE userapi_key_backups SET first_message_index=$1, forwarded_count=$2, is_verified=$3, session_data=$4 " + "WHERE user_id=$5 AND room_id=$6 AND session_id=$7 AND version=$8" const countKeysSQL = "" + - "SELECT COUNT(*) FROM account_e2e_room_keys WHERE user_id = $1 AND version = $2" + "SELECT COUNT(*) FROM userapi_key_backups WHERE user_id = $1 AND version = $2" const selectKeysSQL = "" + - "SELECT room_id, session_id, first_message_index, forwarded_count, is_verified, session_data FROM account_e2e_room_keys " + + "SELECT room_id, session_id, first_message_index, forwarded_count, is_verified, session_data FROM userapi_key_backups " + "WHERE user_id = $1 AND version = $2" const selectKeysByRoomIDSQL = "" + - "SELECT room_id, session_id, first_message_index, forwarded_count, is_verified, session_data FROM account_e2e_room_keys " + + "SELECT room_id, session_id, first_message_index, forwarded_count, is_verified, session_data FROM userapi_key_backups " + "WHERE user_id = $1 AND version = $2 AND room_id = $3" const selectKeysByRoomIDAndSessionIDSQL = "" + - "SELECT room_id, session_id, first_message_index, forwarded_count, is_verified, session_data FROM account_e2e_room_keys " + + "SELECT room_id, session_id, first_message_index, forwarded_count, is_verified, session_data FROM userapi_key_backups " + "WHERE user_id = $1 AND version = $2 AND room_id = $3 AND session_id = $4" type keyBackupStatements struct { diff --git a/userapi/storage/postgres/key_backup_version_table.go b/userapi/storage/postgres/key_backup_version_table.go index e78e4cd51..67c5e5481 100644 --- a/userapi/storage/postgres/key_backup_version_table.go +++ b/userapi/storage/postgres/key_backup_version_table.go @@ -26,40 +26,40 @@ import ( ) const keyBackupVersionTableSchema = ` -CREATE SEQUENCE IF NOT EXISTS account_e2e_room_keys_versions_seq; +CREATE SEQUENCE IF NOT EXISTS userapi_key_backup_versions_seq; -- the metadata for each generation of encrypted e2e session backups -CREATE TABLE IF NOT EXISTS account_e2e_room_keys_versions ( +CREATE TABLE IF NOT EXISTS userapi_key_backup_versions ( user_id TEXT NOT NULL, -- this means no 2 users will ever have the same version of e2e session backups which strictly -- isn't necessary, but this is easy to do rather than SELECT MAX(version)+1. - version BIGINT DEFAULT nextval('account_e2e_room_keys_versions_seq'), + version BIGINT DEFAULT nextval('userapi_key_backup_versions_seq'), algorithm TEXT NOT NULL, auth_data TEXT NOT NULL, etag TEXT NOT NULL, deleted SMALLINT DEFAULT 0 NOT NULL ); -CREATE UNIQUE INDEX IF NOT EXISTS account_e2e_room_keys_versions_idx ON account_e2e_room_keys_versions(user_id, version); +CREATE UNIQUE INDEX IF NOT EXISTS userapi_key_backup_versions_idx ON userapi_key_backup_versions(user_id, version); ` const insertKeyBackupSQL = "" + - "INSERT INTO account_e2e_room_keys_versions(user_id, algorithm, auth_data, etag) VALUES ($1, $2, $3, $4) RETURNING version" + "INSERT INTO userapi_key_backup_versions(user_id, algorithm, auth_data, etag) VALUES ($1, $2, $3, $4) RETURNING version" const updateKeyBackupAuthDataSQL = "" + - "UPDATE account_e2e_room_keys_versions SET auth_data = $1 WHERE user_id = $2 AND version = $3" + "UPDATE userapi_key_backup_versions SET auth_data = $1 WHERE user_id = $2 AND version = $3" const updateKeyBackupETagSQL = "" + - "UPDATE account_e2e_room_keys_versions SET etag = $1 WHERE user_id = $2 AND version = $3" + "UPDATE userapi_key_backup_versions SET etag = $1 WHERE user_id = $2 AND version = $3" const deleteKeyBackupSQL = "" + - "UPDATE account_e2e_room_keys_versions SET deleted=1 WHERE user_id = $1 AND version = $2" + "UPDATE userapi_key_backup_versions SET deleted=1 WHERE user_id = $1 AND version = $2" const selectKeyBackupSQL = "" + - "SELECT algorithm, auth_data, etag, deleted FROM account_e2e_room_keys_versions WHERE user_id = $1 AND version = $2" + "SELECT algorithm, auth_data, etag, deleted FROM userapi_key_backup_versions WHERE user_id = $1 AND version = $2" const selectLatestVersionSQL = "" + - "SELECT MAX(version) FROM account_e2e_room_keys_versions WHERE user_id = $1" + "SELECT MAX(version) FROM userapi_key_backup_versions WHERE user_id = $1" type keyBackupVersionStatements struct { insertKeyBackupStmt *sql.Stmt diff --git a/userapi/storage/postgres/logintoken_table.go b/userapi/storage/postgres/logintoken_table.go index 4de96f839..44c6ca4ae 100644 --- a/userapi/storage/postgres/logintoken_table.go +++ b/userapi/storage/postgres/logintoken_table.go @@ -26,7 +26,7 @@ import ( ) const loginTokenSchema = ` -CREATE TABLE IF NOT EXISTS login_tokens ( +CREATE TABLE IF NOT EXISTS userapi_login_tokens ( -- The random value of the token issued to a user token TEXT NOT NULL PRIMARY KEY, -- When the token expires @@ -37,17 +37,17 @@ CREATE TABLE IF NOT EXISTS login_tokens ( ); -- This index allows efficient garbage collection of expired tokens. -CREATE INDEX IF NOT EXISTS login_tokens_expiration_idx ON login_tokens(token_expires_at); +CREATE INDEX IF NOT EXISTS userapi_login_tokens_expiration_idx ON userapi_login_tokens(token_expires_at); ` const insertLoginTokenSQL = "" + - "INSERT INTO login_tokens(token, token_expires_at, user_id) VALUES ($1, $2, $3)" + "INSERT INTO userapi_login_tokens(token, token_expires_at, user_id) VALUES ($1, $2, $3)" const deleteLoginTokenSQL = "" + - "DELETE FROM login_tokens WHERE token = $1 OR token_expires_at <= $2" + "DELETE FROM userapi_login_tokens WHERE token = $1 OR token_expires_at <= $2" const selectLoginTokenSQL = "" + - "SELECT user_id FROM login_tokens WHERE token = $1 AND token_expires_at > $2" + "SELECT user_id FROM userapi_login_tokens WHERE token = $1 AND token_expires_at > $2" type loginTokenStatements struct { insertStmt *sql.Stmt @@ -78,7 +78,7 @@ func (s *loginTokenStatements) InsertLoginToken(ctx context.Context, txn *sql.Tx // deleteByToken removes the named token. // // As a simple way to garbage-collect stale tokens, we also remove all expired tokens. -// The login_tokens_expiration_idx index should make that efficient. +// The userapi_login_tokens_expiration_idx index should make that efficient. func (s *loginTokenStatements) DeleteLoginToken(ctx context.Context, txn *sql.Tx, token string) error { stmt := sqlutil.TxStmt(txn, s.deleteStmt) res, err := stmt.ExecContext(ctx, token, time.Now().UTC()) diff --git a/userapi/storage/postgres/openid_table.go b/userapi/storage/postgres/openid_table.go index 29c3ddcb4..06ae30d08 100644 --- a/userapi/storage/postgres/openid_table.go +++ b/userapi/storage/postgres/openid_table.go @@ -13,7 +13,7 @@ import ( const openIDTokenSchema = ` -- Stores data about openid tokens issued for accounts. -CREATE TABLE IF NOT EXISTS open_id_tokens ( +CREATE TABLE IF NOT EXISTS userapi_openid_tokens ( -- The value of the token issued to a user token TEXT NOT NULL PRIMARY KEY, -- The Matrix user ID for this account @@ -24,10 +24,10 @@ CREATE TABLE IF NOT EXISTS open_id_tokens ( ` const insertOpenIDTokenSQL = "" + - "INSERT INTO open_id_tokens(token, localpart, token_expires_at_ms) VALUES ($1, $2, $3)" + "INSERT INTO userapi_openid_tokens(token, localpart, token_expires_at_ms) VALUES ($1, $2, $3)" const selectOpenIDTokenSQL = "" + - "SELECT localpart, token_expires_at_ms FROM open_id_tokens WHERE token = $1" + "SELECT localpart, token_expires_at_ms FROM userapi_openid_tokens WHERE token = $1" type openIDTokenStatements struct { insertTokenStmt *sql.Stmt diff --git a/userapi/storage/postgres/profile_table.go b/userapi/storage/postgres/profile_table.go index 6d336eb8e..f686127be 100644 --- a/userapi/storage/postgres/profile_table.go +++ b/userapi/storage/postgres/profile_table.go @@ -27,7 +27,7 @@ import ( const profilesSchema = ` -- Stores data about accounts profiles. -CREATE TABLE IF NOT EXISTS account_profiles ( +CREATE TABLE IF NOT EXISTS userapi_profiles ( -- The Matrix user ID localpart for this account localpart TEXT NOT NULL PRIMARY KEY, -- The display name for this account @@ -38,19 +38,19 @@ CREATE TABLE IF NOT EXISTS account_profiles ( ` const insertProfileSQL = "" + - "INSERT INTO account_profiles(localpart, display_name, avatar_url) VALUES ($1, $2, $3)" + "INSERT INTO userapi_profiles(localpart, display_name, avatar_url) VALUES ($1, $2, $3)" const selectProfileByLocalpartSQL = "" + - "SELECT localpart, display_name, avatar_url FROM account_profiles WHERE localpart = $1" + "SELECT localpart, display_name, avatar_url FROM userapi_profiles WHERE localpart = $1" const setAvatarURLSQL = "" + - "UPDATE account_profiles SET avatar_url = $1 WHERE localpart = $2" + "UPDATE userapi_profiles SET avatar_url = $1 WHERE localpart = $2" const setDisplayNameSQL = "" + - "UPDATE account_profiles SET display_name = $1 WHERE localpart = $2" + "UPDATE userapi_profiles SET display_name = $1 WHERE localpart = $2" const selectProfilesBySearchSQL = "" + - "SELECT localpart, display_name, avatar_url FROM account_profiles WHERE localpart LIKE $1 OR display_name LIKE $1 LIMIT $2" + "SELECT localpart, display_name, avatar_url FROM userapi_profiles WHERE localpart LIKE $1 OR display_name LIKE $1 LIMIT $2" type profilesStatements struct { serverNoticesLocalpart string diff --git a/userapi/storage/postgres/stats_table.go b/userapi/storage/postgres/stats_table.go index c0b317503..20eb0bf46 100644 --- a/userapi/storage/postgres/stats_table.go +++ b/userapi/storage/postgres/stats_table.go @@ -45,7 +45,7 @@ CREATE INDEX IF NOT EXISTS userapi_daily_visits_localpart_timestamp_idx ON usera const countUsersLastSeenAfterSQL = "" + "SELECT COUNT(*) FROM (" + - " SELECT localpart FROM device_devices WHERE last_seen_ts > $1 " + + " SELECT localpart FROM userapi_devices WHERE last_seen_ts > $1 " + " GROUP BY localpart" + " ) u" @@ -62,7 +62,7 @@ R30Users counts the number of 30 day retained users, defined as: const countR30UsersSQL = ` SELECT platform, COUNT(*) FROM ( SELECT users.localpart, platform, users.created_ts, MAX(uip.last_seen_ts) - FROM account_accounts users + FROM userapi_accounts users INNER JOIN (SELECT localpart, last_seen_ts, @@ -75,7 +75,7 @@ SELECT platform, COUNT(*) FROM ( ELSE 'unknown' END AS platform - FROM device_devices + FROM userapi_devices ) uip ON users.localpart = uip.localpart AND users.account_type <> 4 @@ -121,7 +121,7 @@ GROUP BY client_type ` const countUserByAccountTypeSQL = ` -SELECT COUNT(*) FROM account_accounts WHERE account_type = ANY($1) +SELECT COUNT(*) FROM userapi_accounts WHERE account_type = ANY($1) ` // $1 = All non guest AccountType IDs @@ -134,7 +134,7 @@ SELECT user_type, COUNT(*) AS count FROM ( WHEN account_type = $2 AND appservice_id IS NULL THEN 'guest' WHEN account_type = ANY($1) AND appservice_id IS NOT NULL THEN 'bridged' END AS user_type - FROM account_accounts + FROM userapi_accounts WHERE created_ts > $3 ) AS t GROUP BY user_type ` @@ -143,14 +143,14 @@ SELECT user_type, COUNT(*) AS count FROM ( const updateUserDailyVisitsSQL = ` INSERT INTO userapi_daily_visits(localpart, device_id, timestamp, user_agent) SELECT u.localpart, u.device_id, $1, MAX(u.user_agent) - FROM device_devices AS u + FROM userapi_devices AS u LEFT JOIN ( SELECT localpart, device_id, timestamp FROM userapi_daily_visits WHERE timestamp = $1 ) udv ON u.localpart = udv.localpart AND u.device_id = udv.device_id - INNER JOIN device_devices d ON d.localpart = u.localpart - INNER JOIN account_accounts a ON a.localpart = u.localpart + INNER JOIN userapi_devices d ON d.localpart = u.localpart + INNER JOIN userapi_accounts a ON a.localpart = u.localpart WHERE $2 <= d.last_seen_ts AND d.last_seen_ts < $3 AND a.account_type in (1, 3) GROUP BY u.localpart, u.device_id diff --git a/userapi/storage/postgres/storage.go b/userapi/storage/postgres/storage.go index 7d3b9b6a5..c059e3e60 100644 --- a/userapi/storage/postgres/storage.go +++ b/userapi/storage/postgres/storage.go @@ -23,6 +23,7 @@ import ( "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/setup/base" "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/userapi/storage/postgres/deltas" "github.com/matrix-org/dendrite/userapi/storage/shared" // Import the postgres database driver. @@ -36,6 +37,16 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, return nil, err } + m := sqlutil.NewMigrator(db) + m.AddMigrations(sqlutil.Migration{ + Version: "userapi: rename tables", + Up: deltas.UpRenameTables, + Down: deltas.DownRenameTables, + }) + if err = m.Up(base.Context()); err != nil { + return nil, err + } + accountDataTable, err := NewPostgresAccountDataTable(db) if err != nil { return nil, fmt.Errorf("NewPostgresAccountDataTable: %w", err) diff --git a/userapi/storage/postgres/threepid_table.go b/userapi/storage/postgres/threepid_table.go index 63c08d61f..11af76161 100644 --- a/userapi/storage/postgres/threepid_table.go +++ b/userapi/storage/postgres/threepid_table.go @@ -26,7 +26,7 @@ import ( const threepidSchema = ` -- Stores data about third party identifiers -CREATE TABLE IF NOT EXISTS account_threepid ( +CREATE TABLE IF NOT EXISTS userapi_threepids ( -- The third party identifier threepid TEXT NOT NULL, -- The 3PID medium @@ -37,20 +37,20 @@ CREATE TABLE IF NOT EXISTS account_threepid ( PRIMARY KEY(threepid, medium) ); -CREATE INDEX IF NOT EXISTS account_threepid_localpart ON account_threepid(localpart); +CREATE INDEX IF NOT EXISTS userapi_threepid_idx ON userapi_threepids(localpart); ` const selectLocalpartForThreePIDSQL = "" + - "SELECT localpart FROM account_threepid WHERE threepid = $1 AND medium = $2" + "SELECT localpart FROM userapi_threepids WHERE threepid = $1 AND medium = $2" const selectThreePIDsForLocalpartSQL = "" + - "SELECT threepid, medium FROM account_threepid WHERE localpart = $1" + "SELECT threepid, medium FROM userapi_threepids WHERE localpart = $1" const insertThreePIDSQL = "" + - "INSERT INTO account_threepid (threepid, medium, localpart) VALUES ($1, $2, $3)" + "INSERT INTO userapi_threepids (threepid, medium, localpart) VALUES ($1, $2, $3)" const deleteThreePIDSQL = "" + - "DELETE FROM account_threepid WHERE threepid = $1 AND medium = $2" + "DELETE FROM userapi_threepids WHERE threepid = $1 AND medium = $2" type threepidStatements struct { selectLocalpartForThreePIDStmt *sql.Stmt diff --git a/userapi/storage/sqlite3/account_data_table.go b/userapi/storage/sqlite3/account_data_table.go index cfd8568a9..af12decb3 100644 --- a/userapi/storage/sqlite3/account_data_table.go +++ b/userapi/storage/sqlite3/account_data_table.go @@ -25,7 +25,7 @@ import ( const accountDataSchema = ` -- Stores data about accounts data. -CREATE TABLE IF NOT EXISTS account_data ( +CREATE TABLE IF NOT EXISTS userapi_account_datas ( -- The Matrix user ID localpart for this account localpart TEXT NOT NULL, -- The room ID for this data (empty string if not specific to a room) @@ -40,15 +40,15 @@ CREATE TABLE IF NOT EXISTS account_data ( ` const insertAccountDataSQL = ` - INSERT INTO account_data(localpart, room_id, type, content) VALUES($1, $2, $3, $4) + INSERT INTO userapi_account_datas(localpart, room_id, type, content) VALUES($1, $2, $3, $4) ON CONFLICT (localpart, room_id, type) DO UPDATE SET content = $4 ` const selectAccountDataSQL = "" + - "SELECT room_id, type, content FROM account_data WHERE localpart = $1" + "SELECT room_id, type, content FROM userapi_account_datas WHERE localpart = $1" const selectAccountDataByTypeSQL = "" + - "SELECT content FROM account_data WHERE localpart = $1 AND room_id = $2 AND type = $3" + "SELECT content FROM userapi_account_datas WHERE localpart = $1 AND room_id = $2 AND type = $3" type accountDataStatements struct { db *sql.DB diff --git a/userapi/storage/sqlite3/accounts_table.go b/userapi/storage/sqlite3/accounts_table.go index 484e90056..671c1aa04 100644 --- a/userapi/storage/sqlite3/accounts_table.go +++ b/userapi/storage/sqlite3/accounts_table.go @@ -32,7 +32,7 @@ import ( const accountsSchema = ` -- Stores data about accounts. -CREATE TABLE IF NOT EXISTS account_accounts ( +CREATE TABLE IF NOT EXISTS userapi_accounts ( -- The Matrix user ID localpart for this account localpart TEXT NOT NULL PRIMARY KEY, -- When this account was first created, as a unix timestamp (ms resolution). @@ -51,22 +51,22 @@ CREATE TABLE IF NOT EXISTS account_accounts ( ` const insertAccountSQL = "" + - "INSERT INTO account_accounts(localpart, created_ts, password_hash, appservice_id, account_type) VALUES ($1, $2, $3, $4, $5)" + "INSERT INTO userapi_accounts(localpart, created_ts, password_hash, appservice_id, account_type) VALUES ($1, $2, $3, $4, $5)" const updatePasswordSQL = "" + - "UPDATE account_accounts SET password_hash = $1 WHERE localpart = $2" + "UPDATE userapi_accounts SET password_hash = $1 WHERE localpart = $2" const deactivateAccountSQL = "" + - "UPDATE account_accounts SET is_deactivated = 1 WHERE localpart = $1" + "UPDATE userapi_accounts SET is_deactivated = 1 WHERE localpart = $1" const selectAccountByLocalpartSQL = "" + - "SELECT localpart, appservice_id, account_type FROM account_accounts WHERE localpart = $1" + "SELECT localpart, appservice_id, account_type FROM userapi_accounts WHERE localpart = $1" const selectPasswordHashSQL = "" + - "SELECT password_hash FROM account_accounts WHERE localpart = $1 AND is_deactivated = 0" + "SELECT password_hash FROM userapi_accounts WHERE localpart = $1 AND is_deactivated = 0" const selectNewNumericLocalpartSQL = "" + - "SELECT COALESCE(MAX(CAST(localpart AS INT)), 0) FROM account_accounts WHERE CAST(localpart AS INT) <> 0" + "SELECT COALESCE(MAX(CAST(localpart AS INT)), 0) FROM userapi_accounts WHERE CAST(localpart AS INT) <> 0" type accountsStatements struct { db *sql.DB diff --git a/userapi/storage/sqlite3/deltas/20200929203058_is_active.go b/userapi/storage/sqlite3/deltas/20200929203058_is_active.go index e25efc695..9158cb365 100644 --- a/userapi/storage/sqlite3/deltas/20200929203058_is_active.go +++ b/userapi/storage/sqlite3/deltas/20200929203058_is_active.go @@ -8,8 +8,8 @@ import ( func UpIsActive(ctx context.Context, tx *sql.Tx) error { _, err := tx.ExecContext(ctx, ` - ALTER TABLE account_accounts RENAME TO account_accounts_tmp; -CREATE TABLE account_accounts ( + ALTER TABLE userapi_accounts RENAME TO userapi_accounts_tmp; +CREATE TABLE userapi_accounts ( localpart TEXT NOT NULL PRIMARY KEY, created_ts BIGINT NOT NULL, password_hash TEXT, @@ -17,13 +17,13 @@ CREATE TABLE account_accounts ( is_deactivated BOOLEAN DEFAULT 0 ); INSERT - INTO account_accounts ( + INTO userapi_accounts ( localpart, created_ts, password_hash, appservice_id ) SELECT localpart, created_ts, password_hash, appservice_id - FROM account_accounts_tmp + FROM userapi_accounts_tmp ; -DROP TABLE account_accounts_tmp;`) +DROP TABLE userapi_accounts_tmp;`) if err != nil { return fmt.Errorf("failed to execute upgrade: %w", err) } @@ -32,21 +32,21 @@ DROP TABLE account_accounts_tmp;`) func DownIsActive(ctx context.Context, tx *sql.Tx) error { _, err := tx.ExecContext(ctx, ` - ALTER TABLE account_accounts RENAME TO account_accounts_tmp; -CREATE TABLE account_accounts ( + ALTER TABLE userapi_accounts RENAME TO userapi_accounts_tmp; +CREATE TABLE userapi_accounts ( localpart TEXT NOT NULL PRIMARY KEY, created_ts BIGINT NOT NULL, password_hash TEXT, appservice_id TEXT ); INSERT - INTO account_accounts ( + INTO userapi_accounts ( localpart, created_ts, password_hash, appservice_id ) SELECT localpart, created_ts, password_hash, appservice_id - FROM account_accounts_tmp + FROM userapi_accounts_tmp ; -DROP TABLE account_accounts_tmp;`) +DROP TABLE userapi_accounts_tmp;`) if err != nil { return fmt.Errorf("failed to execute downgrade: %w", err) } diff --git a/userapi/storage/sqlite3/deltas/20201001204705_last_seen_ts_ip.go b/userapi/storage/sqlite3/deltas/20201001204705_last_seen_ts_ip.go index 7f7e95d2d..a9224db6b 100644 --- a/userapi/storage/sqlite3/deltas/20201001204705_last_seen_ts_ip.go +++ b/userapi/storage/sqlite3/deltas/20201001204705_last_seen_ts_ip.go @@ -8,8 +8,8 @@ import ( func UpLastSeenTSIP(ctx context.Context, tx *sql.Tx) error { _, err := tx.ExecContext(ctx, ` - ALTER TABLE device_devices RENAME TO device_devices_tmp; - CREATE TABLE device_devices ( + ALTER TABLE userapi_devices RENAME TO userapi_devices_tmp; + CREATE TABLE userapi_devices ( access_token TEXT PRIMARY KEY, session_id INTEGER, device_id TEXT , @@ -22,12 +22,12 @@ func UpLastSeenTSIP(ctx context.Context, tx *sql.Tx) error { UNIQUE (localpart, device_id) ); INSERT - INTO device_devices ( + INTO userapi_devices ( access_token, session_id, device_id, localpart, created_ts, display_name, last_seen_ts, ip, user_agent ) SELECT access_token, session_id, device_id, localpart, created_ts, display_name, created_ts, '', '' - FROM device_devices_tmp; - DROP TABLE device_devices_tmp;`) + FROM userapi_devices_tmp; + DROP TABLE userapi_devices_tmp;`) if err != nil { return fmt.Errorf("failed to execute upgrade: %w", err) } @@ -36,8 +36,8 @@ func UpLastSeenTSIP(ctx context.Context, tx *sql.Tx) error { func DownLastSeenTSIP(ctx context.Context, tx *sql.Tx) error { _, err := tx.ExecContext(ctx, ` -ALTER TABLE device_devices RENAME TO device_devices_tmp; -CREATE TABLE IF NOT EXISTS device_devices ( +ALTER TABLE userapi_devices RENAME TO userapi_devices_tmp; +CREATE TABLE IF NOT EXISTS userapi_devices ( access_token TEXT PRIMARY KEY, session_id INTEGER, device_id TEXT , @@ -47,12 +47,12 @@ CREATE TABLE IF NOT EXISTS device_devices ( UNIQUE (localpart, device_id) ); INSERT -INTO device_devices ( +INTO userapi_devices ( access_token, session_id, device_id, localpart, created_ts, display_name ) SELECT access_token, session_id, device_id, localpart, created_ts, display_name -FROM device_devices_tmp; -DROP TABLE device_devices_tmp;`) +FROM userapi_devices_tmp; +DROP TABLE userapi_devices_tmp;`) if err != nil { return fmt.Errorf("failed to execute downgrade: %w", err) } diff --git a/userapi/storage/sqlite3/deltas/2022021012490600_add_account_type.go b/userapi/storage/sqlite3/deltas/2022021012490600_add_account_type.go index 46532698c..230bc1433 100644 --- a/userapi/storage/sqlite3/deltas/2022021012490600_add_account_type.go +++ b/userapi/storage/sqlite3/deltas/2022021012490600_add_account_type.go @@ -9,8 +9,8 @@ import ( func UpAddAccountType(ctx context.Context, tx *sql.Tx) error { // initially set every account to useraccount, change appservice and guest accounts afterwards // (user = 1, guest = 2, admin = 3, appservice = 4) - _, err := tx.ExecContext(ctx, `ALTER TABLE account_accounts RENAME TO account_accounts_tmp; -CREATE TABLE account_accounts ( + _, err := tx.ExecContext(ctx, `ALTER TABLE userapi_accounts RENAME TO userapi_accounts_tmp; +CREATE TABLE userapi_accounts ( localpart TEXT NOT NULL PRIMARY KEY, created_ts BIGINT NOT NULL, password_hash TEXT, @@ -19,15 +19,15 @@ CREATE TABLE account_accounts ( account_type INTEGER NOT NULL ); INSERT - INTO account_accounts ( + INTO userapi_accounts ( localpart, created_ts, password_hash, appservice_id, account_type ) SELECT localpart, created_ts, password_hash, appservice_id, 1 - FROM account_accounts_tmp + FROM userapi_accounts_tmp ; -UPDATE account_accounts SET account_type = 4 WHERE appservice_id <> ''; -UPDATE account_accounts SET account_type = 2 WHERE localpart GLOB '[0-9]*'; -DROP TABLE account_accounts_tmp;`) +UPDATE userapi_accounts SET account_type = 4 WHERE appservice_id <> ''; +UPDATE userapi_accounts SET account_type = 2 WHERE localpart GLOB '[0-9]*'; +DROP TABLE userapi_accounts_tmp;`) if err != nil { return fmt.Errorf("failed to add column: %w", err) } @@ -35,7 +35,7 @@ DROP TABLE account_accounts_tmp;`) } func DownAddAccountType(ctx context.Context, tx *sql.Tx) error { - _, err := tx.ExecContext(ctx, `ALTER TABLE account_accounts DROP COLUMN account_type;`) + _, err := tx.ExecContext(ctx, `ALTER TABLE userapi_accounts DROP COLUMN account_type;`) if err != nil { return fmt.Errorf("failed to execute downgrade: %w", err) } diff --git a/userapi/storage/sqlite3/deltas/2022101711000000_rename_tables.go b/userapi/storage/sqlite3/deltas/2022101711000000_rename_tables.go new file mode 100644 index 000000000..4ca1dc475 --- /dev/null +++ b/userapi/storage/sqlite3/deltas/2022101711000000_rename_tables.go @@ -0,0 +1,109 @@ +package deltas + +import ( + "context" + "database/sql" + "fmt" + "strings" +) + +var renameTableMappings = map[string]string{ + "account_accounts": "userapi_accounts", + "account_data": "userapi_account_datas", + "device_devices": "userapi_devices", + "account_e2e_room_keys": "userapi_key_backups", + "account_e2e_room_keys_versions": "userapi_key_backup_versions", + "login_tokens": "userapi_login_tokens", + "open_id_tokens": "userapi_openid_tokens", + "account_profiles": "userapi_profiles", + "account_threepid": "userapi_threepids", +} + +var renameIndicesMappings = map[string]string{ + "device_localpart_id_idx": "userapi_device_localpart_id_idx", + "e2e_room_keys_idx": "userapi_key_backups_idx", + "e2e_room_keys_versions_idx": "userapi_key_backups_versions_idx", + "account_e2e_room_keys_versions_idx": "userapi_key_backup_versions_idx", + "login_tokens_expiration_idx": "userapi_login_tokens_expiration_idx", + "account_threepid_localpart": "userapi_threepid_idx", +} + +func UpRenameTables(ctx context.Context, tx *sql.Tx) error { + for old, new := range renameTableMappings { + // SQLite has no "IF EXISTS" so check if the table exists. + var name string + if err := tx.QueryRowContext( + ctx, "SELECT name FROM sqlite_schema WHERE type = 'table' AND name = $1;", old, + ).Scan(&name); err != nil { + if err == sql.ErrNoRows { + continue + } + return err + } + q := fmt.Sprintf( + "ALTER TABLE %s RENAME TO %s;", old, new, + ) + if _, err := tx.ExecContext(ctx, q); err != nil { + return fmt.Errorf("rename table %q to %q error: %w", old, new, err) + } + } + for old, new := range renameIndicesMappings { + var query string + if err := tx.QueryRowContext( + ctx, "SELECT sql FROM sqlite_schema WHERE type = 'index' AND name = $1;", old, + ).Scan(&query); err != nil { + if err == sql.ErrNoRows { + continue + } + return err + } + query = strings.Replace(query, old, new, 1) + if _, err := tx.ExecContext(ctx, fmt.Sprintf("DROP INDEX %s;", old)); err != nil { + return fmt.Errorf("drop index %q to %q error: %w", old, new, err) + } + if _, err := tx.ExecContext(ctx, query); err != nil { + return fmt.Errorf("recreate index %q to %q error: %w", old, new, err) + } + } + return nil +} + +func DownRenameTables(ctx context.Context, tx *sql.Tx) error { + for old, new := range renameTableMappings { + // SQLite has no "IF EXISTS" so check if the table exists. + var name string + if err := tx.QueryRowContext( + ctx, "SELECT name FROM sqlite_schema WHERE type = 'table' AND name = $1;", new, + ).Scan(&name); err != nil { + if err == sql.ErrNoRows { + continue + } + return err + } + q := fmt.Sprintf( + "ALTER TABLE %s RENAME TO %s;", new, old, + ) + if _, err := tx.ExecContext(ctx, q); err != nil { + return fmt.Errorf("rename table %q to %q error: %w", new, old, err) + } + } + for old, new := range renameIndicesMappings { + var query string + if err := tx.QueryRowContext( + ctx, "SELECT sql FROM sqlite_schema WHERE type = 'index' AND name = $1;", new, + ).Scan(&query); err != nil { + if err == sql.ErrNoRows { + continue + } + return err + } + query = strings.Replace(query, new, old, 1) + if _, err := tx.ExecContext(ctx, fmt.Sprintf("DROP INDEX %s;", new)); err != nil { + return fmt.Errorf("drop index %q to %q error: %w", new, old, err) + } + if _, err := tx.ExecContext(ctx, query); err != nil { + return fmt.Errorf("recreate index %q to %q error: %w", new, old, err) + } + } + return nil +} diff --git a/userapi/storage/sqlite3/devices_table.go b/userapi/storage/sqlite3/devices_table.go index 27a7524d6..e53a08062 100644 --- a/userapi/storage/sqlite3/devices_table.go +++ b/userapi/storage/sqlite3/devices_table.go @@ -35,7 +35,7 @@ const devicesSchema = ` -- CREATE SEQUENCE IF NOT EXISTS device_session_id_seq START 1; -- Stores data about devices. -CREATE TABLE IF NOT EXISTS device_devices ( +CREATE TABLE IF NOT EXISTS userapi_devices ( access_token TEXT PRIMARY KEY, session_id INTEGER, device_id TEXT , @@ -51,38 +51,38 @@ CREATE TABLE IF NOT EXISTS device_devices ( ` const insertDeviceSQL = "" + - "INSERT INTO device_devices (device_id, localpart, access_token, created_ts, display_name, session_id, last_seen_ts, ip, user_agent)" + + "INSERT INTO userapi_devices (device_id, localpart, access_token, created_ts, display_name, session_id, last_seen_ts, ip, user_agent)" + " VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)" const selectDevicesCountSQL = "" + - "SELECT COUNT(access_token) FROM device_devices" + "SELECT COUNT(access_token) FROM userapi_devices" const selectDeviceByTokenSQL = "" + - "SELECT session_id, device_id, localpart FROM device_devices WHERE access_token = $1" + "SELECT session_id, device_id, localpart FROM userapi_devices WHERE access_token = $1" const selectDeviceByIDSQL = "" + - "SELECT display_name, last_seen_ts, ip FROM device_devices WHERE localpart = $1 and device_id = $2" + "SELECT display_name, last_seen_ts, ip FROM userapi_devices WHERE localpart = $1 and device_id = $2" const selectDevicesByLocalpartSQL = "" + - "SELECT device_id, display_name, last_seen_ts, ip, user_agent FROM device_devices WHERE localpart = $1 AND device_id != $2 ORDER BY last_seen_ts DESC" + "SELECT device_id, display_name, last_seen_ts, ip, user_agent FROM userapi_devices WHERE localpart = $1 AND device_id != $2 ORDER BY last_seen_ts DESC" const updateDeviceNameSQL = "" + - "UPDATE device_devices SET display_name = $1 WHERE localpart = $2 AND device_id = $3" + "UPDATE userapi_devices SET display_name = $1 WHERE localpart = $2 AND device_id = $3" const deleteDeviceSQL = "" + - "DELETE FROM device_devices WHERE device_id = $1 AND localpart = $2" + "DELETE FROM userapi_devices WHERE device_id = $1 AND localpart = $2" const deleteDevicesByLocalpartSQL = "" + - "DELETE FROM device_devices WHERE localpart = $1 AND device_id != $2" + "DELETE FROM userapi_devices WHERE localpart = $1 AND device_id != $2" const deleteDevicesSQL = "" + - "DELETE FROM device_devices WHERE localpart = $1 AND device_id IN ($2)" + "DELETE FROM userapi_devices WHERE localpart = $1 AND device_id IN ($2)" const selectDevicesByIDSQL = "" + - "SELECT device_id, localpart, display_name, last_seen_ts FROM device_devices WHERE device_id IN ($1) ORDER BY last_seen_ts DESC" + "SELECT device_id, localpart, display_name, last_seen_ts FROM userapi_devices WHERE device_id IN ($1) ORDER BY last_seen_ts DESC" const updateDeviceLastSeen = "" + - "UPDATE device_devices SET last_seen_ts = $1, ip = $2, user_agent = $3 WHERE localpart = $4 AND device_id = $5" + "UPDATE userapi_devices SET last_seen_ts = $1, ip = $2, user_agent = $3 WHERE localpart = $4 AND device_id = $5" type devicesStatements struct { db *sql.DB diff --git a/userapi/storage/sqlite3/key_backup_table.go b/userapi/storage/sqlite3/key_backup_table.go index 81726edf9..7883ffb19 100644 --- a/userapi/storage/sqlite3/key_backup_table.go +++ b/userapi/storage/sqlite3/key_backup_table.go @@ -26,7 +26,7 @@ import ( ) const keyBackupTableSchema = ` -CREATE TABLE IF NOT EXISTS account_e2e_room_keys ( +CREATE TABLE IF NOT EXISTS userapi_key_backups ( user_id TEXT NOT NULL, room_id TEXT NOT NULL, session_id TEXT NOT NULL, @@ -37,31 +37,31 @@ CREATE TABLE IF NOT EXISTS account_e2e_room_keys ( is_verified BOOLEAN NOT NULL, session_data TEXT NOT NULL ); -CREATE UNIQUE INDEX IF NOT EXISTS e2e_room_keys_idx ON account_e2e_room_keys(user_id, room_id, session_id, version); -CREATE INDEX IF NOT EXISTS e2e_room_keys_versions_idx ON account_e2e_room_keys(user_id, version); +CREATE UNIQUE INDEX IF NOT EXISTS e2e_room_keys_idx ON userapi_key_backups(user_id, room_id, session_id, version); +CREATE INDEX IF NOT EXISTS e2e_room_keys_versions_idx ON userapi_key_backups(user_id, version); ` const insertBackupKeySQL = "" + - "INSERT INTO account_e2e_room_keys(user_id, room_id, session_id, version, first_message_index, forwarded_count, is_verified, session_data) " + + "INSERT INTO userapi_key_backups(user_id, room_id, session_id, version, first_message_index, forwarded_count, is_verified, session_data) " + "VALUES ($1, $2, $3, $4, $5, $6, $7, $8)" const updateBackupKeySQL = "" + - "UPDATE account_e2e_room_keys SET first_message_index=$1, forwarded_count=$2, is_verified=$3, session_data=$4 " + + "UPDATE userapi_key_backups SET first_message_index=$1, forwarded_count=$2, is_verified=$3, session_data=$4 " + "WHERE user_id=$5 AND room_id=$6 AND session_id=$7 AND version=$8" const countKeysSQL = "" + - "SELECT COUNT(*) FROM account_e2e_room_keys WHERE user_id = $1 AND version = $2" + "SELECT COUNT(*) FROM userapi_key_backups WHERE user_id = $1 AND version = $2" const selectKeysSQL = "" + - "SELECT room_id, session_id, first_message_index, forwarded_count, is_verified, session_data FROM account_e2e_room_keys " + + "SELECT room_id, session_id, first_message_index, forwarded_count, is_verified, session_data FROM userapi_key_backups " + "WHERE user_id = $1 AND version = $2" const selectKeysByRoomIDSQL = "" + - "SELECT room_id, session_id, first_message_index, forwarded_count, is_verified, session_data FROM account_e2e_room_keys " + + "SELECT room_id, session_id, first_message_index, forwarded_count, is_verified, session_data FROM userapi_key_backups " + "WHERE user_id = $1 AND version = $2 AND room_id = $3" const selectKeysByRoomIDAndSessionIDSQL = "" + - "SELECT room_id, session_id, first_message_index, forwarded_count, is_verified, session_data FROM account_e2e_room_keys " + + "SELECT room_id, session_id, first_message_index, forwarded_count, is_verified, session_data FROM userapi_key_backups " + "WHERE user_id = $1 AND version = $2 AND room_id = $3 AND session_id = $4" type keyBackupStatements struct { diff --git a/userapi/storage/sqlite3/key_backup_version_table.go b/userapi/storage/sqlite3/key_backup_version_table.go index e85e6f08b..37bc13ed1 100644 --- a/userapi/storage/sqlite3/key_backup_version_table.go +++ b/userapi/storage/sqlite3/key_backup_version_table.go @@ -27,7 +27,7 @@ import ( const keyBackupVersionTableSchema = ` -- the metadata for each generation of encrypted e2e session backups -CREATE TABLE IF NOT EXISTS account_e2e_room_keys_versions ( +CREATE TABLE IF NOT EXISTS userapi_key_backup_versions ( user_id TEXT NOT NULL, -- this means no 2 users will ever have the same version of e2e session backups which strictly -- isn't necessary, but this is easy to do rather than SELECT MAX(version)+1. @@ -38,26 +38,26 @@ CREATE TABLE IF NOT EXISTS account_e2e_room_keys_versions ( deleted INTEGER DEFAULT 0 NOT NULL ); -CREATE UNIQUE INDEX IF NOT EXISTS account_e2e_room_keys_versions_idx ON account_e2e_room_keys_versions(user_id, version); +CREATE UNIQUE INDEX IF NOT EXISTS userapi_key_backup_versions_idx ON userapi_key_backup_versions(user_id, version); ` const insertKeyBackupSQL = "" + - "INSERT INTO account_e2e_room_keys_versions(user_id, algorithm, auth_data, etag) VALUES ($1, $2, $3, $4) RETURNING version" + "INSERT INTO userapi_key_backup_versions(user_id, algorithm, auth_data, etag) VALUES ($1, $2, $3, $4) RETURNING version" const updateKeyBackupAuthDataSQL = "" + - "UPDATE account_e2e_room_keys_versions SET auth_data = $1 WHERE user_id = $2 AND version = $3" + "UPDATE userapi_key_backup_versions SET auth_data = $1 WHERE user_id = $2 AND version = $3" const updateKeyBackupETagSQL = "" + - "UPDATE account_e2e_room_keys_versions SET etag = $1 WHERE user_id = $2 AND version = $3" + "UPDATE userapi_key_backup_versions SET etag = $1 WHERE user_id = $2 AND version = $3" const deleteKeyBackupSQL = "" + - "UPDATE account_e2e_room_keys_versions SET deleted=1 WHERE user_id = $1 AND version = $2" + "UPDATE userapi_key_backup_versions SET deleted=1 WHERE user_id = $1 AND version = $2" const selectKeyBackupSQL = "" + - "SELECT algorithm, auth_data, etag, deleted FROM account_e2e_room_keys_versions WHERE user_id = $1 AND version = $2" + "SELECT algorithm, auth_data, etag, deleted FROM userapi_key_backup_versions WHERE user_id = $1 AND version = $2" const selectLatestVersionSQL = "" + - "SELECT MAX(version) FROM account_e2e_room_keys_versions WHERE user_id = $1" + "SELECT MAX(version) FROM userapi_key_backup_versions WHERE user_id = $1" type keyBackupVersionStatements struct { insertKeyBackupStmt *sql.Stmt diff --git a/userapi/storage/sqlite3/logintoken_table.go b/userapi/storage/sqlite3/logintoken_table.go index 78d42029a..2abdcb95e 100644 --- a/userapi/storage/sqlite3/logintoken_table.go +++ b/userapi/storage/sqlite3/logintoken_table.go @@ -32,7 +32,7 @@ type loginTokenStatements struct { } const loginTokenSchema = ` -CREATE TABLE IF NOT EXISTS login_tokens ( +CREATE TABLE IF NOT EXISTS userapi_login_tokens ( -- The random value of the token issued to a user token TEXT NOT NULL PRIMARY KEY, -- When the token expires @@ -43,17 +43,17 @@ CREATE TABLE IF NOT EXISTS login_tokens ( ); -- This index allows efficient garbage collection of expired tokens. -CREATE INDEX IF NOT EXISTS login_tokens_expiration_idx ON login_tokens(token_expires_at); +CREATE INDEX IF NOT EXISTS login_tokens_expiration_idx ON userapi_login_tokens(token_expires_at); ` const insertLoginTokenSQL = "" + - "INSERT INTO login_tokens(token, token_expires_at, user_id) VALUES ($1, $2, $3)" + "INSERT INTO userapi_login_tokens(token, token_expires_at, user_id) VALUES ($1, $2, $3)" const deleteLoginTokenSQL = "" + - "DELETE FROM login_tokens WHERE token = $1 OR token_expires_at <= $2" + "DELETE FROM userapi_login_tokens WHERE token = $1 OR token_expires_at <= $2" const selectLoginTokenSQL = "" + - "SELECT user_id FROM login_tokens WHERE token = $1 AND token_expires_at > $2" + "SELECT user_id FROM userapi_login_tokens WHERE token = $1 AND token_expires_at > $2" func NewSQLiteLoginTokenTable(db *sql.DB) (tables.LoginTokenTable, error) { s := &loginTokenStatements{} @@ -78,7 +78,7 @@ func (s *loginTokenStatements) InsertLoginToken(ctx context.Context, txn *sql.Tx // deleteByToken removes the named token. // // As a simple way to garbage-collect stale tokens, we also remove all expired tokens. -// The login_tokens_expiration_idx index should make that efficient. +// The userapi_login_tokens_expiration_idx index should make that efficient. func (s *loginTokenStatements) DeleteLoginToken(ctx context.Context, txn *sql.Tx, token string) error { stmt := sqlutil.TxStmt(txn, s.deleteStmt) res, err := stmt.ExecContext(ctx, token, time.Now().UTC()) diff --git a/userapi/storage/sqlite3/openid_table.go b/userapi/storage/sqlite3/openid_table.go index d6090e0da..875f1a9a5 100644 --- a/userapi/storage/sqlite3/openid_table.go +++ b/userapi/storage/sqlite3/openid_table.go @@ -13,7 +13,7 @@ import ( const openIDTokenSchema = ` -- Stores data about accounts. -CREATE TABLE IF NOT EXISTS open_id_tokens ( +CREATE TABLE IF NOT EXISTS userapi_openid_tokens ( -- The value of the token issued to a user token TEXT NOT NULL PRIMARY KEY, -- The Matrix user ID for this account @@ -24,10 +24,10 @@ CREATE TABLE IF NOT EXISTS open_id_tokens ( ` const insertOpenIDTokenSQL = "" + - "INSERT INTO open_id_tokens(token, localpart, token_expires_at_ms) VALUES ($1, $2, $3)" + "INSERT INTO userapi_openid_tokens(token, localpart, token_expires_at_ms) VALUES ($1, $2, $3)" const selectOpenIDTokenSQL = "" + - "SELECT localpart, token_expires_at_ms FROM open_id_tokens WHERE token = $1" + "SELECT localpart, token_expires_at_ms FROM userapi_openid_tokens WHERE token = $1" type openIDTokenStatements struct { db *sql.DB diff --git a/userapi/storage/sqlite3/profile_table.go b/userapi/storage/sqlite3/profile_table.go index 3050ff4b5..267daf044 100644 --- a/userapi/storage/sqlite3/profile_table.go +++ b/userapi/storage/sqlite3/profile_table.go @@ -27,7 +27,7 @@ import ( const profilesSchema = ` -- Stores data about accounts profiles. -CREATE TABLE IF NOT EXISTS account_profiles ( +CREATE TABLE IF NOT EXISTS userapi_profiles ( -- The Matrix user ID localpart for this account localpart TEXT NOT NULL PRIMARY KEY, -- The display name for this account @@ -38,19 +38,19 @@ CREATE TABLE IF NOT EXISTS account_profiles ( ` const insertProfileSQL = "" + - "INSERT INTO account_profiles(localpart, display_name, avatar_url) VALUES ($1, $2, $3)" + "INSERT INTO userapi_profiles(localpart, display_name, avatar_url) VALUES ($1, $2, $3)" const selectProfileByLocalpartSQL = "" + - "SELECT localpart, display_name, avatar_url FROM account_profiles WHERE localpart = $1" + "SELECT localpart, display_name, avatar_url FROM userapi_profiles WHERE localpart = $1" const setAvatarURLSQL = "" + - "UPDATE account_profiles SET avatar_url = $1 WHERE localpart = $2" + "UPDATE userapi_profiles SET avatar_url = $1 WHERE localpart = $2" const setDisplayNameSQL = "" + - "UPDATE account_profiles SET display_name = $1 WHERE localpart = $2" + "UPDATE userapi_profiles SET display_name = $1 WHERE localpart = $2" const selectProfilesBySearchSQL = "" + - "SELECT localpart, display_name, avatar_url FROM account_profiles WHERE localpart LIKE $1 OR display_name LIKE $1 LIMIT $2" + "SELECT localpart, display_name, avatar_url FROM userapi_profiles WHERE localpart LIKE $1 OR display_name LIKE $1 LIMIT $2" type profilesStatements struct { db *sql.DB diff --git a/userapi/storage/sqlite3/stats_table.go b/userapi/storage/sqlite3/stats_table.go index 8aa1746c5..35e3c653e 100644 --- a/userapi/storage/sqlite3/stats_table.go +++ b/userapi/storage/sqlite3/stats_table.go @@ -46,7 +46,7 @@ CREATE INDEX IF NOT EXISTS userapi_daily_visits_localpart_timestamp_idx ON usera const countUsersLastSeenAfterSQL = "" + "SELECT COUNT(*) FROM (" + - " SELECT localpart FROM device_devices WHERE last_seen_ts > $1 " + + " SELECT localpart FROM userapi_devices WHERE last_seen_ts > $1 " + " GROUP BY localpart" + " ) u" @@ -63,7 +63,7 @@ R30Users counts the number of 30 day retained users, defined as: const countR30UsersSQL = ` SELECT platform, COUNT(*) FROM ( SELECT users.localpart, platform, users.created_ts, MAX(uip.last_seen_ts) - FROM account_accounts users + FROM userapi_accounts users INNER JOIN (SELECT localpart, last_seen_ts, @@ -76,7 +76,7 @@ SELECT platform, COUNT(*) FROM ( ELSE 'unknown' END AS platform - FROM device_devices + FROM userapi_devices ) uip ON users.localpart = uip.localpart AND users.account_type <> 4 @@ -126,7 +126,7 @@ GROUP BY client_type ` const countUserByAccountTypeSQL = ` -SELECT COUNT(*) FROM account_accounts WHERE account_type IN ($1) +SELECT COUNT(*) FROM userapi_accounts WHERE account_type IN ($1) ` // $1 = Guest AccountType @@ -139,7 +139,7 @@ SELECT user_type, COUNT(*) AS count FROM ( WHEN account_type = $4 AND appservice_id IS NULL THEN 'guest' WHEN account_type IN ($5) AND appservice_id IS NOT NULL THEN 'bridged' END AS user_type - FROM account_accounts + FROM userapi_accounts WHERE created_ts > $8 ) AS t GROUP BY user_type ` @@ -148,14 +148,14 @@ SELECT user_type, COUNT(*) AS count FROM ( const updateUserDailyVisitsSQL = ` INSERT INTO userapi_daily_visits(localpart, device_id, timestamp, user_agent) SELECT u.localpart, u.device_id, $1, MAX(u.user_agent) - FROM device_devices AS u + FROM userapi_devices AS u LEFT JOIN ( SELECT localpart, device_id, timestamp FROM userapi_daily_visits WHERE timestamp = $1 ) udv ON u.localpart = udv.localpart AND u.device_id = udv.device_id - INNER JOIN device_devices d ON d.localpart = u.localpart - INNER JOIN account_accounts a ON a.localpart = u.localpart + INNER JOIN userapi_devices d ON d.localpart = u.localpart + INNER JOIN userapi_accounts a ON a.localpart = u.localpart WHERE $2 <= d.last_seen_ts AND d.last_seen_ts < $3 AND a.account_type in (1, 3) GROUP BY u.localpart, u.device_id diff --git a/userapi/storage/sqlite3/storage.go b/userapi/storage/sqlite3/storage.go index 78b7ce588..dd33dc0cf 100644 --- a/userapi/storage/sqlite3/storage.go +++ b/userapi/storage/sqlite3/storage.go @@ -25,6 +25,7 @@ import ( "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/userapi/storage/shared" + "github.com/matrix-org/dendrite/userapi/storage/sqlite3/deltas" ) // NewDatabase creates a new accounts and profiles database @@ -34,6 +35,16 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, return nil, err } + m := sqlutil.NewMigrator(db) + m.AddMigrations(sqlutil.Migration{ + Version: "userapi: rename tables", + Up: deltas.UpRenameTables, + Down: deltas.DownRenameTables, + }) + if err = m.Up(base.Context()); err != nil { + return nil, err + } + accountDataTable, err := NewSQLiteAccountDataTable(db) if err != nil { return nil, fmt.Errorf("NewSQLiteAccountDataTable: %w", err) diff --git a/userapi/storage/sqlite3/threepid_table.go b/userapi/storage/sqlite3/threepid_table.go index fa174eed5..73af139db 100644 --- a/userapi/storage/sqlite3/threepid_table.go +++ b/userapi/storage/sqlite3/threepid_table.go @@ -27,7 +27,7 @@ import ( const threepidSchema = ` -- Stores data about third party identifiers -CREATE TABLE IF NOT EXISTS account_threepid ( +CREATE TABLE IF NOT EXISTS userapi_threepids ( -- The third party identifier threepid TEXT NOT NULL, -- The 3PID medium @@ -38,20 +38,20 @@ CREATE TABLE IF NOT EXISTS account_threepid ( PRIMARY KEY(threepid, medium) ); -CREATE INDEX IF NOT EXISTS account_threepid_localpart ON account_threepid(localpart); +CREATE INDEX IF NOT EXISTS account_threepid_localpart ON userapi_threepids(localpart); ` const selectLocalpartForThreePIDSQL = "" + - "SELECT localpart FROM account_threepid WHERE threepid = $1 AND medium = $2" + "SELECT localpart FROM userapi_threepids WHERE threepid = $1 AND medium = $2" const selectThreePIDsForLocalpartSQL = "" + - "SELECT threepid, medium FROM account_threepid WHERE localpart = $1" + "SELECT threepid, medium FROM userapi_threepids WHERE localpart = $1" const insertThreePIDSQL = "" + - "INSERT INTO account_threepid (threepid, medium, localpart) VALUES ($1, $2, $3)" + "INSERT INTO userapi_threepids (threepid, medium, localpart) VALUES ($1, $2, $3)" const deleteThreePIDSQL = "" + - "DELETE FROM account_threepid WHERE threepid = $1 AND medium = $2" + "DELETE FROM userapi_threepids WHERE threepid = $1 AND medium = $2" type threepidStatements struct { db *sql.DB diff --git a/userapi/storage/storage_test.go b/userapi/storage/storage_test.go index 1538a8138..8e5b32b6a 100644 --- a/userapi/storage/storage_test.go +++ b/userapi/storage/storage_test.go @@ -16,6 +16,7 @@ import ( "github.com/matrix-org/dendrite/internal/pushrules" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/test" + "github.com/matrix-org/dendrite/test/testrig" "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/dendrite/userapi/storage" "github.com/matrix-org/dendrite/userapi/storage/tables" @@ -29,14 +30,18 @@ var ( ) func mustCreateDatabase(t *testing.T, dbType test.DBType) (storage.Database, func()) { + base, baseclose := testrig.CreateBaseDendrite(t, dbType) connStr, close := test.PrepareDBConnectionString(t, dbType) - db, err := storage.NewUserAPIDatabase(nil, &config.DatabaseOptions{ + db, err := storage.NewUserAPIDatabase(base, &config.DatabaseOptions{ ConnectionString: config.DataSource(connStr), }, "localhost", bcrypt.MinCost, openIDLifetimeMS, loginTokenLifetime, "_server") if err != nil { t.Fatalf("NewUserAPIDatabase returned %s", err) } - return db, close + return db, func() { + close() + baseclose() + } } // Tests storing and getting account data diff --git a/userapi/storage/tables/stats_table_test.go b/userapi/storage/tables/stats_table_test.go index 11521c8b0..c4aec552c 100644 --- a/userapi/storage/tables/stats_table_test.go +++ b/userapi/storage/tables/stats_table_test.go @@ -106,7 +106,7 @@ func mustUpdateDeviceLastSeen( timestamp time.Time, ) { t.Helper() - _, err := db.ExecContext(ctx, "UPDATE device_devices SET last_seen_ts = $1 WHERE localpart = $2", gomatrixserverlib.AsTimestamp(timestamp), localpart) + _, err := db.ExecContext(ctx, "UPDATE userapi_devices SET last_seen_ts = $1 WHERE localpart = $2", gomatrixserverlib.AsTimestamp(timestamp), localpart) if err != nil { t.Fatalf("unable to update device last seen") } @@ -119,7 +119,7 @@ func mustUserUpdateRegistered( localpart string, timestamp time.Time, ) { - _, err := db.ExecContext(ctx, "UPDATE account_accounts SET created_ts = $1 WHERE localpart = $2", gomatrixserverlib.AsTimestamp(timestamp), localpart) + _, err := db.ExecContext(ctx, "UPDATE userapi_accounts SET created_ts = $1 WHERE localpart = $2", gomatrixserverlib.AsTimestamp(timestamp), localpart) if err != nil { t.Fatalf("unable to update device last seen") } diff --git a/userapi/userapi_test.go b/userapi/userapi_test.go index 984fe8854..4417f4dc0 100644 --- a/userapi/userapi_test.go +++ b/userapi/userapi_test.go @@ -25,6 +25,7 @@ import ( "github.com/gorilla/mux" "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/test" + "github.com/matrix-org/dendrite/test/testrig" "github.com/matrix-org/dendrite/userapi" "github.com/matrix-org/dendrite/userapi/inthttp" "github.com/matrix-org/gomatrixserverlib" @@ -48,9 +49,9 @@ func MustMakeInternalAPI(t *testing.T, opts apiTestOpts, dbType test.DBType) (ap if opts.loginTokenLifetime == 0 { opts.loginTokenLifetime = api.DefaultLoginTokenLifetime * time.Millisecond } + base, baseclose := testrig.CreateBaseDendrite(t, dbType) connStr, close := test.PrepareDBConnectionString(t, dbType) - - accountDB, err := storage.NewUserAPIDatabase(nil, &config.DatabaseOptions{ + accountDB, err := storage.NewUserAPIDatabase(base, &config.DatabaseOptions{ ConnectionString: config.DataSource(connStr), }, serverName, bcrypt.MinCost, config.DefaultOpenIDTokenLifetimeMS, opts.loginTokenLifetime, "") if err != nil { @@ -64,9 +65,12 @@ func MustMakeInternalAPI(t *testing.T, opts apiTestOpts, dbType test.DBType) (ap } return &internal.UserInternalAPI{ - DB: accountDB, - ServerName: cfg.Matrix.ServerName, - }, accountDB, close + DB: accountDB, + ServerName: cfg.Matrix.ServerName, + }, accountDB, func() { + close() + baseclose() + } } func TestQueryProfile(t *testing.T) { From 241d5c47dfa9e5cfadc350f688aab30f9e539fbb Mon Sep 17 00:00:00 2001 From: devonh Date: Wed, 19 Oct 2022 10:03:16 +0000 Subject: [PATCH 086/207] Refactor Federation Destination Queues (#2807) This is a refactor of the federation destination queues. It fixes a few things, namely: - actually retry outgoing events with backoff behaviour - obtain enough events from the database to fill messages as much as possible - minimize the amount of running goroutines - use pure timers for backoff - don't restart queue unless necessary - close the background task when backing off - increase max edus in a transaction to match the spec - cleanup timers more aggresively to reduce memory usage - add jitter to backoff timers to reduce resource spikes - add a bunch of tests (with real and fake databases) to ensure everything is working --- federationapi/federationapi.go | 9 +- federationapi/queue/destinationqueue.go | 374 ++++--- federationapi/queue/queue.go | 26 +- federationapi/queue/queue_test.go | 1047 +++++++++++++++++++ federationapi/statistics/statistics.go | 131 ++- federationapi/statistics/statistics_test.go | 19 +- federationapi/storage/shared/storage.go | 4 + go.mod | 2 +- 8 files changed, 1410 insertions(+), 202 deletions(-) create mode 100644 federationapi/queue/queue_test.go diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go index 4a13c9d9b..f6dace702 100644 --- a/federationapi/federationapi.go +++ b/federationapi/federationapi.go @@ -116,17 +116,14 @@ func NewInternalAPI( _ = federationDB.RemoveAllServersFromBlacklist() } - stats := &statistics.Statistics{ - DB: federationDB, - FailuresUntilBlacklist: cfg.FederationMaxRetries, - } + stats := statistics.NewStatistics(federationDB, cfg.FederationMaxRetries+1) js, _ := base.NATS.Prepare(base.ProcessContext, &cfg.Matrix.JetStream) queues := queue.NewOutgoingQueues( federationDB, base.ProcessContext, cfg.Matrix.DisableFederation, - cfg.Matrix.ServerName, federation, rsAPI, stats, + cfg.Matrix.ServerName, federation, rsAPI, &stats, &queue.SigningInfo{ KeyID: cfg.Matrix.KeyID, PrivateKey: cfg.Matrix.PrivateKey, @@ -183,5 +180,5 @@ func NewInternalAPI( } time.AfterFunc(time.Minute, cleanExpiredEDUs) - return internal.NewFederationInternalAPI(federationDB, cfg, rsAPI, federation, stats, caches, queues, keyRing) + return internal.NewFederationInternalAPI(federationDB, cfg, rsAPI, federation, &stats, caches, queues, keyRing) } diff --git a/federationapi/queue/destinationqueue.go b/federationapi/queue/destinationqueue.go index 5cb8cae1f..00e02b2d9 100644 --- a/federationapi/queue/destinationqueue.go +++ b/federationapi/queue/destinationqueue.go @@ -35,7 +35,7 @@ import ( const ( maxPDUsPerTransaction = 50 - maxEDUsPerTransaction = 50 + maxEDUsPerTransaction = 100 maxPDUsInMemory = 128 maxEDUsInMemory = 128 queueIdleTimeout = time.Second * 30 @@ -64,7 +64,6 @@ type destinationQueue struct { pendingPDUs []*queuedPDU // PDUs waiting to be sent pendingEDUs []*queuedEDU // EDUs waiting to be sent pendingMutex sync.RWMutex // protects pendingPDUs and pendingEDUs - interruptBackoff chan bool // interrupts backoff } // Send event adds the event to the pending queue for the destination. @@ -75,6 +74,7 @@ func (oq *destinationQueue) sendEvent(event *gomatrixserverlib.HeaderedEvent, re logrus.Errorf("attempt to send nil PDU with destination %q", oq.destination) return } + // Create a database entry that associates the given PDU NID with // this destination queue. We'll then be able to retrieve the PDU // later. @@ -102,12 +102,12 @@ func (oq *destinationQueue) sendEvent(event *gomatrixserverlib.HeaderedEvent, re oq.overflowed.Store(true) } oq.pendingMutex.Unlock() - // Wake up the queue if it's asleep. - oq.wakeQueueIfNeeded() - select { - case oq.notify <- struct{}{}: - default: + + if !oq.backingOff.Load() { + oq.wakeQueueAndNotify() } + } else { + oq.overflowed.Store(true) } } @@ -147,12 +147,37 @@ func (oq *destinationQueue) sendEDU(event *gomatrixserverlib.EDU, receipt *share oq.overflowed.Store(true) } oq.pendingMutex.Unlock() - // Wake up the queue if it's asleep. - oq.wakeQueueIfNeeded() - select { - case oq.notify <- struct{}{}: - default: + + if !oq.backingOff.Load() { + oq.wakeQueueAndNotify() } + } else { + oq.overflowed.Store(true) + } +} + +// handleBackoffNotifier is registered as the backoff notification +// callback with Statistics. It will wakeup and notify the queue +// if the queue is currently backing off. +func (oq *destinationQueue) handleBackoffNotifier() { + // Only wake up the queue if it is backing off. + // Otherwise there is no pending work for the queue to handle + // so waking the queue would be a waste of resources. + if oq.backingOff.Load() { + oq.wakeQueueAndNotify() + } +} + +// wakeQueueAndNotify ensures the destination queue is running and notifies it +// that there is pending work. +func (oq *destinationQueue) wakeQueueAndNotify() { + // Wake up the queue if it's asleep. + oq.wakeQueueIfNeeded() + + // Notify the queue that there are events ready to send. + select { + case oq.notify <- struct{}{}: + default: } } @@ -161,10 +186,11 @@ func (oq *destinationQueue) sendEDU(event *gomatrixserverlib.EDU, receipt *share // then we will interrupt the backoff, causing any federation // requests to retry. func (oq *destinationQueue) wakeQueueIfNeeded() { - // If we are backing off then interrupt the backoff. + // Clear the backingOff flag and update the backoff metrics if it was set. if oq.backingOff.CompareAndSwap(true, false) { - oq.interruptBackoff <- true + destinationQueueBackingOff.Dec() } + // If we aren't running then wake up the queue. if !oq.running.Load() { // Start the queue. @@ -196,38 +222,54 @@ func (oq *destinationQueue) getPendingFromDatabase() { gotEDUs[edu.receipt.String()] = struct{}{} } + overflowed := false if pduCapacity := maxPDUsInMemory - len(oq.pendingPDUs); pduCapacity > 0 { // We have room in memory for some PDUs - let's request no more than that. - if pdus, err := oq.db.GetPendingPDUs(ctx, oq.destination, pduCapacity); err == nil { + if pdus, err := oq.db.GetPendingPDUs(ctx, oq.destination, maxPDUsInMemory); err == nil { + if len(pdus) == maxPDUsInMemory { + overflowed = true + } for receipt, pdu := range pdus { if _, ok := gotPDUs[receipt.String()]; ok { continue } oq.pendingPDUs = append(oq.pendingPDUs, &queuedPDU{receipt, pdu}) retrieved = true + if len(oq.pendingPDUs) == maxPDUsInMemory { + break + } } } else { logrus.WithError(err).Errorf("Failed to get pending PDUs for %q", oq.destination) } } + if eduCapacity := maxEDUsInMemory - len(oq.pendingEDUs); eduCapacity > 0 { // We have room in memory for some EDUs - let's request no more than that. - if edus, err := oq.db.GetPendingEDUs(ctx, oq.destination, eduCapacity); err == nil { + if edus, err := oq.db.GetPendingEDUs(ctx, oq.destination, maxEDUsInMemory); err == nil { + if len(edus) == maxEDUsInMemory { + overflowed = true + } for receipt, edu := range edus { if _, ok := gotEDUs[receipt.String()]; ok { continue } oq.pendingEDUs = append(oq.pendingEDUs, &queuedEDU{receipt, edu}) retrieved = true + if len(oq.pendingEDUs) == maxEDUsInMemory { + break + } } } else { logrus.WithError(err).Errorf("Failed to get pending EDUs for %q", oq.destination) } } + // If we've retrieved all of the events from the database with room to spare // in memory then we'll no longer consider this queue to be overflowed. - if len(oq.pendingPDUs) < maxPDUsInMemory && len(oq.pendingEDUs) < maxEDUsInMemory { + if !overflowed { oq.overflowed.Store(false) + } else { } // If we've retrieved some events then notify the destination queue goroutine. if retrieved { @@ -238,6 +280,24 @@ func (oq *destinationQueue) getPendingFromDatabase() { } } +// checkNotificationsOnClose checks for any remaining notifications +// and starts a new backgroundSend goroutine if any exist. +func (oq *destinationQueue) checkNotificationsOnClose() { + // NOTE : If we are stopping the queue due to blacklist then it + // doesn't matter if we have been notified of new work since + // this queue instance will be deleted anyway. + if !oq.statistics.Blacklisted() { + select { + case <-oq.notify: + // We received a new notification in between the + // idle timeout firing and stopping the goroutine. + // Immediately restart the queue. + oq.wakeQueueAndNotify() + default: + } + } +} + // backgroundSend is the worker goroutine for sending events. func (oq *destinationQueue) backgroundSend() { // Check if a worker is already running, and if it isn't, then @@ -245,10 +305,17 @@ func (oq *destinationQueue) backgroundSend() { if !oq.running.CompareAndSwap(false, true) { return } + + // Register queue cleanup functions. + // NOTE : The ordering here is very intentional. + defer oq.checkNotificationsOnClose() + defer oq.running.Store(false) + destinationQueueRunning.Inc() defer destinationQueueRunning.Dec() - defer oq.queues.clearQueue(oq) - defer oq.running.Store(false) + + idleTimeout := time.NewTimer(queueIdleTimeout) + defer idleTimeout.Stop() // Mark the queue as overflowed, so we will consult the database // to see if there's anything new to send. @@ -261,59 +328,33 @@ func (oq *destinationQueue) backgroundSend() { oq.getPendingFromDatabase() } + // Reset the queue idle timeout. + if !idleTimeout.Stop() { + select { + case <-idleTimeout.C: + default: + } + } + idleTimeout.Reset(queueIdleTimeout) + // If we have nothing to do then wait either for incoming events, or // until we hit an idle timeout. select { case <-oq.notify: // There's work to do, either because getPendingFromDatabase - // told us there is, or because a new event has come in via - // sendEvent/sendEDU. - case <-time.After(queueIdleTimeout): + // told us there is, a new event has come in via sendEvent/sendEDU, + // or we are backing off and it is time to retry. + case <-idleTimeout.C: // The worker is idle so stop the goroutine. It'll get // restarted automatically the next time we have an event to // send. return case <-oq.process.Context().Done(): // The parent process is shutting down, so stop. + oq.statistics.ClearBackoff() return } - // If we are backing off this server then wait for the - // backoff duration to complete first, or until explicitly - // told to retry. - until, blacklisted := oq.statistics.BackoffInfo() - if blacklisted { - // It's been suggested that we should give up because the backoff - // has exceeded a maximum allowable value. Clean up the in-memory - // buffers at this point. The PDU clean-up is already on a defer. - logrus.Warnf("Blacklisting %q due to exceeding backoff threshold", oq.destination) - oq.pendingMutex.Lock() - for i := range oq.pendingPDUs { - oq.pendingPDUs[i] = nil - } - for i := range oq.pendingEDUs { - oq.pendingEDUs[i] = nil - } - oq.pendingPDUs = nil - oq.pendingEDUs = nil - oq.pendingMutex.Unlock() - return - } - if until != nil && until.After(time.Now()) { - // We haven't backed off yet, so wait for the suggested amount of - // time. - duration := time.Until(*until) - logrus.Debugf("Backing off %q for %s", oq.destination, duration) - oq.backingOff.Store(true) - destinationQueueBackingOff.Inc() - select { - case <-time.After(duration): - case <-oq.interruptBackoff: - } - destinationQueueBackingOff.Dec() - oq.backingOff.Store(false) - } - // Work out which PDUs/EDUs to include in the next transaction. oq.pendingMutex.RLock() pduCount := len(oq.pendingPDUs) @@ -328,99 +369,52 @@ func (oq *destinationQueue) backgroundSend() { toSendEDUs := oq.pendingEDUs[:eduCount] oq.pendingMutex.RUnlock() + // If we didn't get anything from the database and there are no + // pending EDUs then there's nothing to do - stop here. + if pduCount == 0 && eduCount == 0 { + continue + } + // If we have pending PDUs or EDUs then construct a transaction. // Try sending the next transaction and see what happens. - transaction, pc, ec, terr := oq.nextTransaction(toSendPDUs, toSendEDUs) + terr := oq.nextTransaction(toSendPDUs, toSendEDUs) if terr != nil { // We failed to send the transaction. Mark it as a failure. - oq.statistics.Failure() - - } else if transaction { - // If we successfully sent the transaction then clear out - // the pending events and EDUs, and wipe our transaction ID. - oq.statistics.Success() - oq.pendingMutex.Lock() - for i := range oq.pendingPDUs[:pc] { - oq.pendingPDUs[i] = nil + _, blacklisted := oq.statistics.Failure() + if !blacklisted { + // Register the backoff state and exit the goroutine. + // It'll get restarted automatically when the backoff + // completes. + oq.backingOff.Store(true) + destinationQueueBackingOff.Inc() + return + } else { + // Immediately trigger the blacklist logic. + oq.blacklistDestination() + return } - for i := range oq.pendingEDUs[:ec] { - oq.pendingEDUs[i] = nil - } - oq.pendingPDUs = oq.pendingPDUs[pc:] - oq.pendingEDUs = oq.pendingEDUs[ec:] - oq.pendingMutex.Unlock() + } else { + oq.handleTransactionSuccess(pduCount, eduCount) } } } // nextTransaction creates a new transaction from the pending event -// queue and sends it. Returns true if a transaction was sent or -// false otherwise. +// queue and sends it. +// Returns an error if the transaction wasn't sent. func (oq *destinationQueue) nextTransaction( pdus []*queuedPDU, edus []*queuedEDU, -) (bool, int, int, error) { - // If there's no projected transaction ID then generate one. If - // the transaction succeeds then we'll set it back to "" so that - // we generate a new one next time. If it fails, we'll preserve - // it so that we retry with the same transaction ID. - oq.transactionIDMutex.Lock() - if oq.transactionID == "" { - now := gomatrixserverlib.AsTimestamp(time.Now()) - oq.transactionID = gomatrixserverlib.TransactionID(fmt.Sprintf("%d-%d", now, oq.statistics.SuccessCount())) - } - oq.transactionIDMutex.Unlock() - +) error { // Create the transaction. - t := gomatrixserverlib.Transaction{ - PDUs: []json.RawMessage{}, - EDUs: []gomatrixserverlib.EDU{}, - } - t.Origin = oq.origin - t.Destination = oq.destination - t.OriginServerTS = gomatrixserverlib.AsTimestamp(time.Now()) - t.TransactionID = oq.transactionID - - // If we didn't get anything from the database and there are no - // pending EDUs then there's nothing to do - stop here. - if len(pdus) == 0 && len(edus) == 0 { - return false, 0, 0, nil - } - - var pduReceipts []*shared.Receipt - var eduReceipts []*shared.Receipt - - // Go through PDUs that we retrieved from the database, if any, - // and add them into the transaction. - for _, pdu := range pdus { - if pdu == nil || pdu.pdu == nil { - continue - } - // Append the JSON of the event, since this is a json.RawMessage type in the - // gomatrixserverlib.Transaction struct - t.PDUs = append(t.PDUs, pdu.pdu.JSON()) - pduReceipts = append(pduReceipts, pdu.receipt) - } - - // Do the same for pending EDUS in the queue. - for _, edu := range edus { - if edu == nil || edu.edu == nil { - continue - } - t.EDUs = append(t.EDUs, *edu.edu) - eduReceipts = append(eduReceipts, edu.receipt) - } - + t, pduReceipts, eduReceipts := oq.createTransaction(pdus, edus) logrus.WithField("server_name", oq.destination).Debugf("Sending transaction %q containing %d PDUs, %d EDUs", t.TransactionID, len(t.PDUs), len(t.EDUs)) // Try to send the transaction to the destination server. - // TODO: we should check for 500-ish fails vs 400-ish here, - // since we shouldn't queue things indefinitely in response - // to a 400-ish error ctx, cancel := context.WithTimeout(oq.process.Context(), time.Minute*5) defer cancel() _, err := oq.client.SendTransaction(ctx, t) - switch err.(type) { + switch errResponse := err.(type) { case nil: // Clean up the transaction in the database. if pduReceipts != nil { @@ -439,16 +433,128 @@ func (oq *destinationQueue) nextTransaction( oq.transactionIDMutex.Lock() oq.transactionID = "" oq.transactionIDMutex.Unlock() - return true, len(t.PDUs), len(t.EDUs), nil + return nil case gomatrix.HTTPError: // Report that we failed to send the transaction and we // will retry again, subject to backoff. - return false, 0, 0, err + + // TODO: we should check for 500-ish fails vs 400-ish here, + // since we shouldn't queue things indefinitely in response + // to a 400-ish error + code := errResponse.Code + logrus.Debug("Transaction failed with HTTP", code) + return err default: logrus.WithFields(logrus.Fields{ "destination": oq.destination, logrus.ErrorKey: err, }).Debugf("Failed to send transaction %q", t.TransactionID) - return false, 0, 0, err + return err + } +} + +// createTransaction generates a gomatrixserverlib.Transaction from the provided pdus and edus. +// It also returns the associated event receipts so they can be cleaned from the database in +// the case of a successful transaction. +func (oq *destinationQueue) createTransaction( + pdus []*queuedPDU, + edus []*queuedEDU, +) (gomatrixserverlib.Transaction, []*shared.Receipt, []*shared.Receipt) { + // If there's no projected transaction ID then generate one. If + // the transaction succeeds then we'll set it back to "" so that + // we generate a new one next time. If it fails, we'll preserve + // it so that we retry with the same transaction ID. + oq.transactionIDMutex.Lock() + if oq.transactionID == "" { + now := gomatrixserverlib.AsTimestamp(time.Now()) + oq.transactionID = gomatrixserverlib.TransactionID(fmt.Sprintf("%d-%d", now, oq.statistics.SuccessCount())) + } + oq.transactionIDMutex.Unlock() + + t := gomatrixserverlib.Transaction{ + PDUs: []json.RawMessage{}, + EDUs: []gomatrixserverlib.EDU{}, + } + t.Origin = oq.origin + t.Destination = oq.destination + t.OriginServerTS = gomatrixserverlib.AsTimestamp(time.Now()) + t.TransactionID = oq.transactionID + + var pduReceipts []*shared.Receipt + var eduReceipts []*shared.Receipt + + // Go through PDUs that we retrieved from the database, if any, + // and add them into the transaction. + for _, pdu := range pdus { + // These should never be nil. + if pdu == nil || pdu.pdu == nil { + continue + } + // Append the JSON of the event, since this is a json.RawMessage type in the + // gomatrixserverlib.Transaction struct + t.PDUs = append(t.PDUs, pdu.pdu.JSON()) + pduReceipts = append(pduReceipts, pdu.receipt) + } + + // Do the same for pending EDUS in the queue. + for _, edu := range edus { + // These should never be nil. + if edu == nil || edu.edu == nil { + continue + } + t.EDUs = append(t.EDUs, *edu.edu) + eduReceipts = append(eduReceipts, edu.receipt) + } + + return t, pduReceipts, eduReceipts +} + +// blacklistDestination removes all pending PDUs and EDUs that have been cached +// and deletes this queue. +func (oq *destinationQueue) blacklistDestination() { + // It's been suggested that we should give up because the backoff + // has exceeded a maximum allowable value. Clean up the in-memory + // buffers at this point. The PDU clean-up is already on a defer. + logrus.Warnf("Blacklisting %q due to exceeding backoff threshold", oq.destination) + + oq.pendingMutex.Lock() + for i := range oq.pendingPDUs { + oq.pendingPDUs[i] = nil + } + for i := range oq.pendingEDUs { + oq.pendingEDUs[i] = nil + } + oq.pendingPDUs = nil + oq.pendingEDUs = nil + oq.pendingMutex.Unlock() + + // Delete this queue as no more messages will be sent to this + // destination until it is no longer blacklisted. + oq.statistics.AssignBackoffNotifier(nil) + oq.queues.clearQueue(oq) +} + +// handleTransactionSuccess updates the cached event queues as well as the success and +// backoff information for this server. +func (oq *destinationQueue) handleTransactionSuccess(pduCount int, eduCount int) { + // If we successfully sent the transaction then clear out + // the pending events and EDUs, and wipe our transaction ID. + oq.statistics.Success() + oq.pendingMutex.Lock() + for i := range oq.pendingPDUs[:pduCount] { + oq.pendingPDUs[i] = nil + } + for i := range oq.pendingEDUs[:eduCount] { + oq.pendingEDUs[i] = nil + } + oq.pendingPDUs = oq.pendingPDUs[pduCount:] + oq.pendingEDUs = oq.pendingEDUs[eduCount:] + oq.pendingMutex.Unlock() + + if len(oq.pendingPDUs) > 0 || len(oq.pendingEDUs) > 0 { + select { + case oq.notify <- struct{}{}: + default: + } } } diff --git a/federationapi/queue/queue.go b/federationapi/queue/queue.go index 8245aa5bd..68f789e37 100644 --- a/federationapi/queue/queue.go +++ b/federationapi/queue/queue.go @@ -162,23 +162,25 @@ func (oqs *OutgoingQueues) getQueue(destination gomatrixserverlib.ServerName) *d if !ok || oq == nil { destinationQueueTotal.Inc() oq = &destinationQueue{ - queues: oqs, - db: oqs.db, - process: oqs.process, - rsAPI: oqs.rsAPI, - origin: oqs.origin, - destination: destination, - client: oqs.client, - statistics: oqs.statistics.ForServer(destination), - notify: make(chan struct{}, 1), - interruptBackoff: make(chan bool), - signing: oqs.signing, + queues: oqs, + db: oqs.db, + process: oqs.process, + rsAPI: oqs.rsAPI, + origin: oqs.origin, + destination: destination, + client: oqs.client, + statistics: oqs.statistics.ForServer(destination), + notify: make(chan struct{}, 1), + signing: oqs.signing, } + oq.statistics.AssignBackoffNotifier(oq.handleBackoffNotifier) oqs.queues[destination] = oq } return oq } +// clearQueue removes the queue for the provided destination from the +// set of destination queues. func (oqs *OutgoingQueues) clearQueue(oq *destinationQueue) { oqs.queuesMutex.Lock() defer oqs.queuesMutex.Unlock() @@ -332,7 +334,9 @@ func (oqs *OutgoingQueues) RetryServer(srv gomatrixserverlib.ServerName) { if oqs.disabled { return } + oqs.statistics.ForServer(srv).RemoveBlacklist() if queue := oqs.getQueue(srv); queue != nil { + queue.statistics.ClearBackoff() queue.wakeQueueIfNeeded() } } diff --git a/federationapi/queue/queue_test.go b/federationapi/queue/queue_test.go new file mode 100644 index 000000000..6da9e6b30 --- /dev/null +++ b/federationapi/queue/queue_test.go @@ -0,0 +1,1047 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package queue + +import ( + "context" + "encoding/json" + "fmt" + "sync" + "testing" + "time" + + "go.uber.org/atomic" + "gotest.tools/v3/poll" + + "github.com/matrix-org/dendrite/federationapi/api" + "github.com/matrix-org/dendrite/federationapi/statistics" + "github.com/matrix-org/dendrite/federationapi/storage" + "github.com/matrix-org/dendrite/federationapi/storage/shared" + rsapi "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/setup/process" + "github.com/matrix-org/dendrite/test" + "github.com/matrix-org/dendrite/test/testrig" + "github.com/matrix-org/gomatrixserverlib" + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" +) + +func mustCreateFederationDatabase(t *testing.T, dbType test.DBType, realDatabase bool) (storage.Database, *process.ProcessContext, func()) { + if realDatabase { + // Real Database/s + b, baseClose := testrig.CreateBaseDendrite(t, dbType) + connStr, dbClose := test.PrepareDBConnectionString(t, dbType) + db, err := storage.NewDatabase(b, &config.DatabaseOptions{ + ConnectionString: config.DataSource(connStr), + }, b.Caches, b.Cfg.Global.ServerName) + if err != nil { + t.Fatalf("NewDatabase returned %s", err) + } + return db, b.ProcessContext, func() { + dbClose() + baseClose() + } + } else { + // Fake Database + db := createDatabase() + b := struct { + ProcessContext *process.ProcessContext + }{ProcessContext: process.NewProcessContext()} + return db, b.ProcessContext, func() {} + } +} + +func createDatabase() storage.Database { + return &fakeDatabase{ + pendingPDUServers: make(map[gomatrixserverlib.ServerName]struct{}), + pendingEDUServers: make(map[gomatrixserverlib.ServerName]struct{}), + blacklistedServers: make(map[gomatrixserverlib.ServerName]struct{}), + pendingPDUs: make(map[*shared.Receipt]*gomatrixserverlib.HeaderedEvent), + pendingEDUs: make(map[*shared.Receipt]*gomatrixserverlib.EDU), + associatedPDUs: make(map[gomatrixserverlib.ServerName]map[*shared.Receipt]struct{}), + associatedEDUs: make(map[gomatrixserverlib.ServerName]map[*shared.Receipt]struct{}), + } +} + +type fakeDatabase struct { + storage.Database + dbMutex sync.Mutex + pendingPDUServers map[gomatrixserverlib.ServerName]struct{} + pendingEDUServers map[gomatrixserverlib.ServerName]struct{} + blacklistedServers map[gomatrixserverlib.ServerName]struct{} + pendingPDUs map[*shared.Receipt]*gomatrixserverlib.HeaderedEvent + pendingEDUs map[*shared.Receipt]*gomatrixserverlib.EDU + associatedPDUs map[gomatrixserverlib.ServerName]map[*shared.Receipt]struct{} + associatedEDUs map[gomatrixserverlib.ServerName]map[*shared.Receipt]struct{} +} + +var nidMutex sync.Mutex +var nid = int64(0) + +func (d *fakeDatabase) StoreJSON(ctx context.Context, js string) (*shared.Receipt, error) { + d.dbMutex.Lock() + defer d.dbMutex.Unlock() + + var event gomatrixserverlib.HeaderedEvent + if err := json.Unmarshal([]byte(js), &event); err == nil { + nidMutex.Lock() + defer nidMutex.Unlock() + nid++ + receipt := shared.NewReceipt(nid) + d.pendingPDUs[&receipt] = &event + return &receipt, nil + } + + var edu gomatrixserverlib.EDU + if err := json.Unmarshal([]byte(js), &edu); err == nil { + nidMutex.Lock() + defer nidMutex.Unlock() + nid++ + receipt := shared.NewReceipt(nid) + d.pendingEDUs[&receipt] = &edu + return &receipt, nil + } + + return nil, errors.New("Failed to determine type of json to store") +} + +func (d *fakeDatabase) GetPendingPDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, limit int) (pdus map[*shared.Receipt]*gomatrixserverlib.HeaderedEvent, err error) { + d.dbMutex.Lock() + defer d.dbMutex.Unlock() + + pduCount := 0 + pdus = make(map[*shared.Receipt]*gomatrixserverlib.HeaderedEvent) + if receipts, ok := d.associatedPDUs[serverName]; ok { + for receipt := range receipts { + if event, ok := d.pendingPDUs[receipt]; ok { + pdus[receipt] = event + pduCount++ + if pduCount == limit { + break + } + } + } + } + return pdus, nil +} + +func (d *fakeDatabase) GetPendingEDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, limit int) (edus map[*shared.Receipt]*gomatrixserverlib.EDU, err error) { + d.dbMutex.Lock() + defer d.dbMutex.Unlock() + + eduCount := 0 + edus = make(map[*shared.Receipt]*gomatrixserverlib.EDU) + if receipts, ok := d.associatedEDUs[serverName]; ok { + for receipt := range receipts { + if event, ok := d.pendingEDUs[receipt]; ok { + edus[receipt] = event + eduCount++ + if eduCount == limit { + break + } + } + } + } + return edus, nil +} + +func (d *fakeDatabase) AssociatePDUWithDestination(ctx context.Context, transactionID gomatrixserverlib.TransactionID, serverName gomatrixserverlib.ServerName, receipt *shared.Receipt) error { + d.dbMutex.Lock() + defer d.dbMutex.Unlock() + + if _, ok := d.pendingPDUs[receipt]; ok { + if _, ok := d.associatedPDUs[serverName]; !ok { + d.associatedPDUs[serverName] = make(map[*shared.Receipt]struct{}) + } + d.associatedPDUs[serverName][receipt] = struct{}{} + return nil + } else { + return errors.New("PDU doesn't exist") + } +} + +func (d *fakeDatabase) AssociateEDUWithDestination(ctx context.Context, serverName gomatrixserverlib.ServerName, receipt *shared.Receipt, eduType string, expireEDUTypes map[string]time.Duration) error { + d.dbMutex.Lock() + defer d.dbMutex.Unlock() + + if _, ok := d.pendingEDUs[receipt]; ok { + if _, ok := d.associatedEDUs[serverName]; !ok { + d.associatedEDUs[serverName] = make(map[*shared.Receipt]struct{}) + } + d.associatedEDUs[serverName][receipt] = struct{}{} + return nil + } else { + return errors.New("EDU doesn't exist") + } +} + +func (d *fakeDatabase) CleanPDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, receipts []*shared.Receipt) error { + d.dbMutex.Lock() + defer d.dbMutex.Unlock() + + if pdus, ok := d.associatedPDUs[serverName]; ok { + for _, receipt := range receipts { + delete(pdus, receipt) + } + } + + return nil +} + +func (d *fakeDatabase) CleanEDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, receipts []*shared.Receipt) error { + d.dbMutex.Lock() + defer d.dbMutex.Unlock() + + if edus, ok := d.associatedEDUs[serverName]; ok { + for _, receipt := range receipts { + delete(edus, receipt) + } + } + + return nil +} + +func (d *fakeDatabase) GetPendingPDUCount(ctx context.Context, serverName gomatrixserverlib.ServerName) (int64, error) { + d.dbMutex.Lock() + defer d.dbMutex.Unlock() + + var count int64 + if pdus, ok := d.associatedPDUs[serverName]; ok { + count = int64(len(pdus)) + } + return count, nil +} + +func (d *fakeDatabase) GetPendingEDUCount(ctx context.Context, serverName gomatrixserverlib.ServerName) (int64, error) { + d.dbMutex.Lock() + defer d.dbMutex.Unlock() + + var count int64 + if edus, ok := d.associatedEDUs[serverName]; ok { + count = int64(len(edus)) + } + return count, nil +} + +func (d *fakeDatabase) GetPendingPDUServerNames(ctx context.Context) ([]gomatrixserverlib.ServerName, error) { + d.dbMutex.Lock() + defer d.dbMutex.Unlock() + + servers := []gomatrixserverlib.ServerName{} + for server := range d.pendingPDUServers { + servers = append(servers, server) + } + return servers, nil +} + +func (d *fakeDatabase) GetPendingEDUServerNames(ctx context.Context) ([]gomatrixserverlib.ServerName, error) { + d.dbMutex.Lock() + defer d.dbMutex.Unlock() + + servers := []gomatrixserverlib.ServerName{} + for server := range d.pendingEDUServers { + servers = append(servers, server) + } + return servers, nil +} + +func (d *fakeDatabase) AddServerToBlacklist(serverName gomatrixserverlib.ServerName) error { + d.dbMutex.Lock() + defer d.dbMutex.Unlock() + + d.blacklistedServers[serverName] = struct{}{} + return nil +} + +func (d *fakeDatabase) RemoveServerFromBlacklist(serverName gomatrixserverlib.ServerName) error { + d.dbMutex.Lock() + defer d.dbMutex.Unlock() + + delete(d.blacklistedServers, serverName) + return nil +} + +func (d *fakeDatabase) RemoveAllServersFromBlacklist() error { + d.dbMutex.Lock() + defer d.dbMutex.Unlock() + + d.blacklistedServers = make(map[gomatrixserverlib.ServerName]struct{}) + return nil +} + +func (d *fakeDatabase) IsServerBlacklisted(serverName gomatrixserverlib.ServerName) (bool, error) { + d.dbMutex.Lock() + defer d.dbMutex.Unlock() + + isBlacklisted := false + if _, ok := d.blacklistedServers[serverName]; ok { + isBlacklisted = true + } + + return isBlacklisted, nil +} + +type stubFederationRoomServerAPI struct { + rsapi.FederationRoomserverAPI +} + +func (r *stubFederationRoomServerAPI) QueryServerBannedFromRoom(ctx context.Context, req *rsapi.QueryServerBannedFromRoomRequest, res *rsapi.QueryServerBannedFromRoomResponse) error { + res.Banned = false + return nil +} + +type stubFederationClient struct { + api.FederationClient + shouldTxSucceed bool + txCount atomic.Uint32 +} + +func (f *stubFederationClient) SendTransaction(ctx context.Context, t gomatrixserverlib.Transaction) (res gomatrixserverlib.RespSend, err error) { + var result error + if !f.shouldTxSucceed { + result = fmt.Errorf("transaction failed") + } + + f.txCount.Add(1) + return gomatrixserverlib.RespSend{}, result +} + +func mustCreatePDU(t *testing.T) *gomatrixserverlib.HeaderedEvent { + t.Helper() + content := `{"type":"m.room.message"}` + ev, err := gomatrixserverlib.NewEventFromTrustedJSON([]byte(content), false, gomatrixserverlib.RoomVersionV10) + if err != nil { + t.Fatalf("failed to create event: %v", err) + } + return ev.Headered(gomatrixserverlib.RoomVersionV10) +} + +func mustCreateEDU(t *testing.T) *gomatrixserverlib.EDU { + t.Helper() + return &gomatrixserverlib.EDU{Type: gomatrixserverlib.MTyping} +} + +func testSetup(failuresUntilBlacklist uint32, shouldTxSucceed bool, t *testing.T, dbType test.DBType, realDatabase bool) (storage.Database, *stubFederationClient, *OutgoingQueues, *process.ProcessContext, func()) { + db, processContext, close := mustCreateFederationDatabase(t, dbType, realDatabase) + + fc := &stubFederationClient{ + shouldTxSucceed: shouldTxSucceed, + txCount: *atomic.NewUint32(0), + } + rs := &stubFederationRoomServerAPI{} + stats := statistics.NewStatistics(db, failuresUntilBlacklist) + signingInfo := &SigningInfo{ + KeyID: "ed21019:auto", + PrivateKey: test.PrivateKeyA, + ServerName: "localhost", + } + queues := NewOutgoingQueues(db, processContext, false, "localhost", fc, rs, &stats, signingInfo) + + return db, fc, queues, processContext, close +} + +func TestSendPDUOnSuccessRemovedFromDB(t *testing.T) { + t.Parallel() + failuresUntilBlacklist := uint32(16) + destination := gomatrixserverlib.ServerName("remotehost") + db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, true, t, test.DBTypeSQLite, false) + defer close() + defer func() { + pc.ShutdownDendrite() + <-pc.WaitForShutdown() + }() + + ev := mustCreatePDU(t) + err := queues.SendEvent(ev, "localhost", []gomatrixserverlib.ServerName{destination}) + assert.NoError(t, err) + + check := func(log poll.LogT) poll.Result { + if fc.txCount.Load() == 1 { + data, dbErr := db.GetPendingPDUs(pc.Context(), destination, 100) + assert.NoError(t, dbErr) + if len(data) == 0 { + return poll.Success() + } + return poll.Continue("waiting for event to be removed from database. Currently present PDU: %d", len(data)) + } + return poll.Continue("waiting for more send attempts before checking database. Currently %d", fc.txCount.Load()) + } + poll.WaitOn(t, check, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond)) +} + +func TestSendEDUOnSuccessRemovedFromDB(t *testing.T) { + t.Parallel() + failuresUntilBlacklist := uint32(16) + destination := gomatrixserverlib.ServerName("remotehost") + db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, true, t, test.DBTypeSQLite, false) + defer close() + defer func() { + pc.ShutdownDendrite() + <-pc.WaitForShutdown() + }() + + ev := mustCreateEDU(t) + err := queues.SendEDU(ev, "localhost", []gomatrixserverlib.ServerName{destination}) + assert.NoError(t, err) + + check := func(log poll.LogT) poll.Result { + if fc.txCount.Load() == 1 { + data, dbErr := db.GetPendingEDUs(pc.Context(), destination, 100) + assert.NoError(t, dbErr) + if len(data) == 0 { + return poll.Success() + } + return poll.Continue("waiting for event to be removed from database. Currently present EDU: %d", len(data)) + } + return poll.Continue("waiting for more send attempts before checking database. Currently %d", fc.txCount.Load()) + } + poll.WaitOn(t, check, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond)) +} + +func TestSendPDUOnFailStoredInDB(t *testing.T) { + t.Parallel() + failuresUntilBlacklist := uint32(16) + destination := gomatrixserverlib.ServerName("remotehost") + db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, false, t, test.DBTypeSQLite, false) + defer close() + defer func() { + pc.ShutdownDendrite() + <-pc.WaitForShutdown() + }() + + ev := mustCreatePDU(t) + err := queues.SendEvent(ev, "localhost", []gomatrixserverlib.ServerName{destination}) + assert.NoError(t, err) + + check := func(log poll.LogT) poll.Result { + // Wait for 2 backoff attempts to ensure there was adequate time to attempt sending + if fc.txCount.Load() >= 2 { + data, dbErr := db.GetPendingPDUs(pc.Context(), destination, 100) + assert.NoError(t, dbErr) + if len(data) == 1 { + return poll.Success() + } + return poll.Continue("waiting for event to be added to database. Currently present PDU: %d", len(data)) + } + return poll.Continue("waiting for more send attempts before checking database. Currently %d", fc.txCount.Load()) + } + poll.WaitOn(t, check, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond)) +} + +func TestSendEDUOnFailStoredInDB(t *testing.T) { + t.Parallel() + failuresUntilBlacklist := uint32(16) + destination := gomatrixserverlib.ServerName("remotehost") + db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, false, t, test.DBTypeSQLite, false) + defer close() + defer func() { + pc.ShutdownDendrite() + <-pc.WaitForShutdown() + }() + + ev := mustCreateEDU(t) + err := queues.SendEDU(ev, "localhost", []gomatrixserverlib.ServerName{destination}) + assert.NoError(t, err) + + check := func(log poll.LogT) poll.Result { + // Wait for 2 backoff attempts to ensure there was adequate time to attempt sending + if fc.txCount.Load() >= 2 { + data, dbErr := db.GetPendingEDUs(pc.Context(), destination, 100) + assert.NoError(t, dbErr) + if len(data) == 1 { + return poll.Success() + } + return poll.Continue("waiting for event to be added to database. Currently present EDU: %d", len(data)) + } + return poll.Continue("waiting for more send attempts before checking database. Currently %d", fc.txCount.Load()) + } + poll.WaitOn(t, check, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond)) +} + +func TestSendPDUAgainDoesntInterruptBackoff(t *testing.T) { + t.Parallel() + failuresUntilBlacklist := uint32(16) + destination := gomatrixserverlib.ServerName("remotehost") + db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, false, t, test.DBTypeSQLite, false) + defer close() + defer func() { + pc.ShutdownDendrite() + <-pc.WaitForShutdown() + }() + + ev := mustCreatePDU(t) + err := queues.SendEvent(ev, "localhost", []gomatrixserverlib.ServerName{destination}) + assert.NoError(t, err) + + check := func(log poll.LogT) poll.Result { + // Wait for 2 backoff attempts to ensure there was adequate time to attempt sending + if fc.txCount.Load() >= 2 { + data, dbErr := db.GetPendingPDUs(pc.Context(), destination, 100) + assert.NoError(t, dbErr) + if len(data) == 1 { + return poll.Success() + } + return poll.Continue("waiting for event to be added to database. Currently present PDU: %d", len(data)) + } + return poll.Continue("waiting for more send attempts before checking database. Currently %d", fc.txCount.Load()) + } + poll.WaitOn(t, check, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond)) + + fc.shouldTxSucceed = true + ev = mustCreatePDU(t) + err = queues.SendEvent(ev, "localhost", []gomatrixserverlib.ServerName{destination}) + assert.NoError(t, err) + + pollEnd := time.Now().Add(1 * time.Second) + immediateCheck := func(log poll.LogT) poll.Result { + data, dbErr := db.GetPendingPDUs(pc.Context(), destination, 100) + assert.NoError(t, dbErr) + if len(data) == 0 { + return poll.Error(fmt.Errorf("The backoff was interrupted early")) + } + if time.Now().After(pollEnd) { + // Allow more than enough time for the backoff to be interrupted before + // reporting that it wasn't. + return poll.Success() + } + return poll.Continue("waiting for events to be removed from database. Currently present PDU: %d", len(data)) + } + poll.WaitOn(t, immediateCheck, poll.WithTimeout(2*time.Second), poll.WithDelay(100*time.Millisecond)) +} + +func TestSendEDUAgainDoesntInterruptBackoff(t *testing.T) { + t.Parallel() + failuresUntilBlacklist := uint32(16) + destination := gomatrixserverlib.ServerName("remotehost") + db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, false, t, test.DBTypeSQLite, false) + defer close() + defer func() { + pc.ShutdownDendrite() + <-pc.WaitForShutdown() + }() + + ev := mustCreateEDU(t) + err := queues.SendEDU(ev, "localhost", []gomatrixserverlib.ServerName{destination}) + assert.NoError(t, err) + + check := func(log poll.LogT) poll.Result { + // Wait for 2 backoff attempts to ensure there was adequate time to attempt sending + if fc.txCount.Load() >= 2 { + data, dbErr := db.GetPendingEDUs(pc.Context(), destination, 100) + assert.NoError(t, dbErr) + if len(data) == 1 { + return poll.Success() + } + return poll.Continue("waiting for event to be added to database. Currently present EDU: %d", len(data)) + } + return poll.Continue("waiting for more send attempts before checking database. Currently %d", fc.txCount.Load()) + } + poll.WaitOn(t, check, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond)) + + fc.shouldTxSucceed = true + ev = mustCreateEDU(t) + err = queues.SendEDU(ev, "localhost", []gomatrixserverlib.ServerName{destination}) + assert.NoError(t, err) + + pollEnd := time.Now().Add(1 * time.Second) + immediateCheck := func(log poll.LogT) poll.Result { + data, dbErr := db.GetPendingEDUs(pc.Context(), destination, 100) + assert.NoError(t, dbErr) + if len(data) == 0 { + return poll.Error(fmt.Errorf("The backoff was interrupted early")) + } + if time.Now().After(pollEnd) { + // Allow more than enough time for the backoff to be interrupted before + // reporting that it wasn't. + return poll.Success() + } + return poll.Continue("waiting for events to be removed from database. Currently present EDU: %d", len(data)) + } + poll.WaitOn(t, immediateCheck, poll.WithTimeout(2*time.Second), poll.WithDelay(100*time.Millisecond)) +} + +func TestSendPDUMultipleFailuresBlacklisted(t *testing.T) { + t.Parallel() + failuresUntilBlacklist := uint32(2) + destination := gomatrixserverlib.ServerName("remotehost") + db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, false, t, test.DBTypeSQLite, false) + defer close() + defer func() { + pc.ShutdownDendrite() + <-pc.WaitForShutdown() + }() + + ev := mustCreatePDU(t) + err := queues.SendEvent(ev, "localhost", []gomatrixserverlib.ServerName{destination}) + assert.NoError(t, err) + + check := func(log poll.LogT) poll.Result { + if fc.txCount.Load() == failuresUntilBlacklist { + data, dbErr := db.GetPendingPDUs(pc.Context(), destination, 100) + assert.NoError(t, dbErr) + if len(data) == 1 { + if val, _ := db.IsServerBlacklisted(destination); val { + return poll.Success() + } + return poll.Continue("waiting for server to be blacklisted") + } + return poll.Continue("waiting for event to be added to database. Currently present PDU: %d", len(data)) + } + return poll.Continue("waiting for more send attempts before checking database. Currently %d", fc.txCount.Load()) + } + poll.WaitOn(t, check, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond)) +} + +func TestSendEDUMultipleFailuresBlacklisted(t *testing.T) { + t.Parallel() + failuresUntilBlacklist := uint32(2) + destination := gomatrixserverlib.ServerName("remotehost") + db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, false, t, test.DBTypeSQLite, false) + defer close() + defer func() { + pc.ShutdownDendrite() + <-pc.WaitForShutdown() + }() + + ev := mustCreateEDU(t) + err := queues.SendEDU(ev, "localhost", []gomatrixserverlib.ServerName{destination}) + assert.NoError(t, err) + + check := func(log poll.LogT) poll.Result { + if fc.txCount.Load() == failuresUntilBlacklist { + data, dbErr := db.GetPendingEDUs(pc.Context(), destination, 100) + assert.NoError(t, dbErr) + if len(data) == 1 { + if val, _ := db.IsServerBlacklisted(destination); val { + return poll.Success() + } + return poll.Continue("waiting for server to be blacklisted") + } + return poll.Continue("waiting for event to be added to database. Currently present EDU: %d", len(data)) + } + return poll.Continue("waiting for more send attempts before checking database. Currently %d", fc.txCount.Load()) + } + poll.WaitOn(t, check, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond)) +} + +func TestSendPDUBlacklistedWithPriorExternalFailure(t *testing.T) { + t.Parallel() + failuresUntilBlacklist := uint32(2) + destination := gomatrixserverlib.ServerName("remotehost") + db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, false, t, test.DBTypeSQLite, false) + defer close() + defer func() { + pc.ShutdownDendrite() + <-pc.WaitForShutdown() + }() + + queues.statistics.ForServer(destination).Failure() + + ev := mustCreatePDU(t) + err := queues.SendEvent(ev, "localhost", []gomatrixserverlib.ServerName{destination}) + assert.NoError(t, err) + + check := func(log poll.LogT) poll.Result { + if fc.txCount.Load() == failuresUntilBlacklist { + data, dbErr := db.GetPendingPDUs(pc.Context(), destination, 100) + assert.NoError(t, dbErr) + if len(data) == 1 { + if val, _ := db.IsServerBlacklisted(destination); val { + return poll.Success() + } + return poll.Continue("waiting for server to be blacklisted") + } + return poll.Continue("waiting for event to be added to database. Currently present PDU: %d", len(data)) + } + return poll.Continue("waiting for more send attempts before checking database. Currently %d", fc.txCount.Load()) + } + poll.WaitOn(t, check, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond)) +} + +func TestSendEDUBlacklistedWithPriorExternalFailure(t *testing.T) { + t.Parallel() + failuresUntilBlacklist := uint32(2) + destination := gomatrixserverlib.ServerName("remotehost") + db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, false, t, test.DBTypeSQLite, false) + defer close() + defer func() { + pc.ShutdownDendrite() + <-pc.WaitForShutdown() + }() + + queues.statistics.ForServer(destination).Failure() + + ev := mustCreateEDU(t) + err := queues.SendEDU(ev, "localhost", []gomatrixserverlib.ServerName{destination}) + assert.NoError(t, err) + + check := func(log poll.LogT) poll.Result { + if fc.txCount.Load() == failuresUntilBlacklist { + data, dbErr := db.GetPendingEDUs(pc.Context(), destination, 100) + assert.NoError(t, dbErr) + if len(data) == 1 { + if val, _ := db.IsServerBlacklisted(destination); val { + return poll.Success() + } + return poll.Continue("waiting for server to be blacklisted") + } + return poll.Continue("waiting for event to be added to database. Currently present EDU: %d", len(data)) + } + return poll.Continue("waiting for more send attempts before checking database. Currently %d", fc.txCount.Load()) + } + poll.WaitOn(t, check, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond)) +} + +func TestRetryServerSendsPDUSuccessfully(t *testing.T) { + t.Parallel() + failuresUntilBlacklist := uint32(1) + destination := gomatrixserverlib.ServerName("remotehost") + db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, false, t, test.DBTypeSQLite, false) + defer close() + defer func() { + pc.ShutdownDendrite() + <-pc.WaitForShutdown() + }() + + // NOTE : getQueue before sending event to ensure we grab the same queue reference + // before it is blacklisted and deleted. + dest := queues.getQueue(destination) + ev := mustCreatePDU(t) + err := queues.SendEvent(ev, "localhost", []gomatrixserverlib.ServerName{destination}) + assert.NoError(t, err) + + checkBlacklisted := func(log poll.LogT) poll.Result { + if fc.txCount.Load() == failuresUntilBlacklist { + data, dbErr := db.GetPendingPDUs(pc.Context(), destination, 100) + assert.NoError(t, dbErr) + if len(data) == 1 { + if val, _ := db.IsServerBlacklisted(destination); val { + if !dest.running.Load() { + return poll.Success() + } + return poll.Continue("waiting for queue to stop completely") + } + return poll.Continue("waiting for server to be blacklisted") + } + return poll.Continue("waiting for event to be added to database. Currently present PDU: %d", len(data)) + } + return poll.Continue("waiting for more send attempts before checking database. Currently %d", fc.txCount.Load()) + } + poll.WaitOn(t, checkBlacklisted, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond)) + + fc.shouldTxSucceed = true + db.RemoveServerFromBlacklist(destination) + queues.RetryServer(destination) + checkRetry := func(log poll.LogT) poll.Result { + data, dbErr := db.GetPendingPDUs(pc.Context(), destination, 100) + assert.NoError(t, dbErr) + if len(data) == 0 { + return poll.Success() + } + return poll.Continue("waiting for event to be removed from database. Currently present PDU: %d", len(data)) + } + poll.WaitOn(t, checkRetry, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond)) +} + +func TestRetryServerSendsEDUSuccessfully(t *testing.T) { + t.Parallel() + failuresUntilBlacklist := uint32(1) + destination := gomatrixserverlib.ServerName("remotehost") + db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, false, t, test.DBTypeSQLite, false) + defer close() + defer func() { + pc.ShutdownDendrite() + <-pc.WaitForShutdown() + }() + + // NOTE : getQueue before sending event to ensure we grab the same queue reference + // before it is blacklisted and deleted. + dest := queues.getQueue(destination) + ev := mustCreateEDU(t) + err := queues.SendEDU(ev, "localhost", []gomatrixserverlib.ServerName{destination}) + assert.NoError(t, err) + + checkBlacklisted := func(log poll.LogT) poll.Result { + if fc.txCount.Load() == failuresUntilBlacklist { + data, dbErr := db.GetPendingEDUs(pc.Context(), destination, 100) + assert.NoError(t, dbErr) + if len(data) == 1 { + if val, _ := db.IsServerBlacklisted(destination); val { + if !dest.running.Load() { + return poll.Success() + } + return poll.Continue("waiting for queue to stop completely") + } + return poll.Continue("waiting for server to be blacklisted") + } + return poll.Continue("waiting for event to be added to database. Currently present EDU: %d", len(data)) + } + return poll.Continue("waiting for more send attempts before checking database. Currently %d", fc.txCount.Load()) + } + poll.WaitOn(t, checkBlacklisted, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond)) + + fc.shouldTxSucceed = true + db.RemoveServerFromBlacklist(destination) + queues.RetryServer(destination) + checkRetry := func(log poll.LogT) poll.Result { + data, dbErr := db.GetPendingEDUs(pc.Context(), destination, 100) + assert.NoError(t, dbErr) + if len(data) == 0 { + return poll.Success() + } + return poll.Continue("waiting for event to be removed from database. Currently present EDU: %d", len(data)) + } + poll.WaitOn(t, checkRetry, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond)) +} + +func TestSendPDUBatches(t *testing.T) { + t.Parallel() + failuresUntilBlacklist := uint32(16) + destination := gomatrixserverlib.ServerName("remotehost") + + // test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + // db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, true, t, dbType, true) + db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, true, t, test.DBTypeSQLite, false) + defer close() + defer func() { + pc.ShutdownDendrite() + <-pc.WaitForShutdown() + }() + + // Populate database with > maxPDUsPerTransaction + pduMultiplier := uint32(3) + for i := 0; i < maxPDUsPerTransaction*int(pduMultiplier); i++ { + ev := mustCreatePDU(t) + headeredJSON, _ := json.Marshal(ev) + nid, _ := db.StoreJSON(pc.Context(), string(headeredJSON)) + now := gomatrixserverlib.AsTimestamp(time.Now()) + transactionID := gomatrixserverlib.TransactionID(fmt.Sprintf("%d-%d", now, i)) + db.AssociatePDUWithDestination(pc.Context(), transactionID, destination, nid) + } + + ev := mustCreatePDU(t) + err := queues.SendEvent(ev, "localhost", []gomatrixserverlib.ServerName{destination}) + assert.NoError(t, err) + + check := func(log poll.LogT) poll.Result { + if fc.txCount.Load() == pduMultiplier+1 { // +1 for the extra SendEvent() + data, dbErr := db.GetPendingPDUs(pc.Context(), destination, 200) + assert.NoError(t, dbErr) + if len(data) == 0 { + return poll.Success() + } + return poll.Continue("waiting for all events to be removed from database. Currently present PDU: %d", len(data)) + } + return poll.Continue("waiting for the right amount of send attempts before checking database. Currently %d", fc.txCount.Load()) + } + poll.WaitOn(t, check, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond)) + // }) +} + +func TestSendEDUBatches(t *testing.T) { + t.Parallel() + failuresUntilBlacklist := uint32(16) + destination := gomatrixserverlib.ServerName("remotehost") + + // test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + // db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, true, t, dbType, true) + db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, true, t, test.DBTypeSQLite, false) + defer close() + defer func() { + pc.ShutdownDendrite() + <-pc.WaitForShutdown() + }() + + // Populate database with > maxEDUsPerTransaction + eduMultiplier := uint32(3) + for i := 0; i < maxEDUsPerTransaction*int(eduMultiplier); i++ { + ev := mustCreateEDU(t) + ephemeralJSON, _ := json.Marshal(ev) + nid, _ := db.StoreJSON(pc.Context(), string(ephemeralJSON)) + db.AssociateEDUWithDestination(pc.Context(), destination, nid, ev.Type, nil) + } + + ev := mustCreateEDU(t) + err := queues.SendEDU(ev, "localhost", []gomatrixserverlib.ServerName{destination}) + assert.NoError(t, err) + + check := func(log poll.LogT) poll.Result { + if fc.txCount.Load() == eduMultiplier+1 { // +1 for the extra SendEvent() + data, dbErr := db.GetPendingEDUs(pc.Context(), destination, 200) + assert.NoError(t, dbErr) + if len(data) == 0 { + return poll.Success() + } + return poll.Continue("waiting for all events to be removed from database. Currently present EDU: %d", len(data)) + } + return poll.Continue("waiting for the right amount of send attempts before checking database. Currently %d", fc.txCount.Load()) + } + poll.WaitOn(t, check, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond)) + // }) +} + +func TestSendPDUAndEDUBatches(t *testing.T) { + t.Parallel() + failuresUntilBlacklist := uint32(16) + destination := gomatrixserverlib.ServerName("remotehost") + + // test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + // db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, true, t, dbType, true) + db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, true, t, test.DBTypeSQLite, false) + defer close() + defer func() { + pc.ShutdownDendrite() + <-pc.WaitForShutdown() + }() + + // Populate database with > maxEDUsPerTransaction + multiplier := uint32(3) + + for i := 0; i < maxPDUsPerTransaction*int(multiplier)+1; i++ { + ev := mustCreatePDU(t) + headeredJSON, _ := json.Marshal(ev) + nid, _ := db.StoreJSON(pc.Context(), string(headeredJSON)) + now := gomatrixserverlib.AsTimestamp(time.Now()) + transactionID := gomatrixserverlib.TransactionID(fmt.Sprintf("%d-%d", now, i)) + db.AssociatePDUWithDestination(pc.Context(), transactionID, destination, nid) + } + + for i := 0; i < maxEDUsPerTransaction*int(multiplier); i++ { + ev := mustCreateEDU(t) + ephemeralJSON, _ := json.Marshal(ev) + nid, _ := db.StoreJSON(pc.Context(), string(ephemeralJSON)) + db.AssociateEDUWithDestination(pc.Context(), destination, nid, ev.Type, nil) + } + + ev := mustCreateEDU(t) + err := queues.SendEDU(ev, "localhost", []gomatrixserverlib.ServerName{destination}) + assert.NoError(t, err) + + check := func(log poll.LogT) poll.Result { + if fc.txCount.Load() == multiplier+1 { // +1 for the extra SendEvent() + pduData, dbErrPDU := db.GetPendingPDUs(pc.Context(), destination, 200) + assert.NoError(t, dbErrPDU) + eduData, dbErrEDU := db.GetPendingEDUs(pc.Context(), destination, 200) + assert.NoError(t, dbErrEDU) + if len(pduData) == 0 && len(eduData) == 0 { + return poll.Success() + } + return poll.Continue("waiting for all events to be removed from database. Currently present PDU: %d EDU: %d", len(pduData), len(eduData)) + } + return poll.Continue("waiting for the right amount of send attempts before checking database. Currently %d", fc.txCount.Load()) + } + poll.WaitOn(t, check, poll.WithTimeout(5*time.Second), poll.WithDelay(100*time.Millisecond)) + // }) +} + +func TestExternalFailureBackoffDoesntStartQueue(t *testing.T) { + t.Parallel() + failuresUntilBlacklist := uint32(16) + destination := gomatrixserverlib.ServerName("remotehost") + db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, true, t, test.DBTypeSQLite, false) + defer close() + defer func() { + pc.ShutdownDendrite() + <-pc.WaitForShutdown() + }() + + dest := queues.getQueue(destination) + queues.statistics.ForServer(destination).Failure() + + ev := mustCreatePDU(t) + headeredJSON, _ := json.Marshal(ev) + nid, _ := db.StoreJSON(pc.Context(), string(headeredJSON)) + now := gomatrixserverlib.AsTimestamp(time.Now()) + transactionID := gomatrixserverlib.TransactionID(fmt.Sprintf("%d-%d", now, 1)) + db.AssociatePDUWithDestination(pc.Context(), transactionID, destination, nid) + + pollEnd := time.Now().Add(3 * time.Second) + runningCheck := func(log poll.LogT) poll.Result { + if dest.running.Load() || fc.txCount.Load() > 0 { + return poll.Error(fmt.Errorf("The queue was started")) + } + if time.Now().After(pollEnd) { + // Allow more than enough time for the queue to be started in the case + // of backoff triggering it to start. + return poll.Success() + } + return poll.Continue("waiting to ensure queue doesn't start.") + } + poll.WaitOn(t, runningCheck, poll.WithTimeout(4*time.Second), poll.WithDelay(100*time.Millisecond)) +} + +func TestQueueInteractsWithRealDatabasePDUAndEDU(t *testing.T) { + // NOTE : Only one test case against real databases can be run at a time. + t.Parallel() + failuresUntilBlacklist := uint32(1) + destination := gomatrixserverlib.ServerName("remotehost") + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, false, t, dbType, true) + // NOTE : These defers aren't called if go test is killed so the dbs may not get cleaned up. + defer close() + defer func() { + pc.ShutdownDendrite() + <-pc.WaitForShutdown() + }() + + // NOTE : getQueue before sending event to ensure we grab the same queue reference + // before it is blacklisted and deleted. + dest := queues.getQueue(destination) + ev := mustCreatePDU(t) + err := queues.SendEvent(ev, "localhost", []gomatrixserverlib.ServerName{destination}) + assert.NoError(t, err) + + edu := mustCreateEDU(t) + errEDU := queues.SendEDU(edu, "localhost", []gomatrixserverlib.ServerName{destination}) + assert.NoError(t, errEDU) + + checkBlacklisted := func(log poll.LogT) poll.Result { + if fc.txCount.Load() == failuresUntilBlacklist { + pduData, dbErrPDU := db.GetPendingPDUs(pc.Context(), destination, 200) + assert.NoError(t, dbErrPDU) + eduData, dbErrEDU := db.GetPendingEDUs(pc.Context(), destination, 200) + assert.NoError(t, dbErrEDU) + if len(pduData) == 1 && len(eduData) == 1 { + if val, _ := db.IsServerBlacklisted(destination); val { + if !dest.running.Load() { + return poll.Success() + } + return poll.Continue("waiting for queue to stop completely") + } + return poll.Continue("waiting for server to be blacklisted") + } + return poll.Continue("waiting for events to be added to database. Currently present PDU: %d EDU: %d", len(pduData), len(eduData)) + } + return poll.Continue("waiting for more send attempts before checking database. Currently %d", fc.txCount.Load()) + } + poll.WaitOn(t, checkBlacklisted, poll.WithTimeout(10*time.Second), poll.WithDelay(100*time.Millisecond)) + + fc.shouldTxSucceed = true + db.RemoveServerFromBlacklist(destination) + queues.RetryServer(destination) + checkRetry := func(log poll.LogT) poll.Result { + pduData, dbErrPDU := db.GetPendingPDUs(pc.Context(), destination, 200) + assert.NoError(t, dbErrPDU) + eduData, dbErrEDU := db.GetPendingEDUs(pc.Context(), destination, 200) + assert.NoError(t, dbErrEDU) + if len(pduData) == 0 && len(eduData) == 0 { + return poll.Success() + } + return poll.Continue("waiting for events to be removed from database. Currently present PDU: %d EDU: %d", len(pduData), len(eduData)) + } + poll.WaitOn(t, checkRetry, poll.WithTimeout(10*time.Second), poll.WithDelay(100*time.Millisecond)) + }) +} diff --git a/federationapi/statistics/statistics.go b/federationapi/statistics/statistics.go index db6d5c735..2ba99112c 100644 --- a/federationapi/statistics/statistics.go +++ b/federationapi/statistics/statistics.go @@ -2,6 +2,7 @@ package statistics import ( "math" + "math/rand" "sync" "time" @@ -20,12 +21,23 @@ type Statistics struct { servers map[gomatrixserverlib.ServerName]*ServerStatistics mutex sync.RWMutex + backoffTimers map[gomatrixserverlib.ServerName]*time.Timer + backoffMutex sync.RWMutex + // How many times should we tolerate consecutive failures before we // just blacklist the host altogether? The backoff is exponential, // so the max time here to attempt is 2**failures seconds. FailuresUntilBlacklist uint32 } +func NewStatistics(db storage.Database, failuresUntilBlacklist uint32) Statistics { + return Statistics{ + DB: db, + FailuresUntilBlacklist: failuresUntilBlacklist, + backoffTimers: make(map[gomatrixserverlib.ServerName]*time.Timer), + } +} + // ForServer returns server statistics for the given server name. If it // does not exist, it will create empty statistics and return those. func (s *Statistics) ForServer(serverName gomatrixserverlib.ServerName) *ServerStatistics { @@ -45,7 +57,6 @@ func (s *Statistics) ForServer(serverName gomatrixserverlib.ServerName) *ServerS server = &ServerStatistics{ statistics: s, serverName: serverName, - interrupt: make(chan struct{}), } s.servers[serverName] = server s.mutex.Unlock() @@ -64,29 +75,43 @@ func (s *Statistics) ForServer(serverName gomatrixserverlib.ServerName) *ServerS // many times we failed etc. It also manages the backoff time and black- // listing a remote host if it remains uncooperative. type ServerStatistics struct { - statistics *Statistics // - serverName gomatrixserverlib.ServerName // - blacklisted atomic.Bool // is the node blacklisted - backoffStarted atomic.Bool // is the backoff started - backoffUntil atomic.Value // time.Time until this backoff interval ends - backoffCount atomic.Uint32 // number of times BackoffDuration has been called - interrupt chan struct{} // interrupts the backoff goroutine - successCounter atomic.Uint32 // how many times have we succeeded? + statistics *Statistics // + serverName gomatrixserverlib.ServerName // + blacklisted atomic.Bool // is the node blacklisted + backoffStarted atomic.Bool // is the backoff started + backoffUntil atomic.Value // time.Time until this backoff interval ends + backoffCount atomic.Uint32 // number of times BackoffDuration has been called + successCounter atomic.Uint32 // how many times have we succeeded? + backoffNotifier func() // notifies destination queue when backoff completes + notifierMutex sync.Mutex } +const maxJitterMultiplier = 1.4 +const minJitterMultiplier = 0.8 + // duration returns how long the next backoff interval should be. func (s *ServerStatistics) duration(count uint32) time.Duration { - return time.Second * time.Duration(math.Exp2(float64(count))) + // Add some jitter to minimise the chance of having multiple backoffs + // ending at the same time. + jitter := rand.Float64()*(maxJitterMultiplier-minJitterMultiplier) + minJitterMultiplier + duration := time.Millisecond * time.Duration(math.Exp2(float64(count))*jitter*1000) + return duration } // cancel will interrupt the currently active backoff. func (s *ServerStatistics) cancel() { s.blacklisted.Store(false) s.backoffUntil.Store(time.Time{}) - select { - case s.interrupt <- struct{}{}: - default: - } + + s.ClearBackoff() +} + +// AssignBackoffNotifier configures the channel to send to when +// a backoff completes. +func (s *ServerStatistics) AssignBackoffNotifier(notifier func()) { + s.notifierMutex.Lock() + defer s.notifierMutex.Unlock() + s.backoffNotifier = notifier } // Success updates the server statistics with a new successful @@ -95,8 +120,8 @@ func (s *ServerStatistics) cancel() { // we will unblacklist it. func (s *ServerStatistics) Success() { s.cancel() - s.successCounter.Inc() s.backoffCount.Store(0) + s.successCounter.Inc() if s.statistics.DB != nil { if err := s.statistics.DB.RemoveServerFromBlacklist(s.serverName); err != nil { logrus.WithError(err).Errorf("Failed to remove %q from blacklist", s.serverName) @@ -105,13 +130,17 @@ func (s *ServerStatistics) Success() { } // Failure marks a failure and starts backing off if needed. -// The next call to BackoffIfRequired will do the right thing -// after this. It will return the time that the current failure +// It will return the time that the current failure // will result in backoff waiting until, and a bool signalling // whether we have blacklisted and therefore to give up. func (s *ServerStatistics) Failure() (time.Time, bool) { + // Return immediately if we have blacklisted this node. + if s.blacklisted.Load() { + return time.Time{}, true + } + // If we aren't already backing off, this call will start - // a new backoff period. Increase the failure counter and + // a new backoff period, increase the failure counter and // start a goroutine which will wait out the backoff and // unset the backoffStarted flag when done. if s.backoffStarted.CompareAndSwap(false, true) { @@ -122,40 +151,48 @@ func (s *ServerStatistics) Failure() (time.Time, bool) { logrus.WithError(err).Errorf("Failed to add %q to blacklist", s.serverName) } } + s.ClearBackoff() return time.Time{}, true } - go func() { - until, ok := s.backoffUntil.Load().(time.Time) - if ok && !until.IsZero() { - select { - case <-time.After(time.Until(until)): - case <-s.interrupt: - } - s.backoffStarted.Store(false) - } - }() + // We're starting a new back off so work out what the next interval + // will be. + count := s.backoffCount.Load() + until := time.Now().Add(s.duration(count)) + s.backoffUntil.Store(until) + + s.statistics.backoffMutex.Lock() + defer s.statistics.backoffMutex.Unlock() + s.statistics.backoffTimers[s.serverName] = time.AfterFunc(time.Until(until), s.backoffFinished) } - // Check if we have blacklisted this node. - if s.blacklisted.Load() { - return time.Now(), true - } + return s.backoffUntil.Load().(time.Time), false +} - // If we're already backing off and we haven't yet surpassed - // the deadline then return that. Repeated calls to Failure - // within a single backoff interval will have no side effects. - if until, ok := s.backoffUntil.Load().(time.Time); ok && !time.Now().After(until) { - return until, false +// ClearBackoff stops the backoff timer for this destination if it is running +// and removes the timer from the backoffTimers map. +func (s *ServerStatistics) ClearBackoff() { + // If the timer is still running then stop it so it's memory is cleaned up sooner. + s.statistics.backoffMutex.Lock() + defer s.statistics.backoffMutex.Unlock() + if timer, ok := s.statistics.backoffTimers[s.serverName]; ok { + timer.Stop() } + delete(s.statistics.backoffTimers, s.serverName) - // We're either backing off and have passed the deadline, or - // we aren't backing off, so work out what the next interval - // will be. - count := s.backoffCount.Load() - until := time.Now().Add(s.duration(count)) - s.backoffUntil.Store(until) - return until, false + s.backoffStarted.Store(false) +} + +// backoffFinished will clear the previous backoff and notify the destination queue. +func (s *ServerStatistics) backoffFinished() { + s.ClearBackoff() + + // Notify the destinationQueue if one is currently running. + s.notifierMutex.Lock() + defer s.notifierMutex.Unlock() + if s.backoffNotifier != nil { + s.backoffNotifier() + } } // BackoffInfo returns information about the current or previous backoff. @@ -174,6 +211,12 @@ func (s *ServerStatistics) Blacklisted() bool { return s.blacklisted.Load() } +// RemoveBlacklist removes the blacklisted status from the server. +func (s *ServerStatistics) RemoveBlacklist() { + s.cancel() + s.backoffCount.Store(0) +} + // SuccessCount returns the number of successful requests. This is // usually useful in constructing transaction IDs. func (s *ServerStatistics) SuccessCount() uint32 { diff --git a/federationapi/statistics/statistics_test.go b/federationapi/statistics/statistics_test.go index 225350b6d..6aa997f44 100644 --- a/federationapi/statistics/statistics_test.go +++ b/federationapi/statistics/statistics_test.go @@ -7,9 +7,7 @@ import ( ) func TestBackoff(t *testing.T) { - stats := Statistics{ - FailuresUntilBlacklist: 7, - } + stats := NewStatistics(nil, 7) server := ServerStatistics{ statistics: &stats, serverName: "test.com", @@ -36,7 +34,7 @@ func TestBackoff(t *testing.T) { // Get the duration. _, blacklist := server.BackoffInfo() - duration := time.Until(until).Round(time.Second) + duration := time.Until(until) // Unset the backoff, or otherwise our next call will think that // there's a backoff in progress and return the same result. @@ -57,8 +55,17 @@ func TestBackoff(t *testing.T) { // Check if the duration is what we expect. t.Logf("Backoff %d is for %s", i, duration) - if wanted := time.Second * time.Duration(math.Exp2(float64(i))); !blacklist && duration != wanted { - t.Fatalf("Backoff %d should have been %s but was %s", i, wanted, duration) + roundingAllowance := 0.01 + minDuration := time.Millisecond * time.Duration(math.Exp2(float64(i))*minJitterMultiplier*1000-roundingAllowance) + maxDuration := time.Millisecond * time.Duration(math.Exp2(float64(i))*maxJitterMultiplier*1000+roundingAllowance) + var inJitterRange bool + if duration >= minDuration && duration <= maxDuration { + inJitterRange = true + } else { + inJitterRange = false + } + if !blacklist && !inJitterRange { + t.Fatalf("Backoff %d should have been between %s and %s but was %s", i, minDuration, maxDuration, duration) } } } diff --git a/federationapi/storage/shared/storage.go b/federationapi/storage/shared/storage.go index 9e40f311c..6afb313a8 100644 --- a/federationapi/storage/shared/storage.go +++ b/federationapi/storage/shared/storage.go @@ -52,6 +52,10 @@ type Receipt struct { nid int64 } +func NewReceipt(nid int64) Receipt { + return Receipt{nid: nid} +} + func (r *Receipt) String() string { return fmt.Sprintf("%d", r.nid) } diff --git a/go.mod b/go.mod index 911d36c1c..2248e73c6 100644 --- a/go.mod +++ b/go.mod @@ -50,6 +50,7 @@ require ( golang.org/x/term v0.0.0-20220919170432-7a66f970e087 gopkg.in/h2non/bimg.v1 v1.1.9 gopkg.in/yaml.v2 v2.4.0 + gotest.tools/v3 v3.4.0 nhooyr.io/websocket v1.8.7 ) @@ -127,7 +128,6 @@ require ( gopkg.in/macaroon.v2 v2.1.0 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - gotest.tools/v3 v3.4.0 // indirect ) go 1.18 From f3dae0e749ca35b1527fbfcb0371e89d0e9833ab Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 19 Oct 2022 11:40:38 +0100 Subject: [PATCH 087/207] Bump nokogiri from 1.13.6 to 1.13.9 in /docs (#2809) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [nokogiri](https://github.com/sparklemotion/nokogiri) from 1.13.6 to 1.13.9.
Release notes

Sourced from nokogiri's releases.

1.13.9 / 2022-10-18

Security

Dependencies

  • [CRuby] Vendored libxml2 is updated to v2.10.3 from v2.9.14.
  • [CRuby] Vendored libxslt is updated to v1.1.37 from v1.1.35.
  • [CRuby] Vendored zlib is updated from 1.2.12 to 1.2.13. (See LICENSE-DEPENDENCIES.md for details on which packages redistribute this library.)

Fixed

  • [CRuby] Nokogiri::XML::Namespace objects, when compacted, update their internal struct's reference to the Ruby object wrapper. Previously, with GC compaction enabled, a segmentation fault was possible after compaction was triggered. [#2658] (Thanks, @​eightbitraptor and @​peterzhu2118!)
  • [CRuby] Document#remove_namespaces! now defers freeing the underlying xmlNs struct until the Document is GCed. Previously, maintaining a reference to a Namespace object that was removed in this way could lead to a segfault. [#2658]

sha256 checksums:

9b69829561d30c4461ea803baeaf3460e8b145cff7a26ce397119577a4083a02
nokogiri-1.13.9-aarch64-linux.gem
e76ebb4b7b2e02c72b2d1541289f8b0679fb5984867cf199d89b8ef485764956
nokogiri-1.13.9-arm64-darwin.gem
15bae7d08bddeaa898d8e3f558723300137c26a2dc2632a1f89c8574c4467165
nokogiri-1.13.9-java.gem
f6a1dbc7229184357f3129503530af73cc59ceba4932c700a458a561edbe04b9
nokogiri-1.13.9-x64-mingw-ucrt.gem
36d935d799baa4dc488024f71881ff0bc8b172cecdfc54781169c40ec02cbdb3
nokogiri-1.13.9-x64-mingw32.gem
ebaf82aa9a11b8fafb67873d19ee48efb565040f04c898cdce8ca0cd53ff1a12
nokogiri-1.13.9-x86-linux.gem
11789a2a11b28bc028ee111f23311461104d8c4468d5b901ab7536b282504154
nokogiri-1.13.9-x86-mingw32.gem
01830e1646803ff91c0fe94bc768ff40082c6de8cfa563dafd01b3f7d5f9d795
nokogiri-1.13.9-x86_64-darwin.gem
8e93b8adec22958013799c8690d81c2cdf8a90b6f6e8150ab22e11895844d781
nokogiri-1.13.9-x86_64-linux.gem
96f37c1baf0234d3ae54c2c89aef7220d4a8a1b03d2675ff7723565b0a095531
nokogiri-1.13.9.gem

1.13.8 / 2022-07-23

Deprecated

  • XML::Reader#attribute_nodes is deprecated due to incompatibility between libxml2's xmlReader memory semantics and Ruby's garbage collector. Although this method continues to exist for backwards compatibility, it is unsafe to call and may segfault. This method will be removed in a future version of Nokogiri, and callers should use #attribute_hash instead. [#2598]

Improvements

  • XML::Reader#attribute_hash is a new method to safely retrieve the attributes of a node from XML::Reader. [#2598, #2599]

Fixed

... (truncated)

Changelog

Sourced from nokogiri's changelog.

1.13.9 / 2022-10-18

Security

Dependencies

  • [CRuby] Vendored libxml2 is updated to v2.10.3 from v2.9.14.
  • [CRuby] Vendored libxslt is updated to v1.1.37 from v1.1.35.
  • [CRuby] Vendored zlib is updated from 1.2.12 to 1.2.13. (See LICENSE-DEPENDENCIES.md for details on which packages redistribute this library.)

Fixed

  • [CRuby] Nokogiri::XML::Namespace objects, when compacted, update their internal struct's reference to the Ruby object wrapper. Previously, with GC compaction enabled, a segmentation fault was possible after compaction was triggered. [#2658] (Thanks, @​eightbitraptor and @​peterzhu2118!)
  • [CRuby] Document#remove_namespaces! now defers freeing the underlying xmlNs struct until the Document is GCed. Previously, maintaining a reference to a Namespace object that was removed in this way could lead to a segfault. [#2658]

1.13.8 / 2022-07-23

Deprecated

  • XML::Reader#attribute_nodes is deprecated due to incompatibility between libxml2's xmlReader memory semantics and Ruby's garbage collector. Although this method continues to exist for backwards compatibility, it is unsafe to call and may segfault. This method will be removed in a future version of Nokogiri, and callers should use #attribute_hash instead. [#2598]

Improvements

  • XML::Reader#attribute_hash is a new method to safely retrieve the attributes of a node from XML::Reader. [#2598, #2599]

Fixed

  • [CRuby] Calling XML::Reader#attributes is now safe to call. In Nokogiri <= 1.13.7 this method may segfault. [#2598, #2599]

1.13.7 / 2022-07-12

Fixed

XML::Node objects, when compacted, update their internal struct's reference to the Ruby object wrapper. Previously, with GC compaction enabled, a segmentation fault was possible after compaction was triggered. [#2578] (Thanks, @​eightbitraptor!)

Commits
  • 897759c version bump to v1.13.9
  • aeb1ac3 doc: update CHANGELOG
  • c663e49 Merge pull request #2671 from sparklemotion/flavorjones-update-zlib-1.2.13_v1...
  • 212e07d ext: hack to cross-compile zlib v1.2.13 on darwin
  • 76dbc8c dep: update zlib to v1.2.13
  • 24e3a9c doc: update CHANGELOG
  • 4db3b4d Merge pull request #2668 from sparklemotion/flavorjones-namespace-scopes-comp...
  • 73d73d6 fix: Document#remove_namespaces! use-after-free bug
  • 5f58b34 fix: namespace nodes behave properly when compacted
  • b08a858 test: repro namespace_scopes compaction issue
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=nokogiri&package-manager=bundler&previous-version=1.13.6&new-version=1.13.9)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) - `@dependabot use these labels` will set the current labels as the default for future PRs for this repo and language - `@dependabot use these reviewers` will set the current reviewers as the default for future PRs for this repo and language - `@dependabot use these assignees` will set the current assignees as the default for future PRs for this repo and language - `@dependabot use this milestone` will set the current milestone as the default for future PRs for this repo and language You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/matrix-org/dendrite/network/alerts).
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- docs/Gemfile.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock index bc73df728..c7ba43711 100644 --- a/docs/Gemfile.lock +++ b/docs/Gemfile.lock @@ -231,9 +231,9 @@ GEM jekyll-seo-tag (~> 2.1) minitest (5.15.0) multipart-post (2.1.1) - nokogiri (1.13.6-arm64-darwin) + nokogiri (1.13.9-arm64-darwin) racc (~> 1.4) - nokogiri (1.13.6-x86_64-linux) + nokogiri (1.13.9-x86_64-linux) racc (~> 1.4) octokit (4.22.0) faraday (>= 0.9) From c1463db6c9183aa67ef41e7ea85ed36dc5817d18 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 19 Oct 2022 12:03:12 +0100 Subject: [PATCH 088/207] Fix concurrent map write in key server --- keyserver/internal/internal.go | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go index 06fc4987c..d2ea20935 100644 --- a/keyserver/internal/internal.go +++ b/keyserver/internal/internal.go @@ -250,6 +250,7 @@ func (a *KeyInternalAPI) PerformMarkAsStaleIfNeeded(ctx context.Context, req *ap // nolint:gocyclo func (a *KeyInternalAPI) QueryKeys(ctx context.Context, req *api.QueryKeysRequest, res *api.QueryKeysResponse) error { + var respMu sync.Mutex res.DeviceKeys = make(map[string]map[string]json.RawMessage) res.MasterKeys = make(map[string]gomatrixserverlib.CrossSigningKey) res.SelfSigningKeys = make(map[string]gomatrixserverlib.CrossSigningKey) @@ -329,7 +330,7 @@ func (a *KeyInternalAPI) QueryKeys(ctx context.Context, req *api.QueryKeysReques } // attempt to satisfy key queries from the local database first as we should get device updates pushed to us - domainToDeviceKeys = a.remoteKeysFromDatabase(ctx, res, domainToDeviceKeys) + domainToDeviceKeys = a.remoteKeysFromDatabase(ctx, res, &respMu, domainToDeviceKeys) if len(domainToDeviceKeys) > 0 || len(domainToCrossSigningKeys) > 0 { // perform key queries for remote devices a.queryRemoteKeys(ctx, req.Timeout, res, domainToDeviceKeys, domainToCrossSigningKeys) @@ -407,7 +408,7 @@ func (a *KeyInternalAPI) QueryKeys(ctx context.Context, req *api.QueryKeysReques } func (a *KeyInternalAPI) remoteKeysFromDatabase( - ctx context.Context, res *api.QueryKeysResponse, domainToDeviceKeys map[string]map[string][]string, + ctx context.Context, res *api.QueryKeysResponse, respMu *sync.Mutex, domainToDeviceKeys map[string]map[string][]string, ) map[string]map[string][]string { fetchRemote := make(map[string]map[string][]string) for domain, userToDeviceMap := range domainToDeviceKeys { @@ -415,7 +416,7 @@ func (a *KeyInternalAPI) remoteKeysFromDatabase( // we can't safely return keys from the db when all devices are requested as we don't // know if one has just been added. if len(deviceIDs) > 0 { - err := a.populateResponseWithDeviceKeysFromDatabase(ctx, res, userID, deviceIDs) + err := a.populateResponseWithDeviceKeysFromDatabase(ctx, res, respMu, userID, deviceIDs) if err == nil { continue } @@ -542,7 +543,7 @@ func (a *KeyInternalAPI) queryRemoteKeysOnServer( // refresh entries from DB: unlike remoteKeysFromDatabase we know we previously had no device info for this // user so the fact that we're populating all devices here isn't a problem so long as we have devices. respMu.Lock() - err = a.populateResponseWithDeviceKeysFromDatabase(ctx, res, userID, nil) + err = a.populateResponseWithDeviceKeysFromDatabase(ctx, res, respMu, userID, nil) respMu.Unlock() if err != nil { logrus.WithFields(logrus.Fields{ @@ -573,7 +574,7 @@ func (a *KeyInternalAPI) queryRemoteKeysOnServer( // inspecting the failures map though so they can know it's a cached response. for userID, dkeys := range devKeys { // drop the error as it's already a failure at this point - _ = a.populateResponseWithDeviceKeysFromDatabase(ctx, res, userID, dkeys) + _ = a.populateResponseWithDeviceKeysFromDatabase(ctx, res, respMu, userID, dkeys) } // Sytest expects no failures, if we still could retrieve keys, e.g. from local cache @@ -585,7 +586,7 @@ func (a *KeyInternalAPI) queryRemoteKeysOnServer( } func (a *KeyInternalAPI) populateResponseWithDeviceKeysFromDatabase( - ctx context.Context, res *api.QueryKeysResponse, userID string, deviceIDs []string, + ctx context.Context, res *api.QueryKeysResponse, respMu *sync.Mutex, userID string, deviceIDs []string, ) error { keys, err := a.DB.DeviceKeysForUser(ctx, userID, deviceIDs, false) // if we can't query the db or there are fewer keys than requested, fetch from remote. @@ -598,9 +599,11 @@ func (a *KeyInternalAPI) populateResponseWithDeviceKeysFromDatabase( if len(deviceIDs) == 0 && len(keys) == 0 { return fmt.Errorf("DeviceKeysForUser %s returned no keys but wanted all keys, falling back to remote", userID) } + respMu.Lock() if res.DeviceKeys[userID] == nil { res.DeviceKeys[userID] = make(map[string]json.RawMessage) } + respMu.Unlock() for _, key := range keys { if len(key.KeyJSON) == 0 { @@ -610,7 +613,9 @@ func (a *KeyInternalAPI) populateResponseWithDeviceKeysFromDatabase( key.KeyJSON, _ = sjson.SetBytes(key.KeyJSON, "unsigned", struct { DisplayName string `json:"device_display_name,omitempty"` }{key.DisplayName}) + respMu.Lock() res.DeviceKeys[userID][key.DeviceID] = key.KeyJSON + respMu.Unlock() } return nil } From 8cbe14bd6d985ceb2f7c098548a3fbeedfce2d55 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 19 Oct 2022 12:27:34 +0100 Subject: [PATCH 089/207] Fix lock contention --- keyserver/internal/internal.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go index d2ea20935..89621aa87 100644 --- a/keyserver/internal/internal.go +++ b/keyserver/internal/internal.go @@ -542,9 +542,7 @@ func (a *KeyInternalAPI) queryRemoteKeysOnServer( } // refresh entries from DB: unlike remoteKeysFromDatabase we know we previously had no device info for this // user so the fact that we're populating all devices here isn't a problem so long as we have devices. - respMu.Lock() err = a.populateResponseWithDeviceKeysFromDatabase(ctx, res, respMu, userID, nil) - respMu.Unlock() if err != nil { logrus.WithFields(logrus.Fields{ logrus.ErrorKey: err, @@ -568,6 +566,7 @@ func (a *KeyInternalAPI) queryRemoteKeysOnServer( res.Failures[serverName] = map[string]interface{}{ "message": err.Error(), } + respMu.Unlock() // last ditch, use the cache only. This is good for when clients hit /keys/query and the remote server // is down, better to return something than nothing at all. Clients can know about the failure by @@ -578,11 +577,11 @@ func (a *KeyInternalAPI) queryRemoteKeysOnServer( } // Sytest expects no failures, if we still could retrieve keys, e.g. from local cache + respMu.Lock() if len(res.DeviceKeys) > 0 { delete(res.Failures, serverName) } respMu.Unlock() - } func (a *KeyInternalAPI) populateResponseWithDeviceKeysFromDatabase( From e79bfd8fd55781783482cb45ae6d4e78062bb8ac Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Wed, 19 Oct 2022 14:05:39 +0200 Subject: [PATCH 090/207] Get state deltas without filters (#2810) This makes the following changes: - get state deltas without the user supplied filter, so we can actually "calculate" state transitions - closes `stmt` when using SQLite - Adds presence for users who newly joined a room, even if the syncing user already knows about the presence status (should fix https://github.com/matrix-org/complement/pull/516) --- .../postgres/output_room_events_table.go | 77 ++++++++++++------- syncapi/storage/shared/storage_sync.go | 30 ++++++-- .../sqlite3/output_room_events_table.go | 45 +++++++++-- syncapi/streams/stream_pdu.go | 16 ++-- syncapi/streams/stream_presence.go | 3 +- syncapi/sync/request.go | 19 ++--- syncapi/types/provider.go | 5 +- sytest-blacklist | 6 -- sytest-whitelist | 8 +- 9 files changed, 144 insertions(+), 65 deletions(-) diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go index b562e6804..0ecbdf4d2 100644 --- a/syncapi/storage/postgres/output_room_events_table.go +++ b/syncapi/storage/postgres/output_room_events_table.go @@ -28,8 +28,9 @@ import ( "github.com/matrix-org/dendrite/syncapi/types" "github.com/lib/pq" - "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" + + "github.com/matrix-org/dendrite/internal/sqlutil" ) const outputRoomEventsSchema = ` @@ -133,7 +134,7 @@ const updateEventJSONSQL = "" + "UPDATE syncapi_output_room_events SET headered_event_json=$1 WHERE event_id=$2" // In order for us to apply the state updates correctly, rows need to be ordered in the order they were received (id). -const selectStateInRangeSQL = "" + +const selectStateInRangeFilteredSQL = "" + "SELECT event_id, id, headered_event_json, exclude_from_sync, add_state_ids, remove_state_ids, history_visibility" + " FROM syncapi_output_room_events" + " WHERE (id > $1 AND id <= $2) AND (add_state_ids IS NOT NULL OR remove_state_ids IS NOT NULL)" + @@ -146,6 +147,15 @@ const selectStateInRangeSQL = "" + " ORDER BY id ASC" + " LIMIT $9" +// In order for us to apply the state updates correctly, rows need to be ordered in the order they were received (id). +const selectStateInRangeSQL = "" + + "SELECT event_id, id, headered_event_json, exclude_from_sync, add_state_ids, remove_state_ids, history_visibility" + + " FROM syncapi_output_room_events" + + " WHERE (id > $1 AND id <= $2) AND (add_state_ids IS NOT NULL OR remove_state_ids IS NOT NULL)" + + " AND room_id = ANY($3)" + + " ORDER BY id ASC" + + " LIMIT $4" + const deleteEventsForRoomSQL = "" + "DELETE FROM syncapi_output_room_events WHERE room_id = $1" @@ -171,20 +181,21 @@ const selectContextAfterEventSQL = "" + const selectSearchSQL = "SELECT id, event_id, headered_event_json FROM syncapi_output_room_events WHERE id > $1 AND type = ANY($2) ORDER BY id ASC LIMIT $3" type outputRoomEventsStatements struct { - insertEventStmt *sql.Stmt - selectEventsStmt *sql.Stmt - selectEventsWitFilterStmt *sql.Stmt - selectMaxEventIDStmt *sql.Stmt - selectRecentEventsStmt *sql.Stmt - selectRecentEventsForSyncStmt *sql.Stmt - selectEarlyEventsStmt *sql.Stmt - selectStateInRangeStmt *sql.Stmt - updateEventJSONStmt *sql.Stmt - deleteEventsForRoomStmt *sql.Stmt - selectContextEventStmt *sql.Stmt - selectContextBeforeEventStmt *sql.Stmt - selectContextAfterEventStmt *sql.Stmt - selectSearchStmt *sql.Stmt + insertEventStmt *sql.Stmt + selectEventsStmt *sql.Stmt + selectEventsWitFilterStmt *sql.Stmt + selectMaxEventIDStmt *sql.Stmt + selectRecentEventsStmt *sql.Stmt + selectRecentEventsForSyncStmt *sql.Stmt + selectEarlyEventsStmt *sql.Stmt + selectStateInRangeFilteredStmt *sql.Stmt + selectStateInRangeStmt *sql.Stmt + updateEventJSONStmt *sql.Stmt + deleteEventsForRoomStmt *sql.Stmt + selectContextEventStmt *sql.Stmt + selectContextBeforeEventStmt *sql.Stmt + selectContextAfterEventStmt *sql.Stmt + selectSearchStmt *sql.Stmt } func NewPostgresEventsTable(db *sql.DB) (tables.Events, error) { @@ -214,6 +225,7 @@ func NewPostgresEventsTable(db *sql.DB) (tables.Events, error) { {&s.selectRecentEventsStmt, selectRecentEventsSQL}, {&s.selectRecentEventsForSyncStmt, selectRecentEventsForSyncSQL}, {&s.selectEarlyEventsStmt, selectEarlyEventsSQL}, + {&s.selectStateInRangeFilteredStmt, selectStateInRangeFilteredSQL}, {&s.selectStateInRangeStmt, selectStateInRangeSQL}, {&s.updateEventJSONStmt, updateEventJSONSQL}, {&s.deleteEventsForRoomStmt, deleteEventsForRoomSQL}, @@ -240,17 +252,28 @@ func (s *outputRoomEventsStatements) SelectStateInRange( ctx context.Context, txn *sql.Tx, r types.Range, stateFilter *gomatrixserverlib.StateFilter, roomIDs []string, ) (map[string]map[string]bool, map[string]types.StreamEvent, error) { - stmt := sqlutil.TxStmt(txn, s.selectStateInRangeStmt) - senders, notSenders := getSendersStateFilterFilter(stateFilter) - rows, err := stmt.QueryContext( - ctx, r.Low(), r.High(), pq.StringArray(roomIDs), - pq.StringArray(senders), - pq.StringArray(notSenders), - pq.StringArray(filterConvertTypeWildcardToSQL(stateFilter.Types)), - pq.StringArray(filterConvertTypeWildcardToSQL(stateFilter.NotTypes)), - stateFilter.ContainsURL, - stateFilter.Limit, - ) + var rows *sql.Rows + var err error + if stateFilter != nil { + stmt := sqlutil.TxStmt(txn, s.selectStateInRangeFilteredStmt) + senders, notSenders := getSendersStateFilterFilter(stateFilter) + rows, err = stmt.QueryContext( + ctx, r.Low(), r.High(), pq.StringArray(roomIDs), + pq.StringArray(senders), + pq.StringArray(notSenders), + pq.StringArray(filterConvertTypeWildcardToSQL(stateFilter.Types)), + pq.StringArray(filterConvertTypeWildcardToSQL(stateFilter.NotTypes)), + stateFilter.ContainsURL, + stateFilter.Limit, + ) + } else { + stmt := sqlutil.TxStmt(txn, s.selectStateInRangeStmt) + rows, err = stmt.QueryContext( + ctx, r.Low(), r.High(), pq.StringArray(roomIDs), + r.High()-r.Low(), + ) + } + if err != nil { return nil, nil, err } diff --git a/syncapi/storage/shared/storage_sync.go b/syncapi/storage/shared/storage_sync.go index cb61c1c26..1f66ccc0e 100644 --- a/syncapi/storage/shared/storage_sync.go +++ b/syncapi/storage/shared/storage_sync.go @@ -5,10 +5,11 @@ import ( "database/sql" "fmt" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/dendrite/internal/eventutil" "github.com/matrix-org/dendrite/syncapi/types" userapi "github.com/matrix-org/dendrite/userapi/api" - "github.com/matrix-org/gomatrixserverlib" ) type DatabaseTransaction struct { @@ -277,6 +278,7 @@ func (d *DatabaseTransaction) GetBackwardTopologyPos( // exclusive of oldPos, inclusive of newPos, for the rooms in which // the user has new membership events. // A list of joined room IDs is also returned in case the caller needs it. +// nolint:gocyclo func (d *DatabaseTransaction) GetStateDeltas( ctx context.Context, device *userapi.Device, r types.Range, userID string, @@ -311,7 +313,7 @@ func (d *DatabaseTransaction) GetStateDeltas( } // get all the state events ever (i.e. for all available rooms) between these two positions - stateNeeded, eventMap, err := d.OutputEvents.SelectStateInRange(ctx, d.txn, r, stateFilter, allRoomIDs) + stateNeeded, eventMap, err := d.OutputEvents.SelectStateInRange(ctx, d.txn, r, nil, allRoomIDs) if err != nil { if err == sql.ErrNoRows { return nil, nil, nil @@ -326,6 +328,22 @@ func (d *DatabaseTransaction) GetStateDeltas( return nil, nil, err } + // get all the state events ever (i.e. for all available rooms) between these two positions + stateNeededFiltered, eventMapFiltered, err := d.OutputEvents.SelectStateInRange(ctx, d.txn, r, stateFilter, allRoomIDs) + if err != nil { + if err == sql.ErrNoRows { + return nil, nil, nil + } + return nil, nil, err + } + stateFiltered, err := d.fetchStateEvents(ctx, d.txn, stateNeededFiltered, eventMapFiltered) + if err != nil { + if err == sql.ErrNoRows { + return nil, nil, nil + } + return nil, nil, err + } + // find out which rooms this user is peeking, if any. // We do this before joins so any peeks get overwritten peeks, err := d.Peeks.SelectPeeksInRange(ctx, d.txn, userID, device.ID, r) @@ -371,6 +389,7 @@ func (d *DatabaseTransaction) GetStateDeltas( // If our membership is now join but the previous membership wasn't // then this is a "join transition", so we'll insert this room. if prevMembership != membership { + newlyJoinedRooms[roomID] = true // Get the full room state, as we'll send that down for a newly // joined room instead of a delta. var s []types.StreamEvent @@ -383,8 +402,7 @@ func (d *DatabaseTransaction) GetStateDeltas( // Add the information for this room into the state so that // it will get added with all of the rest of the joined rooms. - state[roomID] = s - newlyJoinedRooms[roomID] = true + stateFiltered[roomID] = s } // We won't add joined rooms into the delta at this point as they @@ -395,7 +413,7 @@ func (d *DatabaseTransaction) GetStateDeltas( deltas = append(deltas, types.StateDelta{ Membership: membership, MembershipPos: ev.StreamPosition, - StateEvents: d.StreamEventsToEvents(device, stateStreamEvents), + StateEvents: d.StreamEventsToEvents(device, stateFiltered[roomID]), RoomID: roomID, }) break @@ -407,7 +425,7 @@ func (d *DatabaseTransaction) GetStateDeltas( for _, joinedRoomID := range joinedRoomIDs { deltas = append(deltas, types.StateDelta{ Membership: gomatrixserverlib.Join, - StateEvents: d.StreamEventsToEvents(device, state[joinedRoomID]), + StateEvents: d.StreamEventsToEvents(device, stateFiltered[joinedRoomID]), RoomID: joinedRoomID, NewlyJoined: newlyJoinedRooms[joinedRoomID], }) diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index d6a674b9c..77c692ff0 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -29,8 +29,9 @@ import ( "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" - "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/gomatrixserverlib" + + "github.com/matrix-org/dendrite/internal/sqlutil" ) const outputRoomEventsSchema = ` @@ -189,21 +190,36 @@ func (s *outputRoomEventsStatements) SelectStateInRange( for _, roomID := range roomIDs { inputParams = append(inputParams, roomID) } - stmt, params, err := prepareWithFilters( - s.db, txn, stmtSQL, inputParams, - stateFilter.Senders, stateFilter.NotSenders, - stateFilter.Types, stateFilter.NotTypes, - nil, stateFilter.ContainsURL, stateFilter.Limit, FilterOrderAsc, + var ( + stmt *sql.Stmt + params []any + err error ) + if stateFilter != nil { + stmt, params, err = prepareWithFilters( + s.db, txn, stmtSQL, inputParams, + stateFilter.Senders, stateFilter.NotSenders, + stateFilter.Types, stateFilter.NotTypes, + nil, stateFilter.ContainsURL, stateFilter.Limit, FilterOrderAsc, + ) + } else { + stmt, params, err = prepareWithFilters( + s.db, txn, stmtSQL, inputParams, + nil, nil, + nil, nil, + nil, nil, int(r.High()-r.Low()), FilterOrderAsc, + ) + } if err != nil { return nil, nil, fmt.Errorf("s.prepareWithFilters: %w", err) } + defer internal.CloseAndLogIfError(ctx, stmt, "selectStateInRange: stmt.close() failed") rows, err := stmt.QueryContext(ctx, params...) if err != nil { return nil, nil, err } - defer rows.Close() // nolint: errcheck + defer internal.CloseAndLogIfError(ctx, rows, "selectStateInRange: rows.close() failed") // Fetch all the state change events for all rooms between the two positions then loop each event and: // - Keep a cache of the event by ID (99% of state change events are for the event itself) // - For each room ID, build up an array of event IDs which represents cumulative adds/removes @@ -269,6 +285,7 @@ func (s *outputRoomEventsStatements) SelectMaxEventID( ) (id int64, err error) { var nullableID sql.NullInt64 stmt := sqlutil.TxStmt(txn, s.selectMaxEventIDStmt) + defer internal.CloseAndLogIfError(ctx, stmt, "SelectMaxEventID: stmt.close() failed") err = stmt.QueryRowContext(ctx).Scan(&nullableID) if nullableID.Valid { id = nullableID.Int64 @@ -323,6 +340,7 @@ func (s *outputRoomEventsStatements) InsertEvent( return 0, err } insertStmt := sqlutil.TxStmt(txn, s.insertEventStmt) + defer internal.CloseAndLogIfError(ctx, insertStmt, "InsertEvent: stmt.close() failed") _, err = insertStmt.ExecContext( ctx, streamPos, @@ -367,6 +385,7 @@ func (s *outputRoomEventsStatements) SelectRecentEvents( if err != nil { return nil, false, fmt.Errorf("s.prepareWithFilters: %w", err) } + defer internal.CloseAndLogIfError(ctx, stmt, "selectRecentEvents: stmt.close() failed") rows, err := stmt.QueryContext(ctx, params...) if err != nil { @@ -415,6 +434,8 @@ func (s *outputRoomEventsStatements) SelectEarlyEvents( if err != nil { return nil, fmt.Errorf("s.prepareWithFilters: %w", err) } + defer internal.CloseAndLogIfError(ctx, stmt, "SelectEarlyEvents: stmt.close() failed") + rows, err := stmt.QueryContext(ctx, params...) if err != nil { return nil, err @@ -456,6 +477,8 @@ func (s *outputRoomEventsStatements) SelectEvents( if err != nil { return nil, err } + defer internal.CloseAndLogIfError(ctx, stmt, "SelectEvents: stmt.close() failed") + rows, err := stmt.QueryContext(ctx, params...) if err != nil { return nil, err @@ -558,6 +581,10 @@ func (s *outputRoomEventsStatements) SelectContextBeforeEvent( filter.Types, filter.NotTypes, nil, filter.ContainsURL, filter.Limit, FilterOrderDesc, ) + if err != nil { + return + } + defer internal.CloseAndLogIfError(ctx, stmt, "SelectContextBeforeEvent: stmt.close() failed") rows, err := stmt.QueryContext(ctx, params...) if err != nil { @@ -596,6 +623,10 @@ func (s *outputRoomEventsStatements) SelectContextAfterEvent( filter.Types, filter.NotTypes, nil, filter.ContainsURL, filter.Limit, FilterOrderAsc, ) + if err != nil { + return + } + defer internal.CloseAndLogIfError(ctx, stmt, "SelectContextAfterEvent: stmt.close() failed") rows, err := stmt.QueryContext(ctx, params...) if err != nil { diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go index 613ac434f..9ec2b61cd 100644 --- a/syncapi/streams/stream_pdu.go +++ b/syncapi/streams/stream_pdu.go @@ -194,7 +194,7 @@ func (p *PDUStreamProvider) IncrementalSync( } } var pos types.StreamPosition - if pos, err = p.addRoomDeltaToResponse(ctx, snapshot, req.Device, newRange, delta, &eventFilter, &stateFilter, req.Response); err != nil { + if pos, err = p.addRoomDeltaToResponse(ctx, snapshot, req.Device, newRange, delta, &eventFilter, &stateFilter, req); err != nil { req.Log.WithError(err).Error("d.addRoomDeltaToResponse failed") if err == context.DeadlineExceeded || err == context.Canceled || err == sql.ErrTxDone { return newPos @@ -225,7 +225,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( delta types.StateDelta, eventFilter *gomatrixserverlib.RoomEventFilter, stateFilter *gomatrixserverlib.StateFilter, - res *types.Response, + req *types.SyncRequest, ) (types.StreamPosition, error) { if delta.MembershipPos > 0 && delta.Membership == gomatrixserverlib.Leave { // make sure we don't leak recent events after the leave event. @@ -290,8 +290,10 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( hasMembershipChange := false for _, recentEvent := range recentStreamEvents { if recentEvent.Type() == gomatrixserverlib.MRoomMember && recentEvent.StateKey() != nil { + if membership, _ := recentEvent.Membership(); membership == gomatrixserverlib.Join { + req.MembershipChanges[*recentEvent.StateKey()] = struct{}{} + } hasMembershipChange = true - break } } @@ -318,9 +320,9 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(events, gomatrixserverlib.FormatSync) // If we are limited by the filter AND the history visibility filter // didn't "remove" events, return that the response is limited. - jr.Timeline.Limited = limited && len(events) == len(recentEvents) + jr.Timeline.Limited = (limited && len(events) == len(recentEvents)) || delta.NewlyJoined jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.StateEvents, gomatrixserverlib.FormatSync) - res.Rooms.Join[delta.RoomID] = jr + req.Response.Rooms.Join[delta.RoomID] = jr case gomatrixserverlib.Peek: jr := types.NewJoinResponse() @@ -329,7 +331,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( jr.Timeline.Events = gomatrixserverlib.HeaderedToClientEvents(recentEvents, gomatrixserverlib.FormatSync) jr.Timeline.Limited = limited jr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.StateEvents, gomatrixserverlib.FormatSync) - res.Rooms.Peek[delta.RoomID] = jr + req.Response.Rooms.Peek[delta.RoomID] = jr case gomatrixserverlib.Leave: fallthrough // transitions to leave are the same as ban @@ -342,7 +344,7 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( // didn't "remove" events, return that the response is limited. lr.Timeline.Limited = limited && len(events) == len(recentEvents) lr.State.Events = gomatrixserverlib.HeaderedToClientEvents(delta.StateEvents, gomatrixserverlib.FormatSync) - res.Rooms.Leave[delta.RoomID] = lr + req.Response.Rooms.Leave[delta.RoomID] = lr } return latestPosition, nil diff --git a/syncapi/streams/stream_presence.go b/syncapi/streams/stream_presence.go index 8b87af452..030b7c5d5 100644 --- a/syncapi/streams/stream_presence.go +++ b/syncapi/streams/stream_presence.go @@ -121,7 +121,8 @@ func (p *PresenceStreamProvider) IncrementalSync( prevPresence := pres.(*types.PresenceInternal) currentlyActive := prevPresence.CurrentlyActive() skip := prevPresence.Equals(presence) && currentlyActive && req.Device.UserID != presence.UserID - if skip { + _, membershipChange := req.MembershipChanges[presence.UserID] + if skip && !membershipChange { req.Log.Tracef("Skipping presence, no change (%s)", presence.UserID) continue } diff --git a/syncapi/sync/request.go b/syncapi/sync/request.go index 268ed70c6..620dfdcdb 100644 --- a/syncapi/sync/request.go +++ b/syncapi/sync/request.go @@ -91,15 +91,16 @@ func newSyncRequest(req *http.Request, device userapi.Device, syncDB storage.Dat }) return &types.SyncRequest{ - Context: req.Context(), // - Log: logger, // - Device: &device, // - Response: types.NewResponse(), // Populated by all streams - Filter: filter, // - Since: since, // - Timeout: timeout, // - Rooms: make(map[string]string), // Populated by the PDU stream - WantFullState: wantFullState, // + Context: req.Context(), // + Log: logger, // + Device: &device, // + Response: types.NewResponse(), // Populated by all streams + Filter: filter, // + Since: since, // + Timeout: timeout, // + Rooms: make(map[string]string), // Populated by the PDU stream + WantFullState: wantFullState, // + MembershipChanges: make(map[string]struct{}), // Populated by the PDU stream }, nil } diff --git a/syncapi/types/provider.go b/syncapi/types/provider.go index 378cafe99..9a533002b 100644 --- a/syncapi/types/provider.go +++ b/syncapi/types/provider.go @@ -4,9 +4,10 @@ import ( "context" "time" - userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/sirupsen/logrus" + + userapi "github.com/matrix-org/dendrite/userapi/api" ) type SyncRequest struct { @@ -22,6 +23,8 @@ type SyncRequest struct { // Updated by the PDU stream. Rooms map[string]string // Updated by the PDU stream. + MembershipChanges map[string]struct{} + // Updated by the PDU stream. IgnoredUsers IgnoredUsers } diff --git a/sytest-blacklist b/sytest-blacklist index 634c07cf3..fe48fb791 100644 --- a/sytest-blacklist +++ b/sytest-blacklist @@ -39,12 +39,6 @@ Events in rooms with AS-hosted room aliases are sent to AS server Inviting an AS-hosted user asks the AS server Accesing an AS-hosted room alias asks the AS server -# Flakey, need additional investigation - -Messages that notify from another user increment notification_count -Messages that highlight from another user increment unread highlight count -Notifications can be viewed with GET /notifications - # More flakey Guest users can join guest_access rooms diff --git a/sytest-whitelist b/sytest-whitelist index 93d447d28..1387838f7 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -746,4 +746,10 @@ Existing members see new member's presence Inbound federation can return missing events for joined visibility outliers whose auth_events are in a different room are correctly rejected Messages that notify from another user increment notification_count -Messages that highlight from another user increment unread highlight count \ No newline at end of file +Messages that highlight from another user increment unread highlight count +Newly joined room has correct timeline in incremental sync +When user joins a room the state is included in the next sync +When user joins a room the state is included in a gapped sync +Messages that notify from another user increment notification_count +Messages that highlight from another user increment unread highlight count +Notifications can be viewed with GET /notifications \ No newline at end of file From 6a93858125a2ece1c2cc557e11d34e51a67ada45 Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Thu, 20 Oct 2022 10:45:59 +0200 Subject: [PATCH 091/207] Fix race condition --- federationapi/queue/destinationqueue.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/federationapi/queue/destinationqueue.go b/federationapi/queue/destinationqueue.go index 00e02b2d9..768ed1f2b 100644 --- a/federationapi/queue/destinationqueue.go +++ b/federationapi/queue/destinationqueue.go @@ -21,16 +21,17 @@ import ( "sync" "time" + "github.com/matrix-org/gomatrix" + "github.com/matrix-org/gomatrixserverlib" + "github.com/sirupsen/logrus" + "go.uber.org/atomic" + fedapi "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/federationapi/statistics" "github.com/matrix-org/dendrite/federationapi/storage" "github.com/matrix-org/dendrite/federationapi/storage/shared" "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/process" - "github.com/matrix-org/gomatrix" - "github.com/matrix-org/gomatrixserverlib" - "github.com/sirupsen/logrus" - "go.uber.org/atomic" ) const ( @@ -541,6 +542,8 @@ func (oq *destinationQueue) handleTransactionSuccess(pduCount int, eduCount int) // the pending events and EDUs, and wipe our transaction ID. oq.statistics.Success() oq.pendingMutex.Lock() + defer oq.pendingMutex.Unlock() + for i := range oq.pendingPDUs[:pduCount] { oq.pendingPDUs[i] = nil } @@ -549,7 +552,6 @@ func (oq *destinationQueue) handleTransactionSuccess(pduCount int, eduCount int) } oq.pendingPDUs = oq.pendingPDUs[pduCount:] oq.pendingEDUs = oq.pendingEDUs[eduCount:] - oq.pendingMutex.Unlock() if len(oq.pendingPDUs) > 0 || len(oq.pendingEDUs) > 0 { select { From 539c61b3db4a76729e90a52823be89c32cbeb5ec Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Thu, 20 Oct 2022 12:34:53 +0200 Subject: [PATCH 092/207] Remove test from blacklist --- sytest-blacklist | 4 ---- 1 file changed, 4 deletions(-) diff --git a/sytest-blacklist b/sytest-blacklist index fe48fb791..14edf398a 100644 --- a/sytest-blacklist +++ b/sytest-blacklist @@ -22,10 +22,6 @@ Forgotten room messages cannot be paginated Local device key changes get to remote servers with correct prev_id -# Flakey - -Local device key changes appear in /keys/changes - # we don't support groups Remove group category From b58c9bb094f3a069a4f40bbd6cc4a0ac205afcb6 Mon Sep 17 00:00:00 2001 From: devonh Date: Thu, 20 Oct 2022 15:37:35 +0000 Subject: [PATCH 093/207] Fix flakey queue test (#2818) Ensure both events are added to the database, even if the destination is already blacklisted. --- federationapi/queue/queue_test.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/federationapi/queue/queue_test.go b/federationapi/queue/queue_test.go index 6da9e6b30..40419b91f 100644 --- a/federationapi/queue/queue_test.go +++ b/federationapi/queue/queue_test.go @@ -1004,9 +1004,12 @@ func TestQueueInteractsWithRealDatabasePDUAndEDU(t *testing.T) { err := queues.SendEvent(ev, "localhost", []gomatrixserverlib.ServerName{destination}) assert.NoError(t, err) + // NOTE : The server can be blacklisted before this, so manually inject the event + // into the database. edu := mustCreateEDU(t) - errEDU := queues.SendEDU(edu, "localhost", []gomatrixserverlib.ServerName{destination}) - assert.NoError(t, errEDU) + ephemeralJSON, _ := json.Marshal(edu) + nid, _ := db.StoreJSON(pc.Context(), string(ephemeralJSON)) + db.AssociateEDUWithDestination(pc.Context(), destination, nid, edu.Type, nil) checkBlacklisted := func(log poll.LogT) poll.Result { if fc.txCount.Load() == failuresUntilBlacklist { From 90414912012b274f49894b2819f5e6e393928da9 Mon Sep 17 00:00:00 2001 From: devonh Date: Thu, 20 Oct 2022 15:54:18 +0000 Subject: [PATCH 094/207] Mutex protect query keys response (#2812) --- keyserver/internal/internal.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go index 89621aa87..49ef03054 100644 --- a/keyserver/internal/internal.go +++ b/keyserver/internal/internal.go @@ -472,7 +472,9 @@ func (a *KeyInternalAPI) queryRemoteKeys( close(resultCh) }() - for result := range resultCh { + processResult := func(result *gomatrixserverlib.RespQueryKeys) { + respMu.Lock() + defer respMu.Unlock() for userID, nest := range result.DeviceKeys { res.DeviceKeys[userID] = make(map[string]json.RawMessage) for deviceID, deviceKey := range nest { @@ -495,6 +497,10 @@ func (a *KeyInternalAPI) queryRemoteKeys( // TODO: do we want to persist these somewhere now // that we have fetched them? } + + for result := range resultCh { + processResult(result) + } } func (a *KeyInternalAPI) queryRemoteKeysOnServer( From 73e02463cf6e267fdba950d0d231f98f95bc7994 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 21 Oct 2022 09:19:52 +0100 Subject: [PATCH 095/207] Allow `m.read.private` to clear notifications (#2811) Otherwise if a user switches to private read receipts, they may not be able to clear notification counts. --- userapi/consumers/clientapi.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/userapi/consumers/clientapi.go b/userapi/consumers/clientapi.go index c220d35cb..79f1bf06f 100644 --- a/userapi/consumers/clientapi.go +++ b/userapi/consumers/clientapi.go @@ -81,7 +81,7 @@ func (s *OutputReceiptEventConsumer) onMessage(ctx context.Context, msgs []*nats readPos := msg.Header.Get(jetstream.EventID) evType := msg.Header.Get("type") - if readPos == "" || evType != "m.read" { + if readPos == "" || (evType != "m.read" && evType != "m.read.private") { return true } From 40cfb9a4ea23f1c9214553255feb296c2578b213 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Fri, 21 Oct 2022 10:26:22 +0200 Subject: [PATCH 096/207] Fix `invite -> leave -> join` dance when accepting invites (#2817) As mentioned in https://github.com/matrix-org/dendrite/issues/2361#issuecomment-1139394565 and observed by ourselves, this should fix the odd `invite -> leave -> join` dance when accepting invites. --- syncapi/consumers/roomserver.go | 7 +++++++ syncapi/streams/stream_invite.go | 33 ++++++++++++++++++-------------- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/syncapi/consumers/roomserver.go b/syncapi/consumers/roomserver.go index cfbb05327..f767615c8 100644 --- a/syncapi/consumers/roomserver.go +++ b/syncapi/consumers/roomserver.go @@ -428,6 +428,13 @@ func (s *OutputRoomEventConsumer) onRetireInviteEvent( return } + // Only notify clients about retired invite events, if the user didn't accept the invite. + // The PDU stream will also receive an event about accepting the invitation, so there should + // be a "smooth" transition from invite -> join, and not invite -> leave -> join + if msg.Membership == gomatrixserverlib.Join { + return + } + // Notify any active sync requests that the invite has been retired. s.inviteStream.Advance(pduPos) s.notifier.OnNewInvite(types.StreamingToken{InvitePosition: pduPos}, msg.TargetUserID) diff --git a/syncapi/streams/stream_invite.go b/syncapi/streams/stream_invite.go index 7875ffa35..700f25c10 100644 --- a/syncapi/streams/stream_invite.go +++ b/syncapi/streams/stream_invite.go @@ -74,21 +74,26 @@ func (p *InviteStreamProvider) IncrementalSync( return to } for roomID := range retiredInvites { - if _, ok := req.Response.Rooms.Join[roomID]; !ok { - lr := types.NewLeaveResponse() - h := sha256.Sum256(append([]byte(roomID), []byte(strconv.FormatInt(int64(to), 10))...)) - lr.Timeline.Events = append(lr.Timeline.Events, gomatrixserverlib.ClientEvent{ - // fake event ID which muxes in the to position - EventID: "$" + base64.RawURLEncoding.EncodeToString(h[:]), - OriginServerTS: gomatrixserverlib.AsTimestamp(time.Now()), - RoomID: roomID, - Sender: req.Device.UserID, - StateKey: &req.Device.UserID, - Type: "m.room.member", - Content: gomatrixserverlib.RawJSON(`{"membership":"leave"}`), - }) - req.Response.Rooms.Leave[roomID] = lr + if _, ok := req.Response.Rooms.Invite[roomID]; ok { + continue } + if _, ok := req.Response.Rooms.Join[roomID]; ok { + continue + } + lr := types.NewLeaveResponse() + h := sha256.Sum256(append([]byte(roomID), []byte(strconv.FormatInt(int64(to), 10))...)) + lr.Timeline.Events = append(lr.Timeline.Events, gomatrixserverlib.ClientEvent{ + // fake event ID which muxes in the to position + EventID: "$" + base64.RawURLEncoding.EncodeToString(h[:]), + OriginServerTS: gomatrixserverlib.AsTimestamp(time.Now()), + RoomID: roomID, + Sender: req.Device.UserID, + StateKey: &req.Device.UserID, + Type: "m.room.member", + Content: gomatrixserverlib.RawJSON(`{"membership":"leave"}`), + }) + req.Response.Rooms.Leave[roomID] = lr + } return maxID From e57b30172227c4a0b7de15ba635b20921dedda5e Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Fri, 21 Oct 2022 10:48:25 +0200 Subject: [PATCH 097/207] Set `display_name` and/or `avatar_url` for server notices (#2820) This should fix #2815 by making sure we actually set the `display_name` and/or `avatar_url` and create the needed membership event. To avoid creating a new membership event when starting Dendrite, `SetAvatarURL` and `SetDisplayName` now return a `Changed` value, which also makes the regular endpoints idempotent. --- clientapi/routing/profile.go | 127 ++++++++-------------- clientapi/routing/routing.go | 2 +- clientapi/routing/server_notices.go | 31 +++++- userapi/api/api.go | 12 +- userapi/api/api_trace.go | 2 +- userapi/internal/api.go | 14 ++- userapi/inthttp/client.go | 2 +- userapi/storage/interface.go | 4 +- userapi/storage/postgres/profile_table.go | 36 ++++-- userapi/storage/shared/storage.go | 16 ++- userapi/storage/sqlite3/profile_table.go | 40 +++++-- userapi/storage/storage_test.go | 24 ++-- userapi/storage/tables/interface.go | 4 +- userapi/userapi_test.go | 9 +- 14 files changed, 191 insertions(+), 132 deletions(-) diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index 0685c7352..c9647eb1b 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -19,6 +19,8 @@ import ( "net/http" "time" + "github.com/matrix-org/gomatrixserverlib" + appserviceAPI "github.com/matrix-org/dendrite/appservice/api" "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/dendrite/clientapi/httputil" @@ -27,7 +29,6 @@ import ( "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" userapi "github.com/matrix-org/dendrite/userapi/api" - "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrix" "github.com/matrix-org/util" @@ -126,20 +127,6 @@ func SetAvatarURL( } } - res := &userapi.QueryProfileResponse{} - err = profileAPI.QueryProfile(req.Context(), &userapi.QueryProfileRequest{ - UserID: userID, - }, res) - if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("profileAPI.QueryProfile failed") - return jsonerror.InternalServerError() - } - oldProfile := &authtypes.Profile{ - Localpart: localpart, - DisplayName: res.DisplayName, - AvatarURL: res.AvatarURL, - } - setRes := &userapi.PerformSetAvatarURLResponse{} if err = profileAPI.SetAvatarURL(req.Context(), &userapi.PerformSetAvatarURLRequest{ Localpart: localpart, @@ -148,41 +135,17 @@ func SetAvatarURL( util.GetLogger(req.Context()).WithError(err).Error("profileAPI.SetAvatarURL failed") return jsonerror.InternalServerError() } - - var roomsRes api.QueryRoomsForUserResponse - err = rsAPI.QueryRoomsForUser(req.Context(), &api.QueryRoomsForUserRequest{ - UserID: device.UserID, - WantMembership: "join", - }, &roomsRes) - if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("QueryRoomsForUser failed") - return jsonerror.InternalServerError() - } - - newProfile := authtypes.Profile{ - Localpart: localpart, - DisplayName: oldProfile.DisplayName, - AvatarURL: r.AvatarURL, - } - - events, err := buildMembershipEvents( - req.Context(), roomsRes.RoomIDs, newProfile, userID, cfg, evTime, rsAPI, - ) - switch e := err.(type) { - case nil: - case gomatrixserverlib.BadJSONError: + // No need to build new membership events, since nothing changed + if !setRes.Changed { return util.JSONResponse{ - Code: http.StatusBadRequest, - JSON: jsonerror.BadJSON(e.Error()), + Code: http.StatusOK, + JSON: struct{}{}, } - default: - util.GetLogger(req.Context()).WithError(err).Error("buildMembershipEvents failed") - return jsonerror.InternalServerError() } - if err := api.SendEvents(req.Context(), rsAPI, api.KindNew, events, cfg.Matrix.ServerName, cfg.Matrix.ServerName, nil, true); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("SendEvents failed") - return jsonerror.InternalServerError() + response, err := updateProfile(req.Context(), rsAPI, device, setRes.Profile, userID, cfg, evTime) + if err != nil { + return response } return util.JSONResponse{ @@ -255,47 +218,51 @@ func SetDisplayName( } } - pRes := &userapi.QueryProfileResponse{} - err = profileAPI.QueryProfile(req.Context(), &userapi.QueryProfileRequest{ - UserID: userID, - }, pRes) - if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("profileAPI.QueryProfile failed") - return jsonerror.InternalServerError() - } - oldProfile := &authtypes.Profile{ - Localpart: localpart, - DisplayName: pRes.DisplayName, - AvatarURL: pRes.AvatarURL, - } - + profileRes := &userapi.PerformUpdateDisplayNameResponse{} err = profileAPI.SetDisplayName(req.Context(), &userapi.PerformUpdateDisplayNameRequest{ Localpart: localpart, DisplayName: r.DisplayName, - }, &struct{}{}) + }, profileRes) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("profileAPI.SetDisplayName failed") return jsonerror.InternalServerError() } + // No need to build new membership events, since nothing changed + if !profileRes.Changed { + return util.JSONResponse{ + Code: http.StatusOK, + JSON: struct{}{}, + } + } + response, err := updateProfile(req.Context(), rsAPI, device, profileRes.Profile, userID, cfg, evTime) + if err != nil { + return response + } + + return util.JSONResponse{ + Code: http.StatusOK, + JSON: struct{}{}, + } +} + +func updateProfile( + ctx context.Context, rsAPI api.ClientRoomserverAPI, device *userapi.Device, + profile *authtypes.Profile, + userID string, cfg *config.ClientAPI, evTime time.Time, +) (util.JSONResponse, error) { var res api.QueryRoomsForUserResponse - err = rsAPI.QueryRoomsForUser(req.Context(), &api.QueryRoomsForUserRequest{ + err := rsAPI.QueryRoomsForUser(ctx, &api.QueryRoomsForUserRequest{ UserID: device.UserID, WantMembership: "join", }, &res) if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("QueryRoomsForUser failed") - return jsonerror.InternalServerError() - } - - newProfile := authtypes.Profile{ - Localpart: localpart, - DisplayName: r.DisplayName, - AvatarURL: oldProfile.AvatarURL, + util.GetLogger(ctx).WithError(err).Error("QueryRoomsForUser failed") + return jsonerror.InternalServerError(), err } events, err := buildMembershipEvents( - req.Context(), res.RoomIDs, newProfile, userID, cfg, evTime, rsAPI, + ctx, res.RoomIDs, *profile, userID, cfg, evTime, rsAPI, ) switch e := err.(type) { case nil: @@ -303,21 +270,17 @@ func SetDisplayName( return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.BadJSON(e.Error()), - } + }, e default: - util.GetLogger(req.Context()).WithError(err).Error("buildMembershipEvents failed") - return jsonerror.InternalServerError() + util.GetLogger(ctx).WithError(err).Error("buildMembershipEvents failed") + return jsonerror.InternalServerError(), e } - if err := api.SendEvents(req.Context(), rsAPI, api.KindNew, events, cfg.Matrix.ServerName, cfg.Matrix.ServerName, nil, true); err != nil { - util.GetLogger(req.Context()).WithError(err).Error("SendEvents failed") - return jsonerror.InternalServerError() - } - - return util.JSONResponse{ - Code: http.StatusOK, - JSON: struct{}{}, + if err := api.SendEvents(ctx, rsAPI, api.KindNew, events, cfg.Matrix.ServerName, cfg.Matrix.ServerName, nil, true); err != nil { + util.GetLogger(ctx).WithError(err).Error("SendEvents failed") + return jsonerror.InternalServerError(), err } + return util.JSONResponse{}, nil } // getProfile gets the full profile of a user by querying the database or a diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index ec5ca899e..4ca8e59c5 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -178,7 +178,7 @@ func Setup( // server notifications if cfg.Matrix.ServerNotices.Enabled { logrus.Info("Enabling server notices at /_synapse/admin/v1/send_server_notice") - serverNotificationSender, err := getSenderDevice(context.Background(), userAPI, cfg) + serverNotificationSender, err := getSenderDevice(context.Background(), rsAPI, userAPI, cfg) if err != nil { logrus.WithError(err).Fatal("unable to get account for sending sending server notices") } diff --git a/clientapi/routing/server_notices.go b/clientapi/routing/server_notices.go index 7729eddd8..a6a78061d 100644 --- a/clientapi/routing/server_notices.go +++ b/clientapi/routing/server_notices.go @@ -277,6 +277,7 @@ func (r sendServerNoticeRequest) valid() (ok bool) { // It returns an userapi.Device, which is used for building the event func getSenderDevice( ctx context.Context, + rsAPI api.ClientRoomserverAPI, userAPI userapi.ClientUserAPI, cfg *config.ClientAPI, ) (*userapi.Device, error) { @@ -291,16 +292,32 @@ func getSenderDevice( return nil, err } - // set the avatarurl for the user - res := &userapi.PerformSetAvatarURLResponse{} + // Set the avatarurl for the user + avatarRes := &userapi.PerformSetAvatarURLResponse{} if err = userAPI.SetAvatarURL(ctx, &userapi.PerformSetAvatarURLRequest{ Localpart: cfg.Matrix.ServerNotices.LocalPart, AvatarURL: cfg.Matrix.ServerNotices.AvatarURL, - }, res); err != nil { + }, avatarRes); err != nil { util.GetLogger(ctx).WithError(err).Error("userAPI.SetAvatarURL failed") return nil, err } + profile := avatarRes.Profile + + // Set the displayname for the user + displayNameRes := &userapi.PerformUpdateDisplayNameResponse{} + if err = userAPI.SetDisplayName(ctx, &userapi.PerformUpdateDisplayNameRequest{ + Localpart: cfg.Matrix.ServerNotices.LocalPart, + DisplayName: cfg.Matrix.ServerNotices.DisplayName, + }, displayNameRes); err != nil { + util.GetLogger(ctx).WithError(err).Error("userAPI.SetDisplayName failed") + return nil, err + } + + if displayNameRes.Changed { + profile.DisplayName = cfg.Matrix.ServerNotices.DisplayName + } + // Check if we got existing devices deviceRes := &userapi.QueryDevicesResponse{} err = userAPI.QueryDevices(ctx, &userapi.QueryDevicesRequest{ @@ -310,7 +327,15 @@ func getSenderDevice( return nil, err } + // We've got an existing account, return the first device of it if len(deviceRes.Devices) > 0 { + // If there were changes to the profile, create a new membership event + if displayNameRes.Changed || avatarRes.Changed { + _, err = updateProfile(ctx, rsAPI, &deviceRes.Devices[0], profile, accRes.Account.UserID, cfg, time.Now()) + if err != nil { + return nil, err + } + } return &deviceRes.Devices[0], nil } diff --git a/userapi/api/api.go b/userapi/api/api.go index 66ee9c7c8..eef29144a 100644 --- a/userapi/api/api.go +++ b/userapi/api/api.go @@ -96,7 +96,7 @@ type ClientUserAPI interface { PerformAccountDeactivation(ctx context.Context, req *PerformAccountDeactivationRequest, res *PerformAccountDeactivationResponse) error PerformOpenIDTokenCreation(ctx context.Context, req *PerformOpenIDTokenCreationRequest, res *PerformOpenIDTokenCreationResponse) error SetAvatarURL(ctx context.Context, req *PerformSetAvatarURLRequest, res *PerformSetAvatarURLResponse) error - SetDisplayName(ctx context.Context, req *PerformUpdateDisplayNameRequest, res *struct{}) error + SetDisplayName(ctx context.Context, req *PerformUpdateDisplayNameRequest, res *PerformUpdateDisplayNameResponse) error QueryNotifications(ctx context.Context, req *QueryNotificationsRequest, res *QueryNotificationsResponse) error InputAccountData(ctx context.Context, req *InputAccountDataRequest, res *InputAccountDataResponse) error PerformKeyBackup(ctx context.Context, req *PerformKeyBackupRequest, res *PerformKeyBackupResponse) error @@ -579,7 +579,10 @@ type Notification struct { type PerformSetAvatarURLRequest struct { Localpart, AvatarURL string } -type PerformSetAvatarURLResponse struct{} +type PerformSetAvatarURLResponse struct { + Profile *authtypes.Profile `json:"profile"` + Changed bool `json:"changed"` +} type QueryNumericLocalpartResponse struct { ID int64 @@ -606,6 +609,11 @@ type PerformUpdateDisplayNameRequest struct { Localpart, DisplayName string } +type PerformUpdateDisplayNameResponse struct { + Profile *authtypes.Profile `json:"profile"` + Changed bool `json:"changed"` +} + type QueryLocalpartForThreePIDRequest struct { ThreePID, Medium string } diff --git a/userapi/api/api_trace.go b/userapi/api/api_trace.go index 7e2f69615..90834f7e3 100644 --- a/userapi/api/api_trace.go +++ b/userapi/api/api_trace.go @@ -168,7 +168,7 @@ func (t *UserInternalAPITrace) QueryAccountAvailability(ctx context.Context, req return err } -func (t *UserInternalAPITrace) SetDisplayName(ctx context.Context, req *PerformUpdateDisplayNameRequest, res *struct{}) error { +func (t *UserInternalAPITrace) SetDisplayName(ctx context.Context, req *PerformUpdateDisplayNameRequest, res *PerformUpdateDisplayNameResponse) error { err := t.Impl.SetDisplayName(ctx, req, res) util.GetLogger(ctx).Infof("SetDisplayName req=%+v res=%+v", js(req), js(res)) return err diff --git a/userapi/internal/api.go b/userapi/internal/api.go index 2f7795dfe..63044eedb 100644 --- a/userapi/internal/api.go +++ b/userapi/internal/api.go @@ -170,7 +170,7 @@ func (a *UserInternalAPI) PerformAccountCreation(ctx context.Context, req *api.P return nil } - if err = a.DB.SetDisplayName(ctx, req.Localpart, req.Localpart); err != nil { + if _, _, err = a.DB.SetDisplayName(ctx, req.Localpart, req.Localpart); err != nil { return err } @@ -813,7 +813,10 @@ func (a *UserInternalAPI) QueryPushRules(ctx context.Context, req *api.QueryPush } func (a *UserInternalAPI) SetAvatarURL(ctx context.Context, req *api.PerformSetAvatarURLRequest, res *api.PerformSetAvatarURLResponse) error { - return a.DB.SetAvatarURL(ctx, req.Localpart, req.AvatarURL) + profile, changed, err := a.DB.SetAvatarURL(ctx, req.Localpart, req.AvatarURL) + res.Profile = profile + res.Changed = changed + return err } func (a *UserInternalAPI) QueryNumericLocalpart(ctx context.Context, res *api.QueryNumericLocalpartResponse) error { @@ -847,8 +850,11 @@ func (a *UserInternalAPI) QueryAccountByPassword(ctx context.Context, req *api.Q } } -func (a *UserInternalAPI) SetDisplayName(ctx context.Context, req *api.PerformUpdateDisplayNameRequest, _ *struct{}) error { - return a.DB.SetDisplayName(ctx, req.Localpart, req.DisplayName) +func (a *UserInternalAPI) SetDisplayName(ctx context.Context, req *api.PerformUpdateDisplayNameRequest, res *api.PerformUpdateDisplayNameResponse) error { + profile, changed, err := a.DB.SetDisplayName(ctx, req.Localpart, req.DisplayName) + res.Profile = profile + res.Changed = changed + return err } func (a *UserInternalAPI) QueryLocalpartForThreePID(ctx context.Context, req *api.QueryLocalpartForThreePIDRequest, res *api.QueryLocalpartForThreePIDResponse) error { diff --git a/userapi/inthttp/client.go b/userapi/inthttp/client.go index a375d6caa..aa5d46d9f 100644 --- a/userapi/inthttp/client.go +++ b/userapi/inthttp/client.go @@ -388,7 +388,7 @@ func (h *httpUserInternalAPI) QueryAccountByPassword( func (h *httpUserInternalAPI) SetDisplayName( ctx context.Context, request *api.PerformUpdateDisplayNameRequest, - response *struct{}, + response *api.PerformUpdateDisplayNameResponse, ) error { return httputil.CallInternalRPCAPI( "SetDisplayName", h.apiURL+PerformSetDisplayNamePath, diff --git a/userapi/storage/interface.go b/userapi/storage/interface.go index 02efe7afe..fb12b53af 100644 --- a/userapi/storage/interface.go +++ b/userapi/storage/interface.go @@ -29,8 +29,8 @@ import ( type Profile interface { GetProfileByLocalpart(ctx context.Context, localpart string) (*authtypes.Profile, error) SearchProfiles(ctx context.Context, searchString string, limit int) ([]authtypes.Profile, error) - SetAvatarURL(ctx context.Context, localpart string, avatarURL string) error - SetDisplayName(ctx context.Context, localpart string, displayName string) error + SetAvatarURL(ctx context.Context, localpart string, avatarURL string) (*authtypes.Profile, bool, error) + SetDisplayName(ctx context.Context, localpart string, displayName string) (*authtypes.Profile, bool, error) } type Account interface { diff --git a/userapi/storage/postgres/profile_table.go b/userapi/storage/postgres/profile_table.go index f686127be..2753b23d9 100644 --- a/userapi/storage/postgres/profile_table.go +++ b/userapi/storage/postgres/profile_table.go @@ -44,10 +44,18 @@ const selectProfileByLocalpartSQL = "" + "SELECT localpart, display_name, avatar_url FROM userapi_profiles WHERE localpart = $1" const setAvatarURLSQL = "" + - "UPDATE userapi_profiles SET avatar_url = $1 WHERE localpart = $2" + "UPDATE userapi_profiles AS new" + + " SET avatar_url = $1" + + " FROM userapi_profiles AS old" + + " WHERE new.localpart = $2" + + " RETURNING new.display_name, old.avatar_url <> new.avatar_url" const setDisplayNameSQL = "" + - "UPDATE userapi_profiles SET display_name = $1 WHERE localpart = $2" + "UPDATE userapi_profiles AS new" + + " SET display_name = $1" + + " FROM userapi_profiles AS old" + + " WHERE new.localpart = $2" + + " RETURNING new.avatar_url, old.display_name <> new.display_name" const selectProfilesBySearchSQL = "" + "SELECT localpart, display_name, avatar_url FROM userapi_profiles WHERE localpart LIKE $1 OR display_name LIKE $1 LIMIT $2" @@ -100,16 +108,28 @@ func (s *profilesStatements) SelectProfileByLocalpart( func (s *profilesStatements) SetAvatarURL( ctx context.Context, txn *sql.Tx, localpart string, avatarURL string, -) (err error) { - _, err = s.setAvatarURLStmt.ExecContext(ctx, avatarURL, localpart) - return +) (*authtypes.Profile, bool, error) { + profile := &authtypes.Profile{ + Localpart: localpart, + AvatarURL: avatarURL, + } + var changed bool + stmt := sqlutil.TxStmt(txn, s.setAvatarURLStmt) + err := stmt.QueryRowContext(ctx, avatarURL, localpart).Scan(&profile.DisplayName, &changed) + return profile, changed, err } func (s *profilesStatements) SetDisplayName( ctx context.Context, txn *sql.Tx, localpart string, displayName string, -) (err error) { - _, err = s.setDisplayNameStmt.ExecContext(ctx, displayName, localpart) - return +) (*authtypes.Profile, bool, error) { + profile := &authtypes.Profile{ + Localpart: localpart, + DisplayName: displayName, + } + var changed bool + stmt := sqlutil.TxStmt(txn, s.setDisplayNameStmt) + err := stmt.QueryRowContext(ctx, displayName, localpart).Scan(&profile.AvatarURL, &changed) + return profile, changed, err } func (s *profilesStatements) SelectProfilesBySearch( diff --git a/userapi/storage/shared/storage.go b/userapi/storage/shared/storage.go index 4e28f7b5a..f8b6ad311 100644 --- a/userapi/storage/shared/storage.go +++ b/userapi/storage/shared/storage.go @@ -96,20 +96,24 @@ func (d *Database) GetProfileByLocalpart( // localpart. Returns an error if something went wrong with the SQL query func (d *Database) SetAvatarURL( ctx context.Context, localpart string, avatarURL string, -) error { - return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - return d.Profiles.SetAvatarURL(ctx, txn, localpart, avatarURL) +) (profile *authtypes.Profile, changed bool, err error) { + err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + profile, changed, err = d.Profiles.SetAvatarURL(ctx, txn, localpart, avatarURL) + return err }) + return } // SetDisplayName updates the display name of the profile associated with the given // localpart. Returns an error if something went wrong with the SQL query func (d *Database) SetDisplayName( ctx context.Context, localpart string, displayName string, -) error { - return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - return d.Profiles.SetDisplayName(ctx, txn, localpart, displayName) +) (profile *authtypes.Profile, changed bool, err error) { + err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { + profile, changed, err = d.Profiles.SetDisplayName(ctx, txn, localpart, displayName) + return err }) + return } // SetPassword sets the account password to the given hash. diff --git a/userapi/storage/sqlite3/profile_table.go b/userapi/storage/sqlite3/profile_table.go index 267daf044..b6130a1e3 100644 --- a/userapi/storage/sqlite3/profile_table.go +++ b/userapi/storage/sqlite3/profile_table.go @@ -44,10 +44,12 @@ const selectProfileByLocalpartSQL = "" + "SELECT localpart, display_name, avatar_url FROM userapi_profiles WHERE localpart = $1" const setAvatarURLSQL = "" + - "UPDATE userapi_profiles SET avatar_url = $1 WHERE localpart = $2" + "UPDATE userapi_profiles SET avatar_url = $1 WHERE localpart = $2" + + " RETURNING display_name" const setDisplayNameSQL = "" + - "UPDATE userapi_profiles SET display_name = $1 WHERE localpart = $2" + "UPDATE userapi_profiles SET display_name = $1 WHERE localpart = $2" + + " RETURNING avatar_url" const selectProfilesBySearchSQL = "" + "SELECT localpart, display_name, avatar_url FROM userapi_profiles WHERE localpart LIKE $1 OR display_name LIKE $1 LIMIT $2" @@ -102,18 +104,40 @@ func (s *profilesStatements) SelectProfileByLocalpart( func (s *profilesStatements) SetAvatarURL( ctx context.Context, txn *sql.Tx, localpart string, avatarURL string, -) (err error) { +) (*authtypes.Profile, bool, error) { + profile := &authtypes.Profile{ + Localpart: localpart, + AvatarURL: avatarURL, + } + old, err := s.SelectProfileByLocalpart(ctx, localpart) + if err != nil { + return old, false, err + } + if old.AvatarURL == avatarURL { + return old, false, nil + } stmt := sqlutil.TxStmt(txn, s.setAvatarURLStmt) - _, err = stmt.ExecContext(ctx, avatarURL, localpart) - return + err = stmt.QueryRowContext(ctx, avatarURL, localpart).Scan(&profile.DisplayName) + return profile, true, err } func (s *profilesStatements) SetDisplayName( ctx context.Context, txn *sql.Tx, localpart string, displayName string, -) (err error) { +) (*authtypes.Profile, bool, error) { + profile := &authtypes.Profile{ + Localpart: localpart, + DisplayName: displayName, + } + old, err := s.SelectProfileByLocalpart(ctx, localpart) + if err != nil { + return old, false, err + } + if old.DisplayName == displayName { + return old, false, nil + } stmt := sqlutil.TxStmt(txn, s.setDisplayNameStmt) - _, err = stmt.ExecContext(ctx, displayName, localpart) - return + err = stmt.QueryRowContext(ctx, displayName, localpart).Scan(&profile.AvatarURL) + return profile, true, err } func (s *profilesStatements) SelectProfilesBySearch( diff --git a/userapi/storage/storage_test.go b/userapi/storage/storage_test.go index 8e5b32b6a..354f085fc 100644 --- a/userapi/storage/storage_test.go +++ b/userapi/storage/storage_test.go @@ -382,15 +382,23 @@ func Test_Profile(t *testing.T) { // set avatar & displayname wantProfile.DisplayName = "Alice" - wantProfile.AvatarURL = "mxc://aliceAvatar" - err = db.SetDisplayName(ctx, aliceLocalpart, "Alice") - assert.NoError(t, err, "unable to set displayname") - err = db.SetAvatarURL(ctx, aliceLocalpart, "mxc://aliceAvatar") - assert.NoError(t, err, "unable to set avatar url") - // verify profile - gotProfile, err = db.GetProfileByLocalpart(ctx, aliceLocalpart) - assert.NoError(t, err, "unable to get profile by localpart") + gotProfile, changed, err := db.SetDisplayName(ctx, aliceLocalpart, "Alice") assert.Equal(t, wantProfile, gotProfile) + assert.NoError(t, err, "unable to set displayname") + assert.True(t, changed) + + wantProfile.AvatarURL = "mxc://aliceAvatar" + gotProfile, changed, err = db.SetAvatarURL(ctx, aliceLocalpart, "mxc://aliceAvatar") + assert.NoError(t, err, "unable to set avatar url") + assert.Equal(t, wantProfile, gotProfile) + assert.True(t, changed) + + // Setting the same avatar again doesn't change anything + wantProfile.AvatarURL = "mxc://aliceAvatar" + gotProfile, changed, err = db.SetAvatarURL(ctx, aliceLocalpart, "mxc://aliceAvatar") + assert.NoError(t, err, "unable to set avatar url") + assert.Equal(t, wantProfile, gotProfile) + assert.False(t, changed) // search profiles searchRes, err := db.SearchProfiles(ctx, "Alice", 2) diff --git a/userapi/storage/tables/interface.go b/userapi/storage/tables/interface.go index cc4287997..1b239e442 100644 --- a/userapi/storage/tables/interface.go +++ b/userapi/storage/tables/interface.go @@ -84,8 +84,8 @@ type OpenIDTable interface { type ProfileTable interface { InsertProfile(ctx context.Context, txn *sql.Tx, localpart string) error SelectProfileByLocalpart(ctx context.Context, localpart string) (*authtypes.Profile, error) - SetAvatarURL(ctx context.Context, txn *sql.Tx, localpart string, avatarURL string) (err error) - SetDisplayName(ctx context.Context, txn *sql.Tx, localpart string, displayName string) (err error) + SetAvatarURL(ctx context.Context, txn *sql.Tx, localpart string, avatarURL string) (*authtypes.Profile, bool, error) + SetDisplayName(ctx context.Context, txn *sql.Tx, localpart string, displayName string) (*authtypes.Profile, bool, error) SelectProfilesBySearch(ctx context.Context, searchString string, limit int) ([]authtypes.Profile, error) } diff --git a/userapi/userapi_test.go b/userapi/userapi_test.go index 4417f4dc0..aaa93f45b 100644 --- a/userapi/userapi_test.go +++ b/userapi/userapi_test.go @@ -23,13 +23,14 @@ import ( "time" "github.com/gorilla/mux" + "github.com/matrix-org/gomatrixserverlib" + "golang.org/x/crypto/bcrypt" + "github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/test" "github.com/matrix-org/dendrite/test/testrig" "github.com/matrix-org/dendrite/userapi" "github.com/matrix-org/dendrite/userapi/inthttp" - "github.com/matrix-org/gomatrixserverlib" - "golang.org/x/crypto/bcrypt" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/userapi/api" @@ -83,10 +84,10 @@ func TestQueryProfile(t *testing.T) { if err != nil { t.Fatalf("failed to make account: %s", err) } - if err := accountDB.SetAvatarURL(context.TODO(), "alice", aliceAvatarURL); err != nil { + if _, _, err := accountDB.SetAvatarURL(context.TODO(), "alice", aliceAvatarURL); err != nil { t.Fatalf("failed to set avatar url: %s", err) } - if err := accountDB.SetDisplayName(context.TODO(), "alice", aliceDisplayName); err != nil { + if _, _, err := accountDB.SetDisplayName(context.TODO(), "alice", aliceDisplayName); err != nil { t.Fatalf("failed to set display name: %s", err) } From e98d75fd63103243c5af2a63f2f547e4300adc4d Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 21 Oct 2022 10:15:08 +0100 Subject: [PATCH 098/207] Verify `room_id`, `type`, `sender` and `state_key` field lengths using bytes rather than codepoints (update to matrix-org/gomatrixserverlib@7c772f1, reverts bbb3ade4a2b49cfdaf7ec86ddf079ff7d48e0cf3) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 2248e73c6..7f9bb3897 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 - github.com/matrix-org/gomatrixserverlib v0.0.0-20221018085104-a72a83f0e19a + github.com/matrix-org/gomatrixserverlib v0.0.0-20221021091412-7c772f1b388a github.com/matrix-org/pinecone v0.0.0-20221007145426-3adc85477dd3 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.15 diff --git a/go.sum b/go.sum index a141fc9b4..5cce7e0d8 100644 --- a/go.sum +++ b/go.sum @@ -387,8 +387,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8r4Fzarl4+Y3K5hjothkVW5z7T1dUM11U= github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20221018085104-a72a83f0e19a h1:bQKHk3AWlgm7XhzPhuU3Iw3pUptW5l1DR/1y0o7zCKQ= -github.com/matrix-org/gomatrixserverlib v0.0.0-20221018085104-a72a83f0e19a/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4= +github.com/matrix-org/gomatrixserverlib v0.0.0-20221021091412-7c772f1b388a h1:6rJFN5NBuzZ7h5meYkLtXKa6VFZfDc8oVXHd4SDXr5o= +github.com/matrix-org/gomatrixserverlib v0.0.0-20221021091412-7c772f1b388a/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4= github.com/matrix-org/pinecone v0.0.0-20221007145426-3adc85477dd3 h1:lzkSQvBv8TuqKJCPoVwOVvEnARTlua5rrNy/Qw2Vxeo= github.com/matrix-org/pinecone v0.0.0-20221007145426-3adc85477dd3/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= From 9e4c3171da4e2d6d7b95731e702891513d081b49 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Fri, 21 Oct 2022 12:50:51 +0200 Subject: [PATCH 099/207] Optimize inserting pending PDUs/EDUs (#2821) This optimizes the association of PDUs/EDUs to their destination by inserting all destinations in one transaction. --- federationapi/queue/destinationqueue.go | 88 ++++++-------------- federationapi/queue/queue.go | 35 +++++++- federationapi/queue/queue_test.go | 60 +++++++------ federationapi/storage/interface.go | 7 +- federationapi/storage/shared/storage_edus.go | 25 +++--- federationapi/storage/shared/storage_pdus.go | 24 +++--- federationapi/storage/storage_test.go | 7 +- 7 files changed, 127 insertions(+), 119 deletions(-) diff --git a/federationapi/queue/destinationqueue.go b/federationapi/queue/destinationqueue.go index 768ed1f2b..1b7670e9a 100644 --- a/federationapi/queue/destinationqueue.go +++ b/federationapi/queue/destinationqueue.go @@ -76,40 +76,22 @@ func (oq *destinationQueue) sendEvent(event *gomatrixserverlib.HeaderedEvent, re return } - // Create a database entry that associates the given PDU NID with - // this destination queue. We'll then be able to retrieve the PDU - // later. - if err := oq.db.AssociatePDUWithDestination( - oq.process.Context(), - "", // TODO: remove this, as we don't need to persist the transaction ID - oq.destination, // the destination server name - receipt, // NIDs from federationapi_queue_json table - ); err != nil { - logrus.WithError(err).Errorf("failed to associate PDU %q with destination %q", event.EventID(), oq.destination) - return - } - // Check if the destination is blacklisted. If it isn't then wake - // up the queue. - if !oq.statistics.Blacklisted() { - // If there's room in memory to hold the event then add it to the - // list. - oq.pendingMutex.Lock() - if len(oq.pendingPDUs) < maxPDUsInMemory { - oq.pendingPDUs = append(oq.pendingPDUs, &queuedPDU{ - pdu: event, - receipt: receipt, - }) - } else { - oq.overflowed.Store(true) - } - oq.pendingMutex.Unlock() - - if !oq.backingOff.Load() { - oq.wakeQueueAndNotify() - } + // If there's room in memory to hold the event then add it to the + // list. + oq.pendingMutex.Lock() + if len(oq.pendingPDUs) < maxPDUsInMemory { + oq.pendingPDUs = append(oq.pendingPDUs, &queuedPDU{ + pdu: event, + receipt: receipt, + }) } else { oq.overflowed.Store(true) } + oq.pendingMutex.Unlock() + + if !oq.backingOff.Load() { + oq.wakeQueueAndNotify() + } } // sendEDU adds the EDU event to the pending queue for the destination. @@ -120,41 +102,23 @@ func (oq *destinationQueue) sendEDU(event *gomatrixserverlib.EDU, receipt *share logrus.Errorf("attempt to send nil EDU with destination %q", oq.destination) return } - // Create a database entry that associates the given PDU NID with - // this destination queue. We'll then be able to retrieve the PDU - // later. - if err := oq.db.AssociateEDUWithDestination( - oq.process.Context(), - oq.destination, // the destination server name - receipt, // NIDs from federationapi_queue_json table - event.Type, - nil, // this will use the default expireEDUTypes map - ); err != nil { - logrus.WithError(err).Errorf("failed to associate EDU with destination %q", oq.destination) - return - } - // Check if the destination is blacklisted. If it isn't then wake - // up the queue. - if !oq.statistics.Blacklisted() { - // If there's room in memory to hold the event then add it to the - // list. - oq.pendingMutex.Lock() - if len(oq.pendingEDUs) < maxEDUsInMemory { - oq.pendingEDUs = append(oq.pendingEDUs, &queuedEDU{ - edu: event, - receipt: receipt, - }) - } else { - oq.overflowed.Store(true) - } - oq.pendingMutex.Unlock() - if !oq.backingOff.Load() { - oq.wakeQueueAndNotify() - } + // If there's room in memory to hold the event then add it to the + // list. + oq.pendingMutex.Lock() + if len(oq.pendingEDUs) < maxEDUsInMemory { + oq.pendingEDUs = append(oq.pendingEDUs, &queuedEDU{ + edu: event, + receipt: receipt, + }) } else { oq.overflowed.Store(true) } + oq.pendingMutex.Unlock() + + if !oq.backingOff.Load() { + oq.wakeQueueAndNotify() + } } // handleBackoffNotifier is registered as the backoff notification diff --git a/federationapi/queue/queue.go b/federationapi/queue/queue.go index 68f789e37..328334379 100644 --- a/federationapi/queue/queue.go +++ b/federationapi/queue/queue.go @@ -24,6 +24,7 @@ import ( "github.com/getsentry/sentry-go" "github.com/matrix-org/gomatrixserverlib" "github.com/prometheus/client_golang/prometheus" + "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus" "github.com/tidwall/gjson" @@ -247,11 +248,25 @@ func (oqs *OutgoingQueues) SendEvent( } for destination := range destmap { - if queue := oqs.getQueue(destination); queue != nil { + if queue := oqs.getQueue(destination); queue != nil && !queue.statistics.Blacklisted() { queue.sendEvent(ev, nid) + } else { + delete(destmap, destination) } } + // Create a database entry that associates the given PDU NID with + // this destinations queue. We'll then be able to retrieve the PDU + // later. + if err := oqs.db.AssociatePDUWithDestinations( + oqs.process.Context(), + destmap, + nid, // NIDs from federationapi_queue_json table + ); err != nil { + logrus.WithError(err).Errorf("failed to associate PDUs %q with destinations", nid) + return err + } + return nil } @@ -321,11 +336,27 @@ func (oqs *OutgoingQueues) SendEDU( } for destination := range destmap { - if queue := oqs.getQueue(destination); queue != nil { + if queue := oqs.getQueue(destination); queue != nil && !queue.statistics.Blacklisted() { queue.sendEDU(e, nid) + } else { + delete(destmap, destination) } } + // Create a database entry that associates the given PDU NID with + // this destination queue. We'll then be able to retrieve the PDU + // later. + if err := oqs.db.AssociateEDUWithDestinations( + oqs.process.Context(), + destmap, // the destination server name + nid, // NIDs from federationapi_queue_json table + e.Type, + nil, // this will use the default expireEDUTypes map + ); err != nil { + logrus.WithError(err).Errorf("failed to associate EDU with destinations") + return err + } + return nil } diff --git a/federationapi/queue/queue_test.go b/federationapi/queue/queue_test.go index 40419b91f..a1b280103 100644 --- a/federationapi/queue/queue_test.go +++ b/federationapi/queue/queue_test.go @@ -25,6 +25,10 @@ import ( "go.uber.org/atomic" "gotest.tools/v3/poll" + "github.com/matrix-org/gomatrixserverlib" + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + "github.com/matrix-org/dendrite/federationapi/api" "github.com/matrix-org/dendrite/federationapi/statistics" "github.com/matrix-org/dendrite/federationapi/storage" @@ -34,9 +38,6 @@ import ( "github.com/matrix-org/dendrite/setup/process" "github.com/matrix-org/dendrite/test" "github.com/matrix-org/dendrite/test/testrig" - "github.com/matrix-org/gomatrixserverlib" - "github.com/pkg/errors" - "github.com/stretchr/testify/assert" ) func mustCreateFederationDatabase(t *testing.T, dbType test.DBType, realDatabase bool) (storage.Database, *process.ProcessContext, func()) { @@ -158,30 +159,36 @@ func (d *fakeDatabase) GetPendingEDUs(ctx context.Context, serverName gomatrixse return edus, nil } -func (d *fakeDatabase) AssociatePDUWithDestination(ctx context.Context, transactionID gomatrixserverlib.TransactionID, serverName gomatrixserverlib.ServerName, receipt *shared.Receipt) error { +func (d *fakeDatabase) AssociatePDUWithDestinations(ctx context.Context, destinations map[gomatrixserverlib.ServerName]struct{}, receipt *shared.Receipt) error { d.dbMutex.Lock() defer d.dbMutex.Unlock() if _, ok := d.pendingPDUs[receipt]; ok { - if _, ok := d.associatedPDUs[serverName]; !ok { - d.associatedPDUs[serverName] = make(map[*shared.Receipt]struct{}) + for destination := range destinations { + if _, ok := d.associatedPDUs[destination]; !ok { + d.associatedPDUs[destination] = make(map[*shared.Receipt]struct{}) + } + d.associatedPDUs[destination][receipt] = struct{}{} } - d.associatedPDUs[serverName][receipt] = struct{}{} + return nil } else { return errors.New("PDU doesn't exist") } } -func (d *fakeDatabase) AssociateEDUWithDestination(ctx context.Context, serverName gomatrixserverlib.ServerName, receipt *shared.Receipt, eduType string, expireEDUTypes map[string]time.Duration) error { +func (d *fakeDatabase) AssociateEDUWithDestinations(ctx context.Context, destinations map[gomatrixserverlib.ServerName]struct{}, receipt *shared.Receipt, eduType string, expireEDUTypes map[string]time.Duration) error { d.dbMutex.Lock() defer d.dbMutex.Unlock() if _, ok := d.pendingEDUs[receipt]; ok { - if _, ok := d.associatedEDUs[serverName]; !ok { - d.associatedEDUs[serverName] = make(map[*shared.Receipt]struct{}) + for destination := range destinations { + if _, ok := d.associatedEDUs[destination]; !ok { + d.associatedEDUs[destination] = make(map[*shared.Receipt]struct{}) + } + d.associatedEDUs[destination][receipt] = struct{}{} } - d.associatedEDUs[serverName][receipt] = struct{}{} + return nil } else { return errors.New("EDU doesn't exist") @@ -821,15 +828,15 @@ func TestSendPDUBatches(t *testing.T) { <-pc.WaitForShutdown() }() + destinations := map[gomatrixserverlib.ServerName]struct{}{destination: {}} // Populate database with > maxPDUsPerTransaction pduMultiplier := uint32(3) for i := 0; i < maxPDUsPerTransaction*int(pduMultiplier); i++ { ev := mustCreatePDU(t) headeredJSON, _ := json.Marshal(ev) nid, _ := db.StoreJSON(pc.Context(), string(headeredJSON)) - now := gomatrixserverlib.AsTimestamp(time.Now()) - transactionID := gomatrixserverlib.TransactionID(fmt.Sprintf("%d-%d", now, i)) - db.AssociatePDUWithDestination(pc.Context(), transactionID, destination, nid) + err := db.AssociatePDUWithDestinations(pc.Context(), destinations, nid) + assert.NoError(t, err, "failed to associate PDU with destinations") } ev := mustCreatePDU(t) @@ -865,13 +872,15 @@ func TestSendEDUBatches(t *testing.T) { <-pc.WaitForShutdown() }() + destinations := map[gomatrixserverlib.ServerName]struct{}{destination: {}} // Populate database with > maxEDUsPerTransaction eduMultiplier := uint32(3) for i := 0; i < maxEDUsPerTransaction*int(eduMultiplier); i++ { ev := mustCreateEDU(t) ephemeralJSON, _ := json.Marshal(ev) nid, _ := db.StoreJSON(pc.Context(), string(ephemeralJSON)) - db.AssociateEDUWithDestination(pc.Context(), destination, nid, ev.Type, nil) + err := db.AssociateEDUWithDestinations(pc.Context(), destinations, nid, ev.Type, nil) + assert.NoError(t, err, "failed to associate EDU with destinations") } ev := mustCreateEDU(t) @@ -907,23 +916,23 @@ func TestSendPDUAndEDUBatches(t *testing.T) { <-pc.WaitForShutdown() }() + destinations := map[gomatrixserverlib.ServerName]struct{}{destination: {}} // Populate database with > maxEDUsPerTransaction multiplier := uint32(3) - for i := 0; i < maxPDUsPerTransaction*int(multiplier)+1; i++ { ev := mustCreatePDU(t) headeredJSON, _ := json.Marshal(ev) nid, _ := db.StoreJSON(pc.Context(), string(headeredJSON)) - now := gomatrixserverlib.AsTimestamp(time.Now()) - transactionID := gomatrixserverlib.TransactionID(fmt.Sprintf("%d-%d", now, i)) - db.AssociatePDUWithDestination(pc.Context(), transactionID, destination, nid) + err := db.AssociatePDUWithDestinations(pc.Context(), destinations, nid) + assert.NoError(t, err, "failed to associate PDU with destinations") } for i := 0; i < maxEDUsPerTransaction*int(multiplier); i++ { ev := mustCreateEDU(t) ephemeralJSON, _ := json.Marshal(ev) nid, _ := db.StoreJSON(pc.Context(), string(ephemeralJSON)) - db.AssociateEDUWithDestination(pc.Context(), destination, nid, ev.Type, nil) + err := db.AssociateEDUWithDestinations(pc.Context(), destinations, nid, ev.Type, nil) + assert.NoError(t, err, "failed to associate EDU with destinations") } ev := mustCreateEDU(t) @@ -960,13 +969,12 @@ func TestExternalFailureBackoffDoesntStartQueue(t *testing.T) { dest := queues.getQueue(destination) queues.statistics.ForServer(destination).Failure() - + destinations := map[gomatrixserverlib.ServerName]struct{}{destination: {}} ev := mustCreatePDU(t) headeredJSON, _ := json.Marshal(ev) nid, _ := db.StoreJSON(pc.Context(), string(headeredJSON)) - now := gomatrixserverlib.AsTimestamp(time.Now()) - transactionID := gomatrixserverlib.TransactionID(fmt.Sprintf("%d-%d", now, 1)) - db.AssociatePDUWithDestination(pc.Context(), transactionID, destination, nid) + err := db.AssociatePDUWithDestinations(pc.Context(), destinations, nid) + assert.NoError(t, err, "failed to associate PDU with destinations") pollEnd := time.Now().Add(3 * time.Second) runningCheck := func(log poll.LogT) poll.Result { @@ -988,6 +996,7 @@ func TestQueueInteractsWithRealDatabasePDUAndEDU(t *testing.T) { t.Parallel() failuresUntilBlacklist := uint32(1) destination := gomatrixserverlib.ServerName("remotehost") + destinations := map[gomatrixserverlib.ServerName]struct{}{destination: {}} test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { db, fc, queues, pc, close := testSetup(failuresUntilBlacklist, false, t, dbType, true) // NOTE : These defers aren't called if go test is killed so the dbs may not get cleaned up. @@ -1009,7 +1018,8 @@ func TestQueueInteractsWithRealDatabasePDUAndEDU(t *testing.T) { edu := mustCreateEDU(t) ephemeralJSON, _ := json.Marshal(edu) nid, _ := db.StoreJSON(pc.Context(), string(ephemeralJSON)) - db.AssociateEDUWithDestination(pc.Context(), destination, nid, edu.Type, nil) + err = db.AssociateEDUWithDestinations(pc.Context(), destinations, nid, edu.Type, nil) + assert.NoError(t, err, "failed to associate EDU with destinations") checkBlacklisted := func(log poll.LogT) poll.Result { if fc.txCount.Load() == failuresUntilBlacklist { diff --git a/federationapi/storage/interface.go b/federationapi/storage/interface.go index b8109b432..09098cd1e 100644 --- a/federationapi/storage/interface.go +++ b/federationapi/storage/interface.go @@ -18,9 +18,10 @@ import ( "context" "time" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/dendrite/federationapi/storage/shared" "github.com/matrix-org/dendrite/federationapi/types" - "github.com/matrix-org/gomatrixserverlib" ) type Database interface { @@ -38,8 +39,8 @@ type Database interface { GetPendingPDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, limit int) (pdus map[*shared.Receipt]*gomatrixserverlib.HeaderedEvent, err error) GetPendingEDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, limit int) (edus map[*shared.Receipt]*gomatrixserverlib.EDU, err error) - AssociatePDUWithDestination(ctx context.Context, transactionID gomatrixserverlib.TransactionID, serverName gomatrixserverlib.ServerName, receipt *shared.Receipt) error - AssociateEDUWithDestination(ctx context.Context, serverName gomatrixserverlib.ServerName, receipt *shared.Receipt, eduType string, expireEDUTypes map[string]time.Duration) error + AssociatePDUWithDestinations(ctx context.Context, destinations map[gomatrixserverlib.ServerName]struct{}, receipt *shared.Receipt) error + AssociateEDUWithDestinations(ctx context.Context, destinations map[gomatrixserverlib.ServerName]struct{}, receipt *shared.Receipt, eduType string, expireEDUTypes map[string]time.Duration) error CleanPDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, receipts []*shared.Receipt) error CleanEDUs(ctx context.Context, serverName gomatrixserverlib.ServerName, receipts []*shared.Receipt) error diff --git a/federationapi/storage/shared/storage_edus.go b/federationapi/storage/shared/storage_edus.go index e0c740c11..c796d2f8f 100644 --- a/federationapi/storage/shared/storage_edus.go +++ b/federationapi/storage/shared/storage_edus.go @@ -38,9 +38,9 @@ var defaultExpireEDUTypes = map[string]time.Duration{ // AssociateEDUWithDestination creates an association that the // destination queues will use to determine which JSON blobs to send // to which servers. -func (d *Database) AssociateEDUWithDestination( +func (d *Database) AssociateEDUWithDestinations( ctx context.Context, - serverName gomatrixserverlib.ServerName, + destinations map[gomatrixserverlib.ServerName]struct{}, receipt *Receipt, eduType string, expireEDUTypes map[string]time.Duration, @@ -59,17 +59,18 @@ func (d *Database) AssociateEDUWithDestination( expiresAt = 0 } return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - if err := d.FederationQueueEDUs.InsertQueueEDU( - ctx, // context - txn, // SQL transaction - eduType, // EDU type for coalescing - serverName, // destination server name - receipt.nid, // NID from the federationapi_queue_json table - expiresAt, // The timestamp this EDU will expire - ); err != nil { - return fmt.Errorf("InsertQueueEDU: %w", err) + var err error + for destination := range destinations { + err = d.FederationQueueEDUs.InsertQueueEDU( + ctx, // context + txn, // SQL transaction + eduType, // EDU type for coalescing + destination, // destination server name + receipt.nid, // NID from the federationapi_queue_json table + expiresAt, // The timestamp this EDU will expire + ) } - return nil + return err }) } diff --git a/federationapi/storage/shared/storage_pdus.go b/federationapi/storage/shared/storage_pdus.go index 5a12c388a..dc37d7507 100644 --- a/federationapi/storage/shared/storage_pdus.go +++ b/federationapi/storage/shared/storage_pdus.go @@ -27,23 +27,23 @@ import ( // AssociatePDUWithDestination creates an association that the // destination queues will use to determine which JSON blobs to send // to which servers. -func (d *Database) AssociatePDUWithDestination( +func (d *Database) AssociatePDUWithDestinations( ctx context.Context, - transactionID gomatrixserverlib.TransactionID, - serverName gomatrixserverlib.ServerName, + destinations map[gomatrixserverlib.ServerName]struct{}, receipt *Receipt, ) error { return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - if err := d.FederationQueuePDUs.InsertQueuePDU( - ctx, // context - txn, // SQL transaction - transactionID, // transaction ID - serverName, // destination server name - receipt.nid, // NID from the federationapi_queue_json table - ); err != nil { - return fmt.Errorf("InsertQueuePDU: %w", err) + var err error + for destination := range destinations { + err = d.FederationQueuePDUs.InsertQueuePDU( + ctx, // context + txn, // SQL transaction + "", // transaction ID + destination, // destination server name + receipt.nid, // NID from the federationapi_queue_json table + ) } - return nil + return err }) } diff --git a/federationapi/storage/storage_test.go b/federationapi/storage/storage_test.go index 3b0268e55..6272fd2b1 100644 --- a/federationapi/storage/storage_test.go +++ b/federationapi/storage/storage_test.go @@ -35,6 +35,7 @@ func TestExpireEDUs(t *testing.T) { } ctx := context.Background() + destinations := map[gomatrixserverlib.ServerName]struct{}{"localhost": {}} test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { db, close := mustCreateFederationDatabase(t, dbType) defer close() @@ -43,7 +44,7 @@ func TestExpireEDUs(t *testing.T) { receipt, err := db.StoreJSON(ctx, "{}") assert.NoError(t, err) - err = db.AssociateEDUWithDestination(ctx, "localhost", receipt, gomatrixserverlib.MReceipt, expireEDUTypes) + err = db.AssociateEDUWithDestinations(ctx, destinations, receipt, gomatrixserverlib.MReceipt, expireEDUTypes) assert.NoError(t, err) } // add data without expiry @@ -51,7 +52,7 @@ func TestExpireEDUs(t *testing.T) { assert.NoError(t, err) // m.read_marker gets the default expiry of 24h, so won't be deleted further down in this test - err = db.AssociateEDUWithDestination(ctx, "localhost", receipt, "m.read_marker", expireEDUTypes) + err = db.AssociateEDUWithDestinations(ctx, destinations, receipt, "m.read_marker", expireEDUTypes) assert.NoError(t, err) // Delete expired EDUs @@ -67,7 +68,7 @@ func TestExpireEDUs(t *testing.T) { receipt, err = db.StoreJSON(ctx, "{}") assert.NoError(t, err) - err = db.AssociateEDUWithDestination(ctx, "localhost", receipt, gomatrixserverlib.MDirectToDevice, expireEDUTypes) + err = db.AssociateEDUWithDestinations(ctx, destinations, receipt, gomatrixserverlib.MDirectToDevice, expireEDUTypes) assert.NoError(t, err) err = db.DeleteExpiredEDUs(ctx) From 3cf42a1d64712f057fde0a5a4b3db1cf33ca432d Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Fri, 21 Oct 2022 12:53:04 +0200 Subject: [PATCH 100/207] Add `syncapi_memberships` table tests (#2805) --- syncapi/storage/tables/memberships_test.go | 198 +++++++++++++++++++++ 1 file changed, 198 insertions(+) create mode 100644 syncapi/storage/tables/memberships_test.go diff --git a/syncapi/storage/tables/memberships_test.go b/syncapi/storage/tables/memberships_test.go new file mode 100644 index 000000000..0cee7f5a5 --- /dev/null +++ b/syncapi/storage/tables/memberships_test.go @@ -0,0 +1,198 @@ +package tables_test + +import ( + "context" + "database/sql" + "reflect" + "sort" + "testing" + "time" + + "github.com/matrix-org/gomatrixserverlib" + + "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/setup/config" + "github.com/matrix-org/dendrite/syncapi/storage/postgres" + "github.com/matrix-org/dendrite/syncapi/storage/sqlite3" + "github.com/matrix-org/dendrite/syncapi/storage/tables" + "github.com/matrix-org/dendrite/syncapi/types" + "github.com/matrix-org/dendrite/test" +) + +func newMembershipsTable(t *testing.T, dbType test.DBType) (tables.Memberships, *sql.DB, func()) { + t.Helper() + connStr, close := test.PrepareDBConnectionString(t, dbType) + db, err := sqlutil.Open(&config.DatabaseOptions{ + ConnectionString: config.DataSource(connStr), + }, sqlutil.NewExclusiveWriter()) + if err != nil { + t.Fatalf("failed to open db: %s", err) + } + + var tab tables.Memberships + switch dbType { + case test.DBTypePostgres: + tab, err = postgres.NewPostgresMembershipsTable(db) + case test.DBTypeSQLite: + tab, err = sqlite3.NewSqliteMembershipsTable(db) + } + if err != nil { + t.Fatalf("failed to make new table: %s", err) + } + return tab, db, close +} + +func TestMembershipsTable(t *testing.T) { + + alice := test.NewUser(t) + room := test.NewRoom(t, alice) + + // Create users + var userEvents []*gomatrixserverlib.HeaderedEvent + users := []string{alice.ID} + for _, x := range room.CurrentState() { + if x.StateKeyEquals(alice.ID) { + if _, err := x.Membership(); err == nil { + userEvents = append(userEvents, x) + break + } + } + } + + if len(userEvents) == 0 { + t.Fatalf("didn't find creator membership event") + } + + for i := 0; i < 10; i++ { + u := test.NewUser(t) + users = append(users, u.ID) + + ev := room.CreateAndInsert(t, u, gomatrixserverlib.MRoomMember, map[string]interface{}{ + "membership": "join", + }, test.WithStateKey(u.ID)) + userEvents = append(userEvents, ev) + } + + test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) { + table, _, close := newMembershipsTable(t, dbType) + defer close() + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + for _, ev := range userEvents { + if err := table.UpsertMembership(ctx, nil, ev, types.StreamPosition(ev.Depth()), 1); err != nil { + t.Fatalf("failed to upsert membership: %s", err) + } + } + + testUpsert(t, ctx, table, userEvents[0], alice, room) + testMembershipCount(t, ctx, table, room) + testHeroes(t, ctx, table, alice, room, users) + }) +} + +func testHeroes(t *testing.T, ctx context.Context, table tables.Memberships, user *test.User, room *test.Room, users []string) { + + // Re-slice and sort the expected users + users = users[1:] + sort.Strings(users) + type testCase struct { + name string + memberships []string + wantHeroes []string + } + + testCases := []testCase{ + {name: "no memberships queried", memberships: []string{}}, + {name: "joined memberships queried should be limited", memberships: []string{gomatrixserverlib.Join}, wantHeroes: users[:5]}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got, err := table.SelectHeroes(ctx, nil, room.ID, user.ID, tc.memberships) + if err != nil { + t.Fatalf("unable to select heroes: %s", err) + } + if gotLen := len(got); gotLen != len(tc.wantHeroes) { + t.Fatalf("expected %d heroes, got %d", len(tc.wantHeroes), gotLen) + } + + if !reflect.DeepEqual(got, tc.wantHeroes) { + t.Fatalf("expected heroes to be %+v, got %+v", tc.wantHeroes, got) + } + }) + } +} + +func testMembershipCount(t *testing.T, ctx context.Context, table tables.Memberships, room *test.Room) { + t.Run("membership counts are correct", func(t *testing.T) { + // After 10 events, we should have 6 users (5 create related [incl. one member event], 5 member events = 6 users) + count, err := table.SelectMembershipCount(ctx, nil, room.ID, gomatrixserverlib.Join, 10) + if err != nil { + t.Fatalf("failed to get membership count: %s", err) + } + expectedCount := 6 + if expectedCount != count { + t.Fatalf("expected member count to be %d, got %d", expectedCount, count) + } + + // After 100 events, we should have all 11 users + count, err = table.SelectMembershipCount(ctx, nil, room.ID, gomatrixserverlib.Join, 100) + if err != nil { + t.Fatalf("failed to get membership count: %s", err) + } + expectedCount = 11 + if expectedCount != count { + t.Fatalf("expected member count to be %d, got %d", expectedCount, count) + } + }) +} + +func testUpsert(t *testing.T, ctx context.Context, table tables.Memberships, membershipEvent *gomatrixserverlib.HeaderedEvent, user *test.User, room *test.Room) { + t.Run("upserting works as expected", func(t *testing.T) { + if err := table.UpsertMembership(ctx, nil, membershipEvent, 1, 1); err != nil { + t.Fatalf("failed to upsert membership: %s", err) + } + membership, pos, err := table.SelectMembershipForUser(ctx, nil, room.ID, user.ID, 1) + if err != nil { + t.Fatalf("failed to select membership: %s", err) + } + expectedPos := 1 + if pos != expectedPos { + t.Fatalf("expected pos to be %d, got %d", expectedPos, pos) + } + if membership != gomatrixserverlib.Join { + t.Fatalf("expected membership to be join, got %s", membership) + } + // Create a new event which gets upserted and should not cause issues + ev := room.CreateAndInsert(t, user, gomatrixserverlib.MRoomMember, map[string]interface{}{ + "membership": gomatrixserverlib.Join, + }, test.WithStateKey(user.ID)) + // Insert the same event again, but with different positions, which should get updated + if err = table.UpsertMembership(ctx, nil, ev, 2, 2); err != nil { + t.Fatalf("failed to upsert membership: %s", err) + } + + // Verify the position got updated + membership, pos, err = table.SelectMembershipForUser(ctx, nil, room.ID, user.ID, 10) + if err != nil { + t.Fatalf("failed to select membership: %s", err) + } + expectedPos = 2 + if pos != expectedPos { + t.Fatalf("expected pos to be %d, got %d", expectedPos, pos) + } + if membership != gomatrixserverlib.Join { + t.Fatalf("expected membership to be join, got %s", membership) + } + + // If we can't find a membership, it should default to leave + if membership, _, err = table.SelectMembershipForUser(ctx, nil, room.ID, user.ID, 1); err != nil { + t.Fatalf("failed to select membership: %s", err) + } + if membership != gomatrixserverlib.Leave { + t.Fatalf("expected membership to be leave, got %s", membership) + } + }) +} From 411db6083b8257bfe96663e6bb7ce763609216fa Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Fri, 21 Oct 2022 15:00:51 +0100 Subject: [PATCH 101/207] Version 0.10.4 (#2822) Changelog and version bump. --- CHANGES.md | 18 ++++++++++++++++++ internal/version.go | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index eea2c3c7c..1ed87824a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,23 @@ # Changelog +## Dendrite 0.10.4 (2022-10-21) + +### Features + +* Various tables belonging to the user API will be renamed so that they are namespaced with the `userapi_` prefix + * Note that, after upgrading to this version, you should not revert to an older version of Dendrite as the database changes **will not** be reverted automatically +* The backoff and retry behaviour in the federation API has been refactored and improved + +### Fixes + +* Private read receipt support is now advertised in the client `/versions` endpoint +* Private read receipts will now clear notification counts properly +* A bug where a false `leave` membership transition was inserted into the timeline after accepting an invite has been fixed +* Some panics caused by concurrent map writes in the key server have been fixed +* The sync API now calculates membership transitions from state deltas more accurately +* Transaction IDs are now scoped to endpoints, which should fix some bugs where transaction ID reuse could cause nonsensical cached responses from some endpoints +* The length of the `type`, `sender`, `state_key` and `room_id` fields in events are now verified by number of bytes rather than codepoints after a spec clarification, reverting a change made in Dendrite 0.9.6 + ## Dendrite 0.10.3 (2022-10-14) ### Features diff --git a/internal/version.go b/internal/version.go index c888748a8..5d739a45d 100644 --- a/internal/version.go +++ b/internal/version.go @@ -17,7 +17,7 @@ var build string const ( VersionMajor = 0 VersionMinor = 10 - VersionPatch = 3 + VersionPatch = 4 VersionTag = "" // example: "rc1" ) From 0843bd776e9156b7ae62b504a5c7e8c8b26ff476 Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Mon, 24 Oct 2022 07:10:50 +0200 Subject: [PATCH 102/207] Fix wrong config key --- setup/config/config_global.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/config/config_global.go b/setup/config/config_global.go index 2efae0d5a..784893d24 100644 --- a/setup/config/config_global.go +++ b/setup/config/config_global.go @@ -170,7 +170,7 @@ type ServerNotices struct { // The displayname to be used when sending notices DisplayName string `yaml:"display_name"` // The avatar of this user - AvatarURL string `yaml:"avatar"` + AvatarURL string `yaml:"avatar_url"` // The roomname to be used when creating messages RoomName string `yaml:"room_name"` } From a553fe770575b027809fc0a0f81e6709e6d068df Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Mon, 24 Oct 2022 10:07:50 +0100 Subject: [PATCH 103/207] Fix slow querying of cross-signing signatures --- clientapi/routing/keys.go | 6 +++++- keyserver/internal/internal.go | 7 ++++--- keyserver/storage/postgres/cross_signing_sigs_table.go | 2 +- keyserver/storage/sqlite3/cross_signing_sigs_table.go | 4 ++-- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/clientapi/routing/keys.go b/clientapi/routing/keys.go index 5c3681382..0c12b1117 100644 --- a/clientapi/routing/keys.go +++ b/clientapi/routing/keys.go @@ -99,7 +99,11 @@ func (r *queryKeysRequest) GetTimeout() time.Duration { if r.Timeout == 0 { return 10 * time.Second } - return time.Duration(r.Timeout) * time.Millisecond + timeout := time.Duration(r.Timeout) * time.Millisecond + if timeout > time.Second*20 { + timeout = time.Second * 20 + } + return timeout } func QueryKeys(req *http.Request, keyAPI api.ClientKeyAPI, device *userapi.Device) util.JSONResponse { diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go index 49ef03054..ff0968b27 100644 --- a/keyserver/internal/internal.go +++ b/keyserver/internal/internal.go @@ -257,9 +257,6 @@ func (a *KeyInternalAPI) QueryKeys(ctx context.Context, req *api.QueryKeysReques res.UserSigningKeys = make(map[string]gomatrixserverlib.CrossSigningKey) res.Failures = make(map[string]interface{}) - // get cross-signing keys from the database - a.crossSigningKeysFromDatabase(ctx, req, res) - // make a map from domain to device keys domainToDeviceKeys := make(map[string]map[string][]string) domainToCrossSigningKeys := make(map[string]map[string]struct{}) @@ -336,6 +333,10 @@ func (a *KeyInternalAPI) QueryKeys(ctx context.Context, req *api.QueryKeysReques a.queryRemoteKeys(ctx, req.Timeout, res, domainToDeviceKeys, domainToCrossSigningKeys) } + // Now that we've done the potentially expensive work of asking the federation, + // try filling the cross-signing keys from the database that we know about. + a.crossSigningKeysFromDatabase(ctx, req, res) + // Finally, append signatures that we know about // TODO: This is horrible because we need to round-trip the signature from // JSON, add the signatures and marshal it again, for some reason? diff --git a/keyserver/storage/postgres/cross_signing_sigs_table.go b/keyserver/storage/postgres/cross_signing_sigs_table.go index 8b2a865b9..4536b7d80 100644 --- a/keyserver/storage/postgres/cross_signing_sigs_table.go +++ b/keyserver/storage/postgres/cross_signing_sigs_table.go @@ -42,7 +42,7 @@ CREATE INDEX IF NOT EXISTS keyserver_cross_signing_sigs_idx ON keyserver_cross_s const selectCrossSigningSigsForTargetSQL = "" + "SELECT origin_user_id, origin_key_id, signature FROM keyserver_cross_signing_sigs" + - " WHERE (origin_user_id = $1 OR origin_user_id = target_user_id) AND target_user_id = $2 AND target_key_id = $3" + " WHERE (origin_user_id = $1 OR origin_user_id = $2) AND target_user_id = $2 AND target_key_id = $3" const upsertCrossSigningSigsForTargetSQL = "" + "INSERT INTO keyserver_cross_signing_sigs (origin_user_id, origin_key_id, target_user_id, target_key_id, signature)" + diff --git a/keyserver/storage/sqlite3/cross_signing_sigs_table.go b/keyserver/storage/sqlite3/cross_signing_sigs_table.go index ea431151e..7a153e8fb 100644 --- a/keyserver/storage/sqlite3/cross_signing_sigs_table.go +++ b/keyserver/storage/sqlite3/cross_signing_sigs_table.go @@ -42,7 +42,7 @@ CREATE INDEX IF NOT EXISTS keyserver_cross_signing_sigs_idx ON keyserver_cross_s const selectCrossSigningSigsForTargetSQL = "" + "SELECT origin_user_id, origin_key_id, signature FROM keyserver_cross_signing_sigs" + - " WHERE (origin_user_id = $1 OR origin_user_id = target_user_id) AND target_user_id = $2 AND target_key_id = $3" + " WHERE (origin_user_id = $1 OR origin_user_id = $2) AND target_user_id = $3 AND target_key_id = $4" const upsertCrossSigningSigsForTargetSQL = "" + "INSERT OR REPLACE INTO keyserver_cross_signing_sigs (origin_user_id, origin_key_id, target_user_id, target_key_id, signature)" + @@ -85,7 +85,7 @@ func NewSqliteCrossSigningSigsTable(db *sql.DB) (tables.CrossSigningSigs, error) func (s *crossSigningSigsStatements) SelectCrossSigningSigsForTarget( ctx context.Context, txn *sql.Tx, originUserID, targetUserID string, targetKeyID gomatrixserverlib.KeyID, ) (r types.CrossSigningSigMap, err error) { - rows, err := sqlutil.TxStmt(txn, s.selectCrossSigningSigsForTargetStmt).QueryContext(ctx, originUserID, targetUserID, targetKeyID) + rows, err := sqlutil.TxStmt(txn, s.selectCrossSigningSigsForTargetStmt).QueryContext(ctx, originUserID, targetUserID, targetUserID, targetKeyID) if err != nil { return nil, err } From 7506e3303e78e47a7bea454de1e726c6f6640d2f Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Mon, 24 Oct 2022 17:03:04 +0200 Subject: [PATCH 104/207] Get messages from before user left the room (#2824) This is going to make `Can get rooms/{roomId}/messages for a departed room (SPEC-216)` pass, since we now only grep events from before the user left the room. --- syncapi/routing/messages.go | 33 +++++++++++++++++++++++++-------- sytest-whitelist | 4 +++- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/syncapi/routing/messages.go b/syncapi/routing/messages.go index 8f3ed3f5b..86cf8e736 100644 --- a/syncapi/routing/messages.go +++ b/syncapi/routing/messages.go @@ -83,18 +83,18 @@ func OnIncomingMessagesRequest( defer sqlutil.EndTransactionWithCheck(snapshot, &succeeded, &err) // check if the user has already forgotten about this room - isForgotten, roomExists, err := checkIsRoomForgotten(req.Context(), roomID, device.UserID, rsAPI) + membershipResp, err := getMembershipForUser(req.Context(), roomID, device.UserID, rsAPI) if err != nil { return jsonerror.InternalServerError() } - if !roomExists { + if !membershipResp.RoomExists { return util.JSONResponse{ Code: http.StatusForbidden, JSON: jsonerror.Forbidden("room does not exist"), } } - if isForgotten { + if membershipResp.IsRoomForgotten { return util.JSONResponse{ Code: http.StatusForbidden, JSON: jsonerror.Forbidden("user already forgot about this room"), @@ -195,6 +195,20 @@ func OnIncomingMessagesRequest( } } + // If the user already left the room, grep events from before that + if membershipResp.Membership == gomatrixserverlib.Leave { + var token types.TopologyToken + token, err = snapshot.EventPositionInTopology(req.Context(), membershipResp.EventID) + if err != nil { + return util.JSONResponse{ + Code: http.StatusInternalServerError, + } + } + if backwardOrdering { + from = token + } + } + mReq := messagesReq{ ctx: req.Context(), db: db, @@ -283,17 +297,16 @@ func (m *messagesResp) applyLazyLoadMembers( } } -func checkIsRoomForgotten(ctx context.Context, roomID, userID string, rsAPI api.SyncRoomserverAPI) (forgotten bool, exists bool, err error) { +func getMembershipForUser(ctx context.Context, roomID, userID string, rsAPI api.SyncRoomserverAPI) (resp api.QueryMembershipForUserResponse, err error) { req := api.QueryMembershipForUserRequest{ RoomID: roomID, UserID: userID, } - resp := api.QueryMembershipForUserResponse{} if err := rsAPI.QueryMembershipForUser(ctx, &req, &resp); err != nil { - return false, false, err + return api.QueryMembershipForUserResponse{}, err } - return resp.IsRoomForgotten, resp.RoomExists, nil + return resp, nil } // retrieveEvents retrieves events from the local database for a request on @@ -313,7 +326,11 @@ func (r *messagesReq) retrieveEvents() ( } var events []*gomatrixserverlib.HeaderedEvent - util.GetLogger(r.ctx).WithField("start", start).WithField("end", end).Infof("Fetched %d events locally", len(streamEvents)) + util.GetLogger(r.ctx).WithFields(logrus.Fields{ + "start": r.from, + "end": r.to, + "backwards": r.backwardOrdering, + }).Infof("Fetched %d events locally", len(streamEvents)) // There can be two reasons for streamEvents to be empty: either we've // reached the oldest event in the room (or the most recent one, depending diff --git a/sytest-whitelist b/sytest-whitelist index 1387838f7..e92ae6495 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -752,4 +752,6 @@ When user joins a room the state is included in the next sync When user joins a room the state is included in a gapped sync Messages that notify from another user increment notification_count Messages that highlight from another user increment unread highlight count -Notifications can be viewed with GET /notifications \ No newline at end of file +Notifications can be viewed with GET /notifications +Can get rooms/{roomId}/messages for a departed room (SPEC-216) +Local device key changes appear in /keys/changes \ No newline at end of file From 313cb3fd193397536b069d819f8346d625d82af8 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Tue, 25 Oct 2022 12:39:10 +0200 Subject: [PATCH 105/207] Filter `/members`, return members at given point (#2827) Makes the tests ``` Can get rooms/{roomId}/members at a given point Can filter rooms/{roomId}/members ``` pass, by moving `/members` and `/joined_members` to the SyncAPI. --- clientapi/routing/joined_rooms.go | 52 +++++++++ clientapi/routing/routing.go | 20 ---- docs/caddy/polylith/Caddyfile | 2 +- docs/hiawatha/polylith-sample.conf | 4 +- docs/nginx/polylith-sample.conf | 4 +- {clientapi => syncapi}/routing/memberships.go | 100 ++++++++++-------- syncapi/routing/routing.go | 33 ++++++ syncapi/storage/interface.go | 5 + syncapi/storage/postgres/memberships_table.go | 35 +++++- syncapi/storage/shared/storage_consumer.go | 8 ++ syncapi/storage/sqlite3/memberships_table.go | 32 +++++- syncapi/storage/tables/interface.go | 5 + syncapi/streams/stream_pdu.go | 8 +- syncapi/types/types.go | 3 + sytest-whitelist | 4 +- 15 files changed, 243 insertions(+), 72 deletions(-) create mode 100644 clientapi/routing/joined_rooms.go rename {clientapi => syncapi}/routing/memberships.go (55%) diff --git a/clientapi/routing/joined_rooms.go b/clientapi/routing/joined_rooms.go new file mode 100644 index 000000000..4bb353ea9 --- /dev/null +++ b/clientapi/routing/joined_rooms.go @@ -0,0 +1,52 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routing + +import ( + "net/http" + + "github.com/matrix-org/util" + + "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/roomserver/api" + userapi "github.com/matrix-org/dendrite/userapi/api" +) + +type getJoinedRoomsResponse struct { + JoinedRooms []string `json:"joined_rooms"` +} + +func GetJoinedRooms( + req *http.Request, + device *userapi.Device, + rsAPI api.ClientRoomserverAPI, +) util.JSONResponse { + var res api.QueryRoomsForUserResponse + err := rsAPI.QueryRoomsForUser(req.Context(), &api.QueryRoomsForUserRequest{ + UserID: device.UserID, + WantMembership: "join", + }, &res) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("QueryRoomsForUser failed") + return jsonerror.InternalServerError() + } + if res.RoomIDs == nil { + res.RoomIDs = []string{} + } + return util.JSONResponse{ + Code: http.StatusOK, + JSON: getJoinedRoomsResponse{res.RoomIDs}, + } +} diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 4ca8e59c5..e0e3e33d4 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -950,26 +950,6 @@ func Setup( }), ).Methods(http.MethodPost, http.MethodOptions) - v3mux.Handle("/rooms/{roomID}/members", - httputil.MakeAuthAPI("rooms_members", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) - if err != nil { - return util.ErrorResponse(err) - } - return GetMemberships(req, device, vars["roomID"], false, cfg, rsAPI) - }), - ).Methods(http.MethodGet, http.MethodOptions) - - v3mux.Handle("/rooms/{roomID}/joined_members", - httputil.MakeAuthAPI("rooms_members", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { - vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) - if err != nil { - return util.ErrorResponse(err) - } - return GetMemberships(req, device, vars["roomID"], true, cfg, rsAPI) - }), - ).Methods(http.MethodGet, http.MethodOptions) - v3mux.Handle("/rooms/{roomID}/read_markers", httputil.MakeAuthAPI("rooms_read_markers", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { if r := rateLimits.Limit(req, device); r != nil { diff --git a/docs/caddy/polylith/Caddyfile b/docs/caddy/polylith/Caddyfile index 8aeb9317f..c2d81b49b 100644 --- a/docs/caddy/polylith/Caddyfile +++ b/docs/caddy/polylith/Caddyfile @@ -74,7 +74,7 @@ matrix.example.com { # Change the end of each reverse_proxy line to the correct # address for your various services. @sync_api { - path_regexp /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|context/.*?|relations/.*?|event/.*?))$ + path_regexp /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|.*?_?members|context/.*?|relations/.*?|event/.*?))$ } reverse_proxy @sync_api sync_api:8073 diff --git a/docs/hiawatha/polylith-sample.conf b/docs/hiawatha/polylith-sample.conf index 0093fdcf2..eb1dd4f9a 100644 --- a/docs/hiawatha/polylith-sample.conf +++ b/docs/hiawatha/polylith-sample.conf @@ -23,8 +23,10 @@ VirtualHost { # /_matrix/client/.*/rooms/{roomId}/relations/{eventID} # /_matrix/client/.*/rooms/{roomId}/relations/{eventID}/{relType} # /_matrix/client/.*/rooms/{roomId}/relations/{eventID}/{relType}/{eventType} + # /_matrix/client/.*/rooms/{roomId}/members + # /_matrix/client/.*/rooms/{roomId}/joined_members # to sync_api - ReverseProxy = /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|context/.*?|relations/.*?|event/.*?))$ http://localhost:8073 600 + ReverseProxy = /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|.*?_?members|context/.*?|relations/.*?|event/.*?))$ http://localhost:8073 600 ReverseProxy = /_matrix/client http://localhost:8071 600 ReverseProxy = /_matrix/federation http://localhost:8072 600 ReverseProxy = /_matrix/key http://localhost:8072 600 diff --git a/docs/nginx/polylith-sample.conf b/docs/nginx/polylith-sample.conf index 6e81eb5f2..0ad24509a 100644 --- a/docs/nginx/polylith-sample.conf +++ b/docs/nginx/polylith-sample.conf @@ -33,8 +33,10 @@ server { # /_matrix/client/.*/rooms/{roomId}/relations/{eventID} # /_matrix/client/.*/rooms/{roomId}/relations/{eventID}/{relType} # /_matrix/client/.*/rooms/{roomId}/relations/{eventID}/{relType}/{eventType} + # /_matrix/client/.*/rooms/{roomId}/members + # /_matrix/client/.*/rooms/{roomId}/joined_members # to sync_api - location ~ /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|context/.*?|relations/.*?|event/.*?))$ { + location ~ /_matrix/client/.*?/(sync|user/.*?/filter/?.*|keys/changes|rooms/.*?/(messages|.*?_?members|context/.*?|relations/.*?|event/.*?))$ { proxy_pass http://sync_api:8073; } diff --git a/clientapi/routing/memberships.go b/syncapi/routing/memberships.go similarity index 55% rename from clientapi/routing/memberships.go rename to syncapi/routing/memberships.go index 9bdd8a4f4..b4e342251 100644 --- a/clientapi/routing/memberships.go +++ b/syncapi/routing/memberships.go @@ -18,22 +18,20 @@ import ( "encoding/json" "net/http" - "github.com/matrix-org/dendrite/clientapi/jsonerror" - "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/dendrite/setup/config" - userapi "github.com/matrix-org/dendrite/userapi/api" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" + + "github.com/matrix-org/dendrite/clientapi/jsonerror" + "github.com/matrix-org/dendrite/roomserver/api" + "github.com/matrix-org/dendrite/syncapi/storage" + "github.com/matrix-org/dendrite/syncapi/types" + userapi "github.com/matrix-org/dendrite/userapi/api" ) type getMembershipResponse struct { Chunk []gomatrixserverlib.ClientEvent `json:"chunk"` } -type getJoinedRoomsResponse struct { - JoinedRooms []string `json:"joined_rooms"` -} - // https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-rooms-roomid-joined-members type getJoinedMembersResponse struct { Joined map[string]joinedMember `json:"joined"` @@ -51,19 +49,22 @@ type databaseJoinedMember struct { AvatarURL string `json:"avatar_url"` } -// GetMemberships implements GET /rooms/{roomId}/members +// GetMemberships implements +// +// GET /rooms/{roomId}/members +// GET /rooms/{roomId}/joined_members func GetMemberships( - req *http.Request, device *userapi.Device, roomID string, joinedOnly bool, - _ *config.ClientAPI, - rsAPI api.ClientRoomserverAPI, + req *http.Request, device *userapi.Device, roomID string, + syncDB storage.Database, rsAPI api.SyncRoomserverAPI, + joinedOnly bool, membership, notMembership *string, at string, ) util.JSONResponse { - queryReq := api.QueryMembershipsForRoomRequest{ - JoinedOnly: joinedOnly, - RoomID: roomID, - Sender: device.UserID, + queryReq := api.QueryMembershipForUserRequest{ + RoomID: roomID, + UserID: device.UserID, } - var queryRes api.QueryMembershipsForRoomResponse - if err := rsAPI.QueryMembershipsForRoom(req.Context(), &queryReq, &queryRes); err != nil { + + var queryRes api.QueryMembershipForUserResponse + if err := rsAPI.QueryMembershipForUser(req.Context(), &queryReq, &queryRes); err != nil { util.GetLogger(req.Context()).WithError(err).Error("rsAPI.QueryMembershipsForRoom failed") return jsonerror.InternalServerError() } @@ -75,16 +76,48 @@ func GetMemberships( } } + db, err := syncDB.NewDatabaseSnapshot(req.Context()) + if err != nil { + return jsonerror.InternalServerError() + } + + atToken, err := types.NewTopologyTokenFromString(at) + if err != nil { + if queryRes.HasBeenInRoom && !queryRes.IsInRoom { + // If you have left the room then this will be the members of the room when you left. + atToken, err = db.EventPositionInTopology(req.Context(), queryRes.EventID) + } else { + // If you are joined to the room then this will be the current members of the room. + atToken, err = db.MaxTopologicalPosition(req.Context(), roomID) + } + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("unable to get 'atToken'") + return jsonerror.InternalServerError() + } + } + + eventIDs, err := db.SelectMemberships(req.Context(), roomID, atToken, membership, notMembership) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("db.SelectMemberships failed") + return jsonerror.InternalServerError() + } + + result, err := db.Events(req.Context(), eventIDs) + if err != nil { + util.GetLogger(req.Context()).WithError(err).Error("db.Events failed") + return jsonerror.InternalServerError() + } + if joinedOnly { var res getJoinedMembersResponse res.Joined = make(map[string]joinedMember) - for _, ev := range queryRes.JoinEvents { + for _, ev := range result { var content databaseJoinedMember - if err := json.Unmarshal(ev.Content, &content); err != nil { + if err := json.Unmarshal(ev.Content(), &content); err != nil { util.GetLogger(req.Context()).WithError(err).Error("failed to unmarshal event content") return jsonerror.InternalServerError() } - res.Joined[ev.Sender] = joinedMember(content) + res.Joined[ev.Sender()] = joinedMember(content) } return util.JSONResponse{ Code: http.StatusOK, @@ -93,29 +126,6 @@ func GetMemberships( } return util.JSONResponse{ Code: http.StatusOK, - JSON: getMembershipResponse{queryRes.JoinEvents}, - } -} - -func GetJoinedRooms( - req *http.Request, - device *userapi.Device, - rsAPI api.ClientRoomserverAPI, -) util.JSONResponse { - var res api.QueryRoomsForUserResponse - err := rsAPI.QueryRoomsForUser(req.Context(), &api.QueryRoomsForUserRequest{ - UserID: device.UserID, - WantMembership: "join", - }, &res) - if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("QueryRoomsForUser failed") - return jsonerror.InternalServerError() - } - if res.RoomIDs == nil { - res.RoomIDs = []string{} - } - return util.JSONResponse{ - Code: http.StatusOK, - JSON: getJoinedRoomsResponse{res.RoomIDs}, + JSON: getMembershipResponse{gomatrixserverlib.HeaderedToClientEvents(result, gomatrixserverlib.FormatSync)}, } } diff --git a/syncapi/routing/routing.go b/syncapi/routing/routing.go index 71fa93c1e..bc3ad2384 100644 --- a/syncapi/routing/routing.go +++ b/syncapi/routing/routing.go @@ -172,4 +172,37 @@ func Setup( return Search(req, device, syncDB, fts, nextBatch) }), ).Methods(http.MethodPost, http.MethodOptions) + + v3mux.Handle("/rooms/{roomID}/members", + httputil.MakeAuthAPI("rooms_members", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + var membership, notMembership *string + if req.URL.Query().Has("membership") { + m := req.URL.Query().Get("membership") + membership = &m + } + if req.URL.Query().Has("not_membership") { + m := req.URL.Query().Get("not_membership") + notMembership = &m + } + + at := req.URL.Query().Get("at") + return GetMemberships(req, device, vars["roomID"], syncDB, rsAPI, false, membership, notMembership, at) + }), + ).Methods(http.MethodGet, http.MethodOptions) + + v3mux.Handle("/rooms/{roomID}/joined_members", + httputil.MakeAuthAPI("rooms_members", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + at := req.URL.Query().Get("at") + membership := gomatrixserverlib.Join + return GetMemberships(req, device, vars["roomID"], syncDB, rsAPI, true, &membership, nil, at) + }), + ).Methods(http.MethodGet, http.MethodOptions) } diff --git a/syncapi/storage/interface.go b/syncapi/storage/interface.go index 02d45f801..af4fce44e 100644 --- a/syncapi/storage/interface.go +++ b/syncapi/storage/interface.go @@ -178,6 +178,11 @@ type Database interface { ReIndex(ctx context.Context, limit, afterID int64) (map[int64]gomatrixserverlib.HeaderedEvent, error) UpdateRelations(ctx context.Context, event *gomatrixserverlib.HeaderedEvent) error RedactRelations(ctx context.Context, roomID, redactedEventID string) error + SelectMemberships( + ctx context.Context, + roomID string, pos types.TopologyToken, + membership, notMembership *string, + ) (eventIDs []string, err error) } type Presence interface { diff --git a/syncapi/storage/postgres/memberships_table.go b/syncapi/storage/postgres/memberships_table.go index 939d6b3f5..b555e8456 100644 --- a/syncapi/storage/postgres/memberships_table.go +++ b/syncapi/storage/postgres/memberships_table.go @@ -20,11 +20,12 @@ import ( "fmt" "github.com/lib/pq" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" - "github.com/matrix-org/gomatrixserverlib" ) // The memberships table is designed to track the last time that @@ -69,11 +70,20 @@ const selectHeroesSQL = "" + const selectMembershipBeforeSQL = "" + "SELECT membership, topological_pos FROM syncapi_memberships WHERE room_id = $1 and user_id = $2 AND topological_pos <= $3 ORDER BY topological_pos DESC LIMIT 1" +const selectMembersSQL = ` +SELECT event_id FROM ( + SELECT DISTINCT ON (room_id, user_id) room_id, user_id, event_id, membership FROM syncapi_memberships WHERE room_id = $1 AND topological_pos <= $2 ORDER BY room_id, user_id, stream_pos DESC +) t +WHERE ($3::text IS NULL OR t.membership = $3) + AND ($4::text IS NULL OR t.membership <> $4) +` + type membershipsStatements struct { upsertMembershipStmt *sql.Stmt selectMembershipCountStmt *sql.Stmt selectHeroesStmt *sql.Stmt selectMembershipForUserStmt *sql.Stmt + selectMembersStmt *sql.Stmt } func NewPostgresMembershipsTable(db *sql.DB) (tables.Memberships, error) { @@ -87,6 +97,7 @@ func NewPostgresMembershipsTable(db *sql.DB) (tables.Memberships, error) { {&s.selectMembershipCountStmt, selectMembershipCountSQL}, {&s.selectHeroesStmt, selectHeroesSQL}, {&s.selectMembershipForUserStmt, selectMembershipBeforeSQL}, + {&s.selectMembersStmt, selectMembersSQL}, }.Prepare(db) } @@ -154,3 +165,25 @@ func (s *membershipsStatements) SelectMembershipForUser( } return membership, topologyPos, nil } + +func (s *membershipsStatements) SelectMemberships( + ctx context.Context, txn *sql.Tx, + roomID string, pos types.TopologyToken, + membership, notMembership *string, +) (eventIDs []string, err error) { + stmt := sqlutil.TxStmt(txn, s.selectMembersStmt) + rows, err := stmt.QueryContext(ctx, roomID, pos.Depth, membership, notMembership) + if err != nil { + return + } + var ( + eventID string + ) + for rows.Next() { + if err = rows.Scan(&eventID); err != nil { + return + } + eventIDs = append(eventIDs, eventID) + } + return eventIDs, rows.Err() +} diff --git a/syncapi/storage/shared/storage_consumer.go b/syncapi/storage/shared/storage_consumer.go index bf12203db..23f53d11f 100644 --- a/syncapi/storage/shared/storage_consumer.go +++ b/syncapi/storage/shared/storage_consumer.go @@ -617,3 +617,11 @@ func (d *Database) RedactRelations(ctx context.Context, roomID, redactedEventID return d.Relations.DeleteRelation(ctx, txn, roomID, redactedEventID) }) } + +func (d *Database) SelectMemberships( + ctx context.Context, + roomID string, pos types.TopologyToken, + membership, notMembership *string, +) (eventIDs []string, err error) { + return d.Memberships.SelectMemberships(ctx, nil, roomID, pos, membership, notMembership) +} diff --git a/syncapi/storage/sqlite3/memberships_table.go b/syncapi/storage/sqlite3/memberships_table.go index 0c966fca0..7e54fac17 100644 --- a/syncapi/storage/sqlite3/memberships_table.go +++ b/syncapi/storage/sqlite3/memberships_table.go @@ -20,11 +20,12 @@ import ( "fmt" "strings" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/syncapi/storage/tables" "github.com/matrix-org/dendrite/syncapi/types" - "github.com/matrix-org/gomatrixserverlib" ) // The memberships table is designed to track the last time that @@ -69,12 +70,20 @@ const selectHeroesSQL = "" + const selectMembershipBeforeSQL = "" + "SELECT membership, topological_pos FROM syncapi_memberships WHERE room_id = $1 and user_id = $2 AND topological_pos <= $3 ORDER BY topological_pos DESC LIMIT 1" +const selectMembersSQL = ` +SELECT event_id FROM + ( SELECT event_id, membership FROM syncapi_memberships WHERE room_id = $1 AND topological_pos <= $2 GROUP BY user_id HAVING(max(stream_pos))) t + WHERE ($3 IS NULL OR t.membership = $3) + AND ($4 IS NULL OR t.membership <> $4) +` + type membershipsStatements struct { db *sql.DB upsertMembershipStmt *sql.Stmt selectMembershipCountStmt *sql.Stmt //selectHeroesStmt *sql.Stmt - prepared at runtime due to variadic selectMembershipForUserStmt *sql.Stmt + selectMembersStmt *sql.Stmt } func NewSqliteMembershipsTable(db *sql.DB) (tables.Memberships, error) { @@ -89,6 +98,7 @@ func NewSqliteMembershipsTable(db *sql.DB) (tables.Memberships, error) { {&s.upsertMembershipStmt, upsertMembershipSQL}, {&s.selectMembershipCountStmt, selectMembershipCountSQL}, {&s.selectMembershipForUserStmt, selectMembershipBeforeSQL}, + {&s.selectMembersStmt, selectMembersSQL}, // {&s.selectHeroesStmt, selectHeroesSQL}, - prepared at runtime due to variadic }.Prepare(db) } @@ -170,3 +180,23 @@ func (s *membershipsStatements) SelectMembershipForUser( } return membership, topologyPos, nil } + +func (s *membershipsStatements) SelectMemberships( + ctx context.Context, txn *sql.Tx, + roomID string, pos types.TopologyToken, + membership, notMembership *string, +) (eventIDs []string, err error) { + stmt := sqlutil.TxStmt(txn, s.selectMembersStmt) + rows, err := stmt.QueryContext(ctx, roomID, pos.Depth, membership, notMembership) + if err != nil { + return + } + var eventID string + for rows.Next() { + if err = rows.Scan(&eventID); err != nil { + return + } + eventIDs = append(eventIDs, eventID) + } + return eventIDs, rows.Err() +} diff --git a/syncapi/storage/tables/interface.go b/syncapi/storage/tables/interface.go index e48c050dd..2c4f04ec2 100644 --- a/syncapi/storage/tables/interface.go +++ b/syncapi/storage/tables/interface.go @@ -187,6 +187,11 @@ type Memberships interface { SelectMembershipCount(ctx context.Context, txn *sql.Tx, roomID, membership string, pos types.StreamPosition) (count int, err error) SelectHeroes(ctx context.Context, txn *sql.Tx, roomID, userID string, memberships []string) (heroes []string, err error) SelectMembershipForUser(ctx context.Context, txn *sql.Tx, roomID, userID string, pos int64) (membership string, topologicalPos int, err error) + SelectMemberships( + ctx context.Context, txn *sql.Tx, + roomID string, pos types.TopologyToken, + membership, notMembership *string, + ) (eventIDs []string, err error) } type NotificationData interface { diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go index 9ec2b61cd..707dbe8dc 100644 --- a/syncapi/streams/stream_pdu.go +++ b/syncapi/streams/stream_pdu.go @@ -473,7 +473,13 @@ func (p *PDUStreamProvider) getJoinResponseForCompleteSync( var prevBatch *types.TopologyToken if len(recentStreamEvents) > 0 { var backwardTopologyPos, backwardStreamPos types.StreamPosition - backwardTopologyPos, backwardStreamPos, err = snapshot.PositionInTopology(ctx, recentStreamEvents[0].EventID()) + event := recentStreamEvents[0] + // If this is the beginning of the room, we can't go back further. We're going to return + // the TopologyToken from the last event instead. (Synapse returns the /sync next_Batch) + if event.Type() == gomatrixserverlib.MRoomCreate && event.StateKeyEquals("") { + event = recentStreamEvents[len(recentStreamEvents)-1] + } + backwardTopologyPos, backwardStreamPos, err = snapshot.PositionInTopology(ctx, event.EventID()) if err != nil { return } diff --git a/syncapi/types/types.go b/syncapi/types/types.go index 57ce7b6ff..295187acc 100644 --- a/syncapi/types/types.go +++ b/syncapi/types/types.go @@ -234,6 +234,9 @@ func (t *TopologyToken) StreamToken() StreamingToken { } func (t TopologyToken) String() string { + if t.Depth <= 0 && t.PDUPosition <= 0 { + return "" + } return fmt.Sprintf("t%d_%d", t.Depth, t.PDUPosition) } diff --git a/sytest-whitelist b/sytest-whitelist index e92ae6495..e5e405af6 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -754,4 +754,6 @@ Messages that notify from another user increment notification_count Messages that highlight from another user increment unread highlight count Notifications can be viewed with GET /notifications Can get rooms/{roomId}/messages for a departed room (SPEC-216) -Local device key changes appear in /keys/changes \ No newline at end of file +Local device key changes appear in /keys/changes +Can get rooms/{roomId}/members at a given point +Can filter rooms/{roomId}/members \ No newline at end of file From db6a214b046c83d8cacd00608aa464bd040c4997 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Tue, 25 Oct 2022 12:28:34 +0100 Subject: [PATCH 106/207] Prettify unit test output --- .github/workflows/dendrite.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dendrite.yml b/.github/workflows/dendrite.yml index f8019b3ea..a8271b675 100644 --- a/.github/workflows/dendrite.yml +++ b/.github/workflows/dendrite.yml @@ -109,6 +109,11 @@ jobs: uses: actions/setup-go@v3 with: go-version: ${{ matrix.go }} + - name: Set up gotestfmt + uses: gotesttools/gotestfmt-action@v2 + with: + # Optional: pass GITHUB_TOKEN to avoid rate limiting. + token: ${{ secrets.GITHUB_TOKEN }} - uses: actions/cache@v3 with: path: | @@ -117,7 +122,7 @@ jobs: key: ${{ runner.os }}-go${{ matrix.go }}-test-${{ hashFiles('**/go.sum') }} restore-keys: | ${{ runner.os }}-go${{ matrix.go }}-test- - - run: go test ./... + - run: go test -json -v ./... 2>&1 | gotestfmt env: POSTGRES_HOST: localhost POSTGRES_USER: postgres From 8b7bf5e7d7dbb7d87848156c27666fc2353efeba Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Tue, 25 Oct 2022 15:00:52 +0200 Subject: [PATCH 107/207] Return forbidden if not a member anymore (fix #2802) --- syncapi/routing/memberships.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/syncapi/routing/memberships.go b/syncapi/routing/memberships.go index b4e342251..c9acc5d2b 100644 --- a/syncapi/routing/memberships.go +++ b/syncapi/routing/memberships.go @@ -109,6 +109,12 @@ func GetMemberships( } if joinedOnly { + if !queryRes.IsInRoom { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("You aren't a member of the room and weren't previously a member of the room."), + } + } var res getJoinedMembersResponse res.Joined = make(map[string]joinedMember) for _, ev := range result { From c62ac3d6ad5c60f5f28a0f50bba50f7cbc2436ce Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Tue, 25 Oct 2022 15:15:24 +0200 Subject: [PATCH 108/207] Fix `Current state appears in timeline in private history with many messages after` (#2830) The problem was that we weren't getting enough recent events, as most of them were removed by the history visibility filter. Now we're getting all events between the given input range and re-slice the returned values after applying history visibility. --- syncapi/streams/stream_pdu.go | 18 ++++++++++-------- sytest-whitelist | 3 ++- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go index 707dbe8dc..90cf8ce53 100644 --- a/syncapi/streams/stream_pdu.go +++ b/syncapi/streams/stream_pdu.go @@ -227,14 +227,10 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( stateFilter *gomatrixserverlib.StateFilter, req *types.SyncRequest, ) (types.StreamPosition, error) { - if delta.MembershipPos > 0 && delta.Membership == gomatrixserverlib.Leave { - // make sure we don't leak recent events after the leave event. - // TODO: History visibility makes this somewhat complex to handle correctly. For example: - // TODO: This doesn't work for join -> leave in a single /sync request (see events prior to join). - // TODO: This will fail on join -> leave -> sensitive msg -> join -> leave - // in a single /sync request - // This is all "okay" assuming history_visibility == "shared" which it is by default. - r.To = delta.MembershipPos + + originalLimit := eventFilter.Limit + if r.Backwards { + eventFilter.Limit = int(r.From - r.To) } recentStreamEvents, limited, err := snapshot.RecentEvents( ctx, delta.RoomID, r, @@ -303,6 +299,12 @@ func (p *PDUStreamProvider) addRoomDeltaToResponse( logrus.WithError(err).Error("unable to apply history visibility filter") } + if r.Backwards && len(events) > originalLimit { + // We're going backwards and the events are ordered chronologically, so take the last `limit` events + events = events[len(events)-originalLimit:] + limited = true + } + if len(delta.StateEvents) > 0 { updateLatestPosition(delta.StateEvents[len(delta.StateEvents)-1].EventID()) } diff --git a/sytest-whitelist b/sytest-whitelist index e5e405af6..60610929a 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -756,4 +756,5 @@ Notifications can be viewed with GET /notifications Can get rooms/{roomId}/messages for a departed room (SPEC-216) Local device key changes appear in /keys/changes Can get rooms/{roomId}/members at a given point -Can filter rooms/{roomId}/members \ No newline at end of file +Can filter rooms/{roomId}/members +Current state appears in timeline in private history with many messages after \ No newline at end of file From 2a4c7f45b37a9bcd1a37d42b0668e0c3dfb29762 Mon Sep 17 00:00:00 2001 From: Neboer <43609792+Neboer@users.noreply.github.com> Date: Wed, 26 Oct 2022 17:04:53 +0800 Subject: [PATCH 109/207] Add support for config "auto_join_rooms" (#2823) Add support for config "auto_join_rooms". Now new accounts can join the rooms in config file automatically. ### Pull Request Checklist * [x] I have justified why this PR doesn't need tests. * [x] Pull request includes a [sign off below using a legally identifiable name](https://matrix-org.github.io/dendrite/development/contributing#sign-off) Signed-off-by: `Rubin Poster ` --- dendrite-sample.monolith.yaml | 8 +++++++ dendrite-sample.polylith.yaml | 8 +++++++ roomserver/api/api.go | 1 + setup/config/config_userapi.go | 4 ++++ userapi/internal/api.go | 42 ++++++++++++++++++++++++++++++++++ userapi/userapi.go | 1 + 6 files changed, 64 insertions(+) diff --git a/dendrite-sample.monolith.yaml b/dendrite-sample.monolith.yaml index eadb74a2a..5195c29bc 100644 --- a/dendrite-sample.monolith.yaml +++ b/dendrite-sample.monolith.yaml @@ -310,6 +310,14 @@ user_api: # The default lifetime is 3600000ms (60 minutes). # openid_token_lifetime_ms: 3600000 + # Users who register on this homeserver will automatically be joined to the rooms listed under "auto_join_rooms" option. + # By default, any room aliases included in this list will be created as a publicly joinable room + # when the first user registers for the homeserver. If the room already exists, + # make certain it is a publicly joinable room, i.e. the join rule of the room must be set to 'public'. + # As Spaces are just rooms under the hood, Space aliases may also be used. + auto_join_rooms: + # - "#main:matrix.org" + # Configuration for Opentracing. # See https://github.com/matrix-org/dendrite/tree/master/docs/tracing for information on # how this works and how to set it up. diff --git a/dendrite-sample.polylith.yaml b/dendrite-sample.polylith.yaml index aa7e0cc38..bbbe16fdc 100644 --- a/dendrite-sample.polylith.yaml +++ b/dendrite-sample.polylith.yaml @@ -375,6 +375,14 @@ user_api: # The default lifetime is 3600000ms (60 minutes). # openid_token_lifetime_ms: 3600000 + # Users who register on this homeserver will automatically be joined to the rooms listed under "auto_join_rooms" option. + # By default, any room aliases included in this list will be created as a publicly joinable room + # when the first user registers for the homeserver. If the room already exists, + # make certain it is a publicly joinable room, i.e. the join rule of the room must be set to 'public'. + # As Spaces are just rooms under the hood, Space aliases may also be used. + auto_join_rooms: + # - "#main:matrix.org" + # Configuration for Opentracing. # See https://github.com/matrix-org/dendrite/tree/master/docs/tracing for information on # how this works and how to set it up. diff --git a/roomserver/api/api.go b/roomserver/api/api.go index baf63aa31..403bbe8be 100644 --- a/roomserver/api/api.go +++ b/roomserver/api/api.go @@ -167,6 +167,7 @@ type UserRoomserverAPI interface { QueryCurrentState(ctx context.Context, req *QueryCurrentStateRequest, res *QueryCurrentStateResponse) error QueryMembershipsForRoom(ctx context.Context, req *QueryMembershipsForRoomRequest, res *QueryMembershipsForRoomResponse) error PerformAdminEvacuateUser(ctx context.Context, req *PerformAdminEvacuateUserRequest, res *PerformAdminEvacuateUserResponse) error + PerformJoin(ctx context.Context, req *PerformJoinRequest, res *PerformJoinResponse) error } type FederationRoomserverAPI interface { diff --git a/setup/config/config_userapi.go b/setup/config/config_userapi.go index 97a6d738b..f8ad41d93 100644 --- a/setup/config/config_userapi.go +++ b/setup/config/config_userapi.go @@ -19,6 +19,10 @@ type UserAPI struct { // The Account database stores the login details and account information // for local users. It is accessed by the UserAPI. AccountDatabase DatabaseOptions `yaml:"account_database,omitempty"` + + // Users who register on this homeserver will automatically + // be joined to the rooms listed under this option. + AutoJoinRooms []string `yaml:"auto_join_rooms"` } const DefaultOpenIDTokenLifetimeMS = 3600000 // 60 minutes diff --git a/userapi/internal/api.go b/userapi/internal/api.go index 63044eedb..7b94b3da7 100644 --- a/userapi/internal/api.go +++ b/userapi/internal/api.go @@ -54,6 +54,7 @@ type UserInternalAPI struct { KeyAPI keyapi.UserKeyAPI RSAPI rsapi.UserRoomserverAPI PgClient pushgateway.Client + Cfg *config.UserAPI } func (a *UserInternalAPI) InputAccountData(ctx context.Context, req *api.InputAccountDataRequest, res *api.InputAccountDataResponse) error { @@ -130,6 +131,45 @@ func (a *UserInternalAPI) setFullyRead(ctx context.Context, req *api.InputAccoun return nil } +func postRegisterJoinRooms(cfg *config.UserAPI, acc *api.Account, rsAPI rsapi.UserRoomserverAPI) { + // POST register behaviour: check if the user is a normal user. + // If the user is a normal user, add user to room specified in the configuration "auto_join_rooms". + if acc.AccountType != api.AccountTypeAppService && acc.AppServiceID == "" { + for room := range cfg.AutoJoinRooms { + userID := userutil.MakeUserID(acc.Localpart, cfg.Matrix.ServerName) + err := addUserToRoom(context.Background(), rsAPI, cfg.AutoJoinRooms[room], acc.Localpart, userID) + if err != nil { + logrus.WithFields(logrus.Fields{ + "user_id": userID, + "room": cfg.AutoJoinRooms[room], + }).WithError(err).Errorf("user failed to auto-join room") + } + } + } +} + +// Add user to a room. This function currently working for auto_join_rooms config, +// which can add a newly registered user to a specified room. +func addUserToRoom( + ctx context.Context, + rsAPI rsapi.UserRoomserverAPI, + roomID string, + username string, + userID string, +) error { + addGroupContent := make(map[string]interface{}) + // This make sure the user's username can be displayed correctly. + // Because the newly-registered user doesn't have an avatar, the avatar_url is not needed. + addGroupContent["displayname"] = username + joinReq := rsapi.PerformJoinRequest{ + RoomIDOrAlias: roomID, + UserID: userID, + Content: addGroupContent, + } + joinRes := rsapi.PerformJoinResponse{} + return rsAPI.PerformJoin(ctx, &joinReq, &joinRes) +} + func (a *UserInternalAPI) PerformAccountCreation(ctx context.Context, req *api.PerformAccountCreationRequest, res *api.PerformAccountCreationResponse) error { acc, err := a.DB.CreateAccount(ctx, req.Localpart, req.Password, req.AppServiceID, req.AccountType) if err != nil { @@ -174,6 +214,8 @@ func (a *UserInternalAPI) PerformAccountCreation(ctx context.Context, req *api.P return err } + postRegisterJoinRooms(a.Cfg, acc, a.RSAPI) + res.AccountCreated = true res.Account = acc return nil diff --git a/userapi/userapi.go b/userapi/userapi.go index d26b4e19a..c077248e2 100644 --- a/userapi/userapi.go +++ b/userapi/userapi.go @@ -82,6 +82,7 @@ func NewInternalAPI( RSAPI: rsAPI, DisableTLSValidation: cfg.PushGatewayDisableTLSValidation, PgClient: pgClient, + Cfg: cfg, } receiptConsumer := consumers.NewOutputReceiptEventConsumer( From f6dea712d2e9c71f6ebe61f90e45a142852432e8 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 26 Oct 2022 12:59:19 +0100 Subject: [PATCH 110/207] Initial support for multiple server names (#2829) This PR is the first step towards virtual hosting by laying the groundwork for multiple server names being configured. --- clientapi/auth/password.go | 2 +- clientapi/routing/admin.go | 4 +- clientapi/routing/createroom.go | 25 ++++++-- clientapi/routing/directory.go | 4 +- clientapi/routing/directory_public.go | 3 +- clientapi/routing/login.go | 6 +- clientapi/routing/membership.go | 9 +-- clientapi/routing/openid.go | 2 +- clientapi/routing/profile.go | 34 +++++++++-- clientapi/routing/redaction.go | 3 +- clientapi/routing/register.go | 2 +- clientapi/routing/sendevent.go | 5 +- clientapi/threepid/invites.go | 2 +- clientapi/userutil/userutil.go | 13 +++-- clientapi/userutil/userutil_test.go | 25 ++++++-- federationapi/federationapi.go | 4 +- federationapi/federationapi_keys_test.go | 2 +- federationapi/federationapi_test.go | 1 + federationapi/internal/keys.go | 2 +- federationapi/internal/perform.go | 2 +- federationapi/producers/syncapi.go | 5 +- federationapi/queue/queue_test.go | 2 +- federationapi/routing/routing.go | 50 ++++++++-------- federationapi/storage/postgres/storage.go | 4 +- federationapi/storage/shared/storage.go | 4 +- federationapi/storage/sqlite3/storage.go | 4 +- federationapi/storage/storage.go | 6 +- federationapi/storage/storage_test.go | 2 +- go.mod | 2 +- go.sum | 4 +- roomserver/internal/perform/perform_admin.go | 11 +++- roomserver/internal/perform/perform_invite.go | 4 +- roomserver/internal/perform/perform_join.go | 23 +++++--- roomserver/internal/perform/perform_leave.go | 6 +- roomserver/internal/perform/perform_peek.go | 6 +- roomserver/internal/perform/perform_unpeek.go | 2 +- .../internal/perform/perform_upgrade.go | 37 +++++++----- setup/config/config_global.go | 15 +++++ setup/mscs/msc2836/msc2836.go | 2 +- setup/mscs/msc2946/msc2946.go | 2 +- test/testrig/base.go | 2 + userapi/api/api.go | 31 ++++++++-- userapi/internal/api.go | 58 +++++++++++++------ userapi/internal/api_logintoken.go | 8 +-- userapi/userapi.go | 2 +- userapi/userapi_test.go | 4 +- 46 files changed, 291 insertions(+), 155 deletions(-) diff --git a/clientapi/auth/password.go b/clientapi/auth/password.go index 890b18183..700a72f5d 100644 --- a/clientapi/auth/password.go +++ b/clientapi/auth/password.go @@ -74,7 +74,7 @@ func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, JSON: jsonerror.BadJSON("A password must be supplied."), } } - localpart, err := userutil.ParseUsernameParam(username, &t.Config.Matrix.ServerName) + localpart, _, err := userutil.ParseUsernameParam(username, t.Config.Matrix) if err != nil { return nil, &util.JSONResponse{ Code: http.StatusUnauthorized, diff --git a/clientapi/routing/admin.go b/clientapi/routing/admin.go index 89c269f1a..69bca13be 100644 --- a/clientapi/routing/admin.go +++ b/clientapi/routing/admin.go @@ -70,7 +70,7 @@ func AdminEvacuateUser(req *http.Request, cfg *config.ClientAPI, device *userapi if err != nil { return util.MessageResponse(http.StatusBadRequest, err.Error()) } - if domain != cfg.Matrix.ServerName { + if !cfg.Matrix.IsLocalServerName(domain) { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.MissingArgument("User ID must belong to this server."), @@ -169,7 +169,7 @@ func AdminMarkAsStale(req *http.Request, cfg *config.ClientAPI, keyAPI api.Clien if err != nil { return util.MessageResponse(http.StatusBadRequest, err.Error()) } - if domain == cfg.Matrix.ServerName { + if cfg.Matrix.IsLocalServerName(domain) { return util.JSONResponse{ Code: http.StatusBadRequest, JSON: jsonerror.InvalidParam("Can not mark local device list as stale"), diff --git a/clientapi/routing/createroom.go b/clientapi/routing/createroom.go index 3e837c864..eefe8e24b 100644 --- a/clientapi/routing/createroom.go +++ b/clientapi/routing/createroom.go @@ -169,9 +169,21 @@ func createRoom( asAPI appserviceAPI.AppServiceInternalAPI, evTime time.Time, ) util.JSONResponse { + _, userDomain, err := gomatrixserverlib.SplitID('@', device.UserID) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("gomatrixserverlib.SplitID failed") + return jsonerror.InternalServerError() + } + if !cfg.Matrix.IsLocalServerName(userDomain) { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden(fmt.Sprintf("User domain %q not configured locally", userDomain)), + } + } + // TODO (#267): Check room ID doesn't clash with an existing one, and we // probably shouldn't be using pseudo-random strings, maybe GUIDs? - roomID := fmt.Sprintf("!%s:%s", util.RandomString(16), cfg.Matrix.ServerName) + roomID := fmt.Sprintf("!%s:%s", util.RandomString(16), userDomain) logger := util.GetLogger(ctx) userID := device.UserID @@ -314,7 +326,7 @@ func createRoom( var roomAlias string if r.RoomAliasName != "" { - roomAlias = fmt.Sprintf("#%s:%s", r.RoomAliasName, cfg.Matrix.ServerName) + roomAlias = fmt.Sprintf("#%s:%s", r.RoomAliasName, userDomain) // check it's free TODO: This races but is better than nothing hasAliasReq := roomserverAPI.GetRoomIDForAliasRequest{ Alias: roomAlias, @@ -436,7 +448,7 @@ func createRoom( builder.PrevEvents = []gomatrixserverlib.EventReference{builtEvents[i-1].EventReference()} } var ev *gomatrixserverlib.Event - ev, err = buildEvent(&builder, &authEvents, cfg, evTime, roomVersion) + ev, err = buildEvent(&builder, userDomain, &authEvents, cfg, evTime, roomVersion) if err != nil { util.GetLogger(ctx).WithError(err).Error("buildEvent failed") return jsonerror.InternalServerError() @@ -461,7 +473,7 @@ func createRoom( inputs = append(inputs, roomserverAPI.InputRoomEvent{ Kind: roomserverAPI.KindNew, Event: event, - Origin: cfg.Matrix.ServerName, + Origin: userDomain, SendAsServer: roomserverAPI.DoNotSendToOtherServers, }) } @@ -548,7 +560,7 @@ func createRoom( Event: event, InviteRoomState: inviteStrippedState, RoomVersion: event.RoomVersion, - SendAsServer: string(cfg.Matrix.ServerName), + SendAsServer: string(userDomain), }, &inviteRes); err != nil { util.GetLogger(ctx).WithError(err).Error("PerformInvite failed") return util.JSONResponse{ @@ -591,6 +603,7 @@ func createRoom( // buildEvent fills out auth_events for the builder then builds the event func buildEvent( builder *gomatrixserverlib.EventBuilder, + serverName gomatrixserverlib.ServerName, provider gomatrixserverlib.AuthEventProvider, cfg *config.ClientAPI, evTime time.Time, @@ -606,7 +619,7 @@ func buildEvent( } builder.AuthEvents = refs event, err := builder.Build( - evTime, cfg.Matrix.ServerName, cfg.Matrix.KeyID, + evTime, serverName, cfg.Matrix.KeyID, cfg.Matrix.PrivateKey, roomVersion, ) if err != nil { diff --git a/clientapi/routing/directory.go b/clientapi/routing/directory.go index 836d9e152..33bc63d18 100644 --- a/clientapi/routing/directory.go +++ b/clientapi/routing/directory.go @@ -75,7 +75,7 @@ func DirectoryRoom( if res.RoomID == "" { // If we don't know it locally, do a federation query. // But don't send the query to ourselves. - if domain != cfg.Matrix.ServerName { + if !cfg.Matrix.IsLocalServerName(domain) { fedRes, fedErr := federation.LookupRoomAlias(req.Context(), domain, roomAlias) if fedErr != nil { // TODO: Return 502 if the remote server errored. @@ -127,7 +127,7 @@ func SetLocalAlias( } } - if domain != cfg.Matrix.ServerName { + if !cfg.Matrix.IsLocalServerName(domain) { return util.JSONResponse{ Code: http.StatusForbidden, JSON: jsonerror.Forbidden("Alias must be on local homeserver"), diff --git a/clientapi/routing/directory_public.go b/clientapi/routing/directory_public.go index 8ddb3267a..4ebf2295a 100644 --- a/clientapi/routing/directory_public.go +++ b/clientapi/routing/directory_public.go @@ -62,8 +62,7 @@ func GetPostPublicRooms( } serverName := gomatrixserverlib.ServerName(request.Server) - - if serverName != "" && serverName != cfg.Matrix.ServerName { + if serverName != "" && !cfg.Matrix.IsLocalServerName(serverName) { res, err := federation.GetPublicRoomsFiltered( req.Context(), serverName, int(request.Limit), request.Since, diff --git a/clientapi/routing/login.go b/clientapi/routing/login.go index 6017b5840..7f5a8c4f8 100644 --- a/clientapi/routing/login.go +++ b/clientapi/routing/login.go @@ -68,7 +68,7 @@ func Login( return *authErr } // make a device/access token - authErr2 := completeAuth(req.Context(), cfg.Matrix.ServerName, userAPI, login, req.RemoteAddr, req.UserAgent()) + authErr2 := completeAuth(req.Context(), cfg.Matrix, userAPI, login, req.RemoteAddr, req.UserAgent()) cleanup(req.Context(), &authErr2) return authErr2 } @@ -79,7 +79,7 @@ func Login( } func completeAuth( - ctx context.Context, serverName gomatrixserverlib.ServerName, userAPI userapi.ClientUserAPI, login *auth.Login, + ctx context.Context, cfg *config.Global, userAPI userapi.ClientUserAPI, login *auth.Login, ipAddr, userAgent string, ) util.JSONResponse { token, err := auth.GenerateAccessToken() @@ -88,7 +88,7 @@ func completeAuth( return jsonerror.InternalServerError() } - localpart, err := userutil.ParseUsernameParam(login.Username(), &serverName) + localpart, serverName, err := userutil.ParseUsernameParam(login.Username(), cfg) if err != nil { util.GetLogger(ctx).WithError(err).Error("auth.ParseUsernameParam failed") return jsonerror.InternalServerError() diff --git a/clientapi/routing/membership.go b/clientapi/routing/membership.go index 77f627eb2..94ba17a02 100644 --- a/clientapi/routing/membership.go +++ b/clientapi/routing/membership.go @@ -105,12 +105,13 @@ func sendMembership(ctx context.Context, profileAPI userapi.ClientUserAPI, devic return jsonerror.InternalServerError() } + serverName := device.UserDomain() if err = roomserverAPI.SendEvents( ctx, rsAPI, roomserverAPI.KindNew, []*gomatrixserverlib.HeaderedEvent{event.Event.Headered(roomVer)}, - cfg.Matrix.ServerName, - cfg.Matrix.ServerName, + serverName, + serverName, nil, false, ); err != nil { @@ -271,7 +272,7 @@ func sendInvite( Event: event, InviteRoomState: nil, // ask the roomserver to draw up invite room state for us RoomVersion: event.RoomVersion, - SendAsServer: string(cfg.Matrix.ServerName), + SendAsServer: string(device.UserDomain()), }, &inviteRes); err != nil { util.GetLogger(ctx).WithError(err).Error("PerformInvite failed") return util.JSONResponse{ @@ -341,7 +342,7 @@ func loadProfile( } var profile *authtypes.Profile - if serverName == cfg.Matrix.ServerName { + if cfg.Matrix.IsLocalServerName(serverName) { profile, err = appserviceAPI.RetrieveUserProfile(ctx, userID, asAPI, profileAPI) } else { profile = &authtypes.Profile{} diff --git a/clientapi/routing/openid.go b/clientapi/routing/openid.go index cfb440bea..8e9be7889 100644 --- a/clientapi/routing/openid.go +++ b/clientapi/routing/openid.go @@ -63,7 +63,7 @@ func CreateOpenIDToken( JSON: openIDTokenResponse{ AccessToken: response.Token.Token, TokenType: "Bearer", - MatrixServerName: string(cfg.Matrix.ServerName), + MatrixServerName: string(device.UserDomain()), ExpiresIn: response.Token.ExpiresAtMS / 1000, // convert ms to s }, } diff --git a/clientapi/routing/profile.go b/clientapi/routing/profile.go index c9647eb1b..4d9e1f8a5 100644 --- a/clientapi/routing/profile.go +++ b/clientapi/routing/profile.go @@ -113,12 +113,19 @@ func SetAvatarURL( } } - localpart, _, err := gomatrixserverlib.SplitID('@', userID) + localpart, domain, err := gomatrixserverlib.SplitID('@', userID) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") return jsonerror.InternalServerError() } + if !cfg.Matrix.IsLocalServerName(domain) { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("userID does not belong to a locally configured domain"), + } + } + evTime, err := httputil.ParseTSParam(req) if err != nil { return util.JSONResponse{ @@ -129,8 +136,9 @@ func SetAvatarURL( setRes := &userapi.PerformSetAvatarURLResponse{} if err = profileAPI.SetAvatarURL(req.Context(), &userapi.PerformSetAvatarURLRequest{ - Localpart: localpart, - AvatarURL: r.AvatarURL, + Localpart: localpart, + ServerName: domain, + AvatarURL: r.AvatarURL, }, setRes); err != nil { util.GetLogger(req.Context()).WithError(err).Error("profileAPI.SetAvatarURL failed") return jsonerror.InternalServerError() @@ -204,12 +212,19 @@ func SetDisplayName( } } - localpart, _, err := gomatrixserverlib.SplitID('@', userID) + localpart, domain, err := gomatrixserverlib.SplitID('@', userID) if err != nil { util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed") return jsonerror.InternalServerError() } + if !cfg.Matrix.IsLocalServerName(domain) { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("userID does not belong to a locally configured domain"), + } + } + evTime, err := httputil.ParseTSParam(req) if err != nil { return util.JSONResponse{ @@ -221,6 +236,7 @@ func SetDisplayName( profileRes := &userapi.PerformUpdateDisplayNameResponse{} err = profileAPI.SetDisplayName(req.Context(), &userapi.PerformUpdateDisplayNameRequest{ Localpart: localpart, + ServerName: domain, DisplayName: r.DisplayName, }, profileRes) if err != nil { @@ -261,6 +277,12 @@ func updateProfile( return jsonerror.InternalServerError(), err } + _, domain, err := gomatrixserverlib.SplitID('@', userID) + if err != nil { + util.GetLogger(ctx).WithError(err).Error("gomatrixserverlib.SplitID failed") + return jsonerror.InternalServerError(), err + } + events, err := buildMembershipEvents( ctx, res.RoomIDs, *profile, userID, cfg, evTime, rsAPI, ) @@ -276,7 +298,7 @@ func updateProfile( return jsonerror.InternalServerError(), e } - if err := api.SendEvents(ctx, rsAPI, api.KindNew, events, cfg.Matrix.ServerName, cfg.Matrix.ServerName, nil, true); err != nil { + if err := api.SendEvents(ctx, rsAPI, api.KindNew, events, domain, domain, nil, true); err != nil { util.GetLogger(ctx).WithError(err).Error("SendEvents failed") return jsonerror.InternalServerError(), err } @@ -298,7 +320,7 @@ func getProfile( return nil, err } - if domain != cfg.Matrix.ServerName { + if !cfg.Matrix.IsLocalServerName(domain) { profile, fedErr := federation.LookupProfile(ctx, domain, userID, "") if fedErr != nil { if x, ok := fedErr.(gomatrix.HTTPError); ok { diff --git a/clientapi/routing/redaction.go b/clientapi/routing/redaction.go index a0f3b1152..778a02fd4 100644 --- a/clientapi/routing/redaction.go +++ b/clientapi/routing/redaction.go @@ -131,7 +131,8 @@ func SendRedaction( JSON: jsonerror.NotFound("Room does not exist"), } } - if err = roomserverAPI.SendEvents(context.Background(), rsAPI, roomserverAPI.KindNew, []*gomatrixserverlib.HeaderedEvent{e}, cfg.Matrix.ServerName, cfg.Matrix.ServerName, nil, false); err != nil { + domain := device.UserDomain() + if err = roomserverAPI.SendEvents(context.Background(), rsAPI, roomserverAPI.KindNew, []*gomatrixserverlib.HeaderedEvent{e}, domain, domain, nil, false); err != nil { util.GetLogger(req.Context()).WithError(err).Errorf("failed to SendEvents") return jsonerror.InternalServerError() } diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index 0bda1e488..698d185b4 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -412,7 +412,7 @@ func UserIDIsWithinApplicationServiceNamespace( return false } - if domain != cfg.Matrix.ServerName { + if !cfg.Matrix.IsLocalServerName(domain) { return false } diff --git a/clientapi/routing/sendevent.go b/clientapi/routing/sendevent.go index 114e9088d..bb66cf6fc 100644 --- a/clientapi/routing/sendevent.go +++ b/clientapi/routing/sendevent.go @@ -94,6 +94,7 @@ func SendEvent( // create a mutex for the specific user in the specific room // this avoids a situation where events that are received in quick succession are sent to the roomserver in a jumbled order userID := device.UserID + domain := device.UserDomain() mutex, _ := userRoomSendMutexes.LoadOrStore(roomID+userID, &sync.Mutex{}) mutex.(*sync.Mutex).Lock() defer mutex.(*sync.Mutex).Unlock() @@ -185,8 +186,8 @@ func SendEvent( []*gomatrixserverlib.HeaderedEvent{ e.Headered(verRes.RoomVersion), }, - cfg.Matrix.ServerName, - cfg.Matrix.ServerName, + domain, + domain, txnAndSessionID, false, ); err != nil { diff --git a/clientapi/threepid/invites.go b/clientapi/threepid/invites.go index 9670fecad..99fb8171d 100644 --- a/clientapi/threepid/invites.go +++ b/clientapi/threepid/invites.go @@ -215,7 +215,7 @@ func queryIDServerStoreInvite( } var profile *authtypes.Profile - if serverName == cfg.Matrix.ServerName { + if cfg.Matrix.IsLocalServerName(serverName) { res := &userapi.QueryProfileResponse{} err = userAPI.QueryProfile(ctx, &userapi.QueryProfileRequest{UserID: device.UserID}, res) if err != nil { diff --git a/clientapi/userutil/userutil.go b/clientapi/userutil/userutil.go index 7e909ffad..9be1e9b31 100644 --- a/clientapi/userutil/userutil.go +++ b/clientapi/userutil/userutil.go @@ -17,6 +17,7 @@ import ( "fmt" "strings" + "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrixserverlib" ) @@ -24,23 +25,23 @@ import ( // usernameParam can either be a user ID or just the localpart/username. // If serverName is passed, it is verified against the domain obtained from usernameParam (if present) // Returns error in case of invalid usernameParam. -func ParseUsernameParam(usernameParam string, expectedServerName *gomatrixserverlib.ServerName) (string, error) { +func ParseUsernameParam(usernameParam string, cfg *config.Global) (string, gomatrixserverlib.ServerName, error) { localpart := usernameParam if strings.HasPrefix(usernameParam, "@") { lp, domain, err := gomatrixserverlib.SplitID('@', usernameParam) if err != nil { - return "", errors.New("invalid username") + return "", "", errors.New("invalid username") } - if expectedServerName != nil && domain != *expectedServerName { - return "", errors.New("user ID does not belong to this server") + if !cfg.IsLocalServerName(domain) { + return "", "", errors.New("user ID does not belong to this server") } - localpart = lp + return lp, domain, nil } - return localpart, nil + return localpart, cfg.ServerName, nil } // MakeUserID generates user ID from localpart & server name diff --git a/clientapi/userutil/userutil_test.go b/clientapi/userutil/userutil_test.go index 2628642fb..ccd6647b2 100644 --- a/clientapi/userutil/userutil_test.go +++ b/clientapi/userutil/userutil_test.go @@ -15,6 +15,7 @@ package userutil import ( "testing" + "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrixserverlib" ) @@ -28,7 +29,11 @@ var ( // TestGoodUserID checks that correct localpart is returned for a valid user ID. func TestGoodUserID(t *testing.T) { - lp, err := ParseUsernameParam(goodUserID, &serverName) + cfg := &config.Global{ + ServerName: serverName, + } + + lp, _, err := ParseUsernameParam(goodUserID, cfg) if err != nil { t.Error("User ID Parsing failed for ", goodUserID, " with error: ", err.Error()) @@ -41,7 +46,11 @@ func TestGoodUserID(t *testing.T) { // TestWithLocalpartOnly checks that localpart is returned when usernameParam contains only localpart. func TestWithLocalpartOnly(t *testing.T) { - lp, err := ParseUsernameParam(localpart, &serverName) + cfg := &config.Global{ + ServerName: serverName, + } + + lp, _, err := ParseUsernameParam(localpart, cfg) if err != nil { t.Error("User ID Parsing failed for ", localpart, " with error: ", err.Error()) @@ -54,7 +63,11 @@ func TestWithLocalpartOnly(t *testing.T) { // TestIncorrectDomain checks for error when there's server name mismatch. func TestIncorrectDomain(t *testing.T) { - _, err := ParseUsernameParam(goodUserID, &invalidServerName) + cfg := &config.Global{ + ServerName: invalidServerName, + } + + _, _, err := ParseUsernameParam(goodUserID, cfg) if err == nil { t.Error("Invalid Domain should return an error") @@ -63,7 +76,11 @@ func TestIncorrectDomain(t *testing.T) { // TestBadUserID checks that ParseUsernameParam fails for invalid user ID func TestBadUserID(t *testing.T) { - _, err := ParseUsernameParam(badUserID, &serverName) + cfg := &config.Global{ + ServerName: serverName, + } + + _, _, err := ParseUsernameParam(badUserID, cfg) if err == nil { t.Error("Illegal User ID should return an error") diff --git a/federationapi/federationapi.go b/federationapi/federationapi.go index f6dace702..a58cba1b1 100644 --- a/federationapi/federationapi.go +++ b/federationapi/federationapi.go @@ -69,7 +69,7 @@ func AddPublicRoutes( TopicPresenceEvent: cfg.Matrix.JetStream.Prefixed(jetstream.OutputPresenceEvent), TopicDeviceListUpdate: cfg.Matrix.JetStream.Prefixed(jetstream.InputDeviceListUpdate), TopicSigningKeyUpdate: cfg.Matrix.JetStream.Prefixed(jetstream.InputSigningKeyUpdate), - ServerName: cfg.Matrix.ServerName, + Config: cfg, UserAPI: userAPI, } @@ -107,7 +107,7 @@ func NewInternalAPI( ) api.FederationInternalAPI { cfg := &base.Cfg.FederationAPI - federationDB, err := storage.NewDatabase(base, &cfg.Database, base.Caches, base.Cfg.Global.ServerName) + federationDB, err := storage.NewDatabase(base, &cfg.Database, base.Caches, base.Cfg.Global.IsLocalServerName) if err != nil { logrus.WithError(err).Panic("failed to connect to federation sender db") } diff --git a/federationapi/federationapi_keys_test.go b/federationapi/federationapi_keys_test.go index 85cc43aa5..7ccc02f76 100644 --- a/federationapi/federationapi_keys_test.go +++ b/federationapi/federationapi_keys_test.go @@ -87,6 +87,7 @@ func TestMain(m *testing.M) { cfg.Global.JetStream.StoragePath = config.Path(d) cfg.Global.KeyID = serverKeyID cfg.Global.KeyValidityPeriod = s.validity + cfg.FederationAPI.KeyPerspectives = nil f, err := os.CreateTemp(d, "federation_keys_test*.db") if err != nil { return -1 @@ -207,7 +208,6 @@ func TestRenewalBehaviour(t *testing.T) { // happy at this point that the key that we already have is from the past // then repeating a key fetch should cause us to try and renew the key. // If so, then the new key will end up in our cache. - serverC.renew() res, err = serverA.api.FetchKeys( diff --git a/federationapi/federationapi_test.go b/federationapi/federationapi_test.go index e923143a7..c37bc87c2 100644 --- a/federationapi/federationapi_test.go +++ b/federationapi/federationapi_test.go @@ -164,6 +164,7 @@ func TestFederationAPIJoinThenKeyUpdate(t *testing.T) { func testFederationAPIJoinThenKeyUpdate(t *testing.T, dbType test.DBType) { base, close := testrig.CreateBaseDendrite(t, dbType) base.Cfg.FederationAPI.PreferDirectFetch = true + base.Cfg.FederationAPI.KeyPerspectives = nil defer close() jsctx, _ := base.NATS.Prepare(base.ProcessContext, &base.Cfg.Global.JetStream) defer jetstream.DeleteAllStreams(jsctx, &base.Cfg.Global.JetStream) diff --git a/federationapi/internal/keys.go b/federationapi/internal/keys.go index 2b7a8219a..258bd88bf 100644 --- a/federationapi/internal/keys.go +++ b/federationapi/internal/keys.go @@ -99,7 +99,7 @@ func (s *FederationInternalAPI) handleLocalKeys( results map[gomatrixserverlib.PublicKeyLookupRequest]gomatrixserverlib.PublicKeyLookupResult, ) { for req := range requests { - if req.ServerName != s.cfg.Matrix.ServerName { + if !s.cfg.Matrix.IsLocalServerName(req.ServerName) { continue } if req.KeyID == s.cfg.Matrix.KeyID { diff --git a/federationapi/internal/perform.go b/federationapi/internal/perform.go index 28ec48d7b..1b61ec711 100644 --- a/federationapi/internal/perform.go +++ b/federationapi/internal/perform.go @@ -77,7 +77,7 @@ func (r *FederationInternalAPI) PerformJoin( seenSet := make(map[gomatrixserverlib.ServerName]bool) var uniqueList []gomatrixserverlib.ServerName for _, srv := range request.ServerNames { - if seenSet[srv] || srv == r.cfg.Matrix.ServerName { + if seenSet[srv] || r.cfg.Matrix.IsLocalServerName(srv) { continue } seenSet[srv] = true diff --git a/federationapi/producers/syncapi.go b/federationapi/producers/syncapi.go index 659ff1bcf..7cce13a7d 100644 --- a/federationapi/producers/syncapi.go +++ b/federationapi/producers/syncapi.go @@ -25,6 +25,7 @@ import ( "github.com/nats-io/nats.go" log "github.com/sirupsen/logrus" + "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/jetstream" "github.com/matrix-org/dendrite/syncapi/types" userapi "github.com/matrix-org/dendrite/userapi/api" @@ -39,7 +40,7 @@ type SyncAPIProducer struct { TopicDeviceListUpdate string TopicSigningKeyUpdate string JetStream nats.JetStreamContext - ServerName gomatrixserverlib.ServerName + Config *config.FederationAPI UserAPI userapi.UserInternalAPI } @@ -77,7 +78,7 @@ func (p *SyncAPIProducer) SendToDevice( // device. If the event isn't targeted locally then we can't expand the // wildcard as we don't know about the remote devices, so instead we leave it // as-is, so that the federation sender can send it on with the wildcard intact. - if domain == p.ServerName && deviceID == "*" { + if p.Config.Matrix.IsLocalServerName(domain) && deviceID == "*" { var res userapi.QueryDevicesResponse err = p.UserAPI.QueryDevices(context.TODO(), &userapi.QueryDevicesRequest{ UserID: userID, diff --git a/federationapi/queue/queue_test.go b/federationapi/queue/queue_test.go index a1b280103..7ef4646f7 100644 --- a/federationapi/queue/queue_test.go +++ b/federationapi/queue/queue_test.go @@ -47,7 +47,7 @@ func mustCreateFederationDatabase(t *testing.T, dbType test.DBType, realDatabase connStr, dbClose := test.PrepareDBConnectionString(t, dbType) db, err := storage.NewDatabase(b, &config.DatabaseOptions{ ConnectionString: config.DataSource(connStr), - }, b.Caches, b.Cfg.Global.ServerName) + }, b.Caches, b.Cfg.Global.IsLocalServerName) if err != nil { t.Fatalf("NewDatabase returned %s", err) } diff --git a/federationapi/routing/routing.go b/federationapi/routing/routing.go index e25f9866e..9f16e5093 100644 --- a/federationapi/routing/routing.go +++ b/federationapi/routing/routing.go @@ -124,7 +124,7 @@ func Setup( mu := internal.NewMutexByRoom() v1fedmux.Handle("/send/{txnID}", MakeFedAPI( - "federation_send", cfg.Matrix.ServerName, keys, wakeup, + "federation_send", cfg.Matrix.ServerName, cfg.Matrix.IsLocalServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return Send( httpReq, request, gomatrixserverlib.TransactionID(vars["txnID"]), @@ -134,7 +134,7 @@ func Setup( )).Methods(http.MethodPut, http.MethodOptions) v1fedmux.Handle("/invite/{roomID}/{eventID}", MakeFedAPI( - "federation_invite", cfg.Matrix.ServerName, keys, wakeup, + "federation_invite", cfg.Matrix.ServerName, cfg.Matrix.IsLocalServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) { return util.JSONResponse{ @@ -150,7 +150,7 @@ func Setup( )).Methods(http.MethodPut, http.MethodOptions) v2fedmux.Handle("/invite/{roomID}/{eventID}", MakeFedAPI( - "federation_invite", cfg.Matrix.ServerName, keys, wakeup, + "federation_invite", cfg.Matrix.ServerName, cfg.Matrix.IsLocalServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) { return util.JSONResponse{ @@ -172,7 +172,7 @@ func Setup( )).Methods(http.MethodPost, http.MethodOptions) v1fedmux.Handle("/exchange_third_party_invite/{roomID}", MakeFedAPI( - "exchange_third_party_invite", cfg.Matrix.ServerName, keys, wakeup, + "exchange_third_party_invite", cfg.Matrix.ServerName, cfg.Matrix.IsLocalServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return ExchangeThirdPartyInvite( httpReq, request, vars["roomID"], rsAPI, cfg, federation, @@ -181,7 +181,7 @@ func Setup( )).Methods(http.MethodPut, http.MethodOptions) v1fedmux.Handle("/event/{eventID}", MakeFedAPI( - "federation_get_event", cfg.Matrix.ServerName, keys, wakeup, + "federation_get_event", cfg.Matrix.ServerName, cfg.Matrix.IsLocalServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return GetEvent( httpReq.Context(), request, rsAPI, vars["eventID"], cfg.Matrix.ServerName, @@ -190,7 +190,7 @@ func Setup( )).Methods(http.MethodGet) v1fedmux.Handle("/state/{roomID}", MakeFedAPI( - "federation_get_state", cfg.Matrix.ServerName, keys, wakeup, + "federation_get_state", cfg.Matrix.ServerName, cfg.Matrix.IsLocalServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) { return util.JSONResponse{ @@ -205,7 +205,7 @@ func Setup( )).Methods(http.MethodGet) v1fedmux.Handle("/state_ids/{roomID}", MakeFedAPI( - "federation_get_state_ids", cfg.Matrix.ServerName, keys, wakeup, + "federation_get_state_ids", cfg.Matrix.ServerName, cfg.Matrix.IsLocalServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) { return util.JSONResponse{ @@ -220,7 +220,7 @@ func Setup( )).Methods(http.MethodGet) v1fedmux.Handle("/event_auth/{roomID}/{eventID}", MakeFedAPI( - "federation_get_event_auth", cfg.Matrix.ServerName, keys, wakeup, + "federation_get_event_auth", cfg.Matrix.ServerName, cfg.Matrix.IsLocalServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) { return util.JSONResponse{ @@ -235,7 +235,7 @@ func Setup( )).Methods(http.MethodGet) v1fedmux.Handle("/query/directory", MakeFedAPI( - "federation_query_room_alias", cfg.Matrix.ServerName, keys, wakeup, + "federation_query_room_alias", cfg.Matrix.ServerName, cfg.Matrix.IsLocalServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return RoomAliasToID( httpReq, federation, cfg, rsAPI, fsAPI, @@ -244,7 +244,7 @@ func Setup( )).Methods(http.MethodGet) v1fedmux.Handle("/query/profile", MakeFedAPI( - "federation_query_profile", cfg.Matrix.ServerName, keys, wakeup, + "federation_query_profile", cfg.Matrix.ServerName, cfg.Matrix.IsLocalServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return GetProfile( httpReq, userAPI, cfg, @@ -253,7 +253,7 @@ func Setup( )).Methods(http.MethodGet) v1fedmux.Handle("/user/devices/{userID}", MakeFedAPI( - "federation_user_devices", cfg.Matrix.ServerName, keys, wakeup, + "federation_user_devices", cfg.Matrix.ServerName, cfg.Matrix.IsLocalServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return GetUserDevices( httpReq, keyAPI, vars["userID"], @@ -263,7 +263,7 @@ func Setup( if mscCfg.Enabled("msc2444") { v1fedmux.Handle("/peek/{roomID}/{peekID}", MakeFedAPI( - "federation_peek", cfg.Matrix.ServerName, keys, wakeup, + "federation_peek", cfg.Matrix.ServerName, cfg.Matrix.IsLocalServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) { return util.JSONResponse{ @@ -294,7 +294,7 @@ func Setup( } v1fedmux.Handle("/make_join/{roomID}/{userID}", MakeFedAPI( - "federation_make_join", cfg.Matrix.ServerName, keys, wakeup, + "federation_make_join", cfg.Matrix.ServerName, cfg.Matrix.IsLocalServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) { return util.JSONResponse{ @@ -325,7 +325,7 @@ func Setup( )).Methods(http.MethodGet) v1fedmux.Handle("/send_join/{roomID}/{eventID}", MakeFedAPI( - "federation_send_join", cfg.Matrix.ServerName, keys, wakeup, + "federation_send_join", cfg.Matrix.ServerName, cfg.Matrix.IsLocalServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) { return util.JSONResponse{ @@ -357,7 +357,7 @@ func Setup( )).Methods(http.MethodPut) v2fedmux.Handle("/send_join/{roomID}/{eventID}", MakeFedAPI( - "federation_send_join", cfg.Matrix.ServerName, keys, wakeup, + "federation_send_join", cfg.Matrix.ServerName, cfg.Matrix.IsLocalServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) { return util.JSONResponse{ @@ -374,7 +374,7 @@ func Setup( )).Methods(http.MethodPut) v1fedmux.Handle("/make_leave/{roomID}/{eventID}", MakeFedAPI( - "federation_make_leave", cfg.Matrix.ServerName, keys, wakeup, + "federation_make_leave", cfg.Matrix.ServerName, cfg.Matrix.IsLocalServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) { return util.JSONResponse{ @@ -391,7 +391,7 @@ func Setup( )).Methods(http.MethodGet) v1fedmux.Handle("/send_leave/{roomID}/{eventID}", MakeFedAPI( - "federation_send_leave", cfg.Matrix.ServerName, keys, wakeup, + "federation_send_leave", cfg.Matrix.ServerName, cfg.Matrix.IsLocalServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) { return util.JSONResponse{ @@ -423,7 +423,7 @@ func Setup( )).Methods(http.MethodPut) v2fedmux.Handle("/send_leave/{roomID}/{eventID}", MakeFedAPI( - "federation_send_leave", cfg.Matrix.ServerName, keys, wakeup, + "federation_send_leave", cfg.Matrix.ServerName, cfg.Matrix.IsLocalServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) { return util.JSONResponse{ @@ -447,7 +447,7 @@ func Setup( )).Methods(http.MethodGet) v1fedmux.Handle("/get_missing_events/{roomID}", MakeFedAPI( - "federation_get_missing_events", cfg.Matrix.ServerName, keys, wakeup, + "federation_get_missing_events", cfg.Matrix.ServerName, cfg.Matrix.IsLocalServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) { return util.JSONResponse{ @@ -460,7 +460,7 @@ func Setup( )).Methods(http.MethodPost) v1fedmux.Handle("/backfill/{roomID}", MakeFedAPI( - "federation_backfill", cfg.Matrix.ServerName, keys, wakeup, + "federation_backfill", cfg.Matrix.ServerName, cfg.Matrix.IsLocalServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { if roomserverAPI.IsServerBannedFromRoom(httpReq.Context(), rsAPI, vars["roomID"], request.Origin()) { return util.JSONResponse{ @@ -479,14 +479,14 @@ func Setup( ).Methods(http.MethodGet, http.MethodPost) v1fedmux.Handle("/user/keys/claim", MakeFedAPI( - "federation_keys_claim", cfg.Matrix.ServerName, keys, wakeup, + "federation_keys_claim", cfg.Matrix.ServerName, cfg.Matrix.IsLocalServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return ClaimOneTimeKeys(httpReq, request, keyAPI, cfg.Matrix.ServerName) }, )).Methods(http.MethodPost) v1fedmux.Handle("/user/keys/query", MakeFedAPI( - "federation_keys_query", cfg.Matrix.ServerName, keys, wakeup, + "federation_keys_query", cfg.Matrix.ServerName, cfg.Matrix.IsLocalServerName, keys, wakeup, func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { return QueryDeviceKeys(httpReq, request, keyAPI, cfg.Matrix.ServerName) }, @@ -525,15 +525,15 @@ func ErrorIfLocalServerNotInRoom( // MakeFedAPI makes an http.Handler that checks matrix federation authentication. func MakeFedAPI( - metricsName string, - serverName gomatrixserverlib.ServerName, + metricsName string, serverName gomatrixserverlib.ServerName, + isLocalServerName func(gomatrixserverlib.ServerName) bool, keyRing gomatrixserverlib.JSONVerifier, wakeup *FederationWakeups, f func(*http.Request, *gomatrixserverlib.FederationRequest, map[string]string) util.JSONResponse, ) http.Handler { h := func(req *http.Request) util.JSONResponse { fedReq, errResp := gomatrixserverlib.VerifyHTTPRequest( - req, time.Now(), serverName, keyRing, + req, time.Now(), serverName, isLocalServerName, keyRing, ) if fedReq == nil { return errResp diff --git a/federationapi/storage/postgres/storage.go b/federationapi/storage/postgres/storage.go index 6e208d096..a33fa4a43 100644 --- a/federationapi/storage/postgres/storage.go +++ b/federationapi/storage/postgres/storage.go @@ -36,7 +36,7 @@ type Database struct { } // NewDatabase opens a new database -func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache caching.FederationCache, serverName gomatrixserverlib.ServerName) (*Database, error) { +func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache caching.FederationCache, isLocalServerName func(gomatrixserverlib.ServerName) bool) (*Database, error) { var d Database var err error if d.db, d.writer, err = base.DatabaseConnection(dbProperties, sqlutil.NewDummyWriter()); err != nil { @@ -96,7 +96,7 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, } d.Database = shared.Database{ DB: d.db, - ServerName: serverName, + IsLocalServerName: isLocalServerName, Cache: cache, Writer: d.writer, FederationJoinedHosts: joinedHosts, diff --git a/federationapi/storage/shared/storage.go b/federationapi/storage/shared/storage.go index 6afb313a8..4fabff7d4 100644 --- a/federationapi/storage/shared/storage.go +++ b/federationapi/storage/shared/storage.go @@ -29,7 +29,7 @@ import ( type Database struct { DB *sql.DB - ServerName gomatrixserverlib.ServerName + IsLocalServerName func(gomatrixserverlib.ServerName) bool Cache caching.FederationCache Writer sqlutil.Writer FederationQueuePDUs tables.FederationQueuePDUs @@ -124,7 +124,7 @@ func (d *Database) GetJoinedHostsForRooms(ctx context.Context, roomIDs []string, } if excludeSelf { for i, server := range servers { - if server == d.ServerName { + if d.IsLocalServerName(server) { servers = append(servers[:i], servers[i+1:]...) } } diff --git a/federationapi/storage/sqlite3/storage.go b/federationapi/storage/sqlite3/storage.go index c89cb6bea..e86ac817b 100644 --- a/federationapi/storage/sqlite3/storage.go +++ b/federationapi/storage/sqlite3/storage.go @@ -35,7 +35,7 @@ type Database struct { } // NewDatabase opens a new database -func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache caching.FederationCache, serverName gomatrixserverlib.ServerName) (*Database, error) { +func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache caching.FederationCache, isLocalServerName func(gomatrixserverlib.ServerName) bool) (*Database, error) { var d Database var err error if d.db, d.writer, err = base.DatabaseConnection(dbProperties, sqlutil.NewExclusiveWriter()); err != nil { @@ -95,7 +95,7 @@ func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, } d.Database = shared.Database{ DB: d.db, - ServerName: serverName, + IsLocalServerName: isLocalServerName, Cache: cache, Writer: d.writer, FederationJoinedHosts: joinedHosts, diff --git a/federationapi/storage/storage.go b/federationapi/storage/storage.go index f246b9bc9..142e281ea 100644 --- a/federationapi/storage/storage.go +++ b/federationapi/storage/storage.go @@ -29,12 +29,12 @@ import ( ) // NewDatabase opens a new database -func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache caching.FederationCache, serverName gomatrixserverlib.ServerName) (Database, error) { +func NewDatabase(base *base.BaseDendrite, dbProperties *config.DatabaseOptions, cache caching.FederationCache, isLocalServerName func(gomatrixserverlib.ServerName) bool) (Database, error) { switch { case dbProperties.ConnectionString.IsSQLite(): - return sqlite3.NewDatabase(base, dbProperties, cache, serverName) + return sqlite3.NewDatabase(base, dbProperties, cache, isLocalServerName) case dbProperties.ConnectionString.IsPostgres(): - return postgres.NewDatabase(base, dbProperties, cache, serverName) + return postgres.NewDatabase(base, dbProperties, cache, isLocalServerName) default: return nil, fmt.Errorf("unexpected database type") } diff --git a/federationapi/storage/storage_test.go b/federationapi/storage/storage_test.go index 6272fd2b1..f7408fa9f 100644 --- a/federationapi/storage/storage_test.go +++ b/federationapi/storage/storage_test.go @@ -19,7 +19,7 @@ func mustCreateFederationDatabase(t *testing.T, dbType test.DBType) (storage.Dat connStr, dbClose := test.PrepareDBConnectionString(t, dbType) db, err := storage.NewDatabase(b, &config.DatabaseOptions{ ConnectionString: config.DataSource(connStr), - }, b.Caches, b.Cfg.Global.ServerName) + }, b.Caches, func(server gomatrixserverlib.ServerName) bool { return server == "localhost" }) if err != nil { t.Fatalf("NewDatabase returned %s", err) } diff --git a/go.mod b/go.mod index 7f9bb3897..39dfb0fe1 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 - github.com/matrix-org/gomatrixserverlib v0.0.0-20221021091412-7c772f1b388a + github.com/matrix-org/gomatrixserverlib v0.0.0-20221025142407-17b0be811afa github.com/matrix-org/pinecone v0.0.0-20221007145426-3adc85477dd3 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.15 diff --git a/go.sum b/go.sum index 5cce7e0d8..5e6253860 100644 --- a/go.sum +++ b/go.sum @@ -387,8 +387,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8r4Fzarl4+Y3K5hjothkVW5z7T1dUM11U= github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20221021091412-7c772f1b388a h1:6rJFN5NBuzZ7h5meYkLtXKa6VFZfDc8oVXHd4SDXr5o= -github.com/matrix-org/gomatrixserverlib v0.0.0-20221021091412-7c772f1b388a/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4= +github.com/matrix-org/gomatrixserverlib v0.0.0-20221025142407-17b0be811afa h1:S98DShDv3sn7O4n4HjtJOejypseYVpv1R/XPg+cDnfI= +github.com/matrix-org/gomatrixserverlib v0.0.0-20221025142407-17b0be811afa/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4= github.com/matrix-org/pinecone v0.0.0-20221007145426-3adc85477dd3 h1:lzkSQvBv8TuqKJCPoVwOVvEnARTlua5rrNy/Qw2Vxeo= github.com/matrix-org/pinecone v0.0.0-20221007145426-3adc85477dd3/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= diff --git a/roomserver/internal/perform/perform_admin.go b/roomserver/internal/perform/perform_admin.go index cb6b22d32..6a6d51b0a 100644 --- a/roomserver/internal/perform/perform_admin.go +++ b/roomserver/internal/perform/perform_admin.go @@ -117,6 +117,11 @@ func (r *Admin) PerformAdminEvacuateRoom( PrevEvents: prevEvents, } + _, senderDomain, err := gomatrixserverlib.SplitID('@', fledglingEvent.Sender) + if err != nil { + continue + } + if fledglingEvent.Content, err = json.Marshal(memberContent); err != nil { res.Error = &api.PerformError{ Code: api.PerformErrorBadRequest, @@ -146,8 +151,8 @@ func (r *Admin) PerformAdminEvacuateRoom( inputEvents = append(inputEvents, api.InputRoomEvent{ Kind: api.KindNew, Event: event, - Origin: r.Cfg.Matrix.ServerName, - SendAsServer: string(r.Cfg.Matrix.ServerName), + Origin: senderDomain, + SendAsServer: string(senderDomain), }) res.Affected = append(res.Affected, stateKey) prevEvents = []gomatrixserverlib.EventReference{ @@ -176,7 +181,7 @@ func (r *Admin) PerformAdminEvacuateUser( } return nil } - if domain != r.Cfg.Matrix.ServerName { + if !r.Cfg.Matrix.IsLocalServerName(domain) { res.Error = &api.PerformError{ Code: api.PerformErrorBadRequest, Msg: "Can only evacuate local users using this endpoint", diff --git a/roomserver/internal/perform/perform_invite.go b/roomserver/internal/perform/perform_invite.go index 3fbdf332e..f60247cd7 100644 --- a/roomserver/internal/perform/perform_invite.go +++ b/roomserver/internal/perform/perform_invite.go @@ -70,8 +70,8 @@ func (r *Inviter) PerformInvite( } return nil, nil } - isTargetLocal := domain == r.Cfg.Matrix.ServerName - isOriginLocal := senderDomain == r.Cfg.Matrix.ServerName + isTargetLocal := r.Cfg.Matrix.IsLocalServerName(domain) + isOriginLocal := r.Cfg.Matrix.IsLocalServerName(senderDomain) if !isOriginLocal && !isTargetLocal { res.Error = &api.PerformError{ Code: api.PerformErrorBadRequest, diff --git a/roomserver/internal/perform/perform_join.go b/roomserver/internal/perform/perform_join.go index 262273ff5..9d596ab30 100644 --- a/roomserver/internal/perform/perform_join.go +++ b/roomserver/internal/perform/perform_join.go @@ -92,7 +92,7 @@ func (r *Joiner) performJoin( Msg: fmt.Sprintf("Supplied user ID %q in incorrect format", req.UserID), } } - if domain != r.Cfg.Matrix.ServerName { + if !r.Cfg.Matrix.IsLocalServerName(domain) { return "", "", &rsAPI.PerformError{ Code: rsAPI.PerformErrorBadRequest, Msg: fmt.Sprintf("User %q does not belong to this homeserver", req.UserID), @@ -124,7 +124,7 @@ func (r *Joiner) performJoinRoomByAlias( // Check if this alias matches our own server configuration. If it // doesn't then we'll need to try a federated join. var roomID string - if domain != r.Cfg.Matrix.ServerName { + if !r.Cfg.Matrix.IsLocalServerName(domain) { // The alias isn't owned by us, so we will need to try joining using // a remote server. dirReq := fsAPI.PerformDirectoryLookupRequest{ @@ -172,7 +172,7 @@ func (r *Joiner) performJoinRoomByID( // The original client request ?server_name=... may include this HS so filter that out so we // don't attempt to make_join with ourselves for i := 0; i < len(req.ServerNames); i++ { - if req.ServerNames[i] == r.Cfg.Matrix.ServerName { + if r.Cfg.Matrix.IsLocalServerName(req.ServerNames[i]) { // delete this entry req.ServerNames = append(req.ServerNames[:i], req.ServerNames[i+1:]...) i-- @@ -191,12 +191,19 @@ func (r *Joiner) performJoinRoomByID( // If the server name in the room ID isn't ours then it's a // possible candidate for finding the room via federation. Add // it to the list of servers to try. - if domain != r.Cfg.Matrix.ServerName { + if !r.Cfg.Matrix.IsLocalServerName(domain) { req.ServerNames = append(req.ServerNames, domain) } // Prepare the template for the join event. userID := req.UserID + _, userDomain, err := gomatrixserverlib.SplitID('@', userID) + if err != nil { + return "", "", &rsAPI.PerformError{ + Code: rsAPI.PerformErrorBadRequest, + Msg: fmt.Sprintf("User ID %q is invalid: %s", userID, err), + } + } eb := gomatrixserverlib.EventBuilder{ Type: gomatrixserverlib.MRoomMember, Sender: userID, @@ -247,7 +254,7 @@ func (r *Joiner) performJoinRoomByID( // If we were invited by someone from another server then we can // assume they are in the room so we can join via them. - if inviterDomain != r.Cfg.Matrix.ServerName { + if !r.Cfg.Matrix.IsLocalServerName(inviterDomain) { req.ServerNames = append(req.ServerNames, inviterDomain) forceFederatedJoin = true memberEvent := gjson.Parse(string(inviteEvent.JSON())) @@ -300,7 +307,7 @@ func (r *Joiner) performJoinRoomByID( { Kind: rsAPI.KindNew, Event: event.Headered(buildRes.RoomVersion), - SendAsServer: string(r.Cfg.Matrix.ServerName), + SendAsServer: string(userDomain), }, }, } @@ -323,7 +330,7 @@ func (r *Joiner) performJoinRoomByID( // The room doesn't exist locally. If the room ID looks like it should // be ours then this probably means that we've nuked our database at // some point. - if domain == r.Cfg.Matrix.ServerName { + if r.Cfg.Matrix.IsLocalServerName(domain) { // If there are no more server names to try then give up here. // Otherwise we'll try a federated join as normal, since it's quite // possible that the room still exists on other servers. @@ -348,7 +355,7 @@ func (r *Joiner) performJoinRoomByID( // it will have been overwritten with a room ID by performJoinRoomByAlias. // We should now include this in the response so that the CS API can // return the right room ID. - return req.RoomIDOrAlias, r.Cfg.Matrix.ServerName, nil + return req.RoomIDOrAlias, userDomain, nil } func (r *Joiner) performFederatedJoinRoomByID( diff --git a/roomserver/internal/perform/perform_leave.go b/roomserver/internal/perform/perform_leave.go index 85b659814..49e4b479a 100644 --- a/roomserver/internal/perform/perform_leave.go +++ b/roomserver/internal/perform/perform_leave.go @@ -52,7 +52,7 @@ func (r *Leaver) PerformLeave( if err != nil { return nil, fmt.Errorf("supplied user ID %q in incorrect format", req.UserID) } - if domain != r.Cfg.Matrix.ServerName { + if !r.Cfg.Matrix.IsLocalServerName(domain) { return nil, fmt.Errorf("user %q does not belong to this homeserver", req.UserID) } logger := logrus.WithContext(ctx).WithFields(logrus.Fields{ @@ -85,7 +85,7 @@ func (r *Leaver) performLeaveRoomByID( if serr != nil { return nil, fmt.Errorf("sender %q is invalid", senderUser) } - if senderDomain != r.Cfg.Matrix.ServerName { + if !r.Cfg.Matrix.IsLocalServerName(senderDomain) { return r.performFederatedRejectInvite(ctx, req, res, senderUser, eventID) } // check that this is not a "server notice room" @@ -186,7 +186,7 @@ func (r *Leaver) performLeaveRoomByID( Kind: api.KindNew, Event: event.Headered(buildRes.RoomVersion), Origin: senderDomain, - SendAsServer: string(r.Cfg.Matrix.ServerName), + SendAsServer: string(senderDomain), }, }, } diff --git a/roomserver/internal/perform/perform_peek.go b/roomserver/internal/perform/perform_peek.go index 74d87a5b4..436d137ff 100644 --- a/roomserver/internal/perform/perform_peek.go +++ b/roomserver/internal/perform/perform_peek.go @@ -72,7 +72,7 @@ func (r *Peeker) performPeek( Msg: fmt.Sprintf("Supplied user ID %q in incorrect format", req.UserID), } } - if domain != r.Cfg.Matrix.ServerName { + if !r.Cfg.Matrix.IsLocalServerName(domain) { return "", &api.PerformError{ Code: api.PerformErrorBadRequest, Msg: fmt.Sprintf("User %q does not belong to this homeserver", req.UserID), @@ -104,7 +104,7 @@ func (r *Peeker) performPeekRoomByAlias( // Check if this alias matches our own server configuration. If it // doesn't then we'll need to try a federated peek. var roomID string - if domain != r.Cfg.Matrix.ServerName { + if !r.Cfg.Matrix.IsLocalServerName(domain) { // The alias isn't owned by us, so we will need to try peeking using // a remote server. dirReq := fsAPI.PerformDirectoryLookupRequest{ @@ -154,7 +154,7 @@ func (r *Peeker) performPeekRoomByID( // handle federated peeks // FIXME: don't create an outbound peek if we already have one going. - if domain != r.Cfg.Matrix.ServerName { + if !r.Cfg.Matrix.IsLocalServerName(domain) { // If the server name in the room ID isn't ours then it's a // possible candidate for finding the room via federation. Add // it to the list of servers to try. diff --git a/roomserver/internal/perform/perform_unpeek.go b/roomserver/internal/perform/perform_unpeek.go index 49e9067c9..0d97da4d6 100644 --- a/roomserver/internal/perform/perform_unpeek.go +++ b/roomserver/internal/perform/perform_unpeek.go @@ -67,7 +67,7 @@ func (r *Unpeeker) performUnpeek( Msg: fmt.Sprintf("Supplied user ID %q in incorrect format", req.UserID), } } - if domain != r.Cfg.Matrix.ServerName { + if !r.Cfg.Matrix.IsLocalServerName(domain) { return &api.PerformError{ Code: api.PerformErrorBadRequest, Msg: fmt.Sprintf("User %q does not belong to this homeserver", req.UserID), diff --git a/roomserver/internal/perform/perform_upgrade.go b/roomserver/internal/perform/perform_upgrade.go index d6dc9708c..38abe323c 100644 --- a/roomserver/internal/perform/perform_upgrade.go +++ b/roomserver/internal/perform/perform_upgrade.go @@ -60,6 +60,13 @@ func (r *Upgrader) performRoomUpgrade( ) (string, *api.PerformError) { roomID := req.RoomID userID := req.UserID + _, userDomain, err := gomatrixserverlib.SplitID('@', userID) + if err != nil { + return "", &api.PerformError{ + Code: api.PerformErrorNotAllowed, + Msg: "Error validating the user ID", + } + } evTime := time.Now() // Return an immediate error if the room does not exist @@ -80,7 +87,7 @@ func (r *Upgrader) performRoomUpgrade( // TODO (#267): Check room ID doesn't clash with an existing one, and we // probably shouldn't be using pseudo-random strings, maybe GUIDs? - newRoomID := fmt.Sprintf("!%s:%s", util.RandomString(16), r.Cfg.Matrix.ServerName) + newRoomID := fmt.Sprintf("!%s:%s", util.RandomString(16), userDomain) // Get the existing room state for the old room. oldRoomReq := &api.QueryLatestEventsAndStateRequest{ @@ -107,12 +114,12 @@ func (r *Upgrader) performRoomUpgrade( } // Send the setup events to the new room - if pErr = r.sendInitialEvents(ctx, evTime, userID, newRoomID, string(req.RoomVersion), eventsToMake); pErr != nil { + if pErr = r.sendInitialEvents(ctx, evTime, userID, userDomain, newRoomID, string(req.RoomVersion), eventsToMake); pErr != nil { return "", pErr } // 5. Send the tombstone event to the old room - if pErr = r.sendHeaderedEvent(ctx, tombstoneEvent, string(r.Cfg.Matrix.ServerName)); pErr != nil { + if pErr = r.sendHeaderedEvent(ctx, userDomain, tombstoneEvent, string(userDomain)); pErr != nil { return "", pErr } @@ -122,7 +129,7 @@ func (r *Upgrader) performRoomUpgrade( } // If the old room had a canonical alias event, it should be deleted in the old room - if pErr = r.clearOldCanonicalAliasEvent(ctx, oldRoomRes, evTime, userID, roomID); pErr != nil { + if pErr = r.clearOldCanonicalAliasEvent(ctx, oldRoomRes, evTime, userID, userDomain, roomID); pErr != nil { return "", pErr } @@ -132,7 +139,7 @@ func (r *Upgrader) performRoomUpgrade( } // 6. Restrict power levels in the old room - if pErr = r.restrictOldRoomPowerLevels(ctx, evTime, userID, roomID); pErr != nil { + if pErr = r.restrictOldRoomPowerLevels(ctx, evTime, userID, userDomain, roomID); pErr != nil { return "", pErr } @@ -154,7 +161,7 @@ func (r *Upgrader) getRoomPowerLevels(ctx context.Context, roomID string) (*goma return powerLevelContent, nil } -func (r *Upgrader) restrictOldRoomPowerLevels(ctx context.Context, evTime time.Time, userID, roomID string) *api.PerformError { +func (r *Upgrader) restrictOldRoomPowerLevels(ctx context.Context, evTime time.Time, userID string, userDomain gomatrixserverlib.ServerName, roomID string) *api.PerformError { restrictedPowerLevelContent, pErr := r.getRoomPowerLevels(ctx, roomID) if pErr != nil { return pErr @@ -183,7 +190,7 @@ func (r *Upgrader) restrictOldRoomPowerLevels(ctx context.Context, evTime time.T return resErr } } else { - if resErr = r.sendHeaderedEvent(ctx, restrictedPowerLevelsHeadered, api.DoNotSendToOtherServers); resErr != nil { + if resErr = r.sendHeaderedEvent(ctx, userDomain, restrictedPowerLevelsHeadered, api.DoNotSendToOtherServers); resErr != nil { return resErr } } @@ -223,7 +230,7 @@ func moveLocalAliases(ctx context.Context, return nil } -func (r *Upgrader) clearOldCanonicalAliasEvent(ctx context.Context, oldRoom *api.QueryLatestEventsAndStateResponse, evTime time.Time, userID, roomID string) *api.PerformError { +func (r *Upgrader) clearOldCanonicalAliasEvent(ctx context.Context, oldRoom *api.QueryLatestEventsAndStateResponse, evTime time.Time, userID string, userDomain gomatrixserverlib.ServerName, roomID string) *api.PerformError { for _, event := range oldRoom.StateEvents { if event.Type() != gomatrixserverlib.MRoomCanonicalAlias || !event.StateKeyEquals("") { continue @@ -254,7 +261,7 @@ func (r *Upgrader) clearOldCanonicalAliasEvent(ctx context.Context, oldRoom *api return resErr } } else { - if resErr = r.sendHeaderedEvent(ctx, emptyCanonicalAliasEvent, api.DoNotSendToOtherServers); resErr != nil { + if resErr = r.sendHeaderedEvent(ctx, userDomain, emptyCanonicalAliasEvent, api.DoNotSendToOtherServers); resErr != nil { return resErr } } @@ -495,7 +502,7 @@ func (r *Upgrader) generateInitialEvents(ctx context.Context, oldRoom *api.Query return eventsToMake, nil } -func (r *Upgrader) sendInitialEvents(ctx context.Context, evTime time.Time, userID, newRoomID, newVersion string, eventsToMake []fledglingEvent) *api.PerformError { +func (r *Upgrader) sendInitialEvents(ctx context.Context, evTime time.Time, userID string, userDomain gomatrixserverlib.ServerName, newRoomID, newVersion string, eventsToMake []fledglingEvent) *api.PerformError { var err error var builtEvents []*gomatrixserverlib.HeaderedEvent authEvents := gomatrixserverlib.NewAuthEvents(nil) @@ -519,7 +526,7 @@ func (r *Upgrader) sendInitialEvents(ctx context.Context, evTime time.Time, user builder.PrevEvents = []gomatrixserverlib.EventReference{builtEvents[i-1].EventReference()} } var event *gomatrixserverlib.Event - event, err = r.buildEvent(&builder, &authEvents, evTime, gomatrixserverlib.RoomVersion(newVersion)) + event, err = r.buildEvent(&builder, userDomain, &authEvents, evTime, gomatrixserverlib.RoomVersion(newVersion)) if err != nil { return &api.PerformError{ Msg: fmt.Sprintf("Failed to build new %q event: %s", builder.Type, err), @@ -547,7 +554,7 @@ func (r *Upgrader) sendInitialEvents(ctx context.Context, evTime time.Time, user inputs = append(inputs, api.InputRoomEvent{ Kind: api.KindNew, Event: event, - Origin: r.Cfg.Matrix.ServerName, + Origin: userDomain, SendAsServer: api.DoNotSendToOtherServers, }) } @@ -668,6 +675,7 @@ func createTemporaryPowerLevels(powerLevelContent *gomatrixserverlib.PowerLevelC func (r *Upgrader) sendHeaderedEvent( ctx context.Context, + serverName gomatrixserverlib.ServerName, headeredEvent *gomatrixserverlib.HeaderedEvent, sendAsServer string, ) *api.PerformError { @@ -675,7 +683,7 @@ func (r *Upgrader) sendHeaderedEvent( inputs = append(inputs, api.InputRoomEvent{ Kind: api.KindNew, Event: headeredEvent, - Origin: r.Cfg.Matrix.ServerName, + Origin: serverName, SendAsServer: sendAsServer, }) if err := api.SendInputRoomEvents(ctx, r.URSAPI, inputs, false); err != nil { @@ -689,6 +697,7 @@ func (r *Upgrader) sendHeaderedEvent( func (r *Upgrader) buildEvent( builder *gomatrixserverlib.EventBuilder, + serverName gomatrixserverlib.ServerName, provider gomatrixserverlib.AuthEventProvider, evTime time.Time, roomVersion gomatrixserverlib.RoomVersion, @@ -703,7 +712,7 @@ func (r *Upgrader) buildEvent( } builder.AuthEvents = refs event, err := builder.Build( - evTime, r.Cfg.Matrix.ServerName, r.Cfg.Matrix.KeyID, + evTime, serverName, r.Cfg.Matrix.KeyID, r.Cfg.Matrix.PrivateKey, roomVersion, ) if err != nil { diff --git a/setup/config/config_global.go b/setup/config/config_global.go index 784893d24..825772827 100644 --- a/setup/config/config_global.go +++ b/setup/config/config_global.go @@ -14,6 +14,9 @@ type Global struct { // The name of the server. This is usually the domain name, e.g 'matrix.org', 'localhost'. ServerName gomatrixserverlib.ServerName `yaml:"server_name"` + // The secondary server names, used for virtual hosting. + SecondaryServerNames []gomatrixserverlib.ServerName `yaml:"-"` + // Path to the private key which will be used to sign requests and events. PrivateKeyPath Path `yaml:"private_key"` @@ -120,6 +123,18 @@ func (c *Global) Verify(configErrs *ConfigErrors, isMonolith bool) { c.Cache.Verify(configErrs, isMonolith) } +func (c *Global) IsLocalServerName(serverName gomatrixserverlib.ServerName) bool { + if c.ServerName == serverName { + return true + } + for _, secondaryName := range c.SecondaryServerNames { + if secondaryName == serverName { + return true + } + } + return false +} + type OldVerifyKeys struct { // Path to the private key. PrivateKeyPath Path `yaml:"private_key"` diff --git a/setup/mscs/msc2836/msc2836.go b/setup/mscs/msc2836/msc2836.go index 452b14580..98502f5cb 100644 --- a/setup/mscs/msc2836/msc2836.go +++ b/setup/mscs/msc2836/msc2836.go @@ -132,7 +132,7 @@ func Enable( base.PublicFederationAPIMux.Handle("/unstable/event_relationships", httputil.MakeExternalAPI( "msc2836_event_relationships", func(req *http.Request) util.JSONResponse { fedReq, errResp := gomatrixserverlib.VerifyHTTPRequest( - req, time.Now(), base.Cfg.Global.ServerName, keyRing, + req, time.Now(), base.Cfg.Global.ServerName, base.Cfg.Global.IsLocalServerName, keyRing, ) if fedReq == nil { return errResp diff --git a/setup/mscs/msc2946/msc2946.go b/setup/mscs/msc2946/msc2946.go index a92a16a27..bc9df0f96 100644 --- a/setup/mscs/msc2946/msc2946.go +++ b/setup/mscs/msc2946/msc2946.go @@ -64,7 +64,7 @@ func Enable( fedAPI := httputil.MakeExternalAPI( "msc2946_fed_spaces", func(req *http.Request) util.JSONResponse { fedReq, errResp := gomatrixserverlib.VerifyHTTPRequest( - req, time.Now(), base.Cfg.Global.ServerName, keyRing, + req, time.Now(), base.Cfg.Global.ServerName, base.Cfg.Global.IsLocalServerName, keyRing, ) if fedReq == nil { return errResp diff --git a/test/testrig/base.go b/test/testrig/base.go index 10cc2407b..15fb5c370 100644 --- a/test/testrig/base.go +++ b/test/testrig/base.go @@ -36,6 +36,7 @@ func CreateBaseDendrite(t *testing.T, dbType test.DBType) (*base.BaseDendrite, f Monolithic: true, }) cfg.Global.JetStream.InMemory = true + cfg.FederationAPI.KeyPerspectives = nil switch dbType { case test.DBTypePostgres: cfg.Global.Defaults(config.DefaultOpts{ // autogen a signing key @@ -106,6 +107,7 @@ func Base(cfg *config.Dendrite) (*base.BaseDendrite, nats.JetStreamContext, *nat } cfg.Global.JetStream.InMemory = true cfg.SyncAPI.Fulltext.InMemory = true + cfg.FederationAPI.KeyPerspectives = nil base := base.NewBaseDendrite(cfg, "Tests") js, jc := base.NATS.Prepare(base.ProcessContext, &cfg.Global.JetStream) return base, js, jc diff --git a/userapi/api/api.go b/userapi/api/api.go index eef29144a..8d7f783de 100644 --- a/userapi/api/api.go +++ b/userapi/api/api.go @@ -318,8 +318,9 @@ type QuerySearchProfilesResponse struct { // PerformAccountCreationRequest is the request for PerformAccountCreation type PerformAccountCreationRequest struct { - AccountType AccountType // Required: whether this is a guest or user account - Localpart string // Required: The localpart for this account. Ignored if account type is guest. + AccountType AccountType // Required: whether this is a guest or user account + Localpart string // Required: The localpart for this account. Ignored if account type is guest. + ServerName gomatrixserverlib.ServerName // optional: if not specified, default server name used instead AppServiceID string // optional: the application service ID (not user ID) creating this account, if any. Password string // optional: if missing then this account will be a passwordless account @@ -360,7 +361,8 @@ type PerformLastSeenUpdateResponse struct { // PerformDeviceCreationRequest is the request for PerformDeviceCreation type PerformDeviceCreationRequest struct { Localpart string - AccessToken string // optional: if blank one will be made on your behalf + ServerName gomatrixserverlib.ServerName // optional: if blank, default server name used + AccessToken string // optional: if blank one will be made on your behalf // optional: if nil an ID is generated for you. If set, replaces any existing device session, // which will generate a new access token and invalidate the old one. DeviceID *string @@ -384,7 +386,8 @@ type PerformDeviceCreationResponse struct { // PerformAccountDeactivationRequest is the request for PerformAccountDeactivation type PerformAccountDeactivationRequest struct { - Localpart string + Localpart string + ServerName gomatrixserverlib.ServerName // optional: if blank, default server name used } // PerformAccountDeactivationResponse is the response for PerformAccountDeactivation @@ -434,6 +437,18 @@ type Device struct { AccountType AccountType } +func (d *Device) UserDomain() gomatrixserverlib.ServerName { + _, domain, err := gomatrixserverlib.SplitID('@', d.UserID) + if err != nil { + // This really is catastrophic because it means that someone + // managed to forge a malformed user ID for a device during + // login. + // TODO: Is there a better way to deal with this than panic? + panic(err) + } + return domain +} + // Account represents a Matrix account on this home server. type Account struct { UserID string @@ -577,7 +592,9 @@ type Notification struct { } type PerformSetAvatarURLRequest struct { - Localpart, AvatarURL string + Localpart string + ServerName gomatrixserverlib.ServerName + AvatarURL string } type PerformSetAvatarURLResponse struct { Profile *authtypes.Profile `json:"profile"` @@ -606,7 +623,9 @@ type QueryAccountByPasswordResponse struct { } type PerformUpdateDisplayNameRequest struct { - Localpart, DisplayName string + Localpart string + ServerName gomatrixserverlib.ServerName + DisplayName string } type PerformUpdateDisplayNameResponse struct { diff --git a/userapi/internal/api.go b/userapi/internal/api.go index 7b94b3da7..9ca76965d 100644 --- a/userapi/internal/api.go +++ b/userapi/internal/api.go @@ -46,9 +46,9 @@ import ( type UserInternalAPI struct { DB storage.Database SyncProducer *producers.SyncAPI + Config *config.UserAPI DisableTLSValidation bool - ServerName gomatrixserverlib.ServerName // AppServices is the list of all registered AS AppServices []config.ApplicationService KeyAPI keyapi.UserKeyAPI @@ -62,8 +62,8 @@ func (a *UserInternalAPI) InputAccountData(ctx context.Context, req *api.InputAc if err != nil { return err } - if domain != a.ServerName { - return fmt.Errorf("cannot query profile of remote users: got %s want %s", domain, a.ServerName) + if !a.Config.Matrix.IsLocalServerName(domain) { + return fmt.Errorf("cannot update account data of remote users (server name %s)", domain) } if req.DataType == "" { return fmt.Errorf("data type must not be empty") @@ -104,7 +104,7 @@ func (a *UserInternalAPI) setFullyRead(ctx context.Context, req *api.InputAccoun logrus.WithError(err).Error("UserInternalAPI.setFullyRead: SplitID failure") return nil } - if domain != a.ServerName { + if !a.Config.Matrix.IsLocalServerName(domain) { return nil } @@ -171,6 +171,11 @@ func addUserToRoom( } func (a *UserInternalAPI) PerformAccountCreation(ctx context.Context, req *api.PerformAccountCreationRequest, res *api.PerformAccountCreationResponse) error { + serverName := req.ServerName + if serverName == "" { + serverName = a.Config.Matrix.ServerName + } + // XXXX: Use the server name here acc, err := a.DB.CreateAccount(ctx, req.Localpart, req.Password, req.AppServiceID, req.AccountType) if err != nil { if errors.Is(err, sqlutil.ErrUserExists) { // This account already exists @@ -188,8 +193,8 @@ func (a *UserInternalAPI) PerformAccountCreation(ctx context.Context, req *api.P res.Account = &api.Account{ AppServiceID: req.AppServiceID, Localpart: req.Localpart, - ServerName: a.ServerName, - UserID: fmt.Sprintf("@%s:%s", req.Localpart, a.ServerName), + ServerName: serverName, + UserID: fmt.Sprintf("@%s:%s", req.Localpart, serverName), AccountType: req.AccountType, } return nil @@ -235,6 +240,12 @@ func (a *UserInternalAPI) PerformPasswordUpdate(ctx context.Context, req *api.Pe } func (a *UserInternalAPI) PerformDeviceCreation(ctx context.Context, req *api.PerformDeviceCreationRequest, res *api.PerformDeviceCreationResponse) error { + serverName := req.ServerName + if serverName == "" { + serverName = a.Config.Matrix.ServerName + } + _ = serverName + // XXXX: Use the server name here util.GetLogger(ctx).WithFields(logrus.Fields{ "localpart": req.Localpart, "device_id": req.DeviceID, @@ -259,8 +270,8 @@ func (a *UserInternalAPI) PerformDeviceDeletion(ctx context.Context, req *api.Pe if err != nil { return err } - if domain != a.ServerName { - return fmt.Errorf("cannot PerformDeviceDeletion of remote users: got %s want %s", domain, a.ServerName) + if !a.Config.Matrix.IsLocalServerName(domain) { + return fmt.Errorf("cannot PerformDeviceDeletion of remote users (server name %s)", domain) } deletedDeviceIDs := req.DeviceIDs if len(req.DeviceIDs) == 0 { @@ -392,8 +403,8 @@ func (a *UserInternalAPI) QueryProfile(ctx context.Context, req *api.QueryProfil if err != nil { return err } - if domain != a.ServerName { - return fmt.Errorf("cannot query profile of remote users: got %s want %s", domain, a.ServerName) + if !a.Config.Matrix.IsLocalServerName(domain) { + return fmt.Errorf("cannot query profile of remote users (server name %s)", domain) } prof, err := a.DB.GetProfileByLocalpart(ctx, local) if err != nil { @@ -443,8 +454,8 @@ func (a *UserInternalAPI) QueryDevices(ctx context.Context, req *api.QueryDevice if err != nil { return err } - if domain != a.ServerName { - return fmt.Errorf("cannot query devices of remote users: got %s want %s", domain, a.ServerName) + if !a.Config.Matrix.IsLocalServerName(domain) { + return fmt.Errorf("cannot query devices of remote users (server name %s)", domain) } devs, err := a.DB.GetDevicesByLocalpart(ctx, local) if err != nil { @@ -460,8 +471,8 @@ func (a *UserInternalAPI) QueryAccountData(ctx context.Context, req *api.QueryAc if err != nil { return err } - if domain != a.ServerName { - return fmt.Errorf("cannot query account data of remote users: got %s want %s", domain, a.ServerName) + if !a.Config.Matrix.IsLocalServerName(domain) { + return fmt.Errorf("cannot query account data of remote users (server name %s)", domain) } if req.DataType != "" { var data json.RawMessage @@ -509,10 +520,13 @@ func (a *UserInternalAPI) QueryAccessToken(ctx context.Context, req *api.QueryAc } return err } - localPart, _, err := gomatrixserverlib.SplitID('@', device.UserID) + localPart, domain, err := gomatrixserverlib.SplitID('@', device.UserID) if err != nil { return err } + if !a.Config.Matrix.IsLocalServerName(domain) { + return nil + } acc, err := a.DB.GetAccountByLocalpart(ctx, localPart) if err != nil { return err @@ -547,7 +561,7 @@ func (a *UserInternalAPI) queryAppServiceToken(ctx context.Context, token, appSe AccountType: api.AccountTypeAppService, } - localpart, err := userutil.ParseUsernameParam(appServiceUserID, &a.ServerName) + localpart, _, err := userutil.ParseUsernameParam(appServiceUserID, a.Config.Matrix) if err != nil { return nil, err } @@ -572,8 +586,16 @@ func (a *UserInternalAPI) queryAppServiceToken(ctx context.Context, token, appSe // PerformAccountDeactivation deactivates the user's account, removing all ability for the user to login again. func (a *UserInternalAPI) PerformAccountDeactivation(ctx context.Context, req *api.PerformAccountDeactivationRequest, res *api.PerformAccountDeactivationResponse) error { + serverName := req.ServerName + if serverName == "" { + serverName = a.Config.Matrix.ServerName + } + if !a.Config.Matrix.IsLocalServerName(serverName) { + return fmt.Errorf("server name %q not locally configured", serverName) + } + evacuateReq := &rsapi.PerformAdminEvacuateUserRequest{ - UserID: fmt.Sprintf("@%s:%s", req.Localpart, a.ServerName), + UserID: fmt.Sprintf("@%s:%s", req.Localpart, serverName), } evacuateRes := &rsapi.PerformAdminEvacuateUserResponse{} if err := a.RSAPI.PerformAdminEvacuateUser(ctx, evacuateReq, evacuateRes); err != nil { @@ -584,7 +606,7 @@ func (a *UserInternalAPI) PerformAccountDeactivation(ctx context.Context, req *a } deviceReq := &api.PerformDeviceDeletionRequest{ - UserID: fmt.Sprintf("@%s:%s", req.Localpart, a.ServerName), + UserID: fmt.Sprintf("@%s:%s", req.Localpart, serverName), } deviceRes := &api.PerformDeviceDeletionResponse{} if err := a.PerformDeviceDeletion(ctx, deviceReq, deviceRes); err != nil { diff --git a/userapi/internal/api_logintoken.go b/userapi/internal/api_logintoken.go index f1bf391e4..87f25e5e2 100644 --- a/userapi/internal/api_logintoken.go +++ b/userapi/internal/api_logintoken.go @@ -31,8 +31,8 @@ func (a *UserInternalAPI) PerformLoginTokenCreation(ctx context.Context, req *ap if err != nil { return err } - if domain != a.ServerName { - return fmt.Errorf("cannot create a login token for a remote user: got %s want %s", domain, a.ServerName) + if !a.Config.Matrix.IsLocalServerName(domain) { + return fmt.Errorf("cannot create a login token for a remote user (server name %s)", domain) } tokenMeta, err := a.DB.CreateLoginToken(ctx, &req.Data) if err != nil { @@ -63,8 +63,8 @@ func (a *UserInternalAPI) QueryLoginToken(ctx context.Context, req *api.QueryLog if err != nil { return err } - if domain != a.ServerName { - return fmt.Errorf("cannot return a login token for a remote user: got %s want %s", domain, a.ServerName) + if !a.Config.Matrix.IsLocalServerName(domain) { + return fmt.Errorf("cannot return a login token for a remote user (server name %s)", domain) } if _, err := a.DB.GetAccountByLocalpart(ctx, localpart); err != nil { res.Data = nil diff --git a/userapi/userapi.go b/userapi/userapi.go index c077248e2..e46a8e76e 100644 --- a/userapi/userapi.go +++ b/userapi/userapi.go @@ -76,7 +76,7 @@ func NewInternalAPI( userAPI := &internal.UserInternalAPI{ DB: db, SyncProducer: syncProducer, - ServerName: cfg.Matrix.ServerName, + Config: cfg, AppServices: appServices, KeyAPI: keyAPI, RSAPI: rsAPI, diff --git a/userapi/userapi_test.go b/userapi/userapi_test.go index aaa93f45b..2a43c0bd4 100644 --- a/userapi/userapi_test.go +++ b/userapi/userapi_test.go @@ -66,8 +66,8 @@ func MustMakeInternalAPI(t *testing.T, opts apiTestOpts, dbType test.DBType) (ap } return &internal.UserInternalAPI{ - DB: accountDB, - ServerName: cfg.Matrix.ServerName, + DB: accountDB, + Config: cfg, }, accountDB, func() { close() baseclose() From 5298dd1133948606172944b7e5ec3805ccc72644 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 26 Oct 2022 14:52:33 +0100 Subject: [PATCH 111/207] Update federation API consumers --- federationapi/consumers/keychange.go | 44 ++++++++++++------------- federationapi/consumers/presence.go | 10 +++--- federationapi/consumers/receipts.go | 34 +++++++++---------- federationapi/consumers/sendtodevice.go | 36 ++++++++++---------- federationapi/consumers/typing.go | 32 +++++++++--------- 5 files changed, 78 insertions(+), 78 deletions(-) diff --git a/federationapi/consumers/keychange.go b/federationapi/consumers/keychange.go index 67dfdc1d3..7d1ae0f81 100644 --- a/federationapi/consumers/keychange.go +++ b/federationapi/consumers/keychange.go @@ -35,14 +35,14 @@ import ( // KeyChangeConsumer consumes events that originate in key server. type KeyChangeConsumer struct { - ctx context.Context - jetstream nats.JetStreamContext - durable string - db storage.Database - queues *queue.OutgoingQueues - serverName gomatrixserverlib.ServerName - rsAPI roomserverAPI.FederationRoomserverAPI - topic string + ctx context.Context + jetstream nats.JetStreamContext + durable string + db storage.Database + queues *queue.OutgoingQueues + isLocalServerName func(gomatrixserverlib.ServerName) bool + rsAPI roomserverAPI.FederationRoomserverAPI + topic string } // NewKeyChangeConsumer creates a new KeyChangeConsumer. Call Start() to begin consuming from key servers. @@ -55,14 +55,14 @@ func NewKeyChangeConsumer( rsAPI roomserverAPI.FederationRoomserverAPI, ) *KeyChangeConsumer { return &KeyChangeConsumer{ - ctx: process.Context(), - jetstream: js, - durable: cfg.Matrix.JetStream.Prefixed("FederationAPIKeyChangeConsumer"), - topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputKeyChangeEvent), - queues: queues, - db: store, - serverName: cfg.Matrix.ServerName, - rsAPI: rsAPI, + ctx: process.Context(), + jetstream: js, + durable: cfg.Matrix.JetStream.Prefixed("FederationAPIKeyChangeConsumer"), + topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputKeyChangeEvent), + queues: queues, + db: store, + isLocalServerName: cfg.Matrix.IsLocalServerName, + rsAPI: rsAPI, } } @@ -112,7 +112,7 @@ func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) bool { logger.WithError(err).Error("Failed to extract domain from key change event") return true } - if originServerName != t.serverName { + if !t.isLocalServerName(originServerName) { return true } @@ -141,7 +141,7 @@ func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) bool { // Pack the EDU and marshal it edu := &gomatrixserverlib.EDU{ Type: gomatrixserverlib.MDeviceListUpdate, - Origin: string(t.serverName), + Origin: string(originServerName), } event := gomatrixserverlib.DeviceListUpdateEvent{ UserID: m.UserID, @@ -159,7 +159,7 @@ func (t *KeyChangeConsumer) onDeviceKeyMessage(m api.DeviceMessage) bool { } logger.Debugf("Sending device list update message to %q", destinations) - err = t.queues.SendEDU(edu, t.serverName, destinations) + err = t.queues.SendEDU(edu, originServerName, destinations) return err == nil } @@ -171,7 +171,7 @@ func (t *KeyChangeConsumer) onCrossSigningMessage(m api.DeviceMessage) bool { logrus.WithError(err).Errorf("fedsender key change consumer: user ID parse failure") return true } - if host != gomatrixserverlib.ServerName(t.serverName) { + if !t.isLocalServerName(host) { // Ignore any messages that didn't originate locally, otherwise we'll // end up parroting information we received from other servers. return true @@ -203,7 +203,7 @@ func (t *KeyChangeConsumer) onCrossSigningMessage(m api.DeviceMessage) bool { // Pack the EDU and marshal it edu := &gomatrixserverlib.EDU{ Type: types.MSigningKeyUpdate, - Origin: string(t.serverName), + Origin: string(host), } if edu.Content, err = json.Marshal(output); err != nil { sentry.CaptureException(err) @@ -212,7 +212,7 @@ func (t *KeyChangeConsumer) onCrossSigningMessage(m api.DeviceMessage) bool { } logger.Debugf("Sending cross-signing update message to %q", destinations) - err = t.queues.SendEDU(edu, t.serverName, destinations) + err = t.queues.SendEDU(edu, host, destinations) return err == nil } diff --git a/federationapi/consumers/presence.go b/federationapi/consumers/presence.go index e76103cd3..3445d34a9 100644 --- a/federationapi/consumers/presence.go +++ b/federationapi/consumers/presence.go @@ -38,7 +38,7 @@ type OutputPresenceConsumer struct { durable string db storage.Database queues *queue.OutgoingQueues - ServerName gomatrixserverlib.ServerName + isLocalServerName func(gomatrixserverlib.ServerName) bool topic string outboundPresenceEnabled bool } @@ -56,7 +56,7 @@ func NewOutputPresenceConsumer( jetstream: js, queues: queues, db: store, - ServerName: cfg.Matrix.ServerName, + isLocalServerName: cfg.Matrix.IsLocalServerName, durable: cfg.Matrix.JetStream.Durable("FederationAPIPresenceConsumer"), topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputPresenceEvent), outboundPresenceEnabled: cfg.Matrix.Presence.EnableOutbound, @@ -85,7 +85,7 @@ func (t *OutputPresenceConsumer) onMessage(ctx context.Context, msgs []*nats.Msg log.WithError(err).WithField("user_id", userID).Error("failed to extract domain from receipt sender") return true } - if serverName != t.ServerName { + if !t.isLocalServerName(serverName) { return true } @@ -127,7 +127,7 @@ func (t *OutputPresenceConsumer) onMessage(ctx context.Context, msgs []*nats.Msg edu := &gomatrixserverlib.EDU{ Type: gomatrixserverlib.MPresence, - Origin: string(t.ServerName), + Origin: string(serverName), } if edu.Content, err = json.Marshal(content); err != nil { log.WithError(err).Error("failed to marshal EDU JSON") @@ -135,7 +135,7 @@ func (t *OutputPresenceConsumer) onMessage(ctx context.Context, msgs []*nats.Msg } log.Tracef("sending presence EDU to %d servers", len(joined)) - if err = t.queues.SendEDU(edu, t.ServerName, joined); err != nil { + if err = t.queues.SendEDU(edu, serverName, joined); err != nil { log.WithError(err).Error("failed to send EDU") return false } diff --git a/federationapi/consumers/receipts.go b/federationapi/consumers/receipts.go index 75827cb68..200c06e6c 100644 --- a/federationapi/consumers/receipts.go +++ b/federationapi/consumers/receipts.go @@ -34,13 +34,13 @@ import ( // OutputReceiptConsumer consumes events that originate in the clientapi. type OutputReceiptConsumer struct { - ctx context.Context - jetstream nats.JetStreamContext - durable string - db storage.Database - queues *queue.OutgoingQueues - ServerName gomatrixserverlib.ServerName - topic string + ctx context.Context + jetstream nats.JetStreamContext + durable string + db storage.Database + queues *queue.OutgoingQueues + isLocalServerName func(gomatrixserverlib.ServerName) bool + topic string } // NewOutputReceiptConsumer creates a new OutputReceiptConsumer. Call Start() to begin consuming typing events. @@ -52,13 +52,13 @@ func NewOutputReceiptConsumer( store storage.Database, ) *OutputReceiptConsumer { return &OutputReceiptConsumer{ - ctx: process.Context(), - jetstream: js, - queues: queues, - db: store, - ServerName: cfg.Matrix.ServerName, - durable: cfg.Matrix.JetStream.Durable("FederationAPIReceiptConsumer"), - topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputReceiptEvent), + ctx: process.Context(), + jetstream: js, + queues: queues, + db: store, + isLocalServerName: cfg.Matrix.IsLocalServerName, + durable: cfg.Matrix.JetStream.Durable("FederationAPIReceiptConsumer"), + topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputReceiptEvent), } } @@ -95,7 +95,7 @@ func (t *OutputReceiptConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) log.WithError(err).WithField("user_id", receipt.UserID).Error("failed to extract domain from receipt sender") return true } - if receiptServerName != t.ServerName { + if !t.isLocalServerName(receiptServerName) { return true } @@ -134,14 +134,14 @@ func (t *OutputReceiptConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) edu := &gomatrixserverlib.EDU{ Type: gomatrixserverlib.MReceipt, - Origin: string(t.ServerName), + Origin: string(receiptServerName), } if edu.Content, err = json.Marshal(content); err != nil { log.WithError(err).Error("failed to marshal EDU JSON") return true } - if err := t.queues.SendEDU(edu, t.ServerName, names); err != nil { + if err := t.queues.SendEDU(edu, receiptServerName, names); err != nil { log.WithError(err).Error("failed to send EDU") return false } diff --git a/federationapi/consumers/sendtodevice.go b/federationapi/consumers/sendtodevice.go index 9aec22a3e..9620d1612 100644 --- a/federationapi/consumers/sendtodevice.go +++ b/federationapi/consumers/sendtodevice.go @@ -34,13 +34,13 @@ import ( // OutputSendToDeviceConsumer consumes events that originate in the clientapi. type OutputSendToDeviceConsumer struct { - ctx context.Context - jetstream nats.JetStreamContext - durable string - db storage.Database - queues *queue.OutgoingQueues - ServerName gomatrixserverlib.ServerName - topic string + ctx context.Context + jetstream nats.JetStreamContext + durable string + db storage.Database + queues *queue.OutgoingQueues + isLocalServerName func(gomatrixserverlib.ServerName) bool + topic string } // NewOutputSendToDeviceConsumer creates a new OutputSendToDeviceConsumer. Call Start() to begin consuming send-to-device events. @@ -52,13 +52,13 @@ func NewOutputSendToDeviceConsumer( store storage.Database, ) *OutputSendToDeviceConsumer { return &OutputSendToDeviceConsumer{ - ctx: process.Context(), - jetstream: js, - queues: queues, - db: store, - ServerName: cfg.Matrix.ServerName, - durable: cfg.Matrix.JetStream.Durable("FederationAPIESendToDeviceConsumer"), - topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent), + ctx: process.Context(), + jetstream: js, + queues: queues, + db: store, + isLocalServerName: cfg.Matrix.IsLocalServerName, + durable: cfg.Matrix.JetStream.Durable("FederationAPIESendToDeviceConsumer"), + topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputSendToDeviceEvent), } } @@ -82,7 +82,7 @@ func (t *OutputSendToDeviceConsumer) onMessage(ctx context.Context, msgs []*nats log.WithError(err).WithField("user_id", sender).Error("Failed to extract domain from send-to-device sender") return true } - if originServerName != t.ServerName { + if !t.isLocalServerName(originServerName) { return true } // Extract the send-to-device event from msg. @@ -101,14 +101,14 @@ func (t *OutputSendToDeviceConsumer) onMessage(ctx context.Context, msgs []*nats } // The SyncAPI is already handling sendToDevice for the local server - if destServerName == t.ServerName { + if t.isLocalServerName(destServerName) { return true } // Pack the EDU and marshal it edu := &gomatrixserverlib.EDU{ Type: gomatrixserverlib.MDirectToDevice, - Origin: string(t.ServerName), + Origin: string(originServerName), } tdm := gomatrixserverlib.ToDeviceMessage{ Sender: ote.Sender, @@ -127,7 +127,7 @@ func (t *OutputSendToDeviceConsumer) onMessage(ctx context.Context, msgs []*nats } log.Debugf("Sending send-to-device message into %q destination queue", destServerName) - if err := t.queues.SendEDU(edu, t.ServerName, []gomatrixserverlib.ServerName{destServerName}); err != nil { + if err := t.queues.SendEDU(edu, originServerName, []gomatrixserverlib.ServerName{destServerName}); err != nil { log.WithError(err).Error("failed to send EDU") return false } diff --git a/federationapi/consumers/typing.go b/federationapi/consumers/typing.go index 9c7379136..c66f97519 100644 --- a/federationapi/consumers/typing.go +++ b/federationapi/consumers/typing.go @@ -31,13 +31,13 @@ import ( // OutputTypingConsumer consumes events that originate in the clientapi. type OutputTypingConsumer struct { - ctx context.Context - jetstream nats.JetStreamContext - durable string - db storage.Database - queues *queue.OutgoingQueues - ServerName gomatrixserverlib.ServerName - topic string + ctx context.Context + jetstream nats.JetStreamContext + durable string + db storage.Database + queues *queue.OutgoingQueues + isLocalServerName func(gomatrixserverlib.ServerName) bool + topic string } // NewOutputTypingConsumer creates a new OutputTypingConsumer. Call Start() to begin consuming typing events. @@ -49,13 +49,13 @@ func NewOutputTypingConsumer( store storage.Database, ) *OutputTypingConsumer { return &OutputTypingConsumer{ - ctx: process.Context(), - jetstream: js, - queues: queues, - db: store, - ServerName: cfg.Matrix.ServerName, - durable: cfg.Matrix.JetStream.Durable("FederationAPITypingConsumer"), - topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputTypingEvent), + ctx: process.Context(), + jetstream: js, + queues: queues, + db: store, + isLocalServerName: cfg.Matrix.IsLocalServerName, + durable: cfg.Matrix.JetStream.Durable("FederationAPITypingConsumer"), + topic: cfg.Matrix.JetStream.Prefixed(jetstream.OutputTypingEvent), } } @@ -87,7 +87,7 @@ func (t *OutputTypingConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) _ = msg.Ack() return true } - if typingServerName != t.ServerName { + if !t.isLocalServerName(typingServerName) { return true } @@ -111,7 +111,7 @@ func (t *OutputTypingConsumer) onMessage(ctx context.Context, msgs []*nats.Msg) log.WithError(err).Error("failed to marshal EDU JSON") return true } - if err := t.queues.SendEDU(edu, t.ServerName, names); err != nil { + if err := t.queues.SendEDU(edu, typingServerName, names); err != nil { log.WithError(err).Error("failed to send EDU") return false } From a74aea07144615f20a71f1c2f62f4ba0946d7b54 Mon Sep 17 00:00:00 2001 From: devonh Date: Wed, 26 Oct 2022 16:25:57 +0000 Subject: [PATCH 112/207] Add network interface callback to pinecone build (#2825) Co-authored-by: Neil Alexander --- build/gobind-pinecone/monolith.go | 52 ++++++++++++++++++++++++------- go.mod | 2 +- go.sum | 4 +-- 3 files changed, 43 insertions(+), 15 deletions(-) diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go index 4a96e4bef..adb4e40a6 100644 --- a/build/gobind-pinecone/monolith.go +++ b/build/gobind-pinecone/monolith.go @@ -101,18 +101,46 @@ func (m *DendriteMonolith) SessionCount() int { return len(m.PineconeQUIC.Protocol("matrix").Sessions()) } -func (m *DendriteMonolith) RegisterNetworkInterface(name string, index int, mtu int, up bool, broadcast bool, loopback bool, pointToPoint bool, multicast bool, addrs string) { - m.PineconeMulticast.RegisterInterface(pineconeMulticast.InterfaceInfo{ - Name: name, - Index: index, - Mtu: mtu, - Up: up, - Broadcast: broadcast, - Loopback: loopback, - PointToPoint: pointToPoint, - Multicast: multicast, - Addrs: addrs, - }) +type InterfaceInfo struct { + Name string + Index int + Mtu int + Up bool + Broadcast bool + Loopback bool + PointToPoint bool + Multicast bool + Addrs string +} + +type InterfaceRetriever interface { + CacheCurrentInterfaces() int + GetCachedInterface(index int) *InterfaceInfo +} + +func (m *DendriteMonolith) RegisterNetworkCallback(intfCallback InterfaceRetriever) { + callback := func() []pineconeMulticast.InterfaceInfo { + count := intfCallback.CacheCurrentInterfaces() + intfs := []pineconeMulticast.InterfaceInfo{} + for i := 0; i < count; i++ { + iface := intfCallback.GetCachedInterface(i) + if iface != nil { + intfs = append(intfs, pineconeMulticast.InterfaceInfo{ + Name: iface.Name, + Index: iface.Index, + Mtu: iface.Mtu, + Up: iface.Up, + Broadcast: iface.Broadcast, + Loopback: iface.Loopback, + PointToPoint: iface.PointToPoint, + Multicast: iface.Multicast, + Addrs: iface.Addrs, + }) + } + } + return intfs + } + m.PineconeMulticast.RegisterNetworkCallback(callback) } func (m *DendriteMonolith) SetMulticastEnabled(enabled bool) { diff --git a/go.mod b/go.mod index 39dfb0fe1..be5099fcf 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 github.com/matrix-org/gomatrixserverlib v0.0.0-20221025142407-17b0be811afa - github.com/matrix-org/pinecone v0.0.0-20221007145426-3adc85477dd3 + github.com/matrix-org/pinecone v0.0.0-20221026160848-639feeff74d6 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.15 github.com/nats-io/nats-server/v2 v2.9.3 diff --git a/go.sum b/go.sum index 5e6253860..c7903b0ce 100644 --- a/go.sum +++ b/go.sum @@ -389,8 +389,8 @@ github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= github.com/matrix-org/gomatrixserverlib v0.0.0-20221025142407-17b0be811afa h1:S98DShDv3sn7O4n4HjtJOejypseYVpv1R/XPg+cDnfI= github.com/matrix-org/gomatrixserverlib v0.0.0-20221025142407-17b0be811afa/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4= -github.com/matrix-org/pinecone v0.0.0-20221007145426-3adc85477dd3 h1:lzkSQvBv8TuqKJCPoVwOVvEnARTlua5rrNy/Qw2Vxeo= -github.com/matrix-org/pinecone v0.0.0-20221007145426-3adc85477dd3/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k= +github.com/matrix-org/pinecone v0.0.0-20221026160848-639feeff74d6 h1:nAT5w41Q9uWTSnpKW55/hBwP91j2IFYPDRs0jJ8TyFI= +github.com/matrix-org/pinecone v0.0.0-20221026160848-639feeff74d6/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= From 97491a174b7826c5c9058398e54e4a18c7a5a052 Mon Sep 17 00:00:00 2001 From: devonh Date: Wed, 26 Oct 2022 16:35:01 +0000 Subject: [PATCH 113/207] Associate events in db before queueing them to send (#2833) Fixes a race condition between sending federation events and having them fully associated in the database. --- federationapi/queue/destinationqueue.go | 64 ++++++++++++++----------- federationapi/queue/queue.go | 30 +++++++++--- 2 files changed, 60 insertions(+), 34 deletions(-) diff --git a/federationapi/queue/destinationqueue.go b/federationapi/queue/destinationqueue.go index 1b7670e9a..a638a5742 100644 --- a/federationapi/queue/destinationqueue.go +++ b/federationapi/queue/destinationqueue.go @@ -76,21 +76,25 @@ func (oq *destinationQueue) sendEvent(event *gomatrixserverlib.HeaderedEvent, re return } - // If there's room in memory to hold the event then add it to the - // list. - oq.pendingMutex.Lock() - if len(oq.pendingPDUs) < maxPDUsInMemory { - oq.pendingPDUs = append(oq.pendingPDUs, &queuedPDU{ - pdu: event, - receipt: receipt, - }) - } else { - oq.overflowed.Store(true) - } - oq.pendingMutex.Unlock() + // Check if the destination is blacklisted. If it isn't then wake + // up the queue. + if !oq.statistics.Blacklisted() { + // If there's room in memory to hold the event then add it to the + // list. + oq.pendingMutex.Lock() + if len(oq.pendingPDUs) < maxPDUsInMemory { + oq.pendingPDUs = append(oq.pendingPDUs, &queuedPDU{ + pdu: event, + receipt: receipt, + }) + } else { + oq.overflowed.Store(true) + } + oq.pendingMutex.Unlock() - if !oq.backingOff.Load() { - oq.wakeQueueAndNotify() + if !oq.backingOff.Load() { + oq.wakeQueueAndNotify() + } } } @@ -103,21 +107,25 @@ func (oq *destinationQueue) sendEDU(event *gomatrixserverlib.EDU, receipt *share return } - // If there's room in memory to hold the event then add it to the - // list. - oq.pendingMutex.Lock() - if len(oq.pendingEDUs) < maxEDUsInMemory { - oq.pendingEDUs = append(oq.pendingEDUs, &queuedEDU{ - edu: event, - receipt: receipt, - }) - } else { - oq.overflowed.Store(true) - } - oq.pendingMutex.Unlock() + // Check if the destination is blacklisted. If it isn't then wake + // up the queue. + if !oq.statistics.Blacklisted() { + // If there's room in memory to hold the event then add it to the + // list. + oq.pendingMutex.Lock() + if len(oq.pendingEDUs) < maxEDUsInMemory { + oq.pendingEDUs = append(oq.pendingEDUs, &queuedEDU{ + edu: event, + receipt: receipt, + }) + } else { + oq.overflowed.Store(true) + } + oq.pendingMutex.Unlock() - if !oq.backingOff.Load() { - oq.wakeQueueAndNotify() + if !oq.backingOff.Load() { + oq.wakeQueueAndNotify() + } } } diff --git a/federationapi/queue/queue.go b/federationapi/queue/queue.go index 328334379..b5d0552c6 100644 --- a/federationapi/queue/queue.go +++ b/federationapi/queue/queue.go @@ -247,9 +247,10 @@ func (oqs *OutgoingQueues) SendEvent( return fmt.Errorf("sendevent: oqs.db.StoreJSON: %w", err) } + destQueues := make([]*destinationQueue, 0, len(destmap)) for destination := range destmap { - if queue := oqs.getQueue(destination); queue != nil && !queue.statistics.Blacklisted() { - queue.sendEvent(ev, nid) + if queue := oqs.getQueue(destination); queue != nil { + destQueues = append(destQueues, queue) } else { delete(destmap, destination) } @@ -267,6 +268,14 @@ func (oqs *OutgoingQueues) SendEvent( return err } + // NOTE : PDUs should be associated with destinations before sending + // them, otherwise this is technically a race. + // If the send completes before they are associated then they won't + // get properly cleaned up in the database. + for _, queue := range destQueues { + queue.sendEvent(ev, nid) + } + return nil } @@ -335,20 +344,21 @@ func (oqs *OutgoingQueues) SendEDU( return fmt.Errorf("sendevent: oqs.db.StoreJSON: %w", err) } + destQueues := make([]*destinationQueue, 0, len(destmap)) for destination := range destmap { - if queue := oqs.getQueue(destination); queue != nil && !queue.statistics.Blacklisted() { - queue.sendEDU(e, nid) + if queue := oqs.getQueue(destination); queue != nil { + destQueues = append(destQueues, queue) } else { delete(destmap, destination) } } // Create a database entry that associates the given PDU NID with - // this destination queue. We'll then be able to retrieve the PDU + // these destination queues. We'll then be able to retrieve the PDU // later. if err := oqs.db.AssociateEDUWithDestinations( oqs.process.Context(), - destmap, // the destination server name + destmap, // the destination server names nid, // NIDs from federationapi_queue_json table e.Type, nil, // this will use the default expireEDUTypes map @@ -357,6 +367,14 @@ func (oqs *OutgoingQueues) SendEDU( return err } + // NOTE : EDUs should be associated with destinations before sending + // them, otherwise this is technically a race. + // If the send completes before they are associated then they won't + // get properly cleaned up in the database. + for _, queue := range destQueues { + queue.sendEDU(e, nid) + } + return nil } From 238b6ef2cd5077482b46c57e9f44f05a19fcbfc7 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 26 Oct 2022 18:37:01 +0100 Subject: [PATCH 114/207] Update Yggdrasil demo --- build/docker/Dockerfile.demo-yggdrasil | 25 ++++++ cmd/dendrite-demo-yggdrasil/yggconn/node.go | 86 +++++++++++---------- go.mod | 4 +- go.sum | 8 +- 4 files changed, 75 insertions(+), 48 deletions(-) create mode 100644 build/docker/Dockerfile.demo-yggdrasil diff --git a/build/docker/Dockerfile.demo-yggdrasil b/build/docker/Dockerfile.demo-yggdrasil new file mode 100644 index 000000000..76bf35823 --- /dev/null +++ b/build/docker/Dockerfile.demo-yggdrasil @@ -0,0 +1,25 @@ +FROM docker.io/golang:1.19-alpine AS base + +RUN apk --update --no-cache add bash build-base + +WORKDIR /build + +COPY . /build + +RUN mkdir -p bin +RUN go build -trimpath -o bin/ ./cmd/dendrite-demo-yggdrasil +RUN go build -trimpath -o bin/ ./cmd/create-account +RUN go build -trimpath -o bin/ ./cmd/generate-keys + +FROM alpine:latest +LABEL org.opencontainers.image.title="Dendrite (Yggdrasil demo)" +LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go" +LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite" +LABEL org.opencontainers.image.licenses="Apache-2.0" + +COPY --from=base /build/bin/* /usr/bin/ + +VOLUME /etc/dendrite +WORKDIR /etc/dendrite + +ENTRYPOINT ["/usr/bin/dendrite-demo-yggdrasil"] diff --git a/cmd/dendrite-demo-yggdrasil/yggconn/node.go b/cmd/dendrite-demo-yggdrasil/yggconn/node.go index 83b4cdf9e..6df5fa879 100644 --- a/cmd/dendrite-demo-yggdrasil/yggconn/node.go +++ b/cmd/dendrite-demo-yggdrasil/yggconn/node.go @@ -20,6 +20,7 @@ import ( "encoding/hex" "fmt" "net" + "regexp" "strings" "github.com/matrix-org/gomatrixserverlib" @@ -27,9 +28,9 @@ import ( "github.com/sirupsen/logrus" ironwoodtypes "github.com/Arceliar/ironwood/types" - yggdrasilconfig "github.com/yggdrasil-network/yggdrasil-go/src/config" + "github.com/yggdrasil-network/yggdrasil-go/src/core" yggdrasilcore "github.com/yggdrasil-network/yggdrasil-go/src/core" - yggdrasildefaults "github.com/yggdrasil-network/yggdrasil-go/src/defaults" + "github.com/yggdrasil-network/yggdrasil-go/src/multicast" yggdrasilmulticast "github.com/yggdrasil-network/yggdrasil-go/src/multicast" gologme "github.com/gologme/log" @@ -37,7 +38,6 @@ import ( type Node struct { core *yggdrasilcore.Core - config *yggdrasilconfig.NodeConfig multicast *yggdrasilmulticast.Multicast log *gologme.Logger utpSocket *utp.Socket @@ -57,43 +57,52 @@ func (n *Node) DialerContext(ctx context.Context, _, address string) (net.Conn, func Setup(sk ed25519.PrivateKey, instanceName, storageDirectory, peerURI, listenURI string) (*Node, error) { n := &Node{ - core: &yggdrasilcore.Core{}, - config: yggdrasildefaults.GenerateConfig(), - multicast: &yggdrasilmulticast.Multicast{}, - log: gologme.New(logrus.StandardLogger().Writer(), "", 0), - incoming: make(chan net.Conn), + log: gologme.New(logrus.StandardLogger().Writer(), "", 0), + incoming: make(chan net.Conn), } - options := []yggdrasilcore.SetupOption{ - yggdrasilcore.AdminListenAddress("none"), - } - if listenURI != "" { - options = append(options, yggdrasilcore.ListenAddress(listenURI)) - } - if peerURI != "" { - for _, uri := range strings.Split(peerURI, ",") { - options = append(options, yggdrasilcore.Peer{ - URI: uri, - }) - } - } - - var err error - if n.core, err = yggdrasilcore.New(sk, options...); err != nil { - panic(err) - } n.log.EnableLevel("error") n.log.EnableLevel("warn") n.log.EnableLevel("info") - n.core.SetLogger(n.log) - if n.utpSocket, err = utp.NewSocketFromPacketConnNoClose(n.core); err != nil { - panic(err) + + { + var err error + options := []yggdrasilcore.SetupOption{} + if listenURI != "" { + options = append(options, yggdrasilcore.ListenAddress(listenURI)) + } + if peerURI != "" { + for _, uri := range strings.Split(peerURI, ",") { + options = append(options, yggdrasilcore.Peer{ + URI: uri, + }) + } + } + if n.core, err = core.New(sk[:], n.log, options...); err != nil { + panic(err) + } + n.core.SetLogger(n.log) + + if n.utpSocket, err = utp.NewSocketFromPacketConnNoClose(n.core); err != nil { + panic(err) + } } - if err = n.multicast.Init(n.core, n.config, n.log, nil); err != nil { - panic(err) - } - if err = n.multicast.Start(); err != nil { - panic(err) + + // Setup the multicast module. + { + var err error + options := []multicast.SetupOption{ + multicast.MulticastInterface{ + Regex: regexp.MustCompile(".*"), + Beacon: true, + Listen: true, + Port: 0, + Priority: 0, + }, + } + if n.multicast, err = multicast.New(n.core, n.log, options...); err != nil { + panic(err) + } } n.log.Printf("Public key: %x", n.core.PublicKey()) @@ -114,14 +123,7 @@ func (n *Node) DerivedServerName() string { } func (n *Node) PrivateKey() ed25519.PrivateKey { - sk := make(ed25519.PrivateKey, ed25519.PrivateKeySize) - sb, err := hex.DecodeString(n.config.PrivateKey) - if err == nil { - copy(sk, sb[:]) - } else { - panic(err) - } - return sk + return n.core.PrivateKey() } func (n *Node) PublicKey() ed25519.PublicKey { diff --git a/go.mod b/go.mod index be5099fcf..7c290d784 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,7 @@ module github.com/matrix-org/dendrite require ( - github.com/Arceliar/ironwood v0.0.0-20220903132624-ee60c16bcfcf + github.com/Arceliar/ironwood v0.0.0-20221025225125-45b4281814c2 github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979 github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/MFAshby/stdemuxerhook v1.0.0 @@ -41,7 +41,7 @@ require ( github.com/tidwall/sjson v1.2.5 github.com/uber/jaeger-client-go v2.30.0+incompatible github.com/uber/jaeger-lib v2.4.1+incompatible - github.com/yggdrasil-network/yggdrasil-go v0.4.5-0.20220901155642-4f2abece817c + github.com/yggdrasil-network/yggdrasil-go v0.4.6 go.uber.org/atomic v1.10.0 golang.org/x/crypto v0.0.0-20221012134737-56aed061732a golang.org/x/image v0.0.0-20220902085622-e7cb96979f69 diff --git a/go.sum b/go.sum index c7903b0ce..a0db4823e 100644 --- a/go.sum +++ b/go.sum @@ -38,8 +38,8 @@ dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBr dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4= dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= -github.com/Arceliar/ironwood v0.0.0-20220903132624-ee60c16bcfcf h1:kjPkmDHUTWUma/4tqDl208bOk3jsUEqOJA6TsMZo5Jk= -github.com/Arceliar/ironwood v0.0.0-20220903132624-ee60c16bcfcf/go.mod h1:RP72rucOFm5udrnEzTmIWLRVGQiV/fSUAQXJ0RST/nk= +github.com/Arceliar/ironwood v0.0.0-20221025225125-45b4281814c2 h1:Usab30pNT2i/vZvpXcN9uOr5IO1RZPcUqoGH0DIAPnU= +github.com/Arceliar/ironwood v0.0.0-20221025225125-45b4281814c2/go.mod h1:RP72rucOFm5udrnEzTmIWLRVGQiV/fSUAQXJ0RST/nk= github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979 h1:WndgpSW13S32VLQ3ugUxx2EnnWmgba1kCqPkd4Gk1yQ= github.com/Arceliar/phony v0.0.0-20210209235338-dde1a8dca979/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= @@ -592,8 +592,8 @@ github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17 github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yggdrasil-network/yggdrasil-go v0.4.5-0.20220901155642-4f2abece817c h1:/cTmA6pV2Z20BT/FGSmnb5BmJ8eRbDP0HbCB5IO1aKw= -github.com/yggdrasil-network/yggdrasil-go v0.4.5-0.20220901155642-4f2abece817c/go.mod h1:cIwhYwX9yT9Bcei59O0oOBSaj+kQP+9aVQUMWHh5R00= +github.com/yggdrasil-network/yggdrasil-go v0.4.6 h1:GALUDV9QPz/5FVkbazpkTc9EABHufA556JwUJZr41j4= +github.com/yggdrasil-network/yggdrasil-go v0.4.6/go.mod h1:PBMoAOvQjA9geNEeGyMXA9QgCS6Bu+9V+1VkWM84wpw= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= From fa96811e64302cad7d51b6c20ebbc8abd5105af4 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Thu, 27 Oct 2022 12:12:50 +0200 Subject: [PATCH 115/207] Add scheduled tasks to run tests with race detection (#2814) Needs https://github.com/matrix-org/sytest/pull/1308 to be actually useful. Not sure if we need to run Sytest in all combinations with enabled race detection. Closes https://github.com/matrix-org/dendrite/issues/491 --- .github/workflows/schedules.yaml | 128 +++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 .github/workflows/schedules.yaml diff --git a/.github/workflows/schedules.yaml b/.github/workflows/schedules.yaml new file mode 100644 index 000000000..c07917248 --- /dev/null +++ b/.github/workflows/schedules.yaml @@ -0,0 +1,128 @@ +name: Scheduled + +on: + schedule: + - cron: '0 0 * * *' # every day at midnight + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + # run go test with different go versions + test: + timeout-minutes: 20 + name: Unit tests (Go ${{ matrix.go }}) + runs-on: ubuntu-latest + # Service containers to run with `container-job` + services: + # Label used to access the service container + postgres: + # Docker Hub image + image: postgres:13-alpine + # Provide the password for postgres + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: dendrite + ports: + # Maps tcp port 5432 on service container to the host + - 5432:5432 + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + strategy: + fail-fast: false + matrix: + go: ["1.18", "1.19"] + steps: + - uses: actions/checkout@v3 + - name: Setup go + uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go }} + - uses: actions/cache@v3 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go${{ matrix.go }}-test-race-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go${{ matrix.go }}-test-race- + - run: go test -race ./... + env: + POSTGRES_HOST: localhost + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: dendrite + + # Dummy step to gate other tests on without repeating the whole list + initial-tests-done: + name: Initial tests passed + needs: [test] + runs-on: ubuntu-latest + if: ${{ !cancelled() }} # Run this even if prior jobs were skipped + steps: + - name: Check initial tests passed + uses: re-actors/alls-green@release/v1 + with: + jobs: ${{ toJSON(needs) }} + + # run Sytest in different variations + sytest: + timeout-minutes: 60 + needs: initial-tests-done + name: "Sytest (${{ matrix.label }})" + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - label: SQLite + + - label: SQLite, full HTTP APIs + api: full-http + + - label: PostgreSQL + postgres: postgres + + - label: PostgreSQL, full HTTP APIs + postgres: postgres + api: full-http + container: + image: matrixdotorg/sytest-dendrite:latest + volumes: + - ${{ github.workspace }}:/src + env: + POSTGRES: ${{ matrix.postgres && 1}} + API: ${{ matrix.api && 1 }} + SYTEST_BRANCH: ${{ github.head_ref }} + RACE_DETECTION: 1 + steps: + - uses: actions/checkout@v2 + - name: Run Sytest + run: /bootstrap.sh dendrite + working-directory: /src + - name: Summarise results.tap + if: ${{ always() }} + run: /sytest/scripts/tap_to_gha.pl /logs/results.tap + - name: Sytest List Maintenance + if: ${{ always() }} + run: /src/show-expected-fail-tests.sh /logs/results.tap /src/sytest-whitelist /src/sytest-blacklist + continue-on-error: true # not fatal + - name: Are We Synapse Yet? + if: ${{ always() }} + run: /src/are-we-synapse-yet.py /logs/results.tap -v + continue-on-error: true # not fatal + - name: Upload Sytest logs + uses: actions/upload-artifact@v2 + if: ${{ always() }} + with: + name: Sytest Logs - ${{ job.status }} - (Dendrite, ${{ join(matrix.*, ', ') }}) + path: | + /logs/results.tap + /logs/**/*.log* From a169a9121aa6b2ff3e3fba6ec9777227e349300f Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Thu, 27 Oct 2022 14:18:22 +0200 Subject: [PATCH 116/207] Fix /members (#2837) Fixes a bug introduced in #2827, where the SyncAPI might not have all requested eventIDs, resulting in too few members returned. --- syncapi/routing/memberships.go | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/syncapi/routing/memberships.go b/syncapi/routing/memberships.go index c9acc5d2b..05c7deef0 100644 --- a/syncapi/routing/memberships.go +++ b/syncapi/routing/memberships.go @@ -76,6 +76,13 @@ func GetMemberships( } } + if joinedOnly && !queryRes.IsInRoom { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("You aren't a member of the room and weren't previously a member of the room."), + } + } + db, err := syncDB.NewDatabaseSnapshot(req.Context()) if err != nil { return jsonerror.InternalServerError() @@ -102,19 +109,15 @@ func GetMemberships( return jsonerror.InternalServerError() } - result, err := db.Events(req.Context(), eventIDs) - if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("db.Events failed") + qryRes := &api.QueryEventsByIDResponse{} + if err := rsAPI.QueryEventsByID(req.Context(), &api.QueryEventsByIDRequest{EventIDs: eventIDs}, qryRes); err != nil { + util.GetLogger(req.Context()).WithError(err).Error("rsAPI.QueryEventsByID failed") return jsonerror.InternalServerError() } + result := qryRes.Events + if joinedOnly { - if !queryRes.IsInRoom { - return util.JSONResponse{ - Code: http.StatusForbidden, - JSON: jsonerror.Forbidden("You aren't a member of the room and weren't previously a member of the room."), - } - } var res getJoinedMembersResponse res.Joined = make(map[string]joinedMember) for _, ev := range result { From 444b4bbdb8ce6f26651c4516cb23828a65bdd065 Mon Sep 17 00:00:00 2001 From: Till <2353100+S7evinK@users.noreply.github.com> Date: Thu, 27 Oct 2022 14:40:35 +0200 Subject: [PATCH 117/207] Add AS specific public room list endpoints (#2836) Adds `PUT /_matrix/client/v3/directory/list/appservice/{networkId}/{roomId}` and `DELTE /_matrix/client/v3/directory/list/appservice/{networkId}/{roomId}` support, as well as the ability to filter `/publicRooms` on networkID and including all networks. --- clientapi/routing/directory.go | 45 ++++++++++++- clientapi/routing/directory_public.go | 40 +++++++++--- clientapi/routing/directory_public_test.go | 2 +- clientapi/routing/routing.go | 23 ++++++- federationapi/routing/publicrooms.go | 25 ++++++-- roomserver/api/perform.go | 6 +- roomserver/api/query.go | 7 +- .../internal/perform/perform_publish.go | 2 +- roomserver/internal/query/query.go | 2 +- roomserver/storage/interface.go | 4 +- .../20221027084407_published_appservice.go | 45 +++++++++++++ .../storage/postgres/published_table.go | 55 ++++++++++++---- roomserver/storage/shared/storage.go | 8 +-- .../20221027084407_published_appservice.go | 64 +++++++++++++++++++ roomserver/storage/sqlite3/published_table.go | 55 ++++++++++++---- roomserver/storage/tables/interface.go | 4 +- .../storage/tables/published_table_test.go | 33 ++++++++-- sytest-whitelist | 4 +- 18 files changed, 354 insertions(+), 70 deletions(-) create mode 100644 roomserver/storage/postgres/deltas/20221027084407_published_appservice.go create mode 100644 roomserver/storage/sqlite3/deltas/20221027084407_published_appservice.go diff --git a/clientapi/routing/directory.go b/clientapi/routing/directory.go index 33bc63d18..ce14745aa 100644 --- a/clientapi/routing/directory.go +++ b/clientapi/routing/directory.go @@ -18,14 +18,15 @@ import ( "fmt" "net/http" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" + "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" federationAPI "github.com/matrix-org/dendrite/federationapi/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/setup/config" userapi "github.com/matrix-org/dendrite/userapi/api" - "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" ) type roomDirectoryResponse struct { @@ -318,3 +319,43 @@ func SetVisibility( JSON: struct{}{}, } } + +func SetVisibilityAS( + req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI, dev *userapi.Device, + networkID, roomID string, +) util.JSONResponse { + if dev.AccountType != userapi.AccountTypeAppService { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("Only appservice may use this endpoint"), + } + } + var v roomVisibility + + // If the method is delete, we simply mark the visibility as private + if req.Method == http.MethodDelete { + v.Visibility = "private" + } else { + if reqErr := httputil.UnmarshalJSONRequest(req, &v); reqErr != nil { + return *reqErr + } + } + var publishRes roomserverAPI.PerformPublishResponse + if err := rsAPI.PerformPublish(req.Context(), &roomserverAPI.PerformPublishRequest{ + RoomID: roomID, + Visibility: v.Visibility, + NetworkID: networkID, + AppserviceID: dev.AppserviceID, + }, &publishRes); err != nil { + return jsonerror.InternalAPIError(req.Context(), err) + } + if publishRes.Error != nil { + util.GetLogger(req.Context()).WithError(publishRes.Error).Error("PerformPublish failed") + return publishRes.Error.JSONResponse() + } + + return util.JSONResponse{ + Code: http.StatusOK, + JSON: struct{}{}, + } +} diff --git a/clientapi/routing/directory_public.go b/clientapi/routing/directory_public.go index 4ebf2295a..b1043e994 100644 --- a/clientapi/routing/directory_public.go +++ b/clientapi/routing/directory_public.go @@ -39,14 +39,17 @@ var ( ) type PublicRoomReq struct { - Since string `json:"since,omitempty"` - Limit int16 `json:"limit,omitempty"` - Filter filter `json:"filter,omitempty"` - Server string `json:"server,omitempty"` + Since string `json:"since,omitempty"` + Limit int64 `json:"limit,omitempty"` + Filter filter `json:"filter,omitempty"` + Server string `json:"server,omitempty"` + IncludeAllNetworks bool `json:"include_all_networks,omitempty"` + NetworkID string `json:"third_party_instance_id,omitempty"` } type filter struct { - SearchTerms string `json:"generic_search_term,omitempty"` + SearchTerms string `json:"generic_search_term,omitempty"` + RoomTypes []string `json:"room_types,omitempty"` // TODO: Implement filter on this } // GetPostPublicRooms implements GET and POST /publicRooms @@ -61,6 +64,13 @@ func GetPostPublicRooms( return *fillErr } + if request.IncludeAllNetworks && request.NetworkID != "" { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.InvalidParam("include_all_networks and third_party_instance_id can not be used together"), + } + } + serverName := gomatrixserverlib.ServerName(request.Server) if serverName != "" && !cfg.Matrix.IsLocalServerName(serverName) { res, err := federation.GetPublicRoomsFiltered( @@ -97,7 +107,7 @@ func publicRooms( response := gomatrixserverlib.RespPublicRooms{ Chunk: []gomatrixserverlib.PublicRoom{}, } - var limit int16 + var limit int64 var offset int64 limit = request.Limit if limit == 0 { @@ -114,7 +124,7 @@ func publicRooms( var rooms []gomatrixserverlib.PublicRoom if request.Since == "" { - rooms = refreshPublicRoomCache(ctx, rsAPI, extRoomsProvider) + rooms = refreshPublicRoomCache(ctx, rsAPI, extRoomsProvider, request) } else { rooms = getPublicRoomsFromCache() } @@ -176,7 +186,7 @@ func fillPublicRoomsReq(httpReq *http.Request, request *PublicRoomReq) *util.JSO JSON: jsonerror.BadJSON("limit param is not a number"), } } - request.Limit = int16(limit) + request.Limit = int64(limit) request.Since = httpReq.FormValue("since") request.Server = httpReq.FormValue("server") } else { @@ -204,7 +214,7 @@ func fillPublicRoomsReq(httpReq *http.Request, request *PublicRoomReq) *util.JSO // limit=3&since=6 => G (prev='3', next='') // // A value of '-1' for prev/next indicates no position. -func sliceInto(slice []gomatrixserverlib.PublicRoom, since int64, limit int16) (subset []gomatrixserverlib.PublicRoom, prev, next int) { +func sliceInto(slice []gomatrixserverlib.PublicRoom, since int64, limit int64) (subset []gomatrixserverlib.PublicRoom, prev, next int) { prev = -1 next = -1 @@ -230,6 +240,7 @@ func sliceInto(slice []gomatrixserverlib.PublicRoom, since int64, limit int16) ( func refreshPublicRoomCache( ctx context.Context, rsAPI roomserverAPI.ClientRoomserverAPI, extRoomsProvider api.ExtraPublicRoomsProvider, + request PublicRoomReq, ) []gomatrixserverlib.PublicRoom { cacheMu.Lock() defer cacheMu.Unlock() @@ -238,8 +249,17 @@ func refreshPublicRoomCache( extraRooms = extRoomsProvider.Rooms() } + // TODO: this is only here to make Sytest happy, for now. + ns := strings.Split(request.NetworkID, "|") + if len(ns) == 2 { + request.NetworkID = ns[1] + } + var queryRes roomserverAPI.QueryPublishedRoomsResponse - err := rsAPI.QueryPublishedRooms(ctx, &roomserverAPI.QueryPublishedRoomsRequest{}, &queryRes) + err := rsAPI.QueryPublishedRooms(ctx, &roomserverAPI.QueryPublishedRoomsRequest{ + NetworkID: request.NetworkID, + IncludeAllNetworks: request.IncludeAllNetworks, + }, &queryRes) if err != nil { util.GetLogger(ctx).WithError(err).Error("QueryPublishedRooms failed") return publicRoomsCache diff --git a/clientapi/routing/directory_public_test.go b/clientapi/routing/directory_public_test.go index bb3912b8c..65ad392c2 100644 --- a/clientapi/routing/directory_public_test.go +++ b/clientapi/routing/directory_public_test.go @@ -17,7 +17,7 @@ func TestSliceInto(t *testing.T) { slice := []gomatrixserverlib.PublicRoom{ pubRoom("a"), pubRoom("b"), pubRoom("c"), pubRoom("d"), pubRoom("e"), pubRoom("f"), pubRoom("g"), } - limit := int16(3) + limit := int64(3) testCases := []struct { since int64 wantPrev int diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index e0e3e33d4..22bc77a0b 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -480,7 +480,7 @@ func Setup( return GetVisibility(req, rsAPI, vars["roomID"]) }), ).Methods(http.MethodGet, http.MethodOptions) - // TODO: Add AS support + v3mux.Handle("/directory/list/room/{roomID}", httputil.MakeAuthAPI("directory_list", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) @@ -490,6 +490,27 @@ func Setup( return SetVisibility(req, rsAPI, device, vars["roomID"]) }), ).Methods(http.MethodPut, http.MethodOptions) + v3mux.Handle("/directory/list/appservice/{networkID}/{roomID}", + httputil.MakeAuthAPI("directory_list", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return SetVisibilityAS(req, rsAPI, device, vars["networkID"], vars["roomID"]) + }), + ).Methods(http.MethodPut, http.MethodOptions) + + // Undocumented endpoint + v3mux.Handle("/directory/list/appservice/{networkID}/{roomID}", + httputil.MakeAuthAPI("directory_list", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse { + vars, err := httputil.URLDecodeMapValues(mux.Vars(req)) + if err != nil { + return util.ErrorResponse(err) + } + return SetVisibilityAS(req, rsAPI, device, vars["networkID"], vars["roomID"]) + }), + ).Methods(http.MethodDelete, http.MethodOptions) + v3mux.Handle("/publicRooms", httputil.MakeExternalAPI("public_rooms", func(req *http.Request) util.JSONResponse { return GetPostPublicRooms(req, rsAPI, extRoomsProvider, federation, cfg) diff --git a/federationapi/routing/publicrooms.go b/federationapi/routing/publicrooms.go index 1a54f5a7d..34025932a 100644 --- a/federationapi/routing/publicrooms.go +++ b/federationapi/routing/publicrooms.go @@ -2,24 +2,29 @@ package routing import ( "context" + "fmt" "net/http" "strconv" + "github.com/matrix-org/gomatrixserverlib" + "github.com/matrix-org/util" + "github.com/matrix-org/dendrite/clientapi/httputil" "github.com/matrix-org/dendrite/clientapi/jsonerror" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" - "github.com/matrix-org/gomatrixserverlib" - "github.com/matrix-org/util" ) type PublicRoomReq struct { - Since string `json:"since,omitempty"` - Limit int16 `json:"limit,omitempty"` - Filter filter `json:"filter,omitempty"` + Since string `json:"since,omitempty"` + Limit int16 `json:"limit,omitempty"` + Filter filter `json:"filter,omitempty"` + IncludeAllNetworks bool `json:"include_all_networks,omitempty"` + NetworkID string `json:"third_party_instance_id,omitempty"` } type filter struct { - SearchTerms string `json:"generic_search_term,omitempty"` + SearchTerms string `json:"generic_search_term,omitempty"` + RoomTypes []string `json:"room_types,omitempty"` } // GetPostPublicRooms implements GET and POST /publicRooms @@ -57,8 +62,14 @@ func publicRooms( return nil, err } + if request.IncludeAllNetworks && request.NetworkID != "" { + return nil, fmt.Errorf("include_all_networks and third_party_instance_id can not be used together") + } + var queryRes roomserverAPI.QueryPublishedRoomsResponse - err = rsAPI.QueryPublishedRooms(ctx, &roomserverAPI.QueryPublishedRoomsRequest{}, &queryRes) + err = rsAPI.QueryPublishedRooms(ctx, &roomserverAPI.QueryPublishedRoomsRequest{ + NetworkID: request.NetworkID, + }, &queryRes) if err != nil { util.GetLogger(ctx).WithError(err).Error("QueryPublishedRooms failed") return nil, err diff --git a/roomserver/api/perform.go b/roomserver/api/perform.go index 7a362f969..1442a4b09 100644 --- a/roomserver/api/perform.go +++ b/roomserver/api/perform.go @@ -168,8 +168,10 @@ type PerformBackfillResponse struct { } type PerformPublishRequest struct { - RoomID string - Visibility string + RoomID string + Visibility string + AppserviceID string + NetworkID string } type PerformPublishResponse struct { diff --git a/roomserver/api/query.go b/roomserver/api/query.go index d63c24785..b62907f3c 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -21,8 +21,9 @@ import ( "fmt" "strings" - "github.com/matrix-org/dendrite/clientapi/auth/authtypes" "github.com/matrix-org/gomatrixserverlib" + + "github.com/matrix-org/dendrite/clientapi/auth/authtypes" ) // QueryLatestEventsAndStateRequest is a request to QueryLatestEventsAndState @@ -257,7 +258,9 @@ type QueryRoomVersionForRoomResponse struct { type QueryPublishedRoomsRequest struct { // Optional. If specified, returns whether this room is published or not. - RoomID string + RoomID string + NetworkID string + IncludeAllNetworks bool } type QueryPublishedRoomsResponse struct { diff --git a/roomserver/internal/perform/perform_publish.go b/roomserver/internal/perform/perform_publish.go index 1631fc657..fbbfc3219 100644 --- a/roomserver/internal/perform/perform_publish.go +++ b/roomserver/internal/perform/perform_publish.go @@ -30,7 +30,7 @@ func (r *Publisher) PerformPublish( req *api.PerformPublishRequest, res *api.PerformPublishResponse, ) error { - err := r.DB.PublishRoom(ctx, req.RoomID, req.Visibility == "public") + err := r.DB.PublishRoom(ctx, req.RoomID, req.AppserviceID, req.NetworkID, req.Visibility == "public") if err != nil { res.Error = &api.PerformError{ Msg: err.Error(), diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go index f41132403..0db046a86 100644 --- a/roomserver/internal/query/query.go +++ b/roomserver/internal/query/query.go @@ -702,7 +702,7 @@ func (r *Queryer) QueryPublishedRooms( } return err } - rooms, err := r.DB.GetPublishedRooms(ctx) + rooms, err := r.DB.GetPublishedRooms(ctx, req.NetworkID, req.IncludeAllNetworks) if err != nil { return err } diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go index ee0624b21..094537948 100644 --- a/roomserver/storage/interface.go +++ b/roomserver/storage/interface.go @@ -139,9 +139,9 @@ type Database interface { // Returns an error if the retrieval went wrong. EventsFromIDs(ctx context.Context, eventIDs []string) ([]types.Event, error) // Publish or unpublish a room from the room directory. - PublishRoom(ctx context.Context, roomID string, publish bool) error + PublishRoom(ctx context.Context, roomID, appserviceID, networkID string, publish bool) error // Returns a list of room IDs for rooms which are published. - GetPublishedRooms(ctx context.Context) ([]string, error) + GetPublishedRooms(ctx context.Context, networkID string, includeAllNetworks bool) ([]string, error) // Returns whether a given room is published or not. GetPublishedRoom(ctx context.Context, roomID string) (bool, error) diff --git a/roomserver/storage/postgres/deltas/20221027084407_published_appservice.go b/roomserver/storage/postgres/deltas/20221027084407_published_appservice.go new file mode 100644 index 000000000..be046545a --- /dev/null +++ b/roomserver/storage/postgres/deltas/20221027084407_published_appservice.go @@ -0,0 +1,45 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package deltas + +import ( + "context" + "database/sql" + "fmt" +) + +func UpPulishedAppservice(ctx context.Context, tx *sql.Tx) error { + _, err := tx.ExecContext(ctx, `ALTER TABLE roomserver_published ADD COLUMN IF NOT EXISTS appservice_id TEXT NOT NULL;`) + if err != nil { + return fmt.Errorf("failed to execute upgrade: %w", err) + } + _, err = tx.ExecContext(ctx, `ALTER TABLE roomserver_published ADD COLUMN IF NOT EXISTS network_id TEXT NOT NULL;`) + if err != nil { + return fmt.Errorf("failed to execute upgrade: %w", err) + } + return nil +} + +func DownPublishedAppservice(ctx context.Context, tx *sql.Tx) error { + _, err := tx.ExecContext(ctx, `ALTER TABLE roomserver_published DROP COLUMN IF EXISTS appservice_id;`) + if err != nil { + return fmt.Errorf("failed to execute downgrade: %w", err) + } + _, err = tx.ExecContext(ctx, `ALTER TABLE roomserver_published DROP COLUMN IF EXISTS network_id;`) + if err != nil { + return fmt.Errorf("failed to execute downgrade: %w", err) + } + return nil +} diff --git a/roomserver/storage/postgres/published_table.go b/roomserver/storage/postgres/published_table.go index 56fa02f7b..61caccb0e 100644 --- a/roomserver/storage/postgres/published_table.go +++ b/roomserver/storage/postgres/published_table.go @@ -20,6 +20,7 @@ import ( "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/roomserver/storage/postgres/deltas" "github.com/matrix-org/dendrite/roomserver/storage/tables" ) @@ -27,31 +28,48 @@ const publishedSchema = ` -- Stores which rooms are published in the room directory CREATE TABLE IF NOT EXISTS roomserver_published ( -- The room ID of the room - room_id TEXT NOT NULL PRIMARY KEY, + room_id TEXT NOT NULL, + -- The appservice ID of the room + appservice_id TEXT NOT NULL, + -- The network_id of the room + network_id TEXT NOT NULL, -- Whether it is published or not - published BOOLEAN NOT NULL DEFAULT false + published BOOLEAN NOT NULL DEFAULT false, + PRIMARY KEY (room_id, appservice_id, network_id) ); ` const upsertPublishedSQL = "" + - "INSERT INTO roomserver_published (room_id, published) VALUES ($1, $2) " + - "ON CONFLICT (room_id) DO UPDATE SET published=$2" + "INSERT INTO roomserver_published (room_id, appservice_id, network_id, published) VALUES ($1, $2, $3, $4) " + + "ON CONFLICT (room_id, appservice_id, network_id) DO UPDATE SET published=$4" const selectAllPublishedSQL = "" + - "SELECT room_id FROM roomserver_published WHERE published = $1 ORDER BY room_id ASC" + "SELECT room_id FROM roomserver_published WHERE published = $1 AND CASE WHEN $2 THEN 1=1 ELSE network_id = '' END ORDER BY room_id ASC" + +const selectNetworkPublishedSQL = "" + + "SELECT room_id FROM roomserver_published WHERE published = $1 AND network_id = $2 ORDER BY room_id ASC" const selectPublishedSQL = "" + "SELECT published FROM roomserver_published WHERE room_id = $1" type publishedStatements struct { - upsertPublishedStmt *sql.Stmt - selectAllPublishedStmt *sql.Stmt - selectPublishedStmt *sql.Stmt + upsertPublishedStmt *sql.Stmt + selectAllPublishedStmt *sql.Stmt + selectPublishedStmt *sql.Stmt + selectNetworkPublishedStmt *sql.Stmt } func CreatePublishedTable(db *sql.DB) error { _, err := db.Exec(publishedSchema) - return err + if err != nil { + return err + } + m := sqlutil.NewMigrator(db) + m.AddMigrations(sqlutil.Migration{ + Version: "roomserver: published appservice", + Up: deltas.UpPulishedAppservice, + }) + return m.Up(context.Background()) } func PreparePublishedTable(db *sql.DB) (tables.Published, error) { @@ -61,14 +79,15 @@ func PreparePublishedTable(db *sql.DB) (tables.Published, error) { {&s.upsertPublishedStmt, upsertPublishedSQL}, {&s.selectAllPublishedStmt, selectAllPublishedSQL}, {&s.selectPublishedStmt, selectPublishedSQL}, + {&s.selectNetworkPublishedStmt, selectNetworkPublishedSQL}, }.Prepare(db) } func (s *publishedStatements) UpsertRoomPublished( - ctx context.Context, txn *sql.Tx, roomID string, published bool, + ctx context.Context, txn *sql.Tx, roomID, appserviceID, networkID string, published bool, ) (err error) { stmt := sqlutil.TxStmt(txn, s.upsertPublishedStmt) - _, err = stmt.ExecContext(ctx, roomID, published) + _, err = stmt.ExecContext(ctx, roomID, appserviceID, networkID, published) return } @@ -84,10 +103,18 @@ func (s *publishedStatements) SelectPublishedFromRoomID( } func (s *publishedStatements) SelectAllPublishedRooms( - ctx context.Context, txn *sql.Tx, published bool, + ctx context.Context, txn *sql.Tx, networkID string, published, includeAllNetworks bool, ) ([]string, error) { - stmt := sqlutil.TxStmt(txn, s.selectAllPublishedStmt) - rows, err := stmt.QueryContext(ctx, published) + var rows *sql.Rows + var err error + if networkID != "" { + stmt := sqlutil.TxStmt(txn, s.selectNetworkPublishedStmt) + rows, err = stmt.QueryContext(ctx, published, networkID) + } else { + stmt := sqlutil.TxStmt(txn, s.selectAllPublishedStmt) + rows, err = stmt.QueryContext(ctx, published, includeAllNetworks) + + } if err != nil { return nil, err } diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index e401f17dc..ed86280bf 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -722,9 +722,9 @@ func (d *Database) storeEvent( }, redactionEvent, redactedEventID, err } -func (d *Database) PublishRoom(ctx context.Context, roomID string, publish bool) error { +func (d *Database) PublishRoom(ctx context.Context, roomID, appserviceID, networkID string, publish bool) error { return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { - return d.PublishedTable.UpsertRoomPublished(ctx, txn, roomID, publish) + return d.PublishedTable.UpsertRoomPublished(ctx, txn, roomID, appserviceID, networkID, publish) }) } @@ -732,8 +732,8 @@ func (d *Database) GetPublishedRoom(ctx context.Context, roomID string) (bool, e return d.PublishedTable.SelectPublishedFromRoomID(ctx, nil, roomID) } -func (d *Database) GetPublishedRooms(ctx context.Context) ([]string, error) { - return d.PublishedTable.SelectAllPublishedRooms(ctx, nil, true) +func (d *Database) GetPublishedRooms(ctx context.Context, networkID string, includeAllNetworks bool) ([]string, error) { + return d.PublishedTable.SelectAllPublishedRooms(ctx, nil, networkID, true, includeAllNetworks) } func (d *Database) MissingAuthPrevEvents( diff --git a/roomserver/storage/sqlite3/deltas/20221027084407_published_appservice.go b/roomserver/storage/sqlite3/deltas/20221027084407_published_appservice.go new file mode 100644 index 000000000..cd923b1c1 --- /dev/null +++ b/roomserver/storage/sqlite3/deltas/20221027084407_published_appservice.go @@ -0,0 +1,64 @@ +// Copyright 2022 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package deltas + +import ( + "context" + "database/sql" + "fmt" +) + +func UpPulishedAppservice(ctx context.Context, tx *sql.Tx) error { + _, err := tx.ExecContext(ctx, ` ALTER TABLE roomserver_published RENAME TO roomserver_published_tmp; +CREATE TABLE IF NOT EXISTS roomserver_published ( + room_id TEXT NOT NULL, + appservice_id TEXT NOT NULL, + network_id TEXT NOT NULL, + published BOOLEAN NOT NULL DEFAULT false, + CONSTRAINT unique_published_idx PRIMARY KEY (room_id, appservice_id, network_id) +); +INSERT + INTO roomserver_published ( + room_id, published + ) SELECT + room_id, published + FROM roomserver_published_tmp +; +DROP TABLE roomserver_published_tmp;`) + if err != nil { + return fmt.Errorf("failed to execute upgrade: %w", err) + } + return nil +} + +func DownPublishedAppservice(ctx context.Context, tx *sql.Tx) error { + _, err := tx.ExecContext(ctx, ` ALTER TABLE roomserver_published RENAME TO roomserver_published_tmp; +CREATE TABLE IF NOT EXISTS roomserver_published ( + room_id TEXT NOT NULL PRIMARY KEY, + published BOOLEAN NOT NULL DEFAULT false +); +INSERT + INTO roomserver_published ( + room_id, published + ) SELECT + room_id, published + FROM roomserver_published_tmp +; +DROP TABLE roomserver_published_tmp;`) + if err != nil { + return fmt.Errorf("failed to execute upgrade: %w", err) + } + return nil +} diff --git a/roomserver/storage/sqlite3/published_table.go b/roomserver/storage/sqlite3/published_table.go index 50dfa5492..34666552e 100644 --- a/roomserver/storage/sqlite3/published_table.go +++ b/roomserver/storage/sqlite3/published_table.go @@ -20,6 +20,7 @@ import ( "github.com/matrix-org/dendrite/internal" "github.com/matrix-org/dendrite/internal/sqlutil" + "github.com/matrix-org/dendrite/roomserver/storage/sqlite3/deltas" "github.com/matrix-org/dendrite/roomserver/storage/tables" ) @@ -27,31 +28,49 @@ const publishedSchema = ` -- Stores which rooms are published in the room directory CREATE TABLE IF NOT EXISTS roomserver_published ( -- The room ID of the room - room_id TEXT NOT NULL PRIMARY KEY, + room_id TEXT NOT NULL, + -- The appservice ID of the room + appservice_id TEXT NOT NULL, + -- The network_id of the room + network_id TEXT NOT NULL, -- Whether it is published or not - published BOOLEAN NOT NULL DEFAULT false + published BOOLEAN NOT NULL DEFAULT false, + PRIMARY KEY (room_id, appservice_id, network_id) ); ` const upsertPublishedSQL = "" + - "INSERT OR REPLACE INTO roomserver_published (room_id, published) VALUES ($1, $2)" + "INSERT INTO roomserver_published (room_id, appservice_id, network_id, published) VALUES ($1, $2, $3, $4)" + + " ON CONFLICT (room_id, appservice_id, network_id) DO UPDATE SET published = $4" const selectAllPublishedSQL = "" + - "SELECT room_id FROM roomserver_published WHERE published = $1 ORDER BY room_id ASC" + "SELECT room_id FROM roomserver_published WHERE published = $1 AND CASE WHEN $2 THEN 1=1 ELSE network_id = '' END ORDER BY room_id ASC" + +const selectNetworkPublishedSQL = "" + + "SELECT room_id FROM roomserver_published WHERE published = $1 AND network_id = $2 ORDER BY room_id ASC" const selectPublishedSQL = "" + "SELECT published FROM roomserver_published WHERE room_id = $1" type publishedStatements struct { - db *sql.DB - upsertPublishedStmt *sql.Stmt - selectAllPublishedStmt *sql.Stmt - selectPublishedStmt *sql.Stmt + db *sql.DB + upsertPublishedStmt *sql.Stmt + selectAllPublishedStmt *sql.Stmt + selectPublishedStmt *sql.Stmt + selectNetworkPublishedStmt *sql.Stmt } func CreatePublishedTable(db *sql.DB) error { _, err := db.Exec(publishedSchema) - return err + if err != nil { + return err + } + m := sqlutil.NewMigrator(db) + m.AddMigrations(sqlutil.Migration{ + Version: "roomserver: published appservice", + Up: deltas.UpPulishedAppservice, + }) + return m.Up(context.Background()) } func PreparePublishedTable(db *sql.DB) (tables.Published, error) { @@ -63,14 +82,15 @@ func PreparePublishedTable(db *sql.DB) (tables.Published, error) { {&s.upsertPublishedStmt, upsertPublishedSQL}, {&s.selectAllPublishedStmt, selectAllPublishedSQL}, {&s.selectPublishedStmt, selectPublishedSQL}, + {&s.selectNetworkPublishedStmt, selectNetworkPublishedSQL}, }.Prepare(db) } func (s *publishedStatements) UpsertRoomPublished( - ctx context.Context, txn *sql.Tx, roomID string, published bool, + ctx context.Context, txn *sql.Tx, roomID, appserviceID, networkID string, published bool, ) error { stmt := sqlutil.TxStmt(txn, s.upsertPublishedStmt) - _, err := stmt.ExecContext(ctx, roomID, published) + _, err := stmt.ExecContext(ctx, roomID, appserviceID, networkID, published) return err } @@ -86,10 +106,17 @@ func (s *publishedStatements) SelectPublishedFromRoomID( } func (s *publishedStatements) SelectAllPublishedRooms( - ctx context.Context, txn *sql.Tx, published bool, + ctx context.Context, txn *sql.Tx, networkID string, published, includeAllNetworks bool, ) ([]string, error) { - stmt := sqlutil.TxStmt(txn, s.selectAllPublishedStmt) - rows, err := stmt.QueryContext(ctx, published) + var rows *sql.Rows + var err error + if networkID != "" { + stmt := sqlutil.TxStmt(txn, s.selectNetworkPublishedStmt) + rows, err = stmt.QueryContext(ctx, published, networkID) + } else { + stmt := sqlutil.TxStmt(txn, s.selectAllPublishedStmt) + rows, err = stmt.QueryContext(ctx, published, includeAllNetworks) + } if err != nil { return nil, err } diff --git a/roomserver/storage/tables/interface.go b/roomserver/storage/tables/interface.go index 8be47855f..8d6ca324c 100644 --- a/roomserver/storage/tables/interface.go +++ b/roomserver/storage/tables/interface.go @@ -146,9 +146,9 @@ type Membership interface { } type Published interface { - UpsertRoomPublished(ctx context.Context, txn *sql.Tx, roomID string, published bool) (err error) + UpsertRoomPublished(ctx context.Context, txn *sql.Tx, roomID, appserviceID, networkID string, published bool) (err error) SelectPublishedFromRoomID(ctx context.Context, txn *sql.Tx, roomID string) (published bool, err error) - SelectAllPublishedRooms(ctx context.Context, txn *sql.Tx, published bool) ([]string, error) + SelectAllPublishedRooms(ctx context.Context, txn *sql.Tx, networkdID string, published, includeAllNetworks bool) ([]string, error) } type RedactionInfo struct { diff --git a/roomserver/storage/tables/published_table_test.go b/roomserver/storage/tables/published_table_test.go index fff6dc186..e6289e9b1 100644 --- a/roomserver/storage/tables/published_table_test.go +++ b/roomserver/storage/tables/published_table_test.go @@ -2,16 +2,18 @@ package tables_test import ( "context" + "fmt" "sort" "testing" + "github.com/stretchr/testify/assert" + "github.com/matrix-org/dendrite/internal/sqlutil" "github.com/matrix-org/dendrite/roomserver/storage/postgres" "github.com/matrix-org/dendrite/roomserver/storage/sqlite3" "github.com/matrix-org/dendrite/roomserver/storage/tables" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/test" - "github.com/stretchr/testify/assert" ) func mustCreatePublishedTable(t *testing.T, dbType test.DBType) (tab tables.Published, close func()) { @@ -46,10 +48,12 @@ func TestPublishedTable(t *testing.T) { // Publish some rooms publishedRooms := []string{} + asID := "" + nwID := "" for i := 0; i < 10; i++ { room := test.NewRoom(t, alice) published := i%2 == 0 - err := tab.UpsertRoomPublished(ctx, nil, room.ID, published) + err := tab.UpsertRoomPublished(ctx, nil, room.ID, asID, nwID, published) assert.NoError(t, err) if published { publishedRooms = append(publishedRooms, room.ID) @@ -61,19 +65,36 @@ func TestPublishedTable(t *testing.T) { sort.Strings(publishedRooms) // check that we get the expected published rooms - roomIDs, err := tab.SelectAllPublishedRooms(ctx, nil, true) + roomIDs, err := tab.SelectAllPublishedRooms(ctx, nil, "", true, true) assert.NoError(t, err) assert.Equal(t, publishedRooms, roomIDs) // test an actual upsert room := test.NewRoom(t, alice) - err = tab.UpsertRoomPublished(ctx, nil, room.ID, true) + err = tab.UpsertRoomPublished(ctx, nil, room.ID, asID, nwID, true) assert.NoError(t, err) - err = tab.UpsertRoomPublished(ctx, nil, room.ID, false) + err = tab.UpsertRoomPublished(ctx, nil, room.ID, asID, nwID, false) assert.NoError(t, err) // should now be false, due to the upsert publishedRes, err := tab.SelectPublishedFromRoomID(ctx, nil, room.ID) assert.NoError(t, err) - assert.False(t, publishedRes) + assert.False(t, publishedRes, fmt.Sprintf("expected room %s to be unpublished", room.ID)) + + // network specific test + nwID = "irc" + room = test.NewRoom(t, alice) + err = tab.UpsertRoomPublished(ctx, nil, room.ID, asID, nwID, true) + assert.NoError(t, err) + publishedRooms = append(publishedRooms, room.ID) + sort.Strings(publishedRooms) + // should only return the room for network "irc" + allNWPublished, err := tab.SelectAllPublishedRooms(ctx, nil, nwID, true, true) + assert.NoError(t, err) + assert.Equal(t, []string{room.ID}, allNWPublished) + + // check that we still get all published rooms regardless networkID + roomIDs, err = tab.SelectAllPublishedRooms(ctx, nil, "", true, true) + assert.NoError(t, err) + assert.Equal(t, publishedRooms, roomIDs) }) } diff --git a/sytest-whitelist b/sytest-whitelist index 60610929a..6e4500d06 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -757,4 +757,6 @@ Can get rooms/{roomId}/messages for a departed room (SPEC-216) Local device key changes appear in /keys/changes Can get rooms/{roomId}/members at a given point Can filter rooms/{roomId}/members -Current state appears in timeline in private history with many messages after \ No newline at end of file +Current state appears in timeline in private history with many messages after +AS can publish rooms in their own list +AS and main public room lists are separate \ No newline at end of file From a785532463852796ab1676e1e60ae8f2132eb49d Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Thu, 27 Oct 2022 16:01:51 +0200 Subject: [PATCH 118/207] Fix upgrade appservices --- .../postgres/deltas/20221027084407_published_appservice.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roomserver/storage/postgres/deltas/20221027084407_published_appservice.go b/roomserver/storage/postgres/deltas/20221027084407_published_appservice.go index be046545a..687ee9024 100644 --- a/roomserver/storage/postgres/deltas/20221027084407_published_appservice.go +++ b/roomserver/storage/postgres/deltas/20221027084407_published_appservice.go @@ -21,11 +21,11 @@ import ( ) func UpPulishedAppservice(ctx context.Context, tx *sql.Tx) error { - _, err := tx.ExecContext(ctx, `ALTER TABLE roomserver_published ADD COLUMN IF NOT EXISTS appservice_id TEXT NOT NULL;`) + _, err := tx.ExecContext(ctx, `ALTER TABLE roomserver_published ADD COLUMN IF NOT EXISTS appservice_id TEXT NOT NULL DEFAULT '';`) if err != nil { return fmt.Errorf("failed to execute upgrade: %w", err) } - _, err = tx.ExecContext(ctx, `ALTER TABLE roomserver_published ADD COLUMN IF NOT EXISTS network_id TEXT NOT NULL;`) + _, err = tx.ExecContext(ctx, `ALTER TABLE roomserver_published ADD COLUMN IF NOT EXISTS network_id TEXT NOT NULL DEFAULT '';`) if err != nil { return fmt.Errorf("failed to execute upgrade: %w", err) } From a2706e6498287a5b052ef47413175bf7551b36b1 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 27 Oct 2022 15:34:26 +0100 Subject: [PATCH 119/207] Refactor `claimRemoteKeys` --- federationapi/internal/federationclient.go | 2 +- keyserver/internal/internal.go | 63 ++++++++++------------ 2 files changed, 28 insertions(+), 37 deletions(-) diff --git a/federationapi/internal/federationclient.go b/federationapi/internal/federationclient.go index b8bd5beda..2636b7fa0 100644 --- a/federationapi/internal/federationclient.go +++ b/federationapi/internal/federationclient.go @@ -44,7 +44,7 @@ func (a *FederationInternalAPI) ClaimKeys( ) (gomatrixserverlib.RespClaimKeys, error) { ctx, cancel := context.WithTimeout(ctx, time.Second*30) defer cancel() - ires, err := a.doRequestIfNotBackingOffOrBlacklisted(s, func() (interface{}, error) { + ires, err := a.doRequestIfNotBlacklisted(s, func() (interface{}, error) { return a.federation.ClaimKeys(ctx, s, oneTimeKeys) }) if err != nil { diff --git a/keyserver/internal/internal.go b/keyserver/internal/internal.go index ff0968b27..92ee80d81 100644 --- a/keyserver/internal/internal.go +++ b/keyserver/internal/internal.go @@ -128,58 +128,49 @@ func (a *KeyInternalAPI) PerformClaimKeys(ctx context.Context, req *api.PerformC func (a *KeyInternalAPI) claimRemoteKeys( ctx context.Context, timeout time.Duration, res *api.PerformClaimKeysResponse, domainToDeviceKeys map[string]map[string]map[string]string, ) { - resultCh := make(chan *gomatrixserverlib.RespClaimKeys, len(domainToDeviceKeys)) - // allows us to wait until all federation servers have been poked - var wg sync.WaitGroup - wg.Add(len(domainToDeviceKeys)) - // mutex for failures - var failMu sync.Mutex - util.GetLogger(ctx).WithField("num_servers", len(domainToDeviceKeys)).Info("Claiming remote keys from servers") + var wg sync.WaitGroup // Wait for fan-out goroutines to finish + var mu sync.Mutex // Protects the response struct + var claimed int // Number of keys claimed in total + var failures int // Number of servers we failed to ask + + util.GetLogger(ctx).Infof("Claiming remote keys from %d server(s)", len(domainToDeviceKeys)) + wg.Add(len(domainToDeviceKeys)) - // fan out for d, k := range domainToDeviceKeys { go func(domain string, keysToClaim map[string]map[string]string) { - defer wg.Done() fedCtx, cancel := context.WithTimeout(ctx, timeout) defer cancel() + defer wg.Done() + claimKeyRes, err := a.FedClient.ClaimKeys(fedCtx, gomatrixserverlib.ServerName(domain), keysToClaim) + + mu.Lock() + defer mu.Unlock() + if err != nil { util.GetLogger(ctx).WithError(err).WithField("server", domain).Error("ClaimKeys failed") - failMu.Lock() res.Failures[domain] = map[string]interface{}{ "message": err.Error(), } - failMu.Unlock() + failures++ return } - resultCh <- &claimKeyRes + + for userID, deviceIDToKeys := range claimKeyRes.OneTimeKeys { + res.OneTimeKeys[userID] = make(map[string]map[string]json.RawMessage) + for deviceID, keys := range deviceIDToKeys { + res.OneTimeKeys[userID][deviceID] = keys + claimed += len(keys) + } + } }(d, k) } - // Close the result channel when the goroutines have quit so the for .. range exits - go func() { - wg.Wait() - close(resultCh) - }() - - keysClaimed := 0 - for result := range resultCh { - for userID, nest := range result.OneTimeKeys { - res.OneTimeKeys[userID] = make(map[string]map[string]json.RawMessage) - for deviceID, nest2 := range nest { - res.OneTimeKeys[userID][deviceID] = make(map[string]json.RawMessage) - for keyIDWithAlgo, otk := range nest2 { - keyJSON, err := json.Marshal(otk) - if err != nil { - continue - } - res.OneTimeKeys[userID][deviceID][keyIDWithAlgo] = keyJSON - keysClaimed++ - } - } - } - } - util.GetLogger(ctx).WithField("num_keys", keysClaimed).Info("Claimed remote keys") + wg.Wait() + util.GetLogger(ctx).WithFields(logrus.Fields{ + "num_keys": claimed, + "num_failures": failures, + }).Infof("Claimed remote keys from %d server(s)", len(domainToDeviceKeys)) } func (a *KeyInternalAPI) PerformDeleteKeys(ctx context.Context, req *api.PerformDeleteKeysRequest, res *api.PerformDeleteKeysResponse) error { From f6035822e75ad000869a0e01eff35941280f0250 Mon Sep 17 00:00:00 2001 From: Till Faelligen <2353100+S7evinK@users.noreply.github.com> Date: Fri, 28 Oct 2022 08:17:40 +0200 Subject: [PATCH 120/207] Simplify error checking and check the correct error --- syncapi/streams/stream_pdu.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncapi/streams/stream_pdu.go b/syncapi/streams/stream_pdu.go index 90cf8ce53..87f7a064e 100644 --- a/syncapi/streams/stream_pdu.go +++ b/syncapi/streams/stream_pdu.go @@ -101,7 +101,7 @@ func (p *PDUStreamProvider) CompleteSync( ) if jerr != nil { req.Log.WithError(jerr).Error("p.getJoinResponseForCompleteSync failed") - if err == context.DeadlineExceeded || err == context.Canceled || err == sql.ErrTxDone { + if ctxErr := req.Context.Err(); ctxErr != nil || jerr == sql.ErrTxDone { return from } continue From 0782011f54dca98d96a8c5a78f68569ed045892a Mon Sep 17 00:00:00 2001 From: "X. Ding" Date: Fri, 28 Oct 2022 18:25:01 +0800 Subject: [PATCH 121/207] Add hcaptcha support besides Google ReCaptcha (#2834) ### Pull Request Checklist This PR add support for hcaptcha.com as an alternative to Google ReCaptcha. It also makes possible for user to customize ReCaptcha URL when needed. (Such as use recaptcha.net instead of www.google.com) This feature needs manual test cuz it involves 3rd party _captcha_. Signed-off-by: `Simon Ding ` Co-authored-by: dxl --- clientapi/routing/auth_fallback.go | 18 ++++++++++-------- clientapi/routing/register.go | 4 +++- dendrite-sample.monolith.yaml | 8 +++++++- dendrite-sample.polylith.yaml | 8 +++++++- setup/config/config_clientapi.go | 18 ++++++++++++++++++ 5 files changed, 45 insertions(+), 11 deletions(-) diff --git a/clientapi/routing/auth_fallback.go b/clientapi/routing/auth_fallback.go index abfe830fb..ad870993e 100644 --- a/clientapi/routing/auth_fallback.go +++ b/clientapi/routing/auth_fallback.go @@ -31,8 +31,7 @@ const recaptchaTemplate = ` Authentication - +