From 9bc1c36ff6fbf8de130c7131b67b7a7ce9e31e1e Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 25 Nov 2021 09:46:26 +0000 Subject: [PATCH 1/7] Support connecting to multiple Pinecone static peers in the P2P demos (supply a comma-separated list) --- build/gobind-pinecone/monolith.go | 47 +++++++++++++++-------- cmd/dendrite-demo-pinecone/conn/client.go | 7 +++- cmd/dendrite-demo-pinecone/main.go | 35 +++++++++++------ 3 files changed, 62 insertions(+), 27 deletions(-) diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go index 47ca20dbd..cd2809e46 100644 --- a/build/gobind-pinecone/monolith.go +++ b/build/gobind-pinecone/monolith.go @@ -13,6 +13,7 @@ import ( "net" "net/http" "os" + "strings" "sync" "time" @@ -75,7 +76,7 @@ func (m *DendriteMonolith) BaseURL() string { } func (m *DendriteMonolith) PeerCount(peertype int) int { - return m.PineconeRouter.PeerCount(peertype) + return m.PineconeRouter.PeerCount(pineconeRouter.ConnectionPeerType(peertype)) } func (m *DendriteMonolith) SessionCount() int { @@ -87,15 +88,15 @@ func (m *DendriteMonolith) SetMulticastEnabled(enabled bool) { m.PineconeMulticast.Start() } else { m.PineconeMulticast.Stop() - m.DisconnectType(pineconeRouter.PeerTypeMulticast) + m.DisconnectType(int(pineconeRouter.PeerTypeMulticast)) } } func (m *DendriteMonolith) SetStaticPeer(uri string) { m.staticPeerMutex.Lock() - m.staticPeerURI = uri + m.staticPeerURI = strings.TrimSpace(uri) m.staticPeerMutex.Unlock() - m.DisconnectType(pineconeRouter.PeerTypeRemote) + m.DisconnectType(int(pineconeRouter.PeerTypeRemote)) if uri != "" { go func() { m.staticPeerAttempt <- struct{}{} @@ -105,7 +106,7 @@ func (m *DendriteMonolith) SetStaticPeer(uri string) { func (m *DendriteMonolith) DisconnectType(peertype int) { for _, p := range m.PineconeRouter.Peers() { - if peertype == p.PeerType { + if int(peertype) == p.PeerType { m.PineconeRouter.Disconnect(types.SwitchPortID(p.Port), nil) } } @@ -133,7 +134,11 @@ func (m *DendriteMonolith) Conduit(zone string, peertype int) (*Conduit, error) for i := 1; i <= 10; i++ { logrus.Errorf("Attempting authenticated connect (attempt %d)", i) var err error - conduit.port, err = m.PineconeRouter.AuthenticatedConnect(l, zone, peertype, true) + conduit.port, err = m.PineconeRouter.Connect( + l, + pineconeRouter.ConnectionZone(zone), + pineconeRouter.ConnectionPeerType(peertype), + ) switch err { case io.ErrClosedPipe: logrus.Errorf("Authenticated connect failed due to closed pipe (attempt %d)", i) @@ -195,16 +200,28 @@ func (m *DendriteMonolith) RegisterDevice(localpart, deviceID string) (string, e } func (m *DendriteMonolith) staticPeerConnect() { + connected := map[string]bool{} // URI -> connected? attempt := func() { - if m.PineconeRouter.PeerCount(pineconeRouter.PeerTypeRemote) == 0 { - m.staticPeerMutex.RLock() - uri := m.staticPeerURI - m.staticPeerMutex.RUnlock() - if uri == "" { - return - } - if err := conn.ConnectToPeer(m.PineconeRouter, uri); err != nil { - logrus.WithError(err).Error("Failed to connect to static peer") + m.staticPeerMutex.RLock() + uri := m.staticPeerURI + m.staticPeerMutex.RUnlock() + if uri == "" { + return + } + for k := range connected { + connected[k] = false + } + for _, uri := range strings.Split(uri, ",") { + connected[strings.TrimSpace(uri)] = false + } + for _, info := range m.PineconeRouter.Peers() { + connected[info.URI] = true + } + for k, online := range connected { + if !online { + if err := conn.ConnectToPeer(m.PineconeRouter, k); err != nil { + logrus.WithError(err).Error("Failed to connect to static peer") + } } } } diff --git a/cmd/dendrite-demo-pinecone/conn/client.go b/cmd/dendrite-demo-pinecone/conn/client.go index 40ccb9c02..14b648a3b 100644 --- a/cmd/dendrite-demo-pinecone/conn/client.go +++ b/cmd/dendrite-demo-pinecone/conn/client.go @@ -34,7 +34,12 @@ func ConnectToPeer(pRouter *pineconeRouter.Router, peer string) error { if parent == nil { return fmt.Errorf("failed to wrap connection") } - _, err := pRouter.AuthenticatedConnect(parent, "static", pineconeRouter.PeerTypeRemote, true) + _, err := pRouter.Connect( + parent, + pineconeRouter.ConnectionZone("static"), + pineconeRouter.PeerTypeRemote, + pineconeRouter.ConnectionURI(peer), + ) return err } diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go index 2b56ef35d..8ed3f349e 100644 --- a/cmd/dendrite-demo-pinecone/main.go +++ b/cmd/dendrite-demo-pinecone/main.go @@ -26,6 +26,7 @@ import ( "net" "net/http" "os" + "strings" "time" "github.com/gorilla/mux" @@ -61,7 +62,7 @@ import ( var ( instanceName = flag.String("name", "dendrite-p2p-pinecone", "the name of this P2P demo instance") instancePort = flag.Int("port", 8008, "the port that the client API will listen on") - instancePeer = flag.String("peer", "", "the static Pinecone peer to connect to") + instancePeer = flag.String("peer", "", "the static Pinecone peers to connect to, comma separated-list") instanceListen = flag.String("listen", ":0", "the port Pinecone peers can connect to") ) @@ -109,9 +110,9 @@ func main() { continue } - port, err := pRouter.AuthenticatedConnect(conn, "", pineconeRouter.PeerTypeRemote, true) + port, err := pRouter.Connect(conn, pineconeRouter.PeerTypeRemote) if err != nil { - logrus.WithError(err).Error("pSwitch.AuthenticatedConnect failed") + logrus.WithError(err).Error("pSwitch.Connect failed") continue } @@ -124,14 +125,22 @@ func main() { pMulticast.Start() connectToStaticPeer := func() { + connected := map[string]bool{} // URI -> connected? + for _, uri := range strings.Split(*instancePeer, ",") { + connected[strings.TrimSpace(uri)] = false + } attempt := func() { - if pRouter.PeerCount(pineconeRouter.PeerTypeRemote) == 0 { - uri := *instancePeer - if uri == "" { - return - } - if err := conn.ConnectToPeer(pRouter, uri); err != nil { - logrus.WithError(err).Error("Failed to connect to static peer") + for k := range connected { + connected[k] = false + } + for _, info := range pRouter.Peers() { + connected[info.URI] = true + } + for k, online := range connected { + if !online { + if err := conn.ConnectToPeer(pRouter, k); err != nil { + logrus.WithError(err).Error("Failed to connect to static peer") + } } } } @@ -230,7 +239,11 @@ func main() { return } conn := conn.WrapWebSocketConn(c) - if _, err = pRouter.AuthenticatedConnect(conn, "websocket", pineconeRouter.PeerTypeRemote, true); err != nil { + if _, err = pRouter.Connect( + conn, + pineconeRouter.ConnectionZone("websocket"), + pineconeRouter.PeerTypeRemote, + ); err != nil { logrus.WithError(err).Error("Failed to connect WebSocket peer to Pinecone switch") } }) From 8f2a8384528def758173bb687ca7666516bd68a2 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 25 Nov 2021 09:49:00 +0000 Subject: [PATCH 2/7] Update to matrix-org/pinecone@0cc483b --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9b970c334..ab931f0db 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 github.com/matrix-org/gomatrixserverlib v0.0.0-20211115192839-15a64d244aa2 github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 - github.com/matrix-org/pinecone v0.0.0-20211116111603-febf3501584d + github.com/matrix-org/pinecone v0.0.0-20211125093215-0cc483b8231e github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.8 github.com/morikuni/aec v1.0.0 // indirect diff --git a/go.sum b/go.sum index 2c19e76f0..a723b0c77 100644 --- a/go.sum +++ b/go.sum @@ -997,8 +997,8 @@ github.com/matrix-org/gomatrixserverlib v0.0.0-20211115192839-15a64d244aa2 h1:RF github.com/matrix-org/gomatrixserverlib v0.0.0-20211115192839-15a64d244aa2/go.mod h1:rB8tBUUUo1rzUqpzklRDSooxZ6YMhoaEPx4SO5fGeUc= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 h1:HZCzy4oVzz55e+cOMiX/JtSF2UOY1evBl2raaE7ACcU= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= -github.com/matrix-org/pinecone v0.0.0-20211116111603-febf3501584d h1:V1b6GZVvL95qTkjYSEWH9Pja6c0WcJKBt2MlAILlw+Q= -github.com/matrix-org/pinecone v0.0.0-20211116111603-febf3501584d/go.mod h1:r6dsL+ylE0yXe/7zh8y/Bdh6aBYI1r+u4yZni9A4iyk= +github.com/matrix-org/pinecone v0.0.0-20211125093215-0cc483b8231e h1:28XHlmmlzXdg0NWkDRzcfxOberiS3h+CCq3fFIkdeCY= +github.com/matrix-org/pinecone v0.0.0-20211125093215-0cc483b8231e/go.mod h1:r6dsL+ylE0yXe/7zh8y/Bdh6aBYI1r+u4yZni9A4iyk= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= From 310edd737ca52839ec8094e2847f5746e48bad69 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 25 Nov 2021 10:48:44 +0000 Subject: [PATCH 3/7] Fix P2P demo builds --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ab931f0db..300d22673 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 github.com/matrix-org/gomatrixserverlib v0.0.0-20211115192839-15a64d244aa2 github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 - github.com/matrix-org/pinecone v0.0.0-20211125093215-0cc483b8231e + github.com/matrix-org/pinecone v0.0.0-20211125101824-cc7886682cfd github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.8 github.com/morikuni/aec v1.0.0 // indirect diff --git a/go.sum b/go.sum index a723b0c77..d56241d35 100644 --- a/go.sum +++ b/go.sum @@ -997,8 +997,8 @@ github.com/matrix-org/gomatrixserverlib v0.0.0-20211115192839-15a64d244aa2 h1:RF github.com/matrix-org/gomatrixserverlib v0.0.0-20211115192839-15a64d244aa2/go.mod h1:rB8tBUUUo1rzUqpzklRDSooxZ6YMhoaEPx4SO5fGeUc= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 h1:HZCzy4oVzz55e+cOMiX/JtSF2UOY1evBl2raaE7ACcU= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= -github.com/matrix-org/pinecone v0.0.0-20211125093215-0cc483b8231e h1:28XHlmmlzXdg0NWkDRzcfxOberiS3h+CCq3fFIkdeCY= -github.com/matrix-org/pinecone v0.0.0-20211125093215-0cc483b8231e/go.mod h1:r6dsL+ylE0yXe/7zh8y/Bdh6aBYI1r+u4yZni9A4iyk= +github.com/matrix-org/pinecone v0.0.0-20211125101824-cc7886682cfd h1:/iX6jehN2sO8n4pn63U+7iDoNx18fjC6pQ2RpwyZtKk= +github.com/matrix-org/pinecone v0.0.0-20211125101824-cc7886682cfd/go.mod h1:r6dsL+ylE0yXe/7zh8y/Bdh6aBYI1r+u4yZni9A4iyk= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= From 11f588b0d603517be2c0509335c88b85b88f91f8 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 25 Nov 2021 11:18:01 +0000 Subject: [PATCH 4/7] Fixes for multiple Pinecone peers --- build/gobind-pinecone/monolith.go | 6 +++--- build/gobind-yggdrasil/monolith.go | 2 +- cmd/dendrite-demo-pinecone/conn/client.go | 2 +- cmd/dendrite-demo-pinecone/main.go | 7 +++++-- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go index cd2809e46..9c739db37 100644 --- a/build/gobind-pinecone/monolith.go +++ b/build/gobind-pinecone/monolith.go @@ -76,7 +76,7 @@ func (m *DendriteMonolith) BaseURL() string { } func (m *DendriteMonolith) PeerCount(peertype int) int { - return m.PineconeRouter.PeerCount(pineconeRouter.ConnectionPeerType(peertype)) + return m.PineconeRouter.PeerCount(peertype) } func (m *DendriteMonolith) SessionCount() int { @@ -209,7 +209,7 @@ func (m *DendriteMonolith) staticPeerConnect() { return } for k := range connected { - connected[k] = false + delete(connected, k) } for _, uri := range strings.Split(uri, ",") { connected[strings.TrimSpace(uri)] = false @@ -289,7 +289,7 @@ func (m *DendriteMonolith) Start() { cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-syncapi.db", m.StorageDirectory, prefix)) cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-roomserver.db", m.StorageDirectory, prefix)) cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-keyserver.db", m.StorageDirectory, prefix)) - cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-federationapi.db", m.StorageDirectory, prefix)) + cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-federationsender.db", m.StorageDirectory, prefix)) cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/%s-appservice.db", m.StorageDirectory, prefix)) cfg.MediaAPI.BasePath = config.Path(fmt.Sprintf("%s/media", m.CacheDirectory)) cfg.MediaAPI.AbsBasePath = config.Path(fmt.Sprintf("%s/media", m.CacheDirectory)) diff --git a/build/gobind-yggdrasil/monolith.go b/build/gobind-yggdrasil/monolith.go index bde9fbdfa..07fcf836a 100644 --- a/build/gobind-yggdrasil/monolith.go +++ b/build/gobind-yggdrasil/monolith.go @@ -94,7 +94,7 @@ func (m *DendriteMonolith) Start() { cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-syncapi.db", m.StorageDirectory)) cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-roomserver.db", m.StorageDirectory)) cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-keyserver.db", m.StorageDirectory)) - cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-federationapi.db", m.StorageDirectory)) + cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-federationsender.db", m.StorageDirectory)) cfg.AppServiceAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s/dendrite-p2p-appservice.db", m.StorageDirectory)) cfg.MediaAPI.BasePath = config.Path(fmt.Sprintf("%s/tmp", m.StorageDirectory)) cfg.MediaAPI.AbsBasePath = config.Path(fmt.Sprintf("%s/tmp", m.StorageDirectory)) diff --git a/cmd/dendrite-demo-pinecone/conn/client.go b/cmd/dendrite-demo-pinecone/conn/client.go index 14b648a3b..e3cc0468c 100644 --- a/cmd/dendrite-demo-pinecone/conn/client.go +++ b/cmd/dendrite-demo-pinecone/conn/client.go @@ -37,7 +37,7 @@ func ConnectToPeer(pRouter *pineconeRouter.Router, peer string) error { _, err := pRouter.Connect( parent, pineconeRouter.ConnectionZone("static"), - pineconeRouter.PeerTypeRemote, + pineconeRouter.ConnectionPeerType(pineconeRouter.PeerTypeRemote), pineconeRouter.ConnectionURI(peer), ) return err diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go index 8ed3f349e..cc7238ead 100644 --- a/cmd/dendrite-demo-pinecone/main.go +++ b/cmd/dendrite-demo-pinecone/main.go @@ -110,7 +110,10 @@ func main() { continue } - port, err := pRouter.Connect(conn, pineconeRouter.PeerTypeRemote) + port, err := pRouter.Connect( + conn, + pineconeRouter.ConnectionPeerType(pineconeRouter.PeerTypeRemote), + ) if err != nil { logrus.WithError(err).Error("pSwitch.Connect failed") continue @@ -242,7 +245,7 @@ func main() { if _, err = pRouter.Connect( conn, pineconeRouter.ConnectionZone("websocket"), - pineconeRouter.PeerTypeRemote, + pineconeRouter.ConnectionPeerType(pineconeRouter.PeerTypeRemote), ); err != nil { logrus.WithError(err).Error("Failed to connect WebSocket peer to Pinecone switch") } From 83b9cb4d79123ec89e18a46d5202e74a9e9305b8 Mon Sep 17 00:00:00 2001 From: Ben Yanke Date: Mon, 29 Nov 2021 04:17:16 -0600 Subject: [PATCH 5/7] Updating example to Postgres v14 (#2062) See issue #2052 --- build/docker/docker-compose.deps.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/docker/docker-compose.deps.yml b/build/docker/docker-compose.deps.yml index 0732e1813..aa0651889 100644 --- a/build/docker/docker-compose.deps.yml +++ b/build/docker/docker-compose.deps.yml @@ -3,7 +3,7 @@ services: # PostgreSQL is needed for both polylith and monolith modes. postgres: hostname: postgres - image: postgres:11 + image: postgres:14 restart: always volumes: - ./postgres/create_db.sh:/docker-entrypoint-initdb.d/20-create_db.sh From 388d7a197446bce5493c52d36affecf0cf706296 Mon Sep 17 00:00:00 2001 From: nymori Date: Fri, 3 Dec 2021 01:48:49 -0800 Subject: [PATCH 6/7] Squash username to lowercase at login (#2065) Signed-off-by: Bernard Zhao --- clientapi/auth/password.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/clientapi/auth/password.go b/clientapi/auth/password.go index a66e2fe76..7dd21b3f2 100644 --- a/clientapi/auth/password.go +++ b/clientapi/auth/password.go @@ -17,6 +17,7 @@ package auth import ( "context" "net/http" + "strings" "github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/userutil" @@ -48,7 +49,8 @@ func (t *LoginTypePassword) Request() interface{} { func (t *LoginTypePassword) Login(ctx context.Context, req interface{}) (*Login, *util.JSONResponse) { r := req.(*PasswordRequest) - username := r.Username() + // Squash username to all lowercase letters + username := strings.ToLower(r.Username()) if username == "" { return nil, &util.JSONResponse{ Code: http.StatusUnauthorized, From f9bac2f78aaed91a77cc1fe455a05899be2e2a12 Mon Sep 17 00:00:00 2001 From: S7evinK Date: Fri, 3 Dec 2021 10:49:14 +0100 Subject: [PATCH 7/7] Add missing internal routes (#2064) Signed-off-by: Till Faelligen --- cmd/dendrite-polylith-multi/personalities/federationapi.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/dendrite-polylith-multi/personalities/federationapi.go b/cmd/dendrite-polylith-multi/personalities/federationapi.go index 9b59cf45b..5f87f96be 100644 --- a/cmd/dendrite-polylith-multi/personalities/federationapi.go +++ b/cmd/dendrite-polylith-multi/personalities/federationapi.go @@ -35,6 +35,9 @@ func FederationAPI(base *basepkg.BaseDendrite, cfg *config.Dendrite) { &base.Cfg.MSCs, nil, ) + intAPI := federationapi.NewInternalAPI(base, federation, rsAPI, base.Caches, true) + federationapi.AddInternalRoutes(base.InternalAPIMux, intAPI) + base.SetupAndServeHTTP( base.Cfg.FederationAPI.InternalAPI.Listen, base.Cfg.FederationAPI.ExternalAPI.Listen,